from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf

def image_augmentor(image, input_shape, data_format, output_shape, zoom_size=None,
                    crop_method=None, flip_prob=None, fill_mode='BILINEAR', keep_aspect_ratios=False,
                    constant_values=0., color_jitter_prob=None, rotate=None, ground_truth=None, pad_truth_to=None):

    """
    :param image: HWC or CHW
    :param input_shape: [h, w]
    :param data_format: 'channels_first', 'channels_last'
    :param output_shape: [h, w]
    :param zoom_size: [h, w]
    :param crop_method: 'random', 'center'
    :param flip_prob: [flip_top_down_prob, flip_left_right_prob]
    :param fill_mode: 'CONSTANT', 'NEAREST_NEIGHBOR', 'BILINEAR', 'BICUBIC'
    :param keep_aspect_ratios: True, False
    :param constant_values:
    :param color_jitter_prob: prob of color_jitter
    :param rotate: [prob, min_angle, max_angle]
    :param ground_truth: [ymin, ymax, xmin, xmax, classid]
    :param pad_truth_to: pad ground_truth to size [pad_truth_to, 5] with -1
    :return image: output_shape
    :return ground_truth: [pad_truth_to, 5] [ycenter, xcenter, h, w, class_id]
    """
    if data_format not in ['channels_first', 'channels_last']:
        raise Exception("data_format must in ['channels_first', 'channels_last']!")
    if fill_mode not in ['CONSTANT', 'NEAREST_NEIGHBOR', 'BILINEAR', 'BICUBIC']:
        raise Exception("fill_mode must in ['CONSTANT', 'NEAREST_NEIGHBOR', 'BILINEAR', 'BICUBIC']!")
    if fill_mode == 'CONSTANT' and zoom_size is not None:
        raise Exception("if fill_mode is 'CONSTANT', zoom_size can't be None!")
    if zoom_size is not None:
        if keep_aspect_ratios:
            if constant_values is None:
                raise Exception('please provide constant_values!')
        if not zoom_size[0] >= output_shape[0] and zoom_size[1] >= output_shape[1]:
            raise Exception("output_shape can't greater that zoom_size!")
        if crop_method not in ['random', 'center']:
            raise Exception("crop_method must in ['random', 'center']!")
        if fill_mode is 'CONSTANT' and constant_values is None:
            raise Exception("please provide constant_values!")
    if color_jitter_prob is not None:
        if not 0. <= color_jitter_prob <= 1.:
            raise Exception("color_jitter_prob can't less that 0.0, and can't grater that 1.0")
    if flip_prob is not None:
        if not 0. <= flip_prob[0] <= 1. and 0. <= flip_prob[1] <= 1.:
            raise Exception("flip_prob can't less than 0.0, and can't grater than 1.0")
    if rotate is not None:
        if len(rotate) != 3:
            raise Exception('please provide "rotate" parameter as [rotate_prob, min_angle, max_angle]!')
        if not 0. <= rotate[0] <= 1.:
            raise Exception("rotate prob can't less that 0.0, and can't grater that 1.0")
        if ground_truth is not None:
            if not -5. <= rotate[1] <= 5. and -5. <= rotate[2] <= 5.:
                raise Exception('rotate range must be -5 to 5, otherwise coordinate mapping become imprecise!')
        if not rotate[1] <= rotate[2]:
            raise Exception("rotate[1] can't  grater than rotate[2]")

    if fill_mode == 'CONSTANT':
        keep_aspect_ratios = True
    fill_mode_project = {
        'NEAREST_NEIGHBOR': tf.image.ResizeMethod.NEAREST_NEIGHBOR,
        'BILINEAR': tf.image.ResizeMethod.BILINEAR,
        'BICUBIC': tf.image.ResizeMethod.BICUBIC
    }
    if ground_truth is not None:
        ymin = tf.reshape(ground_truth[:, 0], [-1, 1])
        ymax = tf.reshape(ground_truth[:, 1], [-1, 1])
        xmin = tf.reshape(ground_truth[:, 2], [-1, 1])
        xmax = tf.reshape(ground_truth[:, 3], [-1, 1])
        class_id = tf.reshape(ground_truth[:, 4], [-1, 1])
        yy = (ymin + ymax) / 2.
        xx = (xmin + xmax) / 2.
        hh = ymax - ymin
        ww = xmax -xmin
    image_copy = image
    if data_format == 'channels_first':
        image = tf.transpose(image, [1, 2, 0])
    input_h, input_w, input_c = input_shape[0], input_shape[1], input_shape[2]
    output_h, output_w = output_shape
    if zoom_size is not None:
        zoom_or_output_h, zoom_or_output_w = zoom_size
    else:
        zoom_or_output_h, zoom_or_output_w = output_shape
    if keep_aspect_ratios:
        if fill_mode in ['NEAREST_NEIGHBOR', 'BILINEAR', 'BICUBIC']:
            zoom_ratio = tf.cond(
                tf.less(zoom_or_output_h / input_h, zoom_or_output_w / input_w),
                lambda: tf.cast(zoom_or_output_h / input_h, tf.float32),
                lambda: tf.cast(zoom_or_output_w / input_w, tf.float32)
            )
            resize_h, resize_w = tf.cond(
                tf.less(zoom_or_output_h / input_h, zoom_or_output_w / input_w),
                lambda: (zoom_or_output_h,tf.cast(tf.cast(input_w, tf.float32) * zoom_ratio, tf.int32)),
                lambda: (tf.cast(tf.cast(input_h, tf.float32)*zoom_ratio, tf.int32), zoom_or_output_w)
            )
            image = tf.image.resize_images(
                image, [resize_h, resize_w], fill_mode_project[fill_mode],
                align_corners=True,
            )
            if ground_truth is not None:
                ymin, ymax = ymin * zoom_ratio, ymax * zoom_ratio
                xmin, xmax = xmin * zoom_ratio, xmax * zoom_ratio
            image = tf.pad(
                image, [[0, zoom_or_output_h-resize_h], [0, zoom_or_output_w-resize_w], [0, 0]],
                mode='CONSTANT', constant_values=constant_values
            )
        else:
            image = tf.pad(
                image, [[0, zoom_or_output_h-input_h], [0, zoom_or_output_w-input_w], [0, 0]],
                mode='CONSTANT', constant_values=constant_values
            )
    else:
        image = tf.image.resize_images(
            image, [zoom_or_output_h, zoom_or_output_w], fill_mode_project[fill_mode],
            align_corners=True, preserve_aspect_ratio=False
        )
        if ground_truth is not None:
            zoom_ratio_y = tf.cast(zoom_or_output_h / input_h, tf.float32)
            zoom_ratio_x = tf.cast(zoom_or_output_w / input_w, tf.float32)
            ymin, ymax = ymin * zoom_ratio_y, ymax * zoom_ratio_y
            xmin, xmax = xmin * zoom_ratio_x, xmax * zoom_ratio_x

    if zoom_size is not None:
        if crop_method == 'random':
            random_h = zoom_or_output_h - output_h
            random_w = zoom_or_output_w - output_w
            crop_h = tf.random_uniform([], 0, random_h, tf.int32)
            crop_w = tf.random_uniform([], 0, random_w, tf.int32)
        else:
            crop_h = (zoom_or_output_h - output_h) // 2
            crop_w = (zoom_or_output_w - output_w) // 2
        image = tf.slice(
            image, [crop_h, crop_w, 0], [output_h, output_w, input_c]
        )
        if ground_truth is not None:
            ymin, ymax = ymin - tf.cast(crop_h, tf.float32), ymax - tf.cast(crop_h, tf.float32)
            xmin, xmax = xmin - tf.cast(crop_w, tf.float32), xmax - tf.cast(crop_w, tf.float32)

    if flip_prob is not None:
        flip_td_prob = tf.random_uniform([], 0., 1.)
        flip_lr_prob = tf.random_uniform([], 0., 1.)
        image = tf.cond(
            tf.less(flip_td_prob, flip_prob[0]),
            lambda: tf.reverse(image, [0]),
            lambda: image
        )
        image = tf.cond(
            tf.less(flip_lr_prob, flip_prob[1]),
            lambda: tf.reverse(image, [1]),
            lambda: image
        )
        if ground_truth is not None:
            ymax, ymin = tf.cond(
                tf.less(flip_td_prob, flip_prob[0]),
                lambda: (output_h - ymin -1., output_h - ymax -1.),
                lambda: (ymax, ymin)
            )
            xmax, xmin = tf.cond(
                tf.less(flip_lr_prob, flip_prob[1]),
                lambda: (output_w - xmin -1., output_w - xmax - 1.),
                lambda: (xmax, xmin)
            )
    if color_jitter_prob is not None:
        bcs = tf.random_uniform([3], 0., 1.)
        image = tf.cond(bcs[0] < color_jitter_prob,
                        lambda: tf.image.adjust_brightness(image, tf.random_uniform([], 0., 0.3)),
                        lambda: image
                )
        image = tf.cond(bcs[1] < color_jitter_prob,
                        lambda: tf.image.adjust_contrast(image, tf.random_uniform([], 0.8, 1.2)),
                        lambda: image
                )
        image = tf.cond(bcs[2] < color_jitter_prob,
                        lambda: tf.image.adjust_hue(image, tf.random_uniform([], -0.1, 0.1)),
                        lambda: image
                )

    if rotate is not None:
        angles = tf.random_uniform([], rotate[1], rotate[2]) * 3.1415926 / 180.
        image = tf.contrib.image.rotate(image, angles, 'BILINEAR')
        if ground_truth is not None:
            angles = -angles
            rotate_center_x = (output_w - 1.) / 2.
            rotate_center_y = (output_h - 1.) / 2.
            offset_x = rotate_center_x * (1-tf.cos(angles)) + rotate_center_y * tf.sin(angles)
            offset_y = rotate_center_y * (1-tf.cos(angles)) - rotate_center_x * tf.sin(angles)
            xminymin_x = xmin * tf.cos(angles) - ymin * tf.sin(angles) + offset_x
            xminymin_y = xmin * tf.sin(angles) + ymin * tf.cos(angles) + offset_y
            xmaxymax_x = xmax * tf.cos(angles) - ymax * tf.sin(angles) + offset_x
            xmaxymax_y = xmax * tf.sin(angles) + ymax * tf.cos(angles) + offset_y
            xminymax_x = xmin * tf.cos(angles) - ymax * tf.sin(angles) + offset_x
            xminymax_y = xmin * tf.sin(angles) + ymax * tf.cos(angles) + offset_y
            xmaxymin_x = xmax * tf.cos(angles) - ymin * tf.sin(angles) + offset_x
            xmaxymin_y = xmax * tf.sin(angles) + ymin * tf.cos(angles) + offset_y
            xmin = tf.reduce_min(tf.concat([xminymin_x, xmaxymax_x, xminymax_x, xmaxymin_x], axis=-1), axis=-1, keepdims=True)
            ymin = tf.reduce_min(tf.concat([xminymin_y, xmaxymax_y, xminymax_y, xmaxymin_y], axis=-1), axis=-1, keepdims=True)
            xmax = tf.reduce_max(tf.concat([xminymin_x, xmaxymax_x, xminymax_x, xmaxymin_x], axis=-1), axis=-1, keepdims=True)
            ymax = tf.reduce_max(tf.concat([xminymin_y, xmaxymax_y, xminymax_y, xmaxymin_y], axis=-1), axis=-1, keepdims=True)
    if data_format == 'channels_first':
        image = tf.transpose(image, [2, 0, 1])
    if ground_truth is not None:
        y_center = (ymin + ymax) / 2.
        x_center = (xmin + xmax) / 2.
        y_mask = tf.cast(y_center > 0., tf.float32) * tf.cast(y_center < output_h - 1., tf.float32)
        x_mask = tf.cast(x_center > 0., tf.float32) * tf.cast(x_center < output_w - 1., tf.float32)
        mask = tf.reshape((x_mask * y_mask) > 0., [-1])
        ymin = tf.boolean_mask(ymin, mask)
        xmin = tf.boolean_mask(xmin, mask)
        ymax = tf.boolean_mask(ymax, mask)
        xmax = tf.boolean_mask(xmax, mask)
        class_id = tf.boolean_mask(class_id, mask)
        ymin = tf.where(ymin < 0., ymin - ymin, ymin)
        xmin = tf.where(xmin < 0., xmin - xmin, xmin)
        ymax = tf.where(ymax < 0., ymax - ymax, ymax)
        xmax = tf.where(xmax < 0., xmax - xmax, xmax)
        ymin = tf.where(ymin > output_h - 1., ymin - ymin + output_h - 1., ymin)
        xmin = tf.where(xmin > output_w - 1., xmin - xmin + output_w - 1., xmin)
        ymax = tf.where(ymax > output_h - 1., ymax - ymax + output_h - 1., ymax)
        xmax = tf.where(xmax > output_w - 1., xmax - xmax + output_w - 1., xmax)
        y = (ymin + ymax) / 2.
        x = (xmin + xmax) / 2.
        h = ymax - ymin
        w = xmax - xmin
        ground_truth_ = tf.concat([y, x, h, w, class_id], axis=-1)

        if tf.shape(ground_truth_)[0] == 0:
            if pad_truth_to is not None:
                ground_truth_ = tf.concat([yy, xx, hh, ww, class_id], axis=-1)
                ground_truth = tf.pad(
                                ground_truth_, [[0, pad_truth_to-tf.shape(ground_truth)[0]], [0, 0]],
                                constant_values=-1.0
                            )
            return image_copy, ground_truth
        else:
            if pad_truth_to is not None:
                ground_truth = tf.pad(
                                ground_truth_, [[0, pad_truth_to-tf.shape(ground_truth_)[0]], [0, 0]],
                                constant_values=-1.0
                            )
            return image, ground_truth
    else:
        return image


from django.urls import path
from . import views, apps

app_name = apps.CoreConfig.name

urlpatterns = [
    path('', views.IndexView.as_view(), name='index'),
    # Employee
    path('employee/clearances/create', views.EmployeeCreateClearance.as_view(), name='employee-create-clearance'),
    path('employee/clearances/search', views.EmployeeSearchClearances.as_view(),
         name='employee-search-clearances'),
    path('employee/<int:employee_number>/clearances/', views.EmployeeClearances.as_view(), name='employee-clearances'),
    path('employee/clearances/<int:clearance_number>/', views.EmployeeShowClearance.as_view(),
         name='employee-show-clearance'),
    path('employee/clearances/<int:clearance_number>/print', views.EmployeePrintClearance.as_view(),
         name='employee-print-clearance'),
    # Manager
    path('manager/approvals/', views.ManagerApprovals.as_view(), name='manager-approvals'),
    path('manager/approvals/<int:approval_id>/update', views.ManagerUpdateApproval.as_view(),
         name='manager-update-approval'),

]


# Exercise 5.9 of the book "Python Crash Course"

usernames = ['dave', 'phill', 'john', 'admin', 'joe']
if usernames:
	for username in usernames:
		if username == 'admin':
			print("Hello admin, would you like to see a status report?")
		else:
			print(f"Hello {username.title()}, thank you for logging in again.")
else:
	print("We need to find some users!")


def contains(my_str, substring):
    if substring in my_str:
        print(f"{my_str} contains {substring}")
    else:
        print("Substring not found!")

def flip(my_str, act, start, end):
    start = int(start)
    end = int(end)
    if start <= len(my_str) > end:
        if act == "Upper":
            my_str = my_str[:start] + my_str[start:end].upper() + my_str[end:]
        elif act == "Lower":
            my_str = my_str[:start] + my_str[start:end].lower() + my_str[end:]
    return my_str

def slice(my_str, start, end):
    start = int(start)
    end = int(end)
    if start <= len(my_str) > end:
        my_str = my_str[:start] + my_str[end:]
    return my_str

data = input()
data_input = input()
while not data_input == "Generate":
    if data_input.startswith("Contains"):
        comm, sub = data_input.split(">>>")
        contains(data, sub)
    elif data_input.startswith("Flip"):
        comm, action, start_index, end_index = data_input.split(">>>")
        data = flip(data, action, start_index, end_index)
        print(data)
    elif data_input.startswith("Slice"):
        comm, start_index, end_index = data_input.split(">>>")
        data = slice(data, start_index, end_index)
        print(data)
    data_input = input()
print(f"Your activation key is: {data}")

SoftUni/Python fundamentals/Final exam exercises/01. Activation Keys.py


import gzip
import re
import tqdm
from rationale_net.utils.embedding import get_indices_tensor
from rationale_net.datasets.factory import RegisterDataset
from rationale_net.datasets.abstract_dataset import AbstractDataset
from sklearn.datasets import fetch_20newsgroups
import random
random.seed(0)

SMALL_TRAIN_SIZE = 800
CATEGORIES = ['alt.atheism',
 'comp.graphics',
 'comp.os.ms-windows.misc',
 'comp.sys.ibm.pc.hardware',
 'comp.sys.mac.hardware',
 'comp.windows.x',
 'misc.forsale',
 'rec.autos',
 'rec.motorcycles',
 'rec.sport.baseball',
 'rec.sport.hockey',
 'sci.crypt',
 'sci.electronics',
 'sci.med',
 'sci.space',
 'soc.religion.christian',
 'talk.politics.guns',
 'talk.politics.mideast',
 'talk.politics.misc',
 'talk.religion.misc']

def preprocess_data(data):
    processed_data = []
    for indx, sample in enumerate(data['data']):
        text, label = sample, data['target'][indx]
        label_name = data['target_names'][label]
        text = re.sub('\W+', ' ', text).lower().strip()
        processed_data.append( (text, label, label_name) )
    return processed_data

@RegisterDataset('news_group')
class NewsGroupDataset(AbstractDataset):

    def __init__(self, args, word_to_indx, name, max_length=80):
        self.args = args
        self.args.num_class = 20
        self.name = name
        self.dataset = []
        self.word_to_indx  = word_to_indx
        self.max_length = max_length
        self.class_balance = {}

        if name in ['train', 'dev']:
            data = preprocess_data(fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'), categories=CATEGORIES))
            random.shuffle(data)
            num_train = int(len(data)*.8)
            if name == 'train':
                data = data[:num_train]
            else:
                data = data[num_train:]
        else:
            data = preprocess_data(fetch_20newsgroups(subset='test', remove=('headers', 'footers', 'quotes'), categories=CATEGORIES))

        for indx, _sample in tqdm.tqdm(enumerate(data)):
            sample = self.processLine(_sample)

            if not sample['y'] in self.class_balance:
                self.class_balance[ sample['y'] ] = 0
            self.class_balance[ sample['y'] ] += 1
            self.dataset.append(sample)

        print ("Class balance", self.class_balance)

        if args.class_balance:
            raise NotImplementedError("NewsGroup dataset doesn't support balanced sampling")
        if args.objective == 'mse':
            raise NotImplementedError("News Group does not support Regression objective")

    ## Convert one line from beer dataset to {Text, Tensor, Labels}
    def processLine(self, row):
        text, label, label_name = row
        text = " ".join(text.split()[:self.max_length])
        x =  get_indices_tensor(text.split(), self.word_to_indx, self.max_length)
        sample = {'text':text,'x':x, 'y':label, 'y_name': label_name}
        return sample


# Shivam Kumar Jha
# 17CS30033
# Assignment 3
# ML-2020
import pickle as pkl
import numpy as np
from numpy.linalg import norm

MIN_DIST = 1e10
VEC_IND = 0
IND_IND = 1

class AgglomerativeClustering():
    """
    Agglomerative Clustering
    Recursively merges the pair of clusters that minimally increases
    a given linkage distance.
    """
    def __init__(self, n_clusters=8, affinity='cosine', linkage='single', path="clusters/agglomerative.txt"):
        self._n_clusters = n_clusters
        self._affinity = affinity
        self._linkage = linkage
        self._path = path
        self._dist_mat = None

    @staticmethod
    def cos_sim(X,Y):
        """
        Return cosine similarity between two vectors.
        """
        return (X @ Y.T)/(norm(X)*norm(Y))

    def distance_matrix(self, X):
        """
        Computer matrix of exponential cosine distance between each data point.
        """
        self._dist_mat = np.zeros((X.shape[0], X.shape[0]))
#       calculating cosine distances
        for i in range(X.shape[0]):
            for j in range(X.shape[0]):
                if i!=j:
                    self._dist_mat[i][j] = np.exp(-1*self.cos_sim(X[i],X[j]))
        np.fill_diagonal(self._dist_mat, MIN_DIST)
#         print(self._dist_mat[545][545])
#         print(self._dist_mat.shape)

    def merge_clusters(self, X,Y):
        """
        Merge two clusters and return merged cluster.
        """
#         print(X[IND_IND],Y[IND_IND])
        merged_cluster = [[], []]
        merged_cluster[VEC_IND].extend(X[VEC_IND])
        merged_cluster[VEC_IND].extend(Y[VEC_IND])
        merged_cluster[IND_IND].extend(X[IND_IND])
        merged_cluster[IND_IND].extend(Y[IND_IND])
#         print("merged:",merged_cluster[IND_IND])
        return merged_cluster

    def single_clus_distance(self, X,Y):
        """
        Return single linkage minimum distance.

        Single linkage uses the minimum of the distances between all observations
        of the two sets.
        """
        x_fin, y_fin = None, None
        min_dist = MIN_DIST
#         print(len(X[IND_IND]), len(Y[IND_IND]))
        for i in range(0, len(X[IND_IND])):
#             print(X[IND_IND][i])
            for j in range(0, len(Y[IND_IND])):
#                 print(Y[IND_IND][j])
                if min_dist >= self._dist_mat[X[IND_IND][i]][Y[IND_IND][j]]:
#                     print("dist_mat",X[IND_IND][i],Y[IND_IND][j],self._dist_mat[X[IND_IND][i]][Y[IND_IND][j]])
                    min_dist = self._dist_mat[X[IND_IND][i]][Y[IND_IND][j]]
                    x_fin, y_fin = i,j
#         print("fin",x_fin,y_fin, min_dist)
        return min_dist

    def fit(self, X_arr):
        """
        Method to call to fit data.
        """
        init_clusters = [[[X_arr[i]], [i]] for i in range(X_arr.shape[0])]
        self.distance_matrix(X_arr)
        total_cur_clusters = X_arr.shape[0]
        while True:
            print(f"Number of clusters: {total_cur_clusters}", end='\r')
            clus_to_merge = None
            total_cur_clusters -= 1
            min_dist = MIN_DIST
            for i in range(total_cur_clusters):
                for j in range(i+1, total_cur_clusters):
                    cur_dist = self.single_clus_distance(init_clusters[i], init_clusters[j])
                    if min_dist >= cur_dist:
                        min_dist = cur_dist
                        clus_to_merge = [i,j]
#             print("min dist: ", min_dist, clus_to_merge[0], clus_to_merge[1])
            init_clusters.append(self.merge_clusters(init_clusters[clus_to_merge[0]], init_clusters[clus_to_merge[1]]))
            for index in sorted(clus_to_merge, reverse=True):
                del init_clusters[index]
            if total_cur_clusters==8:
                break
        self._results = init_clusters
#         print(init_clusters)
        self.save()

    def save(self):
        """
        Save the results in a sorted manner to ../clusters/agglomerative.txt
        """
        sorted_results = sorted(self._results, key= lambda x: min(x[IND_IND]))
        sorted_results = [sorted(x[IND_IND]) for x in sorted_results]
#         print(sorted_results)
        with open(self._path, 'w') as f_open:
            for result in sorted_results:
                f_open.write(','.join([str(x) for x in result]))
                f_open.write('\n')

def main():
    with open('data/tdidf_vector.pkl', 'rb') as f_open:
        tfidf_matrix = pkl.load(f_open)
    # tfidf_matrix.toarray().shape
    agglo = AgglomerativeClustering()
    agglo.fit(tfidf_matrix.toarray())

if __name__=="__main__":
    main()


'''
readData_WATS.py
written by: Hannah Holt
last updated: 5/2/18

readData_WATS.py is a program that reads in ASCII data taken from the DE-2 WATS instrument to look at the ETA structure
 using neutral wind and temperature measurements. Each WATS .txt file contains data for one time stamp split between two
consecutive lines. No header lines.

VERSION 2.0 UPDATES
    The main update is the program now reads in two WATS data files and looks at the temperature differences between them. T
    The first file is subtracted from the second file. MSIS features have been removed.

'''

from __future__ import print_function, division
import sys, os
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt
import csv

def print_header():
    print('\n**********************************************')
    print(os.path.basename(__file__))
    print('Written by: Hannah Holt')
    print('Last updated: 6/21/2018')
    print('**********************************************\n')

def plotData(x_values, y_values, filenames, LT_name):
    '''
    This function plots the given x and y values for a given gas and orbit number file.

    :param x_values:    array of values to plot on x axis. This could be lat, lon, LST, etc.
    :param y_values:    array of y values for the specific gas. This will usually be density
    :param filenames:    array of input files for graph title

    :return: plots the data and returns 0
    '''

    print("Done reading files. Plotting...")

    name = []

    # GRAPH LABELING....
    for i in range(0, len(filenames)):
        tmp = filenames[i]
        tmp = tmp[4:]
        tmp = tmp.replace("_orbit", " (#")
        tmp = tmp.replace(".txt", ")")
        tmp = tmp.replace("_", "Day ")
        name.append(tmp)

    y_label = whatplot + ' ' + y_name + ' Difference ' + ' [1/cm^3]'
    x_label = x_name + ' [deg]'
    graphTitle = 'Temperature Change of ' + name[1] + ' - ' + name[0] + '\nLT ~ ' + str(LT_name)

    if average:
        plt.plot(x_values, y_values, label='Raw DE2')
        plt.plot(x_values_avg, y_values_avg, label='Avg')
    else:
        plt.plot(x_values, y_values, label='Raw DE2')

    if x_lower != 0:
        plt.xlim(x_lower, x_upper)
    if y_lower != 0:
        plt.ylim(y_lower, y_upper)

    #plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.title(graphTitle)
    plt.legend()
    plt.grid()
    plt.show()

    return 0

def check_lon(lon_1, lat_1, lon_2, lat_2):
    '''

    :param lon_1: longitude values of first orbit file
    :param lat_1: geographical latitude values of first orbit file
    :param lon_2: "       "          second orbit file
    :param lat_2: "       "          second orbit file
    :return: plots the  latitude of the magnetic equator and two orbits (where they overlap) as a function of longitude
    '''

    # read in the magnetic coordinates
    mag_filename = 'mag_eq_lon_lat.txt'
    mag_lon = []
    mag_lat = []
    with open(mag_filename) as csvfile:
        readCSV = csv.reader(csvfile, delimiter=',')
        for line in readCSV:
            mag_lon.append(float(line[0]))
            mag_lat.append(float(line[1]))
    csvfile.close()

    # plot
    fig, ax = plt.subplots()
    label = []                              # legend labels
    for i in range(0, len(filenames)):
        temp = filenames[i]
        temp = temp[4:]
        temp = temp.replace("_orbit", " (#")
        temp = temp.replace(".txt", ")")
        temp = temp.replace("_", "Day ")
        label.append(temp)

    plt.plot(mag_lon, mag_lat, label='Magnetic Eq.')
    plt.plot(lon_1, lat_1, label=label[0])
    plt.plot(lon_2, lat_2, label=label[1])
    plt.xlabel("Geographic Longitude [deg]")
    plt.ylabel("Geographic Latitude [deg]")
    plt.title('DE 2 Orbit Comparison')
    plt.xlim(-180, 180)
    plt.ylim(-90, 90)
    ax.set_xticks(np.arange(-180, 181, step=20))
    plt.grid()
    plt.legend()
    plt.show()

    return 0

def temp_difference(temp_1, temp_2, lat_1, lat_2, lon_1, lon_2, LST_1, LST_2, split_1, split_2):
    """
    Takes in two temperature arrays and finds the realtive difference between the values for the same latitude

    Arguments:
        temp_1  :   temperature arr of first file [K]
        temp_2  :   temperature arr of second file [K]
        lat_1   :   latitude arr of first file [deg]
        lat_2   :   latitude arr of second file [deg]
        LST_1   :   local time arr of first file
        LST_2   :   local time arr of second file
        split_1 :   LT split integer for first file
        split_2 :   LT split integer for second file

    Returns:
        lat_diff    :   even though the latitudes will be roughly the same for consecutive files, this returns the average
        temp_diff   :   relative difference between the density values
        LT_name     :   the average LT value for the two orbits. Used later for plotting
    """
    # check to see if we want MLAT instead
    if x_name == "Magnetic Latitude":
        print("Coverting to Geomagentic Coordinates...")
        lat_1 = get_MLAT(lat_1, lon_1)
        lat_2 = get_MLAT(lat_2, lon_2)

    # only use the LT around 18
    if LST_1[split_1] > 12:
        length_1 = len(LST_1[split_1:])
        start1 = split_1
    else:
        length_1 = len(LST_1[:split_1])
        start1 = 0
        print("*** UPDATE: Changing 1st file split...")
        print("New LT1 = ", LST_1[start1])

    if LST_2[split_2] > 12:
        length_2 = len(LST_2[split_2:])
        start2 = split_2
    else:
        length_2 = len(LST_2[:split_2])
        start2 = 0
        print("*** UPDATE: Changing 2nd file split...")
        print("New LT2 = ", LST_2[start2])

    # now compare lengths - difference density array will be size of the smaller one
    if length_1 < length_2:
        size = length_1
    else:
        size = length_2

    # print('1at1[start1], lat2[start2] = ', lat_1[start1], lat_2[start2])
    # print('size = ', size)
    # print('start1 start2 =', start1, start2)
    # print('length to the end of 1 =', len(temp_1[start1:]))
    # print('length to the end of 2 =', len(temp_2[start2:]))

    # need to find latitude bin number where they start to overlap
    if lat_1[start1] > lat_2[start2]:
        for i in range(0, length_1):
            if lat_1[start1 + i] <= lat_2[start2]:
                start1 += i
                if len(temp_1[start1:start1+size]) < size:
                    size = len(temp_1[start1:start1+size])                                 # size of overlap shrinks

                break
    else:
        for i in range(0, length_2):
            if lat_2[start2 + i] <= lat_1[start1]:
                start2 += i
                if len(temp_2[start2:start2+size]) < size:
                    size = len(temp_2[start2:start2+size])

                break

    temp_diff = np.zeros(size)                      # difference between densities at same latitude
    lat_overlap = np.zeros(size)
    LT_value = (LST_1[start1] + LST_2[start2])/2

    # print("\nNEW VALUES.....")
    # print('1at1[start1], lat2[start2] = ', lat_1[start1], lat_2[start2])
    # print('size = ', size)
    # print('start1 start2 =', start1, start2)
    # print('length to the end of 1 =', len(temp_1[start1:]))
    # print('length to the end of 2 =', len(temp_2[start2:]))
    #
    # print('lat2[start stop] = ', lat_2[start2: start2+size])
    # print('lat1[start stop] = ', lat_1[start1: start1 + size])

    # now go through and iterate to find the differences
    for i in range(0, (size)):
        #print(i)
        temp_diff[i] = temp_2[start2 + i] - temp_1[start1 + i]

        # DE satellite will have basically the same latitude for the overlapping parts
        lat_overlap[i] = 0.5 * (lat_2[start2 + i] + lat_1[start1 + i])

    if compare:
        plt.plot(lat_1[start1: start1+size], temp_1[start1:start1+size], label='Orbit 7191')
        plt.plot(lat_2[start2: start2+size], temp_2[start2:start2+size], label='Orbit 7153')
        plt.xlabel('Latitude [deg]')
        plt.ylabel('Temperature [K]')
        plt.grid()
        plt.legend()
        plt.show()

    return lat_overlap, temp_diff, LT_value

def delete_bad_values(temp, lat, lon, alt, LST):
    """
    The DE data has some bad temperature values - appearing as zeros in the temperture array after we read everything in
    These must be deleted in order to subtract the two temperature values
    :param temp:    array of temperature values  (with bad values)
    :param lat:
    :param lon:
    :param alt:
    :param LST:
    :return:        arrays for temp, lat, alt, and LST with the bad values deleted
    """

    temp_good, lat_good, lon_good, alt_good, LST_good = [], [], [], [], []

    for i in range(0, len(temp)):
        if temp[i] != 0:
            temp_good.append(temp[i])
            lat_good.append(lat[i])
            lon_good.append(lon[i])
            alt_good.append(alt[i])
            LST_good.append(LST[i])
        else: continue

    return temp_good, lat_good, lon_good, alt_good, LST_good

def find_split(LST_arr):
    '''
    Since DE2 is a polar orbiter, it will drastically change local times (or longitude) when it passes over the poles. We want to find when this happens

    :param LST_arr: the array of LT values for a specific orbit. Find when LT_n+1 - LT_n > x.
    :return: an integer that marks the first location AFTER the split in the array from 0 to len(arr)
    '''

    dLT = 1  # the LT difference cutoff. I arbitrarily choose this
    split = 0
    for i in range(1, len(LST_arr)):
        if (abs(LST_arr[i] - LST_arr[i - 1]) > dLT):
            split = i
            break
    print("LT split = ", LST_arr[split])
    return split

def get_MLAT(DE_lat, DE_lon):
    '''

    :param DE_lat: array of DE geographical latitudes for the orbit
    :param DE_lon: corresponding DE geographical longitudes for each of the latitudes
    :return: MLAT:  an array that gives the corresponding magnetic latitude for each of the longitudes
    '''

    #coordinates for the dip equator
    mag_lon_eq = []
    mag_lat_eq = []

    MLAT = np.zeros(len(DE_lat))        # magnetic latitude values to return
    # lat_values = np.zeros(len(DE_lat))

    mag_filename = 'mag_eq_lon_lat.txt'
    if not os.path.isfile(mag_filename):
        print("ERROR. Magnetic equator file", mag_filename, "does not exist.")
        sys.exit()

    with open(mag_filename) as csvfile:
        readCSV = csv.reader(csvfile, delimiter=',')
        for line in readCSV:
            mag_lon_eq.append(float(line[0]))
            mag_lat_eq.append(float(line[1]))
    csvfile.close()

    for i in range(0, len(DE_lat)):
        for j in range(0, len(mag_lon_eq)):
            if int(DE_lon[i]) == mag_lon_eq[j]:
                MLAT[i] = DE_lat[i] - mag_lat_eq[j]

    if show_mag:
        plt.scatter(mag_lon_eq, mag_lat_eq, label='Mag. Eq.', marker='.')
        plt.scatter(DE_lon, DE_lat, label='GLAT', marker='.')
        plt.scatter(DE_lon, MLAT, label='MLAT', marker = '.')
        plt.xlabel('Longitude [deg]')
        plt.ylabel('Latitude [deg]')
        plt.legend()
        plt.grid()
        plt.ylim(-180, 180)
        plt.xlim(-100, 100)
        plt.show()

    return MLAT

def avg(array, num_points):
    '''
    This function averages the given array over the specified number of points
    :param array: a numpy array of values
    :param num_points: an integer that tells how many points you would like to average over.
    :return: an array with len(array) - num_points + 1 values
    '''
    average = np.zeros(len(array) - num_points + 1)

    for i in range(0, len(average)):
        sum = 0.0
        for j in range(0, num_points):
            sum += array[i+j]
        average[i] = sum/num_points

    return average

# ----------------------- VARIABLES TO CHANGE -------------------------------------------------------
# ----------------------------------------------------------------------------------------------------
directory = './WATSfiles/'
filenames = ['1982_326_orbit7191.txt', '1982_324_orbit7153.txt']
whatplot = 'Temperature'                # can be Temperature
x_name = "Latitude"                     # can be: 'Latitude' or 'Magnetic Latitude'
y_name = "Temperature"                  # can be: Temperature
average = True                          # do you want to plot the average over it
npts = 25                               # number of points to average array over
show_mag = False                        # show plot of geographic to geomagentic coordinates
show_lon_check = True                  # show where the orbits are longitudinally with respect to the mag eq.
compare = True                          # compare the two tempertaure plots before subtracting from each other
x_lower = 40
x_upper = -40
y_upper = 0
y_lower = 0

# ----------------------------------------------------------------------------------------------------

print_header()

delimiter = '\t'  # file delimiter
hour = 3600000  # ms in an hr
numFiles = len(filenames)  # number of files to read

# check for good paths/ variables
for i in range(0, numFiles):
    if not os.path.isfile(directory + filenames[i]):
        print("ERROR. Read file", filenames[i], "does not exist.")
        sys.exit()
if y_name != 'Temperature':
    print('ERROR: Invalid y axis.')
    sys.exit()
if x_name != 'Latitude' and x_name != 'Magnetic Latitude':
    print('ERROR: Invalid x axis.')
    sys.exit()

# read in file
for j in range(0, numFiles):
    fread = open(directory + filenames[j], 'r')
    print('Opening file', filenames[j])

    # get number of lines and initialize data. Time is in UT [ms], densities are 1/cm^3, errors are %
    line_num = 0
    for line in fread:
        line_num += 1
    fread.close()

    data_lines = int(line_num/2)                                # each data time stamp is split between two liness)

    time = np.zeros(data_lines)
    dens = np.zeros(data_lines)                                 # total neutral density [cm^-3]
    temp = np.zeros(data_lines)                                 # corrected neutral temperature [K]
    alt, lat = np.zeros(data_lines), np.zeros(data_lines)       # altitude and geo lat [km] and [deg]
    lon, LST = np.zeros(data_lines), np.zeros(data_lines)       # geo longitude and local solar time [deg] and [hr]

    # ----------- READ IN DATA-----------------------------------------------------------------
    # -----------------------------------------------------------------------------------------
    fread = open(directory + filenames[j], 'r')

    counter = 0
    k = 0

    for line in fread:
        line = line.lstrip(' ')         # remove leading whitespace if we have it

        if not (counter % 2):           # we are on first line of data split k % 2 = 0
            for i in range(0, 10):
                if i == 1:
                    time[k] = float((line.split(delimiter)[i])) / hour        # UT time in hours
                elif i == 4:
                    dens[k] = float(line.split(delimiter)[i])

                # although temperatures have some bad values (inlcuding zeros, we will take care of these
                # later in post processing)
                elif i == 6 and float(line.split(delimiter)[i]) < 9000 and float(line.split(delimiter)[i]) != -9:
                    temp[k] = float(line.split(delimiter)[i])
                else:
                    continue

        else:                               # second line of split
            for i in range(0, 10):
                if i == 3:
                    alt[k] = float(line.split(delimiter)[i])
                elif i == 4:
                    lat[k] = float(line.split(delimiter)[i])
                elif i == 5:
                    lon[k] = float(line.split(delimiter)[i])
                elif i == 6:
                    LST[k] = float(line.split(delimiter)[i])
                else: continue
            k += 1          # only increment k once per two lines
        counter += 1
    fread.close()

    # -----------------------------------------------------------------------------------------

    # ---------------- GET GOOD VALUES --------------------------------------------------------
    # we are on the first file
    if j == 0:
        temp1, lat1, lon1, alt1, LST1 = delete_bad_values(temp, lat, lon, alt, LST)
        split1 = find_split(LST1)
        print("First file split occurred at:", split1, '\n')

    # we are on the second file
    else:
        temp2, lat2, lon2, alt2, LST2 = delete_bad_values(temp, lat, lon, alt, LST)
        split2 = find_split(LST2)
        print("Second file split occurred at:", split2, '\n')

# end j in range (0, numfiles)

# ----------- POST PROCESSING + PLOTTING ---------------------------------------------------
#  -----------------------------------------------------------------------------------------

x_values, y_values, LT_name = temp_difference(temp1, temp2, lat1, lat2, lon1, lon2, LST1, LST2, split1, split2)

if show_lon_check:
    check_lon(lon1, lat1, lon2, lat2)

if average:
    x_values_avg = avg(x_values, npts)
    y_values_avg = avg(y_values, npts)

plotData(x_values, y_values, filenames, LT_name)


#!/usr/bin/python3 -u
import asyncio
import genshin
import json
from datetime import datetime, timezone
import os
import configparser

cache_file = os.path.expanduser("~/.cache/hoyo-stats.json")
config_file = os.path.expanduser("~/.config/hoyo-stats.ini")

# Create cache file if it doesn't exist
if not os.path.exists(config_file):
    f = open(config_file, "a")
    f.write("[settings]\nltuid = \nltoken = \ngenshin_id = \nhsr_id =")
    f.close()
# Load config from cache
config = configparser.ConfigParser()
config.read(config_file)

def config_fail():
    print(json.dumps({"text": "Set up hoyo module in ~/.config/hoyo-stats.json"}))

def time_diff(now, future, rate):
    max_time = future - now
    until_next = (max_time.seconds // 60) % (rate * 40)
    return "{}:{}".format(until_next // 60, until_next % 60)

async def main():
    cookies = {
            "ltuid": config["settings"]["ltuid"],
            "ltoken": config["settings"]["ltoken"]
            }
    # Get current time
    time_now = datetime.now(timezone.utc)

    # Main output dictionary
    main_dict = {}

    try:
        if config["settings"]["genshin_id"]:
            # Get genshin info
            genshin_client = genshin.Client(cookies)
            gn = await genshin_client.get_notes()
            gu = await genshin_client.get_full_genshin_user(
                    config["settings"]["genshin_id"])
            main_dict["Genshin"] = {
                "Resin": gn.current_resin,
                "Until next 40": time_diff(
                    time_now, gn.resin_recovery_time, 8),
                "Dailies completed": "{}/{}".format(
                    gn.completed_commissions + int(gn.claimed_commission_reward),
                    "5"),
                "Remaining boss discounts": gn.remaining_resin_discounts,
                "Realm currency": gn.current_realm_currency,
                "Abyss progress": "{} {}".format(
                    gu.abyss.current.max_floor,
                    gu.abyss.current.total_stars),
            }
    except KeyError:
        pass

    try:
        if config["settings"]["hsr_id"]:
            # Get HSR info
            hsr_client = genshin.Client(cookies, game=genshin.Game.STARRAIL)
            hn = await hsr_client.get_starrail_notes()
            hu = await hsr_client.get_starrail_challenge(
                    config["settings"]["hsr_id"])
            moc_floors = [floor.name for floor in hu.floors]
            main_dict["HSR"] = {
                "Trailblaze power": hn.current_stamina,
                "Until next 40": time_diff(
                    time_now, hn.stamina_recovery_time, 6),
                "Dailies completed": "{}/{}".format(
                    hn.current_train_score // 100,
                    "5"
                    ),
                "Remaining boss discounts": hn.remaining_weekly_discounts,
                "SU weekly score": "{}/{}".format(hn.current_rogue_score,
                                                  hn.max_rogue_score),
                "MoC progress": "{} {}".format(
                    len(moc_floors),
                    hu.total_stars),
            }
    except KeyError:
        pass

    # Write dictionary to json file
    with open(cache_file, "w") as json_file:
        json_file.write(json.dumps(main_dict, indent=4))

# Check modification date
try:
    mod_time = datetime.fromtimestamp(os.path.getmtime(cache_file))
    mod_seconds = (datetime.now() - mod_time).total_seconds()
    if mod_seconds > 60*2:
        asyncio.run(main())
except FileNotFoundError:
    asyncio.run(main())

# Load cache from file
cache = {}
with open(cache_file) as json_file:
    cache = json.load(json_file)

text = []
tooltip = []

if not cache:
    config_fail()
    exit(0)

for game, data in cache.items():

    if game == "Genshin":
        dailies = int(data["Dailies completed"].split("/")[0])
        # Set color based on status
        if dailies < 5 and data["Realm currency"] >= 2000:
            icon = " "
        elif data["Realm currency"] >= 2000:
            icon = " "
        elif dailies < 5:
            icon = " "
        else:
            icon = "  "
        text.append(icon + str(data["Resin"]))
    elif game == "HSR":
        su_points = int(data["SU weekly score"].split("/")[0])
        su_max = int(data["SU weekly score"].split("/")[1])
        dailies = int(data["Dailies completed"].split("/")[0])
        if dailies < 5:
            icon = " "
        elif su_points < su_max and datetime.today().weekday() >= 5:
            icon = " "
        else:
            icon = " "
        text.append(icon + str(data["Trailblaze power"]))

    # Create tooltip
    tooltip.append(
            (""
             "{} Stats\n").format(game) +
            '\n'.join(
                '{}: {}'.format(key, value) for key, value in data.items()))

# Print output as json
print(json.dumps({"text": "  ".join(text), "tooltip": "\n\n".join(tooltip)}))


#DD.MM.YYYY

def check_date(inp_year):
    day, month, year = map(int, inp_year.split('.'))
    if 0 < day < 32 and 0 < month < 13 and 0 < year < 10_000:
        if month != 2:
            if day in __month_dict:
                return month in __month_dict[day]
            else:
                return True
        else:
            if _check_year(year):
                  return day < 29
            else:
                return day < 30

def _check_year(year):
    if year % 4 != 0 or year % 100 == 0 and year % 400 != 0:
        print('Нормальный')
        return  True
    else:
        print('Высокосный')
        return False

__month_dict = {31:[1,3,5,7,8,10,12], 30:[4,6,9,11]}

if __name__ == '__main__':
    print(check_date(input('Введите дату #DD.MM.YYYY: ')))


'''
对那个expect dtype=string，but found dtype=int64的debug。结果是，
one_feature = tf.contrib.layers.sparse_column_with_keys(
            column_name=tag.tag_name,
            keys=tag.tag_set,
            default_value=0,
            combiner='sum',
            #dtype=tf.dtypes.int64
            dtype=tf.dtypes.string
        )
这个函数中设置了keys关键字的话，那么就不应该在调用中再用table了
table = tf.contrib.lookup.index_table_from_tensor(mapping=tag.tag_set, default_value=0)  ## 这里构造了个查找表 ##
one_sparse = tf.SparseTensor(
    indices=split_tag.indices,
    values=table.lookup(split_tag.values),
    dense_shape=split_tag.dense_shape
)

'''

import pickle
import json
import tensorflow as tf
import  numpy as np

GLOBAL_BATCH_SIZE = 2
GLOBAL_START_INDEX = 0
GLOBAL_TOTAL_EXAMPLE_NUM = 3

df_data = pickle.load(open("/home2/data/ttd/zhengquan_test.processed.csv.pkl","rb")) #一个DataFrame
# import pdb
# pdb.set_trace()
df_data = df_data.dropna(how="all", axis=0) # 0 对行进行操作，how='any'只要有一个NA就删除这一行，how='all'必须全部是NA才能删除这一行
#不能用any过滤，否则过滤完了，1000个只剩3个。
df_data['label'] = (df_data['label']).astype(int)
df_data = df_data[df_data['label'].isin([0,1])] #只保留label为0或者1的

#分离X,Y
X_data = df_data.drop(['label'],axis = 1)
# X_data = X_data.values.astype(tf.string)
X_data = X_data.applymap(str)
X_data = X_data.values.astype(np.str)
X_data = X_data.tolist()
Y_data = df_data['label'].values.astype(np.int32)

# data_size=len(X_data)
# indices=np.random.permutation(np.arange(data_size))
# shufflfed_X=X_data[indices]
# shufflfed_Y=Y_data[indices]
# X_batch_data ,Y_batch_data = shufflfed_X[0:GLOBAL_BATCH_SIZE]
# # input_data = csv_s
input_data = X_data
tags = []

class Tag(object):
    def __init__(self,
                 featureNum = 0,
                 featureNumOneline=0,
                 vocab_size=0,
                 embedding_size = 0,
                 tag_set=[],
                 tag_name = ""):
        self.featureNum = featureNum
        self.featureNumOneline = featureNumOneline
        self.vocab_size = vocab_size
        self.embedding_size = embedding_size
        self.tag_set =tag_set
        self.tag_name = tag_name

    def cal_vocabSize_embeddingSize(self,tag_set):
        if self.featureNum <=30:
            self.kind = "one_hot"
            self.wide_or_deep_side = "wide"
            self.vocab_size = self.featureNum
            self.embedding_size = self.featureNum
            self.tag_set = tag_set

        #直接映射成embedding的情况
        elif 30 < self.featureNum <= 10000:
            self.kind = "inteEmbdding"
            self.wide_or_deep_side = "wide+deep"
            self.vocab_size = self.featureNum
            self.embedding_size = 10 #写死的
            self.tag_set = tag_set

        #hash的情况
        else:
            if self.featureNum <= 100000:
                self.kind = "hash"
                self.wide_or_deep_side = "deep"
                self.vocab_size = min(self.featureNum,70000)
                self.embedding_size = 40  # 写死的
                self.tag_set = tag_set if self.vocab_size == self.featureNum else []
            else:
                self.kind = "hash"
                self.wide_or_deep_side = "deep"
                self.vocab_size = min(self.featureNum , max(100000,self.featureNum * 0.4))
                self.embedding_size = 40  # 写死的
                self.tag_set = tag_set if self.vocab_size == self.featureNum else []

tag2value = json.load(open("tag2value.json","r",encoding="utf-8"))
tag2valueOneline = json.load(open('tag2valueOneline.json',"r",encoding="utf-8"))
#type = dict, key是tag，value是这个tag的所有的可能的取值组成的列表
for key in tag2value:
    tag = Tag(
        featureNum=len(tag2value[key]),
        featureNumOneline=len(tag2valueOneline),
        tag_name=key,
    )
    tag.cal_vocabSize_embeddingSize(tag2value[key])
    tags.append(tag)

tag = tags[0]
one_feature = tf.contrib.layers.sparse_column_with_keys(
            column_name=tag.tag_name,
            keys=tag.tag_set,
            default_value=0,
            combiner='sum',
            #dtype=tf.dtypes.int64
            dtype=tf.dtypes.string
        )
res = tf.contrib.layers.one_hot_column(one_feature)

tag = tags[0]
print(tag.tag_set)
table = tf.contrib.lookup.index_table_from_tensor(mapping=tag.tag_set, default_value=0)  ## 这里构造了个查找表 ##
tag.table = table
vocab_size = tag.vocab_size
embedding_size = tag.embedding_size
one_feature = tf.contrib.layers.sparse_column_with_keys(
    column_name=tag.tag_name,
    keys=["Marry","Hello","World"],
    default_value=-1,
    combiner='sum',
    #dtype=tf.dtypes.int64
    dtype=tf.dtypes.string
)
res = tf.contrib.layers.one_hot_column(one_feature)
tag.embedding_res = res

current_batch_data = input_data[GLOBAL_START_INDEX:GLOBAL_START_INDEX+GLOBAL_BATCH_SIZE]

wide_embedding_res_list = []
deep_embedding_res_list = []
# current_batch_data = current_batch_data.tolist()
one_example = current_batch_data[0]
wide_feature_embedding_res_list = []
deep_feature_embedding_res_list = []
one_feature_of_one_example = one_example[0]
one_feature_of_one_example = "Hello|World"
print("one_feature = ",type(one_feature_of_one_example))
print(one_feature_of_one_example)
split_tag = tf.string_split([one_feature_of_one_example], "|")

split_tag_values = split_tag.values

look_up_res = table.lookup(split_tag_values)

one_sparse = tf.SparseTensor(
    indices=split_tag.indices,
    values=split_tag.values,  ## 这里给出了不同值通过表查到的index ##
    #values=look_up_res,
    dense_shape=split_tag.dense_shape
)
current_mapping = {tag.tag_name: one_sparse}
one_feature_embedding_res = tf.feature_column.input_layer(current_mapping, res)
print("tag_name = ", tag.tag_name, "  tag_kind = ", tag.kind)
print("tag_set = ",tag.tag_set)
with tf.Session() as sess:
    sess.run([tf.global_variables_initializer(),tf.tables_initializer()])
    a,b ,c,d= sess.run([split_tag,one_sparse,look_up_res,one_feature_embedding_res])
    print("a = ",a)
    print("type_a = ",type(a[0]))
    print("b = ",b)
    print("c = ",c)
    print("tag.tag_set = ",tag.tag_set)
    print("d = ",d)


from PySide2 import QtGui, QtCore, QtWidgets
from .node_graph_model import NodeGraphModel

class Outline:
    def __init__(self, editor, parent):
        self.editor = editor

        self.model = NodeGraphModel([])

        self.tree_view = QtWidgets.QTreeView(parent=parent)
        self.tree_view.setModel(self.model)

        self.tree_view.clicked.connect(self._onClick)

    def setNodeGraph(self, nodeGraph):
        self.model = NodeGraphModel(nodeGraph.labels)
        self.tree_view.setModel(self.model)
        self.tree_view.setRootIndex(QtCore.QModelIndex())
        self.tree_view.expandAll() # tree view cannot see all nodes until expanded
        self.tree_view.collapseAll()
        self.tree_view.update()

    def _onClick(self, index):
        node = self.model.getNodeFromIndex(index)
        view = node.view
        self.editor.findNode(view)

    def focusView(self, view):
        modelNode = self.model.getNodeFromView(view)
        index = self.model.getIndexFromNode(modelNode)
        if index:
            self.tree_view.setExpanded(index, True)
            self.tree_view.setCurrentIndex(index)


from math import ceil
fin = open("input.txt", 'r')

text = [int(i) for i in fin.read().splitlines()]

n = len(text)
ans1 = 0
ans2 = 0

i = 25
while i < n:
    sums = text[i]
    maps = {}
    valid = False
    for k in text[i-25:i]:
        if sums - k in maps:
            valid = True
            break
        else:
            maps[k] = 0
    i += 1
    if valid == False:
        ans1 = sums
        break

for x,i in enumerate(text):
    sums = i
    valid = False
    for y, j in enumerate(text):
        if y > x:
            sums += j
        if sums == 105950735:
            ans2 = min(text[x:y]) + max(text[x:y])
            valid = True
            break
    if valid:
        break

print("Part 1: ", ans1)
print("Part 2: ", ans2)


class PlacementCalculation:
    raise NotImplementedError

class SimplaDistancePlacement(PlacementCalculation):

    def process(self, env):
        """
        Tries to place a multi-processus application from a given device

        # Application will be deployed on device if possible, else the deployment will be tried on closest devices until all devices are explored

        Args:
            env : Environment
        """

        deployment_success = True
        # Get ordered device distance

        deployed_onto_devices = list()
        deployment_times = list()
        deployment_success = True

        logging.debug(f"Placement procedure from {self.deployment_starting_point}")

        device = env.getDeviceByID(self.deployment_starting_point)
        distance_from_device = {i: device.routing_table[i][1] for i in device.routing_table}
        sorted_distance_from_device = sorted(distance_from_device.items(), key=lambda x: x[1])

        pref_proc = dict()
        for proc in self.application_to_place.processus_list:
            pref_proc[proc.id] = list()
            for dev_id,dev_latency in sorted_distance_from_device:
                device = env.getDeviceByID(dev_id)
                if self.deployable_proc(proc, device):
                    pref_proc[proc.id].append((dev_id, dev_latency))

        matching = dict()
        matching_latency = dict()
        to_match  = self.application_to_place.getAppProcsIDs()

        while len(to_match)!=0:
            proc_id = to_match.pop(0)

            try:
                deployed, deployment_latency  = pref_proc[proc_id].pop(0)
            except IndexError:
                deployment_success = False
                break

            if deployed not in matching.values():
                matching[proc_id] = deployed
                matching_latency[proc_id] = deployment_latency
            else:
                matching_procs = [self.application_to_place.getAppProcByID(proc) for proc,dev in matching.items() if dev == deployed]
                agglomerated = sum(matching_procs)# + proc
                if self.deployable_proc(agglomerated, env.getDeviceByID(dev_id)):
                    matching[proc_id] = deployed
                    matching_latency[proc_id] = deployment_latency
                else:
                    min_proc_deployed = min([self.application_to_place.getAppProcByID(proc) for proc,dev in matching.items() if dev == deployed])
                    if self.application_to_place.getAppProcByID(proc_id) > min_proc_deployed:
                        min_proc_deployed_id = min_proc_deployed.getProcessusID()
                        to_match.append(min_proc_deployed_id)
                        matching[proc_id] = deployed
                        matching_latency[proc_id] = deployment_latency
                        matching.pop(min_proc_deployed_id, None)
                        matching_latency.pop(min_proc_deployed_id, None)
                    else:
                        to_match.append(proc_id)

        if deployment_success:

            prev_time, prev_value = env.count_accepted_application[-1]
            _, prev_tentative = env.count_tentatives[-1]

            for proc_id in self.application_to_place.getAppProcsIDs():
                deployed_onto_devices.append(matching[proc_id])
                deployment_times.append(matching_latency[proc_id])

            logging.info(f"Placement Module : application id : {self.application_to_place.id} , {self.application_to_place.num_procs} processus deployed on {deployed_onto_devices}")

            for i in range(len(deployed_onto_devices)):
                Deploy_Proc("Deployment Proc", self.queue, self.application_to_place, deployed_onto_devices, i, event_time=int((self.time+deployment_times[i])/10)*10, last=(i+1==len(deployed_onto_devices))).add_to_queue()

            if env.current_time == prev_time:
                env.count_accepted_application[-1][1] += 1
                env.count_tentatives[-1][1] += self.tentatives
            else:
                env.count_accepted_application.append([env.current_time, prev_value+1])
                env.count_tentatives.append([env.current_time, prev_tentative+self.tentatives])

        else:
            prev_time, prev_value = env.count_rejected_application[-1]

            logging.info(f"Placement Module : application id : {self.application_to_place.id} , {self.application_to_place.num_procs} processus not deployed")

            if env.current_time == prev_time:
                env.count_rejected_application[-1][1] += 1
            else:
                env.count_rejected_application.append([env.current_time, prev_value+1])

            # We could ask for a retry after 15 mins

            logging.info(f"Placement set back to future time, from {self.time} to {int((self.time+15*60*1000)/10)*10}")
            self.retry(event_time=int((self.time+15*60*1000)/10)*10)

        return deployment_times, deployed_onto_devices


from __future__ import division
from mysql.connector import MySQLConnection, Error
from db_config import connection_dict

import pandas as pd
import numpy as np
import  collections
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn import svm
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.linear_model import LogisticRegression

from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns

def get_training_data():
    data = None
    dict = {}
    try:
        conn = MySQLConnection(**connection_dict)
        cursor = conn.cursor()
        cursor.execute('select NAME, CATEGORY from usproduct where CATEGORY in (select CATEGORY from usproduct where CATEGORY is not null group by(CATEGORY) having count(*)>100)')
        rows = cursor.fetchall()
        data = pd.DataFrame(rows, columns=['name', 'category'])

        cursor.execute('select ID, NAME from category')
        rows = cursor.fetchall()

        df = pd.DataFrame(rows, columns=['id', 'name'])
        for index, row in df.iterrows():
            dict[row['id']] = str(row['name'])

    except Error as e:
        print(e)

    finally:
        cursor.close()
        conn.close()

    return (data, dict)

def process_data(data):
    words = []
    prefix = ''
    for i in range(0, 5):
        for j in range(0, 10):
            words.append(prefix + str(j))
        prefix += '0'
    vectorizer = TfidfVectorizer(stop_words=words)
    X = vectorizer.fit_transform(data['name'])
    print(X.shape)
    Y = data['category']
    return (X, Y, vectorizer)

def plot_result(x, y, title, xlabel, ylabel):
    plt.plot(x, y, 'bo', x, y, 'k')
    plt.ylabel(ylabel, fontsize=36)
    plt.xlabel(xlabel, fontsize=36)
    plt.title(title, fontsize=38)
    plt.tick_params(labelsize=36)
    plt.show()

def model_comparison(X, Y):
    print('Naive Bayes Score:')
    variables = [0.001, 0.01, 0.1, 0.5, 1, 10]
    scores = []

    for alpha in variables:
        nb = MultinomialNB(alpha=alpha)
        score = cross_val_score(nb, X, Y, cv=10)
        print('smoothing parameter alpha=%.3f, score:%.4f'%(alpha, np.mean(score)))
        scores.append(np.mean(score))

    plot_result(variables, scores, 'Multinomial Naive Bayes', 'Alpha', 'Accuracy')

    print('Logistic Regression Score:')
    variables = [0.01, 0.1, 1, 10, 100]
    scores = []
    for c in variables:
        lr = LogisticRegression(C=c, solver='lbfgs', multi_class='auto')
        score = cross_val_score(lr, X, Y, cv=10)
        print('regularization strength C=%.2f, score:%.4f'%(c, np.mean(score)))
        scores.append(np.mean(score))

    plot_result(variables, scores, 'Logistic Regression', 'C', 'Accuracy')

    print('SVM with Linear Kernel Score:')
    variables = [0.01, 0.1, 1, 10, 100]
    scores = []
    for c in variables:
        clf = svm.SVC(kernel='linear', C=c)
        score = cross_val_score(clf, X, Y, cv=10)
        print('penalty parameter C=%.2f, score:%.4f' % (c, np.mean(score)))
        scores.append(np.mean(score))

    plot_result(variables, scores, 'SVM with Linear Kernel', 'C', 'Accuracy')

    print('SVM with rbf Kernel Score:')
    variables = [0.01, 0.1, 1, 10]
    scores = []
    for c in variables:
        clf = svm.SVC(kernel='rbf', C=c, gamma='scale')
        score = cross_val_score(clf, X, Y, cv=10)
        print('penalty parameter C=%.2f, score:%.4f' % (c, np.mean(score)))
        scores.append(np.mean(score))

    plot_result(variables, scores, 'SVM with rbf Kernel', 'C', 'Accuracy')

def show_model_heat_map(X, Y, dict):
    names = Y.drop_duplicates().sort_values().map(dict)
    clf = svm.SVC(kernel='linear', C=1)
    fold = 10
    percs = 0
    kf = KFold(fold, False, 0)

    for train_index, test_index in kf.split(X):
        clf.fit(X[train_index], Y[train_index])

        i = 0
        correct = 0
        total = len(test_index)

        pres = clf.predict(X[test_index])

        pres = list(map(lambda t:dict[t], pres))
        acts = list(map(lambda t:dict[t], Y[test_index]))
        #
        # print(collections.Counter(pres))
        # print(collections.Counter(Y[test_index]))
        #
        # print(Y[test_index].values)

        conf_mat = confusion_matrix(acts, pres)
        fig, ax = plt.subplots(figsize=(10,10))

        sns.set(font_scale=2.5)

        heat_map = sns.heatmap(conf_mat, annot=True, fmt='d', xticklabels=names.values, yticklabels=names.values)
        heat_map.set_xticklabels(heat_map.get_xticklabels(), rotation=0)
        plt.ylabel('Actual')
        plt.xlabel('Predicted')

        for y in pres:
            if y == acts[i]:
                correct += 1
            # else:
            #     print('pre:' + str(y) + ',act:' + str(Y[test_index].values[i]) + ',total:' + str(i))
            i += 1
        percs += correct / total
        print(correct / total)
        plt.show()

    print(percs / fold)

def train_model(X, Y):
    clf = svm.SVC(kernel='linear', C=1)
    clf.fit(X, Y)
    return clf

def get_predict_data():
    data = None
    try:
        conn = MySQLConnection(**connection_dict)
        cursor = conn.cursor()
        cursor.execute(
            'select NAME from promotion')
        rows = cursor.fetchall()
        data = pd.DataFrame(rows, columns=['name'])

    except Error as e:
        print(e)

    finally:
        cursor.close()
        conn.close()

    return data

def predict(clf, X):
    Y = clf.predict(X)
    return Y

def save_predict_data(data, result):
    try:
        conn = MySQLConnection(**connection_dict)
        cursor = conn.cursor()
        cursor.executemany('update promotion set CATEGORY = %s where NAME = %s', zip(result, data))
        conn.commit()

    except Error as e:
        print(e)

    finally:
        cursor.close()
        conn.close()

if __name__ == '__main__':
    data, dict = get_training_data()
    if data is not None and len(dict) > 0:
        X, Y, vectorizer = process_data(data)
        # model_comparison(X, Y)
        # show_model_heat_map(X, Y, dict)

        print('start train')
        clf = train_model(X, Y)

        test = get_predict_data()
        if test is not None:
            print('start predict')
            result = predict(clf, vectorizer.transform(test['name']))
            print('start save')
            save_predict_data(test['name'].values, list(map(lambda r:int(r), result)))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import networkx as nx
from collections import deque

class UnionFindTree:
    def __init__(self, iterable):
        self.parent = {v: v for v in iterable}
        self.rank = {v: 0 for v in iterable}

    def union(self, x, y):
        x_root = self.find(x)
        y_root = self.find(y)
        if x_root == y_root:
            return
        if self.rank[x_root] < self.rank[y_root]:
            self.parent[x_root] = y_root
        else:
            self.parent[y_root] = x_root
            if self.rank[x_root] == self.rank[y_root]:
                self.rank[x_root] += 1

    def find(self, x):
        if self.parent[x] == x:
            return x
        else:
            self.parent[x] = self.find(self.parent[x])
            return self.parent[x]

    def same_check(self, x, y):
        return self.find(x) == self.find(y)

class Graph:
    def __init__(self):
        self.vertex = set()
        self.edge = {}
        self.adjacent = {}

    def __repr__(self):
        return str(self.edge)

    def add_edge(self, tail, head, weight):
        if tail != head:  # self loop排除
            self.vertex.add(tail)
            self.vertex.add(head)
            self.edge[tail, head] = weight
            if tail not in self.adjacent:
                self.adjacent[tail] = [head]
            else:
                self.adjacent[tail].append(head)
            if head not in self.adjacent:
                self.adjacent[head] = [tail]
            else:
                self.adjacent[head].append(tail)

    def add_weighted_edges_from(self, weights):
        for weight in weights:
            self.add_edge(*weight)

    def remove_edge(self, tail, head):
        del self.edge[tail, head]
        self.adjacent[tail].remove(head)
        self.adjacent[head].remove(tail)

    def total_weight(self):
        return sum(self.edge.values())

def is_tree(G, start):
    """幅優先探索で巡回路判定やろうとしたけど、
    うまくいかないので使わないこと
    """
    connected = {v: False for v in G.vertex}
    connected[start] = True
    queue = deque([start])
    u = -1
    while queue:
        v = queue.popleft()
        for w in G.adjacent[v]:
            if not connected[w]:
                connected[w] = True
                queue.append(w)
            elif w != u:
                return False
        u = v
    return True

def kruskal(G):
    """
    クラスカル法。最小全域木を返す。
    UnionFind木を用いて巡回路判定を行う。
    :param G: Graph
    :return: minimum spanning tree
    """
    H = Graph()
    UFT = UnionFindTree(G.vertex)
    n = len(G.vertex) - 1
    edges = sorted(G.edge.items(), key=lambda x: x[1])
    print('edges: ', edges)
    i = 0
    for (u, v), weight in edges:
        if UFT.same_check(u, v):
            print('not tree!')
        else:
            H.add_edge(u, v, weight)
            UFT.union(u, v)
            print('tree!')
            i += 1
        if i >= n:
            break
    print('parents: ', UFT.parent)
    print('rank: ', UFT.rank)
    return H

if __name__ == '__main__':
    G = Graph()
    weights = [[1, 3, 5], [1, 2, 3], [2, 3, 1], [3, 4, 5], [4, 5, 9], [3, 5, 2]]
    G.add_weighted_edges_from(weights)
    print(kruskal(G))

    G = Graph()
    weights = [[1, 2, 2], [1, 3, 1], [2, 3, 1], [2, 4, 3], [2, 5, 5], [3, 5, 3], [4, 5, 4]]
    G.add_weighted_edges_from(weights)
    print(kruskal(G))

    G = Graph()
    weights = [[1, 2, 33],
               [2, 3, 16],
               [1, 4, 98],
               [2, 5, 9],
               [3, 6, 84],
               [4, 5, 73],
               [5, 6, 49],
               [4, 7, 18],
               [5, 8, 61],
               [6, 9, 58],
               [7, 8, 64],
               [8, 9, 98]]
    G.add_weighted_edges_from(weights)
    print(kruskal(G))


import cmd
import sys
import os
import getpass
from pyfiglet import Figlet
import matplotlib.pyplot as plt

class CoolerParams:
    """
    Class for functions related to modelling cooler.
    """

    def __init__(self):
        self.energy = 0
        self.questions = ["What is the internal volume of the cooler? [m^3]",
                          "What is the average heat capacity of the item stored in the cooler? [Cp]",
                          "How many kgs of that item are there? [kg]",
                          "What is the density of that item? [kg/m^3]",
                          "What temperature is the cooler set at? [degrees C]",
                          "What is the convection heat transfer coefficient in the cooler? [W/m^2 C]",
                          "What is the emissivity of the item stored in the cooler?",
                          "What is the thickness of the cooler inner surface [m]",
                          "What is the k of the cooler inner surface? [W/m C]",
                          "What is the thickness of the cooler walls? [m]",
                          "What is the k of the cooler walls? [W/m C]",
                          "What is the thickness of the cooler outer surface? [m]",
                          "What is the k of the cooler outer surface? [W/m C]",
                          "What is the convection heat transfer coefficient outside the cooler? [W/m^2 C]",
                          "What is the emissivity of the cooler outer surface?",
                          "What is the ambient temperature outside the cooler?"
                          ]
        self.item_temperature = [20, 25, 30, 35]  # [deg C] from best case to worst case, temperature of inserted item
        self.ambient_humidity = [0, 0.25, 0.5, 0.75, 1.0]  # RH from best to worst, humidity of ambient air

    def get_cooler_params(self):
        """Asks a series of questions from user to get information on fridge in question.

        :return:
        """
        cooler_params = []
        print("Welcome to Cooler Params Questions. \n Let's define a small cubic volume as an estimate on how much energy we'll need to cool it. \n")
        for i in range(len(self.questions)):
            while True:
                try:
                    print(self.questions[i])
                    question_answer = float(input("\n"))
                    cooler_params.append(question_answer)
                    break
                except ValueError:
                    print("I didn't get that, try again.")

        return cooler_params


import datetime
import itertools
import json
import os

from chapter import Chapter, EPUBChapter, DuoKanChapter, PDFChapter, TXTChapter
from connector import Connector
from formatter import ChapterFormatter
from setting import SortType, OutlineType, ExportBook, ExportSetting

def _get_chapter(package_type, file_path, use_duokan_notes, outline_type):
    '''根据文件类型创建相应 chapter 对象
    package_type: str
      File type
    file_path: str
      Path to book
    use_duokan_notes: boolean
      If true, then file_path is the path to exported duokan notes
    '''
    chapter = None

    if not file_path:
        return chapter

    if use_duokan_notes:
        return DuoKanChapter(file_path)

    if package_type == 'EPUB':
        chapter = EPUBChapter(file_path)
    elif package_type == 'PDF':
        chapter = PDFChapter(file_path)
    elif package_type == 'TXT':
        chapter = TXTChapter(file_path)

    prefix = ''
    if outline_type == OutlineType.ORIGIN:
        pass
    elif outline_type == OutlineType.MD:
        prefix = '#'
    elif outline_type == OutlineType.ORG:
        prefix = '*'

    chapter = ChapterFormatter(prefix, chapter)

    return chapter

def _get_chapter_name(chapter, package_type, index, annotations, file_path, use_duokan_notes):
    arg = index

    if package_type == 'EPUB':
        arg = json.loads(annotations[0][1])[0]['chapter_id']
    elif package_type == 'PDF':
        arg = index
    elif package_type == 'TXT':
        arg = annotations[0][3]

    if use_duokan_notes:
        arg = annotations[0]

    chapter_name = chapter.getChapterName(arg)

    if file_path != '' and use_duokan_notes == False:
        chapter_name = chapter.getAncestorChapterNames(arg, True)

    return chapter_name

def get_annotations_in_book(exportBook, exportSetting):
    connector = Connector(exportSetting.db_path)

    annotations = connector.find_annotations_in_book(exportBook.book_id)

    if len(annotations) <= 0:
        return ''

    package_type = annotations[0][4]

    index_key = ''
    if package_type == 'PDF':
        index_key = 'fixed_index'
    elif package_type == 'TXT':
        index_key = 'byte_offset'
    elif package_type == 'EPUB':
        index_key = 'chapter_index'

    chapter = _get_chapter(package_type, exportBook.file_path, exportBook.use_duokan_notes,
                           exportSetting.outline_type)

    if exportSetting.sort_type == SortType.CHAPTER:
        annotations.sort(key=lambda annotation: json.loads(annotation[1])[0][index_key])
    elif exportSetting.sort_type == SortType.TIME:
        annotations.sort(key=lambda annotation: int(annotation[0]))
    #     chapter       chapter
    # [[annotation], [annotation], ...]
    annotations_by_chapter = itertools.groupby(annotations, key=lambda annotation: json.loads(annotation[1])[0][index_key])

    # chapter_name1 and chapter_name2 may be the same in PDF case
    # PDF case is groupby fixed_index which is the annotation page not chapter page
    # [[(chapter_name1, annotation_with_time)], [(chapter_name2, annotation_with_time)], ...]
    annotations_by_chapter_with_chapter_name = []
    for index, annotations in annotations_by_chapter:
        annotations = list(annotations)

        if exportBook.file_path:
            # If not specific file path, then use info in
            # annotations.annotation_range as chapter name
            chapter_name = _get_chapter_name(chapter, package_type, index, annotations,
                                             exportBook.file_path, exportBook.use_duokan_notes)
        else:
            chapter_name = str(index)

        annotation_by_chapter_with_chapter_name = []
        for annotation in annotations:
            added_time = datetime.datetime.utcfromtimestamp(annotation[0]/1000).strftime(exportSetting.time_format)
            note_text = json.loads(annotation[2])['note_text']
            annotation_sample = annotation[3]

            result = added_time + '\n' + annotation_sample + '\n'

            if len(note_text):
                result += '注：' + note_text + '\n'

            annotation_by_chapter_with_chapter_name.append((chapter_name, result))

        annotations_by_chapter_with_chapter_name.append(annotation_by_chapter_with_chapter_name)

    # Merge same chapter name in different list
    # [[(chapter_name, annotation_with_time)], [(chapter_name, annotation_with_time)], ...]
    annotations_by_chapter_name = []
    i = 0
    while i < len(annotations_by_chapter_with_chapter_name):
        annotations_i = annotations_by_chapter_with_chapter_name[i]
        annotation_by_chapter_name = annotations_i

        j = i + 1

        while j < len(annotations_by_chapter_with_chapter_name):
            annotations_j = annotations_by_chapter_with_chapter_name[j]

            if ''.join(annotations_i[0][0]) == ''.join(annotations_j[0][0]):
                annotation_by_chapter_name += annotations_j
                j = j + 1
            else:
                break

        annotations_by_chapter_name.append(annotation_by_chapter_name)
        i = j

    result = ''

    if exportBook.file_path != '' and exportBook.use_duokan_notes == False:
        known_chapter_names = set()
        for annotation_by_chapter_name in annotations_by_chapter_name:
            chapter_names = annotation_by_chapter_name[0][0]
            unknown_chapter_names = [chapter_name for chapter_name in chapter_names
                                     if chapter_name not in known_chapter_names]
            known_chapter_names.update(unknown_chapter_names)

            result += '\n' + '\n'.join(reversed(unknown_chapter_names)) + '\n'
            for chapter_name, annotation in annotation_by_chapter_name:
                result += annotation
    else:
        for annotation_by_chapter_name in annotations_by_chapter_name:
            result += '\n' + annotation_by_chapter_name[0][0] + '\n'
            for chapter_name, annotation in annotation_by_chapter_name:
                result += annotation

    return result.strip()

def export_annotations_in_book(exportBook, exportSetting):
    file_dir = exportSetting.export_dir
    file_name = exportBook.file_name

    file_extension = '.txt'
    if exportSetting.outline_type == OutlineType.ORIGIN:
        pass
    elif exportSetting.outline_type == OutlineType.MD:
        file_extension = '.md'
    elif exportSetting.outline_type == OutlineType.ORG:
        file_extension = '.org'

    file_path = os.path.join(file_dir, file_name + file_extension)

    content = get_annotations_in_book(exportBook, exportSetting)

    content = file_name + '\n' + exportBook.author + '\n\n' + content

    with open(file_path, 'w') as f:
        f.write(content)


from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer

'''
This is an example showing how to train a chat bot using the
ChatterBot ListTrainer.
'''

chatbot2 = ChatBot('Example Bot',
        storage_adapter='chatterbot.storage.SQLStorageAdapter',
        logic_adapters=[
        {
        'import_path': 'chatterbot.logic.BestMatch',
        'default_response': 'I am sorry, but I do not understand.',
        'maximum_similarity_threshold': 0.90
        },
        {
        'import_path': 'chatterbot.logic.MathematicalEvaluation',
        }
        ],
        preprocessors=['chatterbot.preprocessors.clean_whitespace']
)

# Start by training our bot with the ChatterBot corpus data
trainer = ListTrainer(chatbot2)

trainer.train([
    'Hello, how are you?',
    'I am doing well.',
    'That is good to hear.',
    'Thank you'
])

# You can train with a second list of data to add response variations

trainer.train([
    'Hello, how are you?',
    'I am great.',
    'That is awesome.',
    'Thanks'
])

# Now let's get a response to a greeting
while True:
    try:
        text = input("Enter Query: ")
        response = chatbot2.get_response(text)
        print(response)
        # print("\n")

    except Exception as e:
        raise e
        break


import copy
from dtk.utils.Campaign.CampaignClass import *

def add_ATSB(cb, start=0, coverage=0.15, kill_cfg=[], duration=180, duration_std_dev=14,
             nodeIDs=[], node_property_restrictions=[]):

    cfg_species = [x for x in cb.get_param('Vector_Species_Names') if cb.get_param('Vector_Species_Params')[x]['Vector_Sugar_Feeding_Frequency'] != 'VECTOR_SUGAR_FEEDING_NONE']
    atsb_master = WaningEffectBoxExponential(Initial_Effect=0.0337*coverage,
                                             Box_Duration=180,
                                             Decay_Time_Constant=30)
    dummy_kill_cfg = WaningEffectConstant(Initial_Effect=0)

    # default killing cfg
    killing_cfg = [{ 'Species' : sp,
                     'Killing_Config' : atsb_master} for sp in cfg_species]

    # if user has specified a kill cfg, just use dicts rather than CampaignClasses
    if kill_cfg :
        # if user-inputed killing cfg is dict and species not specified, make a list
        if isinstance(kill_cfg, dict) :
            kill_cfg['Killing_Config']['Initial_Effect'] *= coverage
            if 'Species' not in kill_cfg :
                killing_cfg = [{ 'Species' : sp,
                                 'Killing_Config' : kill_cfg['Killing_Config']} for sp in cfg_species]
            else :
                killing_cfg = [kill_cfg]
        # if user-inputed killing cfg is list, check if all cfg_species are listed. if not, put in a dummy no-effect kill_cfg.
        elif isinstance(kill_cfg, list) :
            for x in kill_cfg :
                if 'Species' not in x :
                    raise ValueError('Each config in SugarTrap killing config list must contain species name')
                x['Killing_Config']['Initial_Effect'] *= coverage
            listed_sp = [x['Species'] for x in kill_cfg]
            if any([x not in cfg_species for x in listed_sp]) :
                raise ValueError('A targeted SugarTrap species is not a sugar-feeding species in config')
            killing_cfg = [x for x in kill_cfg if x['Species'] in cfg_species]
        else :
            raise ValueError('Invalid SugarTrap killing config')

    atsb_config = SugarTrap(
        Cost_To_Consumer=3.75,
        Killing_Config_Per_Species=killing_cfg,
        Expiration_Distribution_Type="GAUSSIAN_DURATION",
        Expiration_Period_Mean=duration,
        Expiration_Period_Std_Dev=duration_std_dev
    )

    node_cfg = NodeSetNodeList(Node_List=nodeIDs) if nodeIDs else NodeSetAll()

    ATSB_event = CampaignEvent(
        Start_Day=start,
        Event_Coordinator_Config=StandardInterventionDistributionEventCoordinator(Intervention_Config=atsb_config),
        Intervention_Name="Attractive Toxic Sugar Bait",
        Demographic_Coverage=1,
        Node_Property_Restrictions=node_property_restrictions,
        Nodeset_Config=node_cfg
    )

    cb.add_event(ATSB_event)

def add_topical_repellent(config_builder, start, coverage_by_ages, cost=0, initial_blocking=0.95, duration=0.3,
                          repetitions=1, interval=1, nodeIDs=[]):

    repellent = {   "class": "SimpleIndividualRepellent",
                    "Event_Name": "Individual Repellent",
                    "Blocking_Config": {
                        "Initial_Effect": initial_blocking,
                        "Decay_Time_Constant": duration,
                        "class": "WaningEffectBox"
                    },
                    "Cost_To_Consumer": cost
    }

    for coverage_by_age in coverage_by_ages:

        repellent_event = { "class" : "CampaignEvent",
                          "Start_Day": start,
                          "Event_Coordinator_Config": {
                              "class": "StandardInterventionDistributionEventCoordinator",
                              "Target_Residents_Only" : 0,
                              "Demographic_Coverage": coverage_by_age["coverage"],
                              "Intervention_Config": repellent,
                              "Number_Repetitions": repetitions,
                              "Timesteps_Between_Repetitions": interval
                          }
                        }

        if all([k in coverage_by_age.keys() for k in ['min','max']]):
            repellent_event["Event_Coordinator_Config"].update({
                   "Target_Demographic": "ExplicitAgeRanges",
                   "Target_Age_Min": coverage_by_age["min"],
                   "Target_Age_Max": coverage_by_age["max"]})

        if not nodeIDs:
            repellent_event["Nodeset_Config"] = { "class": "NodeSetAll" }
        else:
            repellent_event["Nodeset_Config"] = { "class": "NodeSetNodeList", "Node_List": nodeIDs }

        if 'birth' in coverage_by_age.keys() and coverage_by_age['birth']:
            birth_triggered_intervention = {
                "class": "BirthTriggeredIV",
                "Duration": coverage_by_age.get('duration', -1), # default to forever if  duration not specified
                "Demographic_Coverage": coverage_by_age["coverage"],
                "Actual_IndividualIntervention_Config": itn_bednet_w_event #itn_bednet
            }

            repellent_event["Event_Coordinator_Config"]["Intervention_Config"] = birth_triggered_intervention
            repellent_event["Event_Coordinator_Config"].pop("Demographic_Coverage")
            repellent_event["Event_Coordinator_Config"].pop("Target_Residents_Only")

        config_builder.add_event(repellent_event)

def add_ors_node(config_builder, start, coverage=1, initial_killing=0.95, duration=30, cost=0,
                 nodeIDs=[]):

    ors_config = {  "Reduction_Config": {
                        "Decay_Time_Constant": 365,
                        "Initial_Effect": 0,
                        "class": "WaningEffectBox"
                    },
                    "Habitat_Target": "ALL_HABITATS",
                    "Cost_To_Consumer": cost,
                    "Killing_Config": {
                        "Decay_Time_Constant": duration,
                        "Initial_Effect": initial_killing*coverage,
                        "class": "WaningEffectExponential"
                    },
                    "Spray_Kill_Target": "SpaceSpray_FemalesAndMales",
                    "class": "SpaceSpraying"
                }

    ORS_event = {   "Event_Coordinator_Config": {
                        "Intervention_Config": ors_config,
                        "class": "NodeEventCoordinator"
                    },
                    "Nodeset_Config": {
                        "class": "NodeSetAll"
                    },
                    "Start_Day": start,
                    "Event_Name": "Outdoor Residual Spray",
                    "class": "CampaignEvent"
                }

    if nodeIDs:
        ORS_event["Nodeset_Config"] = { "class": "NodeSetNodeList", "Node_List": nodeIDs }

    config_builder.add_event(ORS_event)

def add_larvicide(config_builder, start, coverage=1, initial_killing=1.0, duration=30, cost=0,
                  habitat_target="ALL_HABITATS", nodeIDs=[]):

    larvicide_config = {  "Blocking_Config": {
                        "Decay_Time_Constant": 365,
                        "Initial_Effect": 0,
                        "class": "WaningEffectBox"
                    },
                    "Habitat_Target": habitat_target,
                    "Cost_To_Consumer": cost,
                    "Killing_Config": {
                        "Decay_Time_Constant": duration,
                        "Initial_Effect": initial_killing*coverage,
                        "class": "WaningEffectBox"
                    },
                    "class": "Larvicides"
                }

    larvicide_event = {   "Event_Coordinator_Config": {
                        "Intervention_Config": larvicide_config,
                        "class": "NodeEventCoordinator"
                    },
                    "Nodeset_Config": {
                        "class": "NodeSetAll"
                    },
                    "Start_Day": start,
                    "Event_Name": "Larvicide",
                    "class": "CampaignEvent"
                }

    if nodeIDs:
        larvicide_event["Nodeset_Config"] = { "class": "NodeSetNodeList", "Node_List": nodeIDs }

    config_builder.add_event(larvicide_event)

def add_eave_tubes(config_builder, start, coverage=1, initial_killing=1.0, killing_duration=180,
                   initial_blocking=1.0, blocking_duration=730, outdoor_killing_discount=0.3, cost=0,
                   nodeIDs=[]):

    indoor_config = {   "class": "IRSHousingModification",
                        "Killing_Config": {
                            "Decay_Time_Constant": killing_duration,
                            "Initial_Effect": initial_killing,
                            "class": "WaningEffectExponential"
                        },
                        "Blocking_Config": {
                            "Decay_Time_Constant": blocking_duration,
                            "Initial_Effect": initial_blocking,
                            "class": "WaningEffectExponential"
                        },
                        "Cost_To_Consumer": cost
                        }

    indoor_event = {"class": "CampaignEvent",
                    "Start_Day": start,
                    "Nodeset_Config": {
                        "class": "NodeSetAll"
                    },
                    "Event_Coordinator_Config": {
                        "class": "StandardInterventionDistributionEventCoordinator",
                        "Demographic_Coverage": coverage,
                        "Target_Demographic": "Everyone",
                        "Intervention_Config": indoor_config
                    }
                    }

    if nodeIDs:
        indoor_event["Nodeset_Config"] = { "class": "NodeSetNodeList", "Node_List": nodeIDs }

    config_builder.add_event(indoor_event)
    add_ors_node(config_builder, start, coverage=coverage,
                 initial_killing=initial_killing*outdoor_killing_discount,
                 duration=killing_duration, cost=cost,
                 nodeIDs=nodeIDs)


import numpy as np
from numba import jit
import cv2
from openBlur.blur_2 import numpyBlur

@jit
def numbaBlur(filename, pictureName):
    """
        Blurs a picture using numba implementation
        Uses same implementiation as the python method
        but compiles using jit from Numba
        returns array values
    """
    image = cv2.imread(filename)
    image = image.astype('float')
    PicA2 = np.zeros(np.shape(image))
    H_len, W_len, C_len = image.shape
    for h in range(H_len-1):
        for w in range(W_len-1):
            for c in range(C_len):
                PicA2[h][w][c] = (image[h][w][c]
                                + image[h-1][w][c]
                                + image[h+1][w][c]
                                + image[h][w-1][c]
                                + image[h][w+1][c]
                                + image[h-1][w-1][c]
                                + image[h-1][w+1][c]
                                + image[h+1][w-1][c]
                                + image[h+1][w+1][c]) / 9
    PicA2 = PicA2.astype('uint8')
    cv2.imwrite(pictureName, PicA2)
    return PicA2


"""implement the fips project registry"""

import yaml

registry = None

#-------------------------------------------------------------------------------
def load(fips_dir) :
    """load registry on demand"""
    global registry
    if registry is None :
        with open(fips_dir + '/registry.yml', 'r') as f :
            registry = yaml.load(f)

#-------------------------------------------------------------------------------
def exists(fips_dir, proj_name) :
    """check if a project exists in the registry

    :param fips_dir:    absolute path to fips
    :param proj_name:   short-name of project
    :returns:           True if project is in the registry
    """
    load(fips_dir)
    return proj_name in registry

#-------------------------------------------------------------------------------
def lookup_url(fips_dir, proj_name) :
    """lookup git url for project name, return None if not found

    :param fips_dir:    absolute path to fips
    :param proj_name:   project name
    :returns:           git url from registry, or None
    """
    load(fips_dir)
    if proj_name in registry :
        return registry[proj_name]
    else :
        return None

#-------------------------------------------------------------------------------
def get_url(fips_dir, name_or_url) :
    """Checks if name_or_url is in the registry, if yes, return url
    from registry, otherwise return name_or_url. This is useful
    if a parameter can be either a project name or a valid URL, and
    must be converted into an URL.

    :param fips_dir:    the absolute fips directory
    :param name_or_url: a project-name or URL
    :returns:           URL from registry, or the original name_or_url param
    """
    load(fips_dir)
    if name_or_url in registry :
        return registry[name_or_url]
    else :
        return name_or_url


import os
import sys
import torch

metric_type = ["ndcg", "err"]
# metric_type = ["ndcg"]
metric_position = [3, 5, 10]
max_length = 0
# max_label = 2  # or 4 for five fold dataset
max_label = 4  # or 4 for five fold dataset

def safe_div(numerator, denominator):
    return torch.where(
        torch.eq(denominator, 0),
        torch.zeros_like(numerator),
        torch.div(numerator, denominator),
    )

def read_one_line(label_fin):
    label_line = label_fin.readline()
    if label_line == "":
        return None, None
    arr = label_line.strip().split(" ")
    qid = arr[0]
    initial_label_list = [float(i) for i in arr[1:]]

    return qid, initial_label_list

def prepare_one_label_set(label_path, output_path, set_name):
    global metric_position, metric_type
    global max_length
    global max_label
    qid_list = []
    label_fin = open(label_path + set_name + ".labels")

    ## read all label lists
    print(f"Read {set_name} input label lists...")
    label_lists = []
    qid, label_list = read_one_line(label_fin)
    while qid is not None:
        qid_list.append(qid)
        label_lists.append(label_list)
        max_length = max(len(label_list), max_length)
        qid, label_list = read_one_line(label_fin)
    label_fin.close()

    ## pad with zero
    print("Padding...")
    for i in range(len(label_lists)):
        label_lists[i] += [0.0 for _ in range(max_length - len(label_lists[i]))]

    ## calculate metric for each initial rank
    print("Calculate metrics...")
    metric_map = {}  # e.g. {"ndcg": {3: 0.3, 5: 0.5, 10: 0.7}}
    labels_tensor = torch.tensor(label_lists)
    for metric in metric_type:
        if metric == "ndcg":
            discount = 1 / torch.log2(torch.arange(max_length) + 2)
            dcg = (torch.pow(2, labels_tensor) - 1) * discount
            dcg = torch.cumsum(dcg, dim=1)
            max_dcg = (
                torch.pow(2, labels_tensor.sort(descending=True, dim=1)[0]) - 1
            ) * discount
            max_dcg = torch.cumsum(max_dcg, dim=1)
            ndcg = safe_div(dcg, max_dcg).mean(dim=0)
            metric_map[metric] = {}
            for pos in metric_position:
                metric_map[metric][pos] = ndcg[pos-1].item()
        elif metric == "err":
            relevance = (torch.pow(2, labels_tensor) - 1) / pow(2, max_label)
            non_rel = torch.cumprod(1.0 - relevance, dim=1) / (1.0 - relevance)
            reciprocal_rank = 1.0 / torch.arange(start=1, end=max_length + 1)
            mask = [
                torch.ge(reciprocal_rank, 1.0 / n).type(torch.float32)
                for n in metric_position
            ]
            reciprocal_rank_topn = [reciprocal_rank * top_n_mask for top_n_mask in mask]
            # ERR has a shape of [batch_size, 1]
            err = [
                torch.sum(
                    relevance * non_rel * reciprocal_rank,
                    dim=1,
                    keepdim=True,
                )
                for reciprocal_rank in reciprocal_rank_topn
            ]
            err = torch.stack(err, dim=0).mean(dim=1)
            metric_map[metric] = {}
            i = 0
            for pos in metric_position:
                metric_map[metric][pos] = err[i].item()
                i += 1

    ## write metric
    print("Write to metric file")
    metric_fout = open(output_path + set_name + ".metric", "w")
    line = ""
    for metric in metric_type:
        line += " ".join(
            [f"{metric}_{pos}:{value}" for pos, value in metric_map[metric].items()]
        )
        line += "\n"
    metric_fout.write(line)

    metric_fout.close()

def main():
    DATA_PATH = sys.argv[1]  # for svm, point to `tmp_data` folder
    OUTPUT_PATH = sys.argv[2]  # for svm, point to `tmp_data` folder
    SET_NAME = ["train", "test", "valid"]

    print(f"Metric Type: {' '.join(metric for metric in metric_type)}")
    print(f"Position: {' '.join(str(pos) for pos in metric_position)}")
    for set_name in SET_NAME:
        if not os.path.exists(OUTPUT_PATH + set_name + "/"):
            os.makedirs(OUTPUT_PATH + set_name + "/")
        prepare_one_label_set(
            DATA_PATH + set_name + "/",
            OUTPUT_PATH + set_name + "/",
            set_name,
        )

if __name__ == "__main__":
    main()


import pickle
import matplotlib.pyplot as plt
import numpy as np
import os
from PIL import Image

data_file_path = os.path.abspath('../environments/simulated_data/')
data_file_name = 'pendulum_pixel_2022-09-05-12-37-35.pkl'
data_file = os.path.abspath(os.path.join(data_file_path, data_file_name))

with open(data_file, 'rb') as f:
    data = pickle.load(f)

print(data.keys())

pixel_trajectory = data['pixel_trajectories'][2]

frames = []
for i in range(pixel_trajectory.shape[0]):
    if i % 10 == 0: # Plot every 10th frame
        frames.append(Image.fromarray(np.uint8(pixel_trajectory[i] * 255)).resize((500,500)))
frame_one = frames[0]
frame_one.save("my_awesome.gif", format="GIF", append_images=frames,
            save_all=True, duration=100, loop=0)


from .granulometria import *
from .cartaplasticidad import *
from .valoresdeentrada import *

import matplotlib.pyplot as plt
import matplotlib.path as mpath
import numpy as np
import pandas as pd
def ClasificacionSuelos():

    if A==2:
      porcentaje_finos= 6
      if porcentaje_finos<5:
        cu=4
        cc=2
        if cu>3 and cc>1 and cc<3:
          print("Grava bien gradada")
        else:
          print("Grava mal gradada")
      else :
        b="si"
        if b=="si" or b=="no":
          if b=="si":
            print( "Grava Limosa")
          else:
            print("Grava Arcillosa")
        else:
          print("Error")

    if A==1:
      porcentaje_finos= 10
      if porcentaje_finos<5:
        cu=4
        cc=3
        if cu>=6 and cc>1 and cc<3:
          print("Arena bien gradada")
        else:
          print("Arena mal gradada")
      else :
        b="si"
        if b=="si" or b=="no":
          if b=="si":
            print( "Arena Limosa")
          else:
            print("Arena Arcillosa")
        else:
          print("Error")

    if A==3:
        cartaplasticidad()

tamiz_200 = 49

if tamiz_200 < 50:
    porcentaje_tamiz_4 = 10
    if porcentaje_tamiz_4 > 40:
        A=print("Es una Arena")
        A=1
    else:
        A=print("Es una grava")
        A=2
else:
    A=print("Ir a la carta de plasticidad")
    A=3

ClasificacionSuelos()


# type: ignore
import board
import digitalio
import time

led = digitalio.DigitalInOut(board.GP15)
led2 = digitalio.DigitalInOut(board.GP16)
led.direction = digitalio.Direction.OUTPUT
led2.direction = digitalio.Direction.OUTPUT
button = digitalio.DigitalInOut(board.GP0)
button.direction = digitalio.Direction.INPUT
button.pull = digitalio.Pull.DOWN

while True:
  print(button.value)

  if(button.value == True):
    for x in range(10, 0, -1):
      time.sleep(.25)
      print (x)
      time.sleep(.25)
      led.value = True
      time.sleep(.25)
      led.value = False
      time.sleep(.25)
    time.sleep(.25)
    print("liftoff")
    while True:
      led2.value = True