import random
from string import ascii_letters
from copy import deepcopy
import time
import threading
from datetime import datetime
from tkinter import *
import math

#Size sensors parameters
NB_COLUMNS = 6
NB_LINES = 3

#Size of the real tapis (mètre)
WIDTH_CARPET = 2.0
HEIGHT_CARPET = 1.0

#Limite minimum pour que le capteur soit pertinent
MIN_RELEVANT = 50

#Size of GUI
WIDTH_GUI = 600
HEIGHT_GUI = 300

CASE_WIDTH = WIDTH_GUI / NB_COLUMNS
CASE_HEIGHT = HEIGHT_GUI / NB_LINES

##################################################################################################################################
#                                                                                #
#                                                          Classe Data                                                           #
#                                                                                #
##################################################################################################################################
class Data:
    """docstring for Grid"""
    def __init__(self, data):

    	#Récupération des données brutes
        self.listCases = deepcopy(data)

        #Création de la grille
        self.grid = []
        for i in range(NB_LINES):
        	self.grid.append([])
        	for j in range(NB_COLUMNS):
        		self.grid[i].append(0)

        for i in range(NB_LINES*NB_COLUMNS):
            self.grid[i // NB_COLUMNS][i % NB_COLUMNS] = deepcopy(int(self.listCases[i]))

        #Test de la pertinence de la donnée : y a -t -il quelqu'un sur le tapis
        self.relevant = False
        for i in range(NB_LINES*NB_COLUMNS):
            if self.listCases[i] > MIN_RELEVANT:
                self.relevant = True

    def getRedCases(self):
        l = []
        for i in range(NB_LINES):
            for j in range(NB_COLUMNS):
                if self.grid[i][j] > MIN_RELEVANT:
                    l.append((i,j))
        return l

    #Affichage de la grille
    def __str__(self):
    	p = []
    	for i in range(NB_LINES):
    		l = []
    		for j in range(NB_COLUMNS):
    			l.append(str(self.grid[i][j]))
    		s = "\t".join(l)
    		p.append(s)
    	return "\n".join(p)

##################################################################################################################################
#                                                                                #
#                                                          Classe Foot                                                           #
#                                                                                #
##################################################################################################################################

def degToRad(deg):
	return deg*0.0174533

#Rotation d'un point autour de (cx,cy) d'angle angle
def rotate_point(cx,cy,angle,x,y):
	s = math.sin(degToRad(angle));
	c = math.cos(degToRad(angle));

	#translate point back to origin:
	x1 = x - cx;
	y1 = y - cy;

	#rotate point
	xnew = x1 * c - y1 * s;
	ynew = x1 * s + y1 * c;

	#translate point back:
	x1 = xnew + cx;
	y1 = ynew + cy;
	return x1,y1

class Foot:
	"""docstring for Foot
		t : instant auquel est associé le pied
		x : coordonnée selon x du pied
		y : coordonnée selon y du pied
		footWidth : largeur du pied
		footHeight : hauteur du pied
		angle : angle de rotation du pied
	"""
	def __init__(self, t,x,y,footWidth,footHeight, angle):
		self.t = t
		self.x = x
		self.y = y
		self.footWidth = footWidth
		self.footHeight = footHeight
		self.angle = angle

		#A B
		#C D

		self.xG = self.x + footWidth/2.0
		self.yG = self.y + footHeight/2.0

		self.xA, self.yA = rotate_point(self.xG,self.yG,self.angle,x,y)
		self.xB, self.yB = rotate_point(self.xG,self.yG,self.angle,x + footWidth,y)
		self.xC, self.yC = rotate_point(self.xG,self.yG,self.angle,x,y + footHeight)
		self.xD, self.yD = rotate_point(self.xG,self.yG,self.angle,x + footWidth,y + footHeight)

	def printGui(self,canva):
		canva.delete("AB_1")
		canva.delete("BD_1")
		canva.delete("DC_1")
		canva.delete("CA_1")
		canva.create_line(self.xA, self.yA, self.xB,self.yB, fill="green",tags="AB_1")
		canva.create_line(self.xB, self.yB, self.xD,self.yD, fill="yellow",tags="BD_1")
		canva.create_line(self.xD, self.yD, self.xC,self.yC, fill="blue",tags="DC_1")
		canva.create_line(self.xC, self.yC, self.xA,self.yA, fill="white",tags="CA_1")
		canva.update()

##################################################################################################################################
#                                                                                #
#                                                         Parseur                                                                #
#                                                                                #
##################################################################################################################################

#Tableau de données
st_datas = []
st_datas_relevant = []

#Parseur
while True:
	try:
		st_datas.append(Data(list(map(int,eval(input())))))
	except EOFError:
		break

serieStart = False
for i in range(len(st_datas)):
	if serieStart and not st_datas[i].relevant:
		break
	if st_datas[i].relevant:
		st_datas_relevant.append(st_datas[i])
		serieStart = True

######################################################################################################################
#                                                                                #
#                                                          Algo génétique                                            #
#                                                                                #
######################################################################################################################

#min inclus et max exclus
def getRandom(n_min,n_max):
    return int(random.uniform(n_min,n_max))

#Probabilié de mutation
CHANCE_TO_MUTATE = 0.1

#Proportion des meilleurs scores gardés pour la prochaine génération
GRADED_RETAIN_PERCENT = 0.2

#Probabilité de garder un génome mal gradé
CHANCE_RETAIN_NONGRATED = 0.05

#Taille de notre population
POPULATION_COUNT = 100

#Nombre de générations à faire
GENERATION_COUNT_MAX = 1000

#Nombre dindividus gardé au bout d'une génération
GRADED_INDIVIDUAL_RETAIN_COUNT = int(POPULATION_COUNT * GRADED_RETAIN_PERCENT)

#Liste d'individu qui représente la population initiale
populationInitiale = []

class Individu:
	"""
	foots : liste de foot
	"""
	def __init__(self,footHeight,footWidth):
		#Liste des pieds à l'instant t
		self.foots = []
		self.footHeight = footHeight
		self.footWidth = footWidth

	def generateRandomIndividu(self):
		for i in range(len(st_datas_relevant)):

			angleMin, angleMax, xMin, xMax, yMin, yMax = self.getBorneNextFoot(i-1);

			#On tire au hasard l'angle (en degré)
			angle = getRandom(angleMin,angleMax)

			#On tire au hasard la position du centre de gravité
			x = getRandom(xMin,xMax)
			y = getRandom(yMin,yMax)

			self.foots.append(Foot(i,x,y,self.footWidth,self.footHeight,angle))

	def getBorneNextFoot(self, i):
		if i >= 0:
			footPrec = self.foots[i]

			GAP_ANGLE = 10
			GAP_DISTANCE = 50

			#Angle min
			angleMin = footPrec.angle - GAP_ANGLE
			while angleMin < 0:
				angleMin += 360;

			#Angle max
			angleMax = footPrec.angle + GAP_ANGLE
			while angleMax < 0:
				angleMax += 360;

			#X min
			xMin = footPrec.x - GAP_DISTANCE
			if xMin < 0:
				xMin = 0

			#X max
			xMax = footPrec.x + GAP_DISTANCE
			if xMax > WIDTH_GUI:
				xMax = WIDTH_GUI

			#yMin
			yMin = footPrec.y - GAP_DISTANCE
			if yMin < 0:
				yMin = 0

			#yMax
			yMax = footPrec.y + GAP_DISTANCE
			if yMax > HEIGHT_GUI:
				yMax = HEIGHT_GUI

			return angleMin,angleMax,xMin,xMax,yMin,yMax
		else:
			return 0,360,0,WIDTH_GUI,0,HEIGHT_GUI;

def isPointInRectangle(x,y,ax,ay,bx,by,dx,dy):
	# Pseudo code
	# Corners in ax,ay,bx,by,dx,dy
	# Point in x, y

	bax = bx - ax
	bay = by - ay
	dax = dx - ax
	day = dy - ay

	if ((x - ax) * bax + (y - ay) * bay < 0.0):
		return False
	if ((x - bx) * bax + (y - by) * bay > 0.0):
		return False
	if ((x - ax) * dax + (y - ay) * day < 0.0):
		return False
	if ((x - dx) * dax + (y - dy) * day > 0.0):
		return False

	return True

##################################################################################################################################
#                                                                                #
#                                                         Tapis Gui                                                              #
#                                                                                #
##################################################################################################################################

def getDist(x1,y1,x2,y2):
    return math.sqrt((x1-x2)*(x1-x2) + (y1 - y2)*(y1 - y2))

class TapisGui(Frame):

    def __init__(self, fenetre, **kwargs):
        Frame.__init__(self, fenetre, width=768, height=576, **kwargs)
        self.pack(fill=BOTH)

        self.w = Canvas(self, width=WIDTH_GUI, height=HEIGHT_GUI, bd=0,relief='ridge',)
        self.w.pack()

        self.draw_grid()
        #Foot(0.0,50.0,20.0,70.0,33.0, 270.0).printGui(self.w)
        #individu = Individu(38,15)
        #individu.generateRandomIndividu()
        #for i in range(len(st_datas_relevant)):
        #    self.printData(st_datas_relevant[i].listCases)
        #    individu.foots[i].printGui(self.w)
        #    time.sleep(0.1)

        for i in range(1):
            MARGE = 100
            x = getRandom(MARGE , WIDTH_GUI - MARGE)
            y = getRandom(MARGE , HEIGHT_GUI - MARGE)

            angle = getRandom(0,360)

            footHeight = 400 + getRandom(-100,100)
            footWidth = 150 + getRandom(-50,50)

            foot = Foot(0,x,y,footWidth,footHeight,angle)

            l = []

            cTotal = 0

            for j in range(NB_LINES*NB_COLUMNS):
                line = j // NB_COLUMNS
                col = j % NB_COLUMNS

                xMin = col*CASE_WIDTH
                yMin = line*CASE_HEIGHT

                xMax = xMin + CASE_WIDTH
                yMax = yMin + CASE_HEIGHT

                c = 0
                d = getDist(xMin,yMin,foot.xG,foot.yG)
                d = min(d,getDist(xMin,yMax,foot.xG,foot.yG))
                d = min(d,getDist(xMax,yMin,foot.xG,foot.yG))
                d = min(d,getDist(xMax,yMax,foot.xG,foot.yG))
                if d < 2*max(footHeight,footWidth):
                    for x in range(int(xMin),int(xMax+1)):

                        for y in range(int(yMin),int(yMax+1)):
                            if isPointInRectangle(x,y,foot.xA,foot.yA,foot.xB,foot.yB,foot.xD,foot.yD):
                                c += 1
                #print(c)
                cTotal += c
                x0 = 750.0
                y0 = 250.0

                a = y0 / x0
                val = 0
                if c == 0:
                    val = getRandom(0,20)
                else:
                    val = int(min(250 - getRandom(0,20),a*c))
                l.append(val)
            if cTotal > 50:
                self.printData(l)
                foot.printGui(self.w)
                print(l)
                print('1')

    def draw_grid(self):
        rect = []
        a=0
        for j in range (0,NB_LINES):
            for i in range(0,NB_COLUMNS):
                coord = i*(WIDTH_GUI/NB_COLUMNS*1.0), \
                j*(HEIGHT_GUI/NB_LINES*1.0),\
                (i+1)*(WIDTH_GUI/NB_COLUMNS*1.0), (j+1)*(HEIGHT_GUI/NB_LINES*1.0)

                rect.append(self.w.create_rectangle(coord,fill="red",tags="rect"+str(a)))

                self.w.create_text(coord[0]+WIDTH_GUI/NB_COLUMNS*1.0/2.0,\
                coord[1]+HEIGHT_GUI/NB_LINES*1.0/2.0,text="0%",fill="white",tags="text"+str(a))
                #self.tag_bind(rect[len(rect)-1] , '<ButtonPress-1>', onObjectClick)
                a+=1

    def printData(self, line_input):

        for i in range(0,NB_LINES*NB_COLUMNS):
            r = 1
            g = 1
            b = 1

            val = int(line_input[i])

            rgb = val, g, b
            Hex = '#%02x%02x%02x' % rgb

            #print i,'=>',rgb
            self.w.itemconfig("rect"+str(i),fill=str(Hex))
            self.w.itemconfig("text"+str(i),text=str(val))

        self.w.update()

##################################################################################################################################
#                                                                                #
#                                                          Main                                                                  #
#                                                                                #
##################################################################################################################################

fenetre = Tk()
tapis_view = TapisGui(fenetre)

tapis_view.mainloop()


import cv2
import numpy as np
import tensorflow as tf
from tensorflow import keras
from skimage.transform import resize
from time import time

"""Constants"""
#Loading classifiers
face_cascade = cv2.CascadeClassifier('./data/utils/haarcascade_frontalface_default.xml')
eyePair_cascade = cv2.CascadeClassifier('./data/utils/haarcascades_haarcascade_mcs_eyepair_big.xml')
deepgaze = keras.models.load_model('./data/model/model_vastai.h5')

#Video imput
cap = cv2.VideoCapture('./data/video/driving_short.mp4')

classification = ['center', 'down', 'down-left', 'down-right', 'left', 'right', 'up', 'up-left', 'up-right']

# #For FPS
lastFrameTime = time()

"""Functions"""
#Eye cropping function
def ReturnEyePairFunc(frameImage):
	gray = cv2.cvtColor(frameImage, cv2.COLOR_BGR2GRAY)
	faces = face_cascade.detectMultiScale(gray, 1.3, 5)

	for x, y, w, h in faces: # face points
		roiGray = gray[y:y + h, x:x + w]
		roiColor = frameImage[y:y + h, x:x + w]
		eyePairs = eyePair_cascade.detectMultiScale(roiGray)

		for (ex, ey, ew, eh) in eyePairs: # eye_pair points
			roiEyes = roiColor[ex:ex + ew + 35, ey-240: ey + eh + 30]
			return roiEyes
	return frameImage

"""Processing"""
while True:
    ret, frame = cap.read()
    if ret == False:
        break

    #Processing the image
    picture = ReturnEyePairFunc(frame)
    if(picture.shape[1] == 0):
        continue
    gray_picture = cv2.cvtColor(picture, cv2.COLOR_BGR2GRAY)
    resized_image = cv2.resize(gray_picture, (300, 135), interpolation = cv2.INTER_AREA)
    #cv2.imshow("Eyes", resized_image)

    #Applying the prediction
    predictions = deepgaze.predict(np.array( [resized_image] ))
    #print("{}".format(predictions))

    #Getting the prediction (highest probability value)
    x = predictions
    max = -999
    for i in range(9):
        if(max < x[0][i]):
            max = x[0][i]
            k = i
    #print("Value: {};      Class: {}.\n".format(min, classification[k]))

    #FPS text
    cv2.putText(frame, "FPS: %.2f" %(1/(time() - lastFrameTime)), (530, 20), cv2.FONT_HERSHEY_PLAIN, 0.9, (0, 255, 0), 1)
    lastFrameTime = time()
    #Prediction text
    cv2.putText(frame, "Direction: {}".format(classification[k]), (10, 20), cv2.FONT_HERSHEY_PLAIN, 0.9, (0, 255, 0), 1)

    #Show frame
    cv2.imshow("Frame", frame)

    #ESC key will stop the processing
    key = cv2.waitKey(1)
    if key == 27:
        break

# Cleanup the cap and close any open windows
cap.release()
cv2.destroyAllWindows()


import logging

from django.contrib.auth import get_user_model

from .utils import headers_from_request
from .models import get_or_create_lti_user, LTIToolConsumer
from .oauth import validate_lti_launch

logger = logging.getLogger(__name__)

class DevLTILaunchBackend:
    @staticmethod
    def authenticate(dev_lti_user=None):
        return dev_lti_user.user if dev_lti_user else None

    @staticmethod
    def get_user(user_id):
        um = get_user_model()
        try:
            return um.objects.get(pk=user_id)
        except um.DoesNotExist:
            return None

class LTILaunchBackend:
    required_keys = {'oauth_consumer_key',
                     'user_id'}

    def authenticate(self, launch_request=None):
        result = None
        if launch_request:
            if launch_request.POST.keys() >= self.required_keys:
                result = self._find_lti_user(launch_request)
        return result

    @staticmethod
    def _find_lti_user(launch_request):
        consumer_key = launch_request.POST['oauth_consumer_key']
        lti_user_id = launch_request.POST['user_id']
        tool_guid = launch_request.POST.get(
            'tool_consumer_instance_guid', '')
        result = None
        try:
            consumer = LTIToolConsumer.objects.get(
                oauth_consumer_key=consumer_key)
        except LTIToolConsumer.DoesNotExist:
            logger.error("no LTI consumer found for OAuth consumer key '%s'",
                         consumer_key)
        else:
            guid_mismatch = consumer.tool_consumer_instance_guid != tool_guid
            if consumer.match_guid_and_consumer and guid_mismatch:
                logger.error(
                    "OAuth consumer key '%s' and tool consumer instance GUID "
                    "'%s' do not match", consumer_key, tool_guid)
            else:
                is_valid, req = validate_lti_launch(
                    consumer,
                    launch_request.build_absolute_uri(),
                    launch_request.body,
                    headers_from_request(launch_request))
                logger.debug("request: %s", req)
                if not is_valid:
                    logger.error(
                        "LTI launch not valid for OAuth consumer key '%s',"
                        " user_id '%s'", consumer_key, lti_user_id)
                else:
                    result = get_or_create_lti_user(
                        consumer, launch_request.POST).user
        return result

    @staticmethod
    def get_user(user_id):
        um = get_user_model()
        try:
            return um.objects.get(pk=user_id)
        except um.DoesNotExist:
            return None


import numpy as np
from numpy import zeros, int8, log
# from pylab import random
import pickle
import random
import sys
import jieba
import re
import time
import codecs
import argparse
import operator
import collections
# import ipdb

ap = argparse.ArgumentParser()
ap.add_argument("-d", "--dataset", required=True,
    help="path dataset")
ap.add_argument("-s", "--size", required=True,
    help="mini batch size")
ap.add_argument("-e", "--epochs", required=True,
    help="nb of epochs")
args = vars(ap.parse_args())

def initializeParameters():
    for i in range(0, N):
        normalization = sum(lamda[i, :])
        for j in range(0, K):
            lamda[i, j] /= normalization;

    for i in range(0, K):
        normalization = sum(theta[i, :])
        for j in range(0, M):
            theta[i, j] /= normalization;

def preprocessing(datasetFilePath, stopwordsFilePath):

    # read the stopwords file
    file = codecs.open(stopwordsFilePath, 'r', 'utf-8')
    stopwords = [line.strip() for line in file]
    file.close()

    # read the documents
    # file = codecs.open(datasetFilePath, 'r', 'utf-8')
    file = codecs.open(datasetFilePath, 'r', encoding="latin-1")
    documents = [document.strip() for document in file]
    file.close()

    # number of documents
    N = len(documents)

    wordCounts = [];
    word2id = {}
    id2word = {}
    currentId = 0;
    # generate the word2id and id2word maps and count the number of times of words showing up in documents
    for document in documents:
        segList = jieba.cut(document)
        wordCount = {}
        for word in segList:
            word = word.lower().strip()
            if len(word) > 1 and not re.search('[0-9]', word) and word not in stopwords:
                if word not in word2id.keys():
                    word2id[word] = currentId;
                    id2word[currentId] = word;
                    currentId += 1;
                if word in wordCount:
                    wordCount[word] += 1
                else:
                    wordCount[word] = 1
        wordCounts.append(wordCount);

    word2id = collections.OrderedDict(sorted(word2id.items(), key=operator.itemgetter(1)))
    MAX_RESULTS = 300
    word2id = dict(word2id.items()[:MAX_RESULTS])
    # length of dictionary
    M = len(word2id)

    # generate the document-word matrix
    X = zeros([N, M], int8)
    for word in word2id.keys():
        j = word2id[word]
        for i in range(0, N):
            if word in wordCounts[i]:
                X[i, j] = wordCounts[i][word];

    return N, M, word2id, id2word, X

def EStep():
    for i in range(0, N):
        for j in range(0, M):
            denominator = 0;
            for k in range(0, K):
                p[i, j, k] = theta[k, j] * lamda[i, k];
                denominator += p[i, j, k];
            if denominator == 0:
                for k in range(0, K):
                    p[i, j, k] = 0;
            else:
                for k in range(0, K):
                    p[i, j, k] /= denominator;

def MStep():
    # update theta
    for k in range(0, K):
        denominator = 0
        for j in range(0, M):
            theta[k, j] = 0
            for i in range(0, N):
                theta[k, j] += X[i, j] * p[i, j, k]
            denominator += theta[k, j]
        if denominator == 0:
            for j in range(0, M):
                theta[k, j] = 1.0 / M
        else:
            for j in range(0, M):
                theta[k, j] /= denominator

    # update lamda
    for i in range(0, N):
        for k in range(0, K):
            lamda[i, k] = 0
            denominator = 0
            for j in range(0, M):
                lamda[i, k] += X[i, j] * p[i, j, k]
                # lamda[i, k] += 1
                denominator += X[i, j];
            if denominator == 0:
                lamda[i, k] = 1.0 / K
            else:
                lamda[i, k] /= denominator

def SAGAStep(index_i, index_j):
    # compute new and old individual h_i
    oldh = h
    for i in index_i:
        for j in range(0, M):
            denominator = 0;
            for k in range(0, K):
                h[i, j, k] = theta[k, j] * lamda[i, k];
                denominator += h[i, j, k];
            if denominator == 0:
                for k in range(0, K):
                    h[i, j, k] = 0;
            else:
                for k in range(0, K):
                    h[i, j, k] /= denominator;

    # update theta
    for k in range(0, K):
        denominator = 0
        for j in range(0, M):
            tmp = 0
            for i in index_i:
                tmp += X[i, j] *(h[i, j, k] - oldh[i, j, k])
            Vsomme[k,j] = Hsomme[k,j] + N*tmp
            Ssomme[k,j] = (1-rho_saga)*Ssomme[k,j] + rho_saga*Vsomme[k,j]
            theta[k, j] = Ssomme[k, j]
            denominator += theta[k, j]
        if denominator == 0:
            for j in range(0, M):
                theta[k, j] = 1.0 / M
        else:
            for j in range(0, M):
                theta[k, j] /= denominator

    # update lamda
    for i in range(0, N):
        for k in range(0, K):
            lamda[i, k] = 0
            denominator = 0
            for j in range(0, M):
                lamda[i, k] += X[i, j] * p[i, j, k]
                denominator += X[i, j];
            if denominator == 0:
                lamda[i, k] = 1.0 / K
            else:
                lamda[i, k] /= denominator

    for k in range(0, K):
        for j in range(0, M):
            for i in index_j:
                Hsomme[k,j] += X[i, j] *(theta[k, j] * lamda[i, k] - listofthetas[i][k, j] * listoflamdas[i, k]);
    #save all parameters per individual
    for i in index_j:
        listofthetas[i] = theta
        for k in range(0, K):
            listoflamdas[i,k] = lamda[i,k]

# calculate the log likelihood
def LogLikelihood():
    loglikelihood = 0
    for i in range(0, N):
        for j in range(0, M):
            tmp = 0
            for k in range(0, K):
                tmp += theta[k, j] * lamda[i, k]
            if tmp > 0:
                loglikelihood += X[i, j] * log(tmp)
    return loglikelihood

# output the params of model and top words of topics to files
def output():
    # document-topic distribution
    file = codecs.open(docTopicDist,'w','utf-8')
    for i in range(0, N):
        tmp = ''
        for j in range(0, K):
            tmp += str(lamda[i, j]) + ' '
        file.write(tmp + '\n')
    file.close()

    # topic-word distribution
    file = codecs.open(topicWordDist,'w','utf-8')
    for i in range(0, K):
        tmp = ''
        for j in range(0, M):
            tmp += str(theta[i, j]) + ' '
        file.write(tmp + '\n')
    file.close()

    # dictionary
    file = codecs.open(dictionary,'w','utf-8')
    for i in range(0, M):
        file.write(id2word[i] + '\n')
    file.close()

    # top words of each topic
    file = codecs.open(topicWords,'w','utf-8')
    for i in range(0, K):
        topicword = []
        ids = theta[i, :].argsort()
        for j in ids:
            topicword.insert(0, id2word[j])
        tmp = ''
        for word in topicword[0:min(topicWordsNum, len(topicword))]:
            tmp += word + ' '
        file.write(tmp + '\n')
    file.close()

# datasetFilePath = 'dataset1.txt'
# datasetFilePath = 'dataset10k.txt'
# mini_batch_size = 4

datasetFilePath = args["dataset"]
mini_batch_size = int(args["size"])

stopwordsFilePath = 'stopwords.dic'
K = 10    # number of topic
nb_epochs = int(args["epochs"])
maxIteration = 30
threshold = 10.0
topicWordsNum = 10
docTopicDist = 'docTopicDistribution.txt'
topicWordDist = 'topicWordDistribution.txt'
dictionary = 'dictionary.dic'
topicWords = 'topics.txt'
if(len(sys.argv) == 11):
    datasetFilePath = sys.argv[1]
    stopwordsFilePath = sys.argv[2]
    K = int(sys.argv[3])
    maxIteration = int(sys.argv[4])
    threshold = float(sys.argv[5])
    topicWordsNum = int(sys.argv[6])
    docTopicDist = sys.argv[7]
    topicWordDist = sys.argv[8]
    dictionary = sys.argv[9]
    topicWords = sys.argv[10]

# preprocessing
N, M, word2id, id2word, X = preprocessing(datasetFilePath, stopwordsFilePath)
print(N)
print(M)
print(X.shape)

#LIST OF INDICES FOR INCREMENTAL METHODS
seed0 = 333888
indices = [x for x in range(N)]
list_indices_i = []
for epoch in range(0, nb_epochs):
    indices = [x for x in range(N)]
    random.seed(seed0*(epoch+3))
    random.shuffle(indices)
    list_indices_i.append(indices)

list_indices_j = []
for epoch in range(0, nb_epochs):
    indices_j = list_indices_i[epoch]
    random.seed(1999*(epoch+1))
    random.shuffle(indices_j)
    list_indices_j.append(indices_j)

seed0 = 333888
indices = [x for x in range(N)]
list_indices_i = []
for epoch in range(0, nb_epochs):
    indices = [x for x in range(N)]
    random.seed(seed0*(epoch+30))
    random.shuffle(indices)
    list_indices_i.append(indices)

rho_saga = 0.01
#REINITIALIZE
lamda = np.random.sample([N, K])
theta = np.random.sample([K, M])
p = zeros([N, M, K])
initializeParameters()
oldLoglikelihood = 1
newLoglikelihood = 1

### SAGA EM
objectiveSAGA = []
epoch = 0
Hsomme = zeros([K, M])
Ssomme = zeros([K, M])
EStep()
MStep()
newLoglikelihood = LogLikelihood()
print("[", time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())), "] ", epoch+1, " epoch  ", str(newLoglikelihood))
objectiveSAGA.append(newLoglikelihood)
oldLoglikelihood = newLoglikelihood

listofthetas = {}
for i in range(N):
    listofthetas[i] = theta
listoflamdas = lamda

h = p
oldh = p
for k in range(0, K):
    denominator = 0
    for j in range(0, M):
        Hsomme[k, j] = 0
        for i in range(0, N):
            Hsomme[k, j] += X[i, j] * h[i, j, k]

Ssomme = Hsomme
Vsomme = Hsomme

for epoch in range(1, nb_epochs):
    mini_batches_i = [list_indices_i[epoch][k:k+mini_batch_size] for k in range(0, N, mini_batch_size)]
    mini_batches_j = [list_indices_j[epoch][k:k+mini_batch_size] for k in range(0, N, mini_batch_size)]
    for m in range(mini_batch_size):
        SAGAStep(mini_batches_i[m],mini_batches_j[m])
        MStep()
    newLoglikelihood = LogLikelihood()
    print("[", time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())), "] ", epoch+1, " epoch  ", str(newLoglikelihood))
    # if(oldLoglikelihood != 1 and newLoglikelihood - oldLoglikelihood < threshold):
    #     break
    objectiveSAGA.append(newLoglikelihood)
    oldLoglikelihood = newLoglikelihood

with open('losses/sagaloss', 'wb') as fp:
    pickle.dump(objectiveSAGA, fp)

if __name__ == '__main__':
    output()


#Name: Yash Raja
#Date: October 31st, 2017
#File Name: 4_3.py
#Description
#Test Cases:
import random

heads = 0
tails = 0

for z in range (20):
    choice = random.choice("HT")
    if choice == "H":
        heads += 1
        print("H")
    if choice == "T":
        tails += 1
        print("T")
print("The amount of heads flipped are", heads)


# Copyright 2020 The Forte Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import List, Tuple, Dict, Union, Hashable, Iterable, Optional

class Vocabulary:
    r"""This class will store "Elements" that are added, assign "Ids" to them
    and return "Representations" if queried. These three are the main concepts
    in this class.

    1. Element: Any hash-able instance that the user want to store.
    2. Id: Each element will have an unique Id, which is an integer.
    3. Representation: according to the configuration, the representation for
       an element could be an integer (in this case, would be "Id"), or
       an one-hot vector (in this case, would be a list of integer).

    There are two special elements.

    1. One is <PAD> element, which will be mapped into Id of 0 or -1 and have
       different representation according to different setting.
    2. The other one is <UNK> element, which, if added into the vocabulary,
       will be the default element if the queried element is not found.

    Here is a table on how our Vocabulary class behavior under different
    settings. Element0 means the first element that is added to the vocabulary.
    Elements added later will be element1, element2 and so on. They will follow
    the same behavior as element0. For readability, they are not
    listed in the table.

    .. list-table:: Vocabulary Behavior under different settings.

        * - `vocab_method`
          - raw (handle outside)
          - indexing
          - indexing
          - one-hot
          - one-hot
        * - `need_pad`
          - assume False
          - True
          - False
          - True
          - False
        * - `get_pad_value`
          - None
          - 0
          - None
          - [0,0,0]
          - None
        * - `inner_mapping`
          - None
          - 0:pad 1:element0
          - 0:element0
          - -1:<PAD> 0:element0
          - 0:element0
        * - `element2repr`
          - raise Error
          - pad->0 element0->1
          - element0->0
          - <PAD>->[0,0,0] element0->[1,0,0]
          - element0->[1,0,0]
        * - `id2element`
          - raise Error
          - 0->pad 1->element0
          - 0->element0
          - -1 -> <PAD> 0->element0 (be careful)
          - 0->element0

    Args:
        method (str): The method to represent element in vocabulary.
        need_pad (bool): Whether to add <PAD> element in vocabulary.
        use_unk (bool): Whether to add <UNK> element in vocabulary.
            Elements that are not found in vocabulary will be directed
            to <UNK> element.
        pad_value (int): Value used by <PAD> element. Default is 0.
        unk_value (int): Value used by <UNK> element. Default is 1.

    Attributes:
        method (str): Same as above.
        need_pad (bool): Same as above.
        use_unk (bool): Same as above.
        pad_value (int): Same as above.
        unk_value (int): Same as above.
        next_id (int): The id that will be used when next element is added.
        element2id_dict (dict): This stores the mapping from element to id.
        id2element_dict (dict): This stores the mapping from id to element.
    """
    PAD_ELEMENT = "<PAD>"
    UNK_ELEMENT = "<UNK>"

    # TODO: It is better to add a generic type for this class, now every element
    #   is simply empty.
    def __init__(self, method: str, need_pad: bool, use_unk: bool,
                 pad_value: Optional[int] = None,
                 unk_value: Optional[int] = None):
        self.method = method
        self.need_pad = need_pad
        self.use_unk = use_unk

        self.element2id_dict: Dict = dict()
        self.id2element_dict: Dict = dict()

        self.next_id = 0
        if method == "one-hot" and need_pad:
            pad_value = -1
            logging.warning(
                "Cannot use 0 as pad id if one-hot method is used. "
                "Chaning pad id to -1!")
        if need_pad:
            if not pad_value:
                self.add_element(Vocabulary.PAD_ELEMENT)
            else:
                self.add_special_element(Vocabulary.PAD_ELEMENT, pad_value)

        if use_unk:
            if not unk_value:
                self.add_element(Vocabulary.UNK_ELEMENT)
            else:
                self.add_special_element(Vocabulary.UNK_ELEMENT, unk_value)

    def add_special_element(self, element: Hashable, element_id: int):
        r"""
        Add special_elements, such as PAD and UNK.

        Args:
            element (Hashable): The element to be added.
            element_id (int): The ID assigned to the element.
        """
        if element_id in self.id2element_dict:
            raise ValueError(
                    "ID %d is alreay being used in Vocabulary. " % element_id)
        if element not in self.element2id_dict:
            self.element2id_dict[element] = element_id
            self.id2element_dict[element_id] = element

    def add_element(self, element: Hashable):
        r"""This function will add element to the vocabulary.

        Args:
            element (Hashable): The element to be added.
        """
        while self.next_id in self.id2element_dict:
            self.next_id += 1
        if element not in self.element2id_dict:
            self.element2id_dict[element] = self.next_id
            self.id2element_dict[self.next_id] = element
            self.next_id += 1

    def id2element(self, idx: int) -> Hashable:
        r"""This function will map id to element.

        Args:
            idx (int): The queried id of element.

        Returns:
            The corresponding element if exist. Check the behavior
            of this function under different setting in the documentation.

        Raises:
            KeyError: If the id is not found.
        """
        return self.id2element_dict[idx]

    def element2repr(
            self, element: Hashable) -> Union[int, List[int]]:
        r"""This function will map element to representation.

        Args:
            element (Hashable): The queried element.

        Returns:
            Union[int, List[int]]: The corresponding representation
            of the element. Check the behavior of this function
            under different setting in the documentation.

        Raises:
            KeyError: If element is not found and vocabulary does
                not use <UNK> element.
        """
        if self.use_unk:
            idx = self.element2id_dict.get(
                element, self.element2id_dict[Vocabulary.UNK_ELEMENT])
        else:
            idx = self.element2id_dict[element]

        if self.method == "indexing":
            return idx
        else:
            vec_size = len(self.element2id_dict)
            if self.need_pad:
                vec_size -= 1
            vec = [0 for _ in range(vec_size)]
            if idx != -1:
                vec[idx] = 1
            return vec

    def __len__(self) -> int:
        r"""This function return the size of vocabulary.

        Returns:
            int: The number of elements, including
                <PAD>, <UNK>.
        """
        return len(self.element2id_dict)

    def has_element(self, element: Hashable) -> bool:
        r"""This function checks whether an element is added to vocabulary.

        Args:
            element (Hashable): The queried element.

        Returns:
            bool: Whether element is found.
        """
        return element in self.element2id_dict

    def items(self) -> Iterable[Tuple[Hashable, int]]:
        r"""This function will loop over the (element, id) pair inside this
        class.

        Returns:
            Iterable[Tuple]: Iterables of (element, id) pair.
        """
        return self.element2id_dict.items()

    def get_dict(self) -> Dict[Hashable, int]:
        r"""This function will get the inner mapping from element to id.

        Returns:
            dict: The maintained mapping from element to id.
        """
        return self.element2id_dict

    def get_pad_value(self) -> Union[None, int, List[int]]:
        r"""This function will get the PAD element for the vocabulary.

        Returns:
            Union[None, int, List[int]]: The PAD element. Check
            the behavior of this function in the documentation.
        """
        if self.need_pad:
            return self.element2repr(self.PAD_ELEMENT)
        return None


"""Attention module."""
import typing

import torch
import torch.nn as nn
import torch.nn.functional as F

class Attention(nn.Module):
    """
    Attention module.

    :param input_size: Size of input.
    :param mask: An integer to mask the invalid values. Defaults to 0.

    Examples:
        >>> import torch
        >>> attention = Attention(input_size=10)
        >>> x = torch.randn(4, 5, 10)
        >>> x.shape
        torch.Size([4, 5, 10])
        >>> attention(x).shape
        torch.Size([4, 5])

    """

    def __init__(self, input_size: int = 100, mask: int = 0):
        """Attention constructor."""
        super().__init__()
        self.linear = nn.Linear(input_size, 1, bias=False)
        self.mask = mask

    def forward(self, x):
        """Perform attention on the input."""
        x = self.linear(x).squeeze(dim=-1)
        mask = (x != self.mask)
        x = x.masked_fill(mask == self.mask, -float('inf'))
        return F.softmax(x, dim=-1)

class BidirectionalAttention(nn.Module):
    """Computing the soft attention between two sequence."""

    def __init__(self):
        """Init."""
        super().__init__()

    def forward(self, v1, v1_mask, v2, v2_mask):
        """Forward."""
        similarity_matrix = v1.bmm(v2.transpose(2, 1).contiguous())

        v2_v1_attn = F.softmax(
            similarity_matrix.masked_fill(
                v1_mask.unsqueeze(2), -1e-7), dim=1)
        v1_v2_attn = F.softmax(
            similarity_matrix.masked_fill(
                v2_mask.unsqueeze(1), -1e-7), dim=2)

        attended_v1 = v1_v2_attn.bmm(v2)
        attended_v2 = v2_v1_attn.transpose(1, 2).bmm(v1)

        attended_v1.masked_fill_(v1_mask.unsqueeze(2), 0)
        attended_v2.masked_fill_(v2_mask.unsqueeze(2), 0)

        return attended_v1, attended_v2


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import tensorflow as tf

class pyramidial_BLSTM(tf.keras.layers.Layer):
    '''
    Implementation of Pyramidal Bidirectional Long Short-Term Memory cell
    using Keras from Tensorflow:

    - https://arxiv.org/pdf/1508.01211.pdf
    - https://www.youtube.com/watch?v=WfA6OGtM4UI

    '''
    def __init__(self, dimension):
        '''
        Creation of the BLSTM cell:
        '''
        super().__init__()
        self.dimension = dimension
        self.LSTM = tf.keras.layers.LSTM(self.dimension, return_sequences=True)
        self.BLSTM = tf.keras.layers.Bidirectional(self.LSTM)

    def call(self, x):
        y = self.BLSTM(x)
        # If dimension of input is odd, add null adapted vector at the end of the input
        if tf.shape(x)[1] % 2 == 1:
          y = tf.keras.layers.ZeroPadding1D(padding=(0, 1))(y)
        #Concatenation
        y = tf.keras.layers.Reshape(target_shape=(-1,4*self.dimension))(y)
        return y

def Listen(X):
    dimension = len(X)
    inputs = tf.keras.Input(shape=(None, dimension))
    X = pyramidial_BLSTM(dimension//4)(inputs)
    X = pyramidial_BLSTM(dimension//4)(X)
    h = pyramidial_BLSTM(dimension//4)(X)

    model = tf.keras.Model(inputs, h, name="Listen Encoder")
    model.summary()


from GoogleNews import GoogleNews
from wordcloud import WordCloud
#for wordcloud to work you need to install microsoft c++ Build Tools --> VS Download Page
import matplotlib.pyplot as plt

googlenews = GoogleNews("en", "m" )
#Parameters: language and timeframe (day, month, year)

googlenews.search("WTI Price")

googlenews.getpage(1)
#first Result Page

headlines = googlenews.gettext()
#only show Headlines (pictures or links are also available)

text = ' '.join([str(elem) for elem in headlines])
#convert to string, Because Wordcloud expects string like object

#Generate the Wordcloud
wordcloud = WordCloud(max_font_size=50).generate(text)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

googlenews.clear()


from die import Die
import pygal

# create a two D6 dice
die_1 = Die(6)
die_2 = Die(6)
die_3 = Die(6)

# make some rolls
results = []
for roll_num in range(1000):
    result = die_1.roll() + die_2.roll() + die_3.roll()
    results.append(result)

# analyze the results
frequencies = []
max_result = die_1.num_sides + die_2.num_sides + die_3.num_sides
for value in range(3, max_result+1):
    frequency = results.count(value)
    frequencies.append(frequency)

# visualize the results
hist = pygal.Bar()

hist.title = 'Results of rolling three D6 1000 times'
hist.x_labels = []
for x in range(3, max_result+1):
    hist.x_labels.append(str(x))
hist.x_title = 'Result'
hist.y_title = 'Frequency of Result'

hist.add('D6 + D6 + D6', frequencies)
hist.render_to_file('die_visual.svg')

# "" """ ''


import os
import os.path
import torch
from torchvision import transforms
import numpy as np
import scipy.misc as m
import glob
import torch.utils.data as data

from torch.utils import data

class Crackloader(data.Dataset):

    def __init__(self,root,transform=None):

        self.root = root
        # self.transform = transform
        self.train_set_path = self.make_dataset(root)
        self.img_transforms = transforms.Compose(
            [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    def __len__(self):

        return len(self.train_set_path)

    def __getitem__(self, index):

        img_path,lbl_path = self.train_set_path[index]

        img = m.imread(img_path, mode='RGB')
        img = np.array(img, dtype=np.uint8)
        img = transforms.ToTensor()(img)
        # img = self.img_transforms(img)
        # print("img:{}".format(img.dtype))
        lbl = m.imread(lbl_path)

        lbl = np.array(lbl, dtype=np.uint8)

        lbl = np.expand_dims(lbl, axis= 0)
        # lbl = transforms.ToTensor()(lbl)

        lbl = torch.FloatTensor(lbl/255)
        # print("lable:{}".format(lbl.dtype))
        # if not self.transform:
        #     img, lbl = self.transform(img, lbl)

        return img, lbl

    def make_dataset(self,root):
        dataset = []
        images_base = os.path.join(root, "Train_image")
        Lable_base = os.path.join(root, "Lable_image")

        for img_path in glob.glob(os.path.join(images_base,"*.jpg")):
            lb_path = os.path.join(Lable_base, os.path.basename(img_path)[:-4] + ".png")
            dataset.append([img_path,lb_path])
        return dataset


import pickle as pk
from sklearn.preprocessing import LabelEncoder
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler

def load_data(path):
    """

    :param path: string
    :return: DataFrame
    """
    with open(path, 'rb') as f:
        data = pk.load(f)
    return data

def clean(data):
    """
    Cleaning some features that have lots of NaN
    :param data:DataFrame
    :return: DataFrame
    """
    def drop_col(df, col_name, cutoff):
        n = len(df)
        cnt = df[col_name].count()
        if (float(cnt)/n) < cutoff:
            df.drop(col_name, axis=1, inplace=True)

    col_names = list(data.columns)
    for i in col_names:
        drop_col(data, i, 0.8)

    data.fillna(method='bfill', inplace=True)# float16 can't use fillna with method

    return data

def ToNumeric(data):
    """
    Converting non-numeric to numeric
    :param data: DataFrame
    :return: DataFrame
    """
    le = LabelEncoder()
    columns = data.columns
    for i in columns:
        if data[i].dtype == 'object':
            print('convert: ' + str(i))
            data[i] = le.fit_transform(data[i])

    return data

def divide_label(data):
    """

    :param data: DataFrame
    :return: DataFrame, List
    """
    label = list(data['isFraud'].values)
    data = data.drop(['isFraud'], axis=1)

    return data, label

def fill_numbers(data):
    """
    对数值属性进行填充
    :param props:
    :return:
    """
    start_mem_usg = data.memory_usage().sum() / 1024 ** 2
    print("Memory usage of properties dataframe is :", start_mem_usg, " MB")

    for col in data.columns:
        if data[col].dtype != object:

            # Print current column type
            # print("******************************")
            # print("Column: ",col)
            # print("dtype before: ",props[col].dtype)

            # make variables for Int, max and min
            #mn = data[col].min()
            mn = data[col].max()
            # Integer does not support NA, therefore, NA needs to be filled
            if not np.isfinite(data[col]).all():
                data[col].fillna(mn + 1, inplace=True)

                # Print final result
    print("___MEMORY USAGE AFTER COMPLETION:___")
    mem_usg = data.memory_usage().sum() / 1024 ** 2
    print("Memory usage is: ", mem_usg, " MB")
    print("This is ", 100 * mem_usg / start_mem_usg, "% of the initial size")
    return data

def fill_dummy_category(data):
    """
    填充非数值型数据
    :param props:
    :return:
    """
    need_drop = []# 统计被编码的变量
    need_add = pd.DataFrame()# 暂时存放dummy后的数据
    for col in data.columns:
        if data[col].dtype == object:  # Exclude strings
            need_drop.append(col)
            dummy = pd.get_dummies(data[col], dummy_na=True, prefix='%s_' % col)
            need_add = pd.concat((need_add, dummy), axis=1)

    props = data.drop(need_drop, axis=1)
    props = pd.concat((props, need_add), axis=1)
    return props

def handle_dt(data):
    """
    对TransactionDT数据进行处理
    :param data:
    :return:
    """
    data['TransactionDT'] = data['TransactionDT'].apply(lambda x: (x + 59) // 60 // 60 % 24)

    '''
    t1 = 3.421434e+07
    t2 = 8.640000e+04

    t1-t2, 24*60*60

    #out
    (34127940.0, 86400)
    '''

    def judge_zone(dt_hour):
        zone = [(7, 12), (12, 17), (17, 23)]  # other is (23, 7)
        # ch = 3
        for idx, zn in enumerate(zone):
            if dt_hour >= zn[0] and dt_hour < zn[1]:
                return idx
        return 3

    data['TransactionDT'] = data['TransactionDT'].apply(lambda x: judge_zone(x))
    # print(data['TransactionDT'].describe())

    dummy = pd.get_dummies(data['TransactionDT'], dummy_na=False, prefix='%s_' % 'TransactionDT')
    data = data.drop(['TransactionDT'], axis=1)
    data = pd.concat((data, dummy), axis=1)

    return data

def baseline_data_preprocess():
    from features import extraction_card_variational_encoder
    df_train_identity = pd.read_csv('./train_identity.csv')
    df_train_transaction = pd.read_csv('./train_transaction.csv')

    df_test_identity = pd.read_csv('./test_identity.csv')
    df_test_transaction = pd.read_csv('./test_transaction.csv')

    df_train = pd.merge(df_train_transaction, df_train_identity, on=['TransactionID'], how='left')
    df_test = pd.merge(df_test_transaction, df_test_identity, on=['TransactionID'], how='left')

    print('merge后: '+str(len(df_test)))
    cutoff = 0.9
    card = ['card1', 'card2', 'card3', 'card4', 'card5', 'card6']
    for i in df_train.columns:
        n = len(df_train)
        cnt = df_train[i].count()
        if (float(cnt) / n) < cutoff and (i not in card):
            #print('del: '+ i)
            df_train.drop(i, axis=1, inplace=True)
            df_test.drop(i, axis=1, inplace=True)

    print('drop_null后: ' + str(len(df_test)))
    df_train = fill_numbers(df_train)
    df_test = fill_numbers(df_test)
    print('fill_number后: '+str(len(df_test)))

    use_top_email = 8
    if 'P_emaildomain' in df_train.columns:
        P_top_email_cols = (df_train['P_emaildomain'].value_counts(dropna=False) /\
                            df_train.shape[0] * 100)[:use_top_email]
        df_train['P_emaildomain'] = df_train['P_emaildomain'].apply(
            lambda x: x if x in P_top_email_cols.index else 'others_Pemail')
        df_test['P_emaildomain'] = df_test['P_emaildomain'].apply(
            lambda x: x if x in P_top_email_cols.index else 'others_Pemail')
    if 'R_emaildomain' in df_train.columns:
        R_top_email_cols = (df_train['R_emaildomain'].value_counts(dropna=False) /\
                            df_train.shape[0] * 100)[:use_top_email]
        df_train['R_emaildomain'] = df_train['R_emaildomain'].apply(
            lambda x: x if x in R_top_email_cols.index else 'others_Remail')
        df_test['R_emaildomain'] = df_test['R_emaildomain'].apply(
            lambda x: x if x in R_top_email_cols.index else 'others_Remail')

    if 'DeviceInfo' in df_train.columns:
        use_top_deviceInfo = 9
        use_top_deviceInfo_cols = (df_train['DeviceInfo'].value_counts(dropna=False)/\
                                   df_train.shape[0] * 100)[:use_top_deviceInfo]

        df_train['DeviceInfo'] = df_train['DeviceInfo'].apply(lambda x: x if x in use_top_deviceInfo_cols.index else 'others_deviceInfo')
        df_test['DeviceInfo'] = df_test['DeviceInfo'].apply(
            lambda x: x if x in use_top_deviceInfo_cols.index else 'others_deviceInfo')
    if 'id_31' in df_train.columns:
        use_top_id31 = 9
        use_top_id31_cols = (df_train['id_31'].value_counts(dropna=False)/\
                             df_train.shape[0] * 100)[:use_top_id31]

        df_train['id_31'] = df_train['id_31'].apply(lambda x: x if x in use_top_id31_cols.index else 'others_id31')
        df_test['id_31'] = df_test['id_31'].apply(lambda x: x if x in use_top_id31_cols.index else 'others_id31')

    print('dummy后: ' + str(len(df_test)))
    df_train = fill_dummy_category(df_train)
    df_test = fill_dummy_category(df_test)
    print('fill_category后: ' + str(len(df_test)))

    print(df_train.head(10))
    print(df_test.head(10))

    #df_train, label = divide_label(df_train)
    df_train.drop(['card6__debit or credit'], axis=1, inplace=True)
    print('start: ' + str(len(df_test)))
    # for i in df_train.columns:
    #    if 'card' in i:
    #        print(i)

    # for i in df_test.columns:
    #    if 'card' in i:
    #        print(i)

    tmp_train_card = []
    tmp_test_card = []
    for i in df_train.columns:
        if 'card' in i:
            tmp_train_card.append(df_train[i].values)
            tmp_test_card.append(df_test[i].values)

    train_card_data = []
    for i in range(len(tmp_train_card[0])):
        temp = []
        for j in range(len(tmp_train_card)):
            temp.append(tmp_train_card[j][i])
        train_card_data.append(temp)
    train_card_data = MinMaxScaler().fit_transform(train_card_data)
    train_card_features = extraction_card_variational_encoder(train_card_data)
    print('training card extracted succeed')

    divied_list = []
    k = 3
    for i in range(k):
        tmp = []
        for j in range(len(train_card_features)):
            tmp.append(train_card_features[j][i])
        divied_list.append(tmp)
    print(len(divied_list))
    divied_dict = {
        'card_feature1': divied_list[0],
        'card_feature2': divied_list[1],
        'card_feature3': divied_list[2]
    }
    df_train_card_features = pd.DataFrame(divied_dict, columns=['card_feature1', 'card_feature2', 'card_feature3'])
    df_train = pd.concat([df_train, df_train_card_features], axis=1)
    # print(df_train.head(10))

    test_card_data = []
    for i in range(len(tmp_test_card[0])):
        temp = []
        for j in range(len(tmp_test_card)):
            temp.append(tmp_test_card[j][i])
        test_card_data.append(temp)
    test_card_data = MinMaxScaler().fit_transform(test_card_data)
    test_card_features = extraction_card_variational_encoder(test_card_data)
    print('testing card extracted succeed')

    divied_list = []
    k = 3
    for i in range(k):
        tmp = []
        for j in range(len(test_card_features)):
            tmp.append(test_card_features[j][i])
        divied_list.append(tmp)
    divied_dict = {
        'card_feature1': divied_list[0],
        'card_feature2': divied_list[1],
        'card_feature3': divied_list[2]
    }
    df_test_card_features = pd.DataFrame(divied_dict, columns=['card_feature1', 'card_feature2', 'card_feature3'])
    df_test = pd.concat([df_test, df_test_card_features], axis=1)
    print('提取特征后: ' + str(len(df_test)))
    for i in df_test.columns:
        if i not in df_train.columns:
            print(i)

    with open('train_data_90_extract.pkl', 'wb') as wb:
        pk.dump(df_train, wb)

    with open('test_data_90_extract.pkl', 'wb') as wb:
        pk.dump(df_test, wb)

if __name__ == '__main__':

    #pd_reader1 = pd.read_csv('./train_transaction.csv')  # TODO: change file path
    #train_trainsaction = pd.DataFrame(pd_reader1)

    #train_trainsaction = clean(train_trainsaction)
    #with open('train_transaction_drop.pkl', 'wb') as wb: # TODO: change file path
    #    pk.dump(train_trainsaction, wb)

    #train_trainsaction = ToNumeric(train_trainsaction)
    #with open('train_transaction_drop_byte.pkl', 'wb') as wb: # TODO: change file path
    #    pk.dump(train_trainsaction, wb)

    # print(type(train_trainsaction))
    # print(train_trainsaction['P_emaildomain'])
    # trainsaction_data, trainsaction_label = divide_label(train_trainsaction)
    # print(trainsaction_data[:2])
    # print(trainsaction_label)

    #train_trainsaction = pk.load(open('./train_transaction_drop_byte.pkl', 'rb'))
    #trains_data, trains_label = divide_label(train_trainsaction)
    #print(trains_data[:2])
    #print(trains_label[:2])

    #pd_reader2 = pd.read_csv('./train_identity.csv')
    #train_identity = pd.DataFrame(pd_reader2)
    #train_identity = clean(train_identity)
    #with open('train_identity_drop.pkl', 'wb') as wb:
    #    pk.dump(train_identity, wb)
    #train_identity = ToNumeric(train_identity)
    #with open('train_identity_drop_byte.pkl', 'wb') as wb:
    #    pk.dump(train_identity, wb)

    """
    pd_reader3 = pd.read_csv('./test_transaction.csv')
    test_transaction = pd.DataFrame(pd_reader3)
    test_transaction = clean(test_transaction)
    test_transaction = ToNumeric(test_transaction)
    with open('test_transaction.pkl', 'wb') as wb:
        pk.dump(test_transaction, wb)

    pd_reader4 = pd.read_csv('./sample_submission.csv')
    label = pd.DataFrame(pd_reader4)
    label = clean(label)
    label = ToNumeric(label)
    with open('test_label.pkl', 'wb') as wb:
        pk.dump(label, wb)
    """

    #baseline_data_preprocess()


from pathlib import Path
import numpy as np
import pandas as pd
import torch
from apex import amp
from tqdm import tqdm
from scripts.utils import empty_cuda_cache, reduce_tensor, one_hot_embedding
from scripts.tb_helper import simple_result, normalize_tensor2tensor, normalize_tensor2numpy
import heapq
import random
import torch.nn.functional as F
from torchvision.utils import save_image
import matplotlib.pyplot as plt
import pdb

class Learning():
    def __init__(self,
                 distrib_config,
                 optimizer,
                 loss_fn,
                 evaluator,
                 device,
                 n_epoches,
                 scheduler,
                 accumulation_step,
                 early_stopping,
                 logger,
                 tb_logger,
                 best_checkpoint_folder,
                 checkpoints_history_folder,
                 checkpoints_topk,
                 calculation_name
        ):
        self.distrib_config = distrib_config
        self.logger = logger
        self.tb_logger = tb_logger
        self.optimizer = optimizer
        self.loss_fn = loss_fn
        self.evaluator = evaluator
        self.device = device
        self.epoch = 0
        self.n_epoches = n_epoches
        self.scheduler = scheduler
        self.accumulation_step = accumulation_step
        self.early_stopping = early_stopping
        self.calculation_name = calculation_name
        self.best_checkpoint_path = Path(
            best_checkpoint_folder,
            '{}.pth'.format(self.calculation_name)
        )
        self.checkpoints_history_folder = checkpoints_history_folder
        self.score_heap = [(0., Path('nothing'))]
        self.summary_file = Path(self.checkpoints_history_folder, 'summary.csv')
        if self.summary_file.is_file():
            self.best_score = pd.read_csv(self.summary_file).MIoU.max()
            if self.distrib_config['LOCAL_RANK'] == 0:
                logger.info('Pretrained best score is {:.5}'.format(self.best_score))
        else:
            self.best_score = 0
        self.best_score = 0
        self.best_epoch = -1
        self.checkpoints_topk = checkpoints_topk
        # empty_cuda_cache()

    def train_epoch(self, model, loader):
        tqdm_loader = tqdm(loader)
        current_loss_mean = 0.
        for idx, batch in enumerate(tqdm_loader):
            if (idx + 1) % self.accumulation_step == 0:
                self.optimizer.zero_grad()

            image, target = batch['image'], batch['label']
            loss = self.batch_train(model, image, target)
            current_loss_mean = (current_loss_mean * idx + loss.item()) / (idx + 1)

            if (idx + 1) % self.accumulation_step == 0:
                self.optimizer.step()

            tqdm_loader.set_description(f'loss: {current_loss_mean:.4f} lr: {self.optimizer.param_groups[0]["lr"]:.6f}')
        # empty_cuda_cache()
        return current_loss_mean

    def batch_train(self, model, batch_imgs, batch_labels):
        batch_imgs = batch_imgs.to(device=self.device)
        batch_labels = batch_labels.to(device=self.device)
        batch_pred = model(batch_imgs)

        if self.epoch < 40:
            loss = self.loss_fn(batch_pred, batch_labels) / self.accumulation_step
        elif 40 < self.epoch < 80:
            t_is = 1.0 - (self.epoch / 80)
            corrected_label = self.correct_label(batch_pred, batch_labels, t_is)
            loss = self.loss_fn(batch_pred, corrected_label) / self.accumulation_step
        else:
            corrected_label = self.correct_label(batch_pred, batch_labels, t_is=0.0001)
            loss = self.loss_fn(batch_pred, corrected_label) / self.accumulation_step

        loss.backward()
        # with amp.scale_loss(loss, self.optimizer, loss_id=0) as scaled_loss:
        #     scaled_loss.backward()
        return loss

    def correct_label(self, batch_pred1, batch_labels, t_is=0.1):
        batch_labels = batch_labels.long()

        with torch.no_grad():
            logp1 = F.log_softmax(batch_pred1, dim=1)

            if t_is:
                mean_logp = logp1.max(1)[0]
                t_mean = mean_logp[batch_labels == 255].mean()
                t_max = mean_logp[batch_labels == 255].max()
                rate = (t_max - t_mean)
                t = t_mean + rate * t_is

                corrected = torch.where((batch_labels == 255) & (mean_logp > t), logp1.argmax(1), batch_labels)
            else:
                corrected = torch.where((batch_labels == 255), logp1.argmax(1), batch_labels)
        return corrected.data

    def valid_epoch(self, model, loader):
        tqdm_loader = tqdm(loader)
        current_loss_mean = 0.
        for idx, batch in enumerate(tqdm_loader):
            image, target = batch['image'], batch['label'].cuda()
            with torch.no_grad():
                pred = self.batch_valid(model, image)
            loss = self.loss_fn(pred, target)
            current_loss_mean = (current_loss_mean * idx + loss.item()) / (idx + 1)

            pred = pred.data.cpu().numpy()
            target = target.cpu().numpy()
            pred = np.argmax(pred, axis=1)
            self.evaluator.add_batch(target, pred)

            tqdm_loader.set_description(f'loss: {current_loss_mean:.4f}')

        return current_loss_mean

    def batch_valid(self, model, batch_imgs):
        batch_imgs = batch_imgs.to(device=self.device)
        batch_pred = model(batch_imgs)
        return batch_pred

    def process_summary(self, valid_loss):
        Loss = valid_loss
        Acc = self.evaluator.Pixel_Accuracy()
        MIoU = self.evaluator.Mean_Intersection_over_Union()
        FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()

        epoch_summary = pd.DataFrame(
            data=[[self.epoch, Loss, MIoU, Acc, FWIoU]],
            columns=['epoch', 'Loss', 'MIoU', 'Acc', 'FWIoU']
        )

        if self.distrib_config['LOCAL_RANK'] == 0:
            self.logger.info(f'Epoch {self.epoch}: \t Loss: {Loss:.6f}, MIoU: {MIoU:.6f},'
                             f' Acc: {Acc:.6f}, FWIoU: {FWIoU:.6f}')
            self.tb_logger.add_scalar('Valid/Loss', Loss, self.epoch)
            self.tb_logger.add_scalar('Valid/MIoU', MIoU, self.epoch)
            self.tb_logger.add_scalar('Valid/Acc', Acc, self.epoch)
            self.tb_logger.add_scalar('Valid/FWIoU', FWIoU, self.epoch)

            if not self.summary_file.is_file():
                epoch_summary.to_csv(self.summary_file, index=False)
            else:
                summary = pd.read_csv(self.summary_file)
                summary = summary.append(epoch_summary)
                summary.to_csv(self.summary_file, index=False)

        return MIoU

    @staticmethod
    def get_state_dict(model):
        if type(model) == torch.nn.DataParallel:
            state_dict = model.module.state_dict()
        else:
            state_dict = model.state_dict()
        return state_dict

    def post_processing(self, score, model):
        if score > self.best_score:
            self.best_score = score
            self.best_epoch = self.epoch
            if self.distrib_config['LOCAL_RANK'] == 0:
                torch.save(self.get_state_dict(model), self.best_checkpoint_path)
                self.logger.info('best model: {} epoch - {:.5}'.format(self.epoch, score))

        if self.distrib_config['LOCAL_RANK'] == 0:
            if self.score_heap[0][0] < score:
                checkpoints_history_path = Path(
                    self.checkpoints_history_folder,
                    '{}_epoch{}.pth'.format(self.calculation_name, self.epoch)
                )
                torch.save(self.get_state_dict(model), checkpoints_history_path)
                heapq.heappush(self.score_heap, (score, checkpoints_history_path))
                if len(self.score_heap) > self.checkpoints_topk:
                    _, removing_checkpoint_path = heapq.heappop(self.score_heap)
                    removing_checkpoint_path.exists() and removing_checkpoint_path.unlink()
                    self.logger.info('Removed checkpoint is {}'.format(removing_checkpoint_path))

        if self.scheduler.__class__.__name__ == 'ReduceLROnPlateau':
            self.scheduler.step(score)
        else:
            self.scheduler.step()

    def inference(self, model, loader):
        empty_cuda_cache()
        state_dict = torch.load(self.best_checkpoint_path)
        model.load_state_dict(state_dict)
        tqdm_loader = tqdm(loader)
        for idx, batch in enumerate(tqdm_loader):
            with torch.no_grad():
                batch_imgs = batch[0].to(device=self.device, non_blocking=True)
                batch_pred = model(batch_imgs).cpu()
                batch_pred = F.softmax(batch_pred, dim=1)[:, 1, ...]
                for pred_idx, pred in enumerate(batch_pred):
                    save_image(pred, self.checkpoints_history_folder / f'{pred_idx}.tif')

    def run_train(self, model, train_dataloader, valid_dataloader):
        # pdb.set_trace()
        model.to(self.device)
        # model, self.optimizer = amp.initialize(model, self.optimizer, opt_level='O1')
        for self.epoch in range(self.n_epoches):
            if self.distrib_config['LOCAL_RANK'] == 0:
                self.logger.info(f'Epoch {self.epoch}: \t start training....')
                self.evaluator.reset()
            model.train()
            train_loss_mean = self.train_epoch(model, train_dataloader)
            if self.distrib_config['LOCAL_RANK'] == 0:
                self.logger.info(f'Epoch {self.epoch}: \t Calculated train loss: {train_loss_mean:.5f}')
                self.tb_logger.add_scalar('Train/Loss', train_loss_mean)

            if self.distrib_config['LOCAL_RANK'] == 0:
                self.logger.info(f'Epoch {self.epoch}: \t start validation....')
            model.eval()
            valid_loss = self.valid_epoch(model, valid_dataloader)
            selected_score = self.process_summary(valid_loss)

            self.post_processing(selected_score, model)

            if self.epoch - self.best_epoch > self.early_stopping:
                if self.distrib_config['LOCAL_RANK'] == 0:
                    self.logger.info('EARLY STOPPING')
                break

        if self.distrib_config['LOCAL_RANK'] == 0:
            self.tb_logger.close()

        # self.inference(model, valid_dataloader)
        empty_cuda_cache()
        return self.best_epoch, self.best_score


#!/usr/bin/python
# -*- coding: UTF-8 -*-

import pandas as pd
import talib
from basic.stock import Stock

class ETLHandler:

    def __init__(self, stock):
        self._stock = stock;

    # 清洗分红数据
    def _clean_bonus(self):
        bonusData = self._stock.read(fold = Stock.FOLD_SRC, file_name=Stock.BONUS_DIVIDENDS);
        bonusData = bonusData[bonusData['fhgk.pxr'] != '--'];
        bonusData.drop(["fhgk.dshrq", "fhgk.gddhrq", "fhgk.ssrq", "fhgk.fhfasm", "fhgk.gqdjr", "fhgk.fajd", "fhgk.glzfl", "fhgk.fhl"], axis=1, inplace=True);
        self._stock.write(bonusData, fold = Stock.FOLD_ETL, file_name=Stock.BONUS_DIVIDENDS);
        return ;

    # 清洗K线数据
    def _clean_kl(self):
        stockData = self._stock.read(fold=Stock.FOLD_SRC, file_name=Stock.KL_PRICE_DAY);
        stockData = stockData[stockData['Volume'] != 0];
        stockData.drop(["Adj Close"], axis=1, inplace=True);
        self._stock.write(stockData, fold = Stock.FOLD_ETL, file_name=Stock.KL_PRICE_DAY);
        return ;

    def _clean(self):
        self._clean_bonus();
        self._clean_kl();
        return ;

    def _merge(self):
        bonusData = self._stock.read(fold = Stock.FOLD_ETL, file_name=Stock.BONUS_DIVIDENDS);
        stockData = self._stock.read(fold = Stock.FOLD_ETL, file_name=Stock.KL_PRICE_DAY);

        # 连接数据
        stockData = stockData.merge(bonusData, how="left", left_on="Date", right_on="fhgk.pxr");
        stockData["fhgk.js"].fillna(10, inplace=True);
        stockData["fhgk.pxs"].fillna(0, inplace=True);
        stockData["fhgk.sgs"].fillna(0, inplace=True);
        stockData["fhgk.zgs"].fillna(0, inplace=True);
        stockData["fhgk.pgs"].fillna(0, inplace=True);

        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_CURRENT, file_name=Stock.KL_PRICE_DAY);
        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_FORWARD, file_name=Stock.KL_PRICE_DAY);
        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_BACKWARD, file_name=Stock.KL_PRICE_DAY);
        return;

    def _reorg(self, dataframe):
        dataframe["Open"] = dataframe["AdjOpen"];
        dataframe["High"] = dataframe["AdjHigh"];
        dataframe["Low"] = dataframe["AdjLow"];
        dataframe["Close"] = dataframe["AdjClose"];

        dataframe.drop(["AdjOpen", "AdjHigh", "AdjLow", "AdjClose"], axis=1, inplace=True);
        return;

    # 向前精确复权
    def _forwardPrice(self, curPriceSeries, prePriceSeries, preAdjPriceSeries, preRow):
        preAdjPrice = preAdjPriceSeries.values[0];
        curPrice = curPriceSeries.values[0];
        prePrice = prePriceSeries.values[0];

        curAdjPrice = preAdjPrice / ((prePrice * (1 + (preRow["fhgk.sgs"] + preRow["fhgk.zgs"]) / preRow["fhgk.js"]) +
                                      preRow["fhgk.pxs"] / preRow["fhgk.js"]) / curPrice);
        return curAdjPrice;

    def _forwardAdjust(self):
        stockData = self._stock.read(fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_FORWARD, file_name=Stock.KL_PRICE_DAY);
        stockData['AdjOpen'] = pd.Series(index=stockData.index);
        stockData['AdjHigh'] = pd.Series(index=stockData.index);
        stockData['AdjLow'] = pd.Series(index=stockData.index);
        stockData['AdjClose'] = pd.Series(index=stockData.index);

        i = 0;
        while (i <= stockData.__len__() - 1):
            row = stockData.iloc[[i]];
            if (i == 0):
                stockData.set_value(i, "AdjOpen", row["Open"]);
                stockData.set_value(i, "AdjHigh", row["High"]);
                stockData.set_value(i, "AdjLow", row["Low"]);
                stockData.set_value(i, "AdjClose", row["Close"]);
            else:
                preRow = stockData.iloc[[i - 1]];
                curRowAdjOpen = self._forwardPrice(row.Open, preRow.Open, preRow.AdjOpen, preRow);
                curRowAdjHigh = self._forwardPrice(row.High, preRow.High, preRow.AdjHigh, preRow);
                curRowAdjLow = self._forwardPrice(row.Low, preRow.Low, preRow.AdjLow, preRow);
                curRowAdjClose1 = self._forwardPrice(row.Close, preRow.Close, preRow.AdjClose1, preRow);

                stockData.set_value(i, "AdjOpen", curRowAdjOpen);
                stockData.set_value(i, "AdjHigh", curRowAdjHigh);
                stockData.set_value(i, "AdjLow", curRowAdjLow);
                stockData.set_value(i, "AdjClose", curRowAdjClose1);

            i = i + 1;

        self._reorg(stockData);

        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_FORWARD, file_name=Stock.KL_PRICE_DAY);
        return;

    # 向后精确复权
    def _backwardPrice(self, curPriceSeries, prePriceSeries, preAdjPriceSeries, curRow):
        preAdjPrice = preAdjPriceSeries.values[0];
        curPrice = curPriceSeries.values[0];
        prePrice = prePriceSeries.values[0];
        curAdjPrice = preAdjPrice * ((curPrice * (
        1 + (curRow["fhgk.sgs"] + curRow["fhgk.zgs"] + curRow["fhgk.pgs"]) / curRow["fhgk.js"]) + curRow["fhgk.pxs"] /
                                      curRow["fhgk.js"]) / prePrice);
        return curAdjPrice;

    def _backwardAdjust(self):
        # calculatesort_values
        stockData = self._stock.read(fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_BACKWARD, file_name=Stock.KL_PRICE_DAY);
        stockData['AdjOpen'] = pd.Series(index=stockData.index);
        stockData['AdjHigh'] = pd.Series(index=stockData.index);
        stockData['AdjLow'] = pd.Series(index=stockData.index);
        stockData['AdjClose'] = pd.Series(index=stockData.index);

        i = stockData.__len__() - 1;
        while (i >= 0):
            row = stockData.iloc[[i]];
            if (i == stockData.__len__() - 1):
                stockData.set_value(i, "AdjOpen", row["Open"]);
                stockData.set_value(i, "AdjHigh", row["High"]);
                stockData.set_value(i, "AdjLow", row["Low"]);
                stockData.set_value(i, "AdjClose", row["Close"]);
            else:
                preRow = stockData.iloc[[i + 1]];
                curRowAdjOpen = self._backwardPrice(row.Open, preRow.Open, preRow.AdjOpen, row);
                curRowAdjHigh = self._backwardPrice(row.High, preRow.High, preRow.AdjHigh, row);
                curRowAdjLow = self._backwardPrice(row.Low, preRow.Low, preRow.AdjLow, row);
                curRowAdjClose1 = self._backwardPrice(row.Close, preRow.Close, preRow.AdjClose1, row);

                stockData.set_value(i, "AdjOpen", curRowAdjOpen);
                stockData.set_value(i, "AdjHigh", curRowAdjHigh);
                stockData.set_value(i, "AdjLow", curRowAdjLow);
                stockData.set_value(i, "AdjClose", curRowAdjClose1);

            i = i - 1;

        self._reorg(stockData);

        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=Stock.KL_ADJ_BACKWARD, file_name=Stock.KL_PRICE_DAY);
        return;

    def _adjust(self):
        self._forwardAdjust();
        self._backwardAdjust();
        return;

    # 计算MA
    def _ma(self, stockData, ma_list):
        # ma_list = [5, 10, 20, 30, 60];

        # calculate MA
        for ma in ma_list:
            stockData['MA_' + str(ma)] = pd.rolling_mean(stockData['Close'], ma);

        return;

    # 计算EMA(ExpMA)
    def _ema(self, stockData, ma_list):
        # ma_list = [5, 10, 20, 30, 60];
        for ma in ma_list:
            stockData['EMA_' + str(ma)] = pd.ewma(stockData['Close'], span=ma)
        return;

    # 计算 BOLL
    def _boll(self, stockData, n):
        sd = pd.rolling_std(stockData['Close'], n);
        stockData['BOLL_MID'] = pd.rolling_mean(stockData['Close'], n);
        stockData['BOLL_UP'] = stockData['BOLL_MID'] + 2 * sd;
        stockData['BOLL_DOWN'] = stockData['BOLL_MID'] - 2 * sd;
        return;

    def _macd(self, stockData, fastPeriod, slowPeriod, signalPeriod):
        macdDIFF, macdDEA, macd = talib.MACDEXT(stockData['Close'].values, fastperiod=fastPeriod, fastmatype=1,
                                                slowperiod=slowPeriod, slowmatype=1,
                                                signalperiod=signalPeriod, signalmatype=1);
        macd = macd * 2;
        stockData['DIF_' + str(fastPeriod) + '_' + str(slowPeriod) + '_' + str(signalPeriod)] = macdDIFF;
        stockData['DEA_' + str(fastPeriod) + '_' + str(slowPeriod) + '_' + str(signalPeriod)] = macdDEA;
        stockData['BAR_' + str(fastPeriod) + '_' + str(slowPeriod) + '_' + str(signalPeriod)] = macd;
        return;

    def _kdj(self, stockData, signalPeriod):
        lowList = pd.rolling_min(stockData['Low'], signalPeriod);
        lowList.fillna(value=pd.expanding_min(stockData['Low']), inplace=True);
        highList = pd.rolling_max(stockData['High'], signalPeriod);
        highList.fillna(value=pd.expanding_max(stockData['High']), inplace=True);
        rsv = (stockData['Close'] - lowList) / (highList - lowList) * 100;

        stockData['KDJ_K'] = pd.ewma(rsv, com=2);
        stockData['KDJ_D'] = pd.ewma(stockData['KDJ_K'], com=2);
        stockData['KDJ_J'] = 3 * stockData['KDJ_D'] - 2 * stockData['KDJ_K'];
        return;

    def _statistic(self, adjust_type):
        stockData = self._stock.read(fold=Stock.FOLD_ETL, type=adjust_type, file_name=Stock.KL_PRICE_DAY);
        self._ma(stockData, ma_list= [5, 10, 20, 30, 60]);
        self._ema(stockData, ma_list=[5, 10, 20, 30, 60]);
        self._boll(stockData, n=20);
        self._macd(stockData, fastPeriod=12, slowPeriod=26, signalPeriod=9);
        self._kdj(stockData, signalPeriod=9);

        self._stock.write(stockData, fold=Stock.FOLD_ETL, type=adjust_type, file_name=Stock.KL_PRICE_DAY);
        return;

    def handle(self):
        self._clean();
        self._merge();
        self._adjust();
        self._statistic(Stock.KL_ADJ_CURRENT);
        self._statistic(Stock.KL_ADJ_FORWARD);
        self._statistic(Stock.KL_ADJ_BACKWARD);
        return;


# 1. Write a program to take the input of  number of seconds from the user, and  convert the number of seconds to hours, minutes, and seconds.

# For example, if seconds = 8000, then the output should be as follows:

# Hours=2

# Minutes=13

# Seconds=20

second = int(input("How many seconds? "))
hours = 0
minute = 0

while (second > 3600):
    hours += 1
    second -= 3600
6
while(second > 60):
    minute += 1
    second -= 60

print("hours = {}".format(hours))
print("minute = {}".format(minute))
print("second = {}".format(second))

# 2. Suppose you want to deposit a certain amount of money into a savings account and leave it alone to draw interest for the next 10 years. At the end of 10 years, you would like to have $10,000 in the account. How much do you need to deposit today to make that happen? Rate of interest is 2.99. You can use the following formula to find out:

p = round(10000/((1+0.0299)**10), 2)
print("you need to deposit ${}".format(p))
# The terms in the formula are as follows:

# P is the present value, or the amount that you need to deposit today.

# F is the future value that you want in the account. (In this case, F is $10,000.)

# r is the annual interest rate.

# n is the number of years that you plan to let the money sit in the account.


import argparse
import os
# Initialize parser
parser = argparse.ArgumentParser()

parser.add_argument("-i", "--input_vocab_file", help = "Vocab file from which constituent compound mapping will be extracted",required=True)
parser.add_argument("-o","--output_mapping_file",help = "Constituent compound Mapping file",required=True)
# Read arguments from command line
args = parser.parse_args()

with open(args.input_vocab_file,'r') as file:
        vocab_words = file.readlines()

constituent_compound_mapping_dict = dict()
#downloading jar for splitting the compounds
jar_download_cmd = 'wget https://repo1.maven.org/maven2/de/danielnaber/jwordsplitter/4.4/jwordsplitter-4.4.jar'
os.system(jar_download_cmd)
# jwordsplitter allows max 70 characters
vocab_words = list(filter(lambda x : len(x) < 70,vocab_words))
with open('filtered_vocab_words','w') as file:
     for word in vocab_words:
         file.write(word+"\n")
#running jword splitter on the vocab file
run_splitter_cmd =  'java -jar jwordsplitter-4.4.jar filtered_vocab_words > splitted_words'
os.system(run_splitter_cmd)
with open('splitted_words','r') as file:
      splitted_words = file.readlines()
for i in range(len(vocab_words)):
      if ',' in splitted_words[i]:
          constituents = [ word.strip() for word in splitted_words[i].split(',') ]
          for constituent in constituents:
               compounds = constituent_compound_mapping_dict.get(constituent)
               if compounds == None:
                  constituent_compound_mapping_dict[constituent] = [vocab_words[i].strip()]
               else:
                  compounds.append(vocab_words[i].strip())

os.system('rm -rf jwordsplitter-4.4.jar')
os.system('rm -rf splitted_words')
os.system('rm -rf filtered_vocab_words')

with open(args.output_mapping_file,'w') as file:
        lines = []
        for constituent,compounds in constituent_compound_mapping_dict.items():
                        line = constituent+" "
                        line = line + " ".join(compounds)
                        lines.append(line+"\n")
        file.writelines(lines)

print("Constituent Compound Mapping file created.")


def encrypt(text,key):
    data = ''
    text = text[::-1]
    print(text)
    for i in range(len(text)):
        char = text[i]
        if (char.isupper()):
           data += chr((ord(char)+key-65) %26 +65)

        else:
           data += chr((ord(char)+key-97)%26+97)
    return(data)
sample = input("Enter Plain text : ")
key1 = int(input("Enter key : "))
print("Ciper test is",encrypt(sample,key1))


# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# This file was copied from project [https://gitee.com/mindspore/models/tree/master/official/cv/PointNet2]
"""network definition"""

import mindspore.nn as nn
import mindspore.ops as P
from mindspore.nn.loss.loss import LossBase
from mindspore.ops import functional as F

from src.layers import Dense
from src.pointnet2_utils import PointNetSetAbstraction

class PointNet2(nn.Cell):
    """PointNet2"""

    def __init__(self, num_class, normal_channel=False):
        super(PointNet2, self).__init__()
        in_channel = 6 if normal_channel else 3
        self.normal_channel = normal_channel

        self.sa1 = PointNetSetAbstraction(npoint=512, radius=0.2, nsample=32,
                                          in_channel=in_channel, mlp=[64, 64, 128],
                                          group_all=False)
        self.sa2 = PointNetSetAbstraction(npoint=128, radius=0.4, nsample=64,
                                          in_channel=128 + 3, mlp=[128, 128, 256],
                                          group_all=False)
        self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None,
                                          in_channel=256 + 3, mlp=[256, 512, 1024],
                                          group_all=True)

        self.fc1 = Dense(1024, 512)
        self.bn1 = nn.BatchNorm1d(512)
        self.drop1 = nn.Dropout(p=0.4)
        self.fc2 = Dense(512, 256)
        self.bn2 = nn.BatchNorm1d(256)
        self.drop2 = nn.Dropout(p=0.5)
        self.fc3 = Dense(256, num_class)

        self.relu = P.ReLU()
        self.reshape = P.Reshape()
        self.log_softmax = P.LogSoftmax()
        self.transpose = P.Transpose()

    def construct(self, xyz):
        """
        construct method
        """
        if self.normal_channel:
            norm = self.transpose(xyz[:, :, 3:], (0, 2, 1))
            xyz = xyz[:, :, :3]
        else:
            norm = None
        l1_xyz, l1_points = self.sa1(xyz, norm)  # [B, 3, 512], [B, 128, 512]
        l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)  # [B, 3, 128], [B, 256, 128]
        _, l3_points = self.sa3(l2_xyz, l2_points)  # [B, 3, 1], [B, 1024, 1]
        x = self.reshape(l3_points, (-1, 1024))
        x = self.drop1(self.relu(self.bn1(self.fc1(x))))
        x = self.drop2(self.relu(self.bn2(self.fc2(x))))
        x = self.fc3(x)
        x = self.log_softmax(x)
        return x

class NLLLoss(LossBase):
    """NLL loss"""

    def __init__(self, reduction='mean'):
        super(NLLLoss, self).__init__(reduction)
        self.one_hot = P.OneHot()
        self.reduce_sum = P.ReduceSum()

    def construct(self, logits, label):
        """
        construct method
        """
        label_one_hot = self.one_hot(label, F.shape(logits)[-1], F.scalar_to_array(1.0), F.scalar_to_array(0.0))
        loss = self.reduce_sum(-1.0 * logits * label_one_hot, (1,))
        return loss


import bpy, mathutils, math, gpu
from mathutils import Vector, Matrix
from bgl import *
from gpu_extras.batch import batch_for_shader
from ... preferences import get_preferences

# --- SHADER --- #
DRAW_HANDLER = None

class DATA:
    running = False
    obj = None
    obj_name = ""
    points = []

    shader = None

    tri_color = (1,.75,0,.25)
    tri_shader = None
    tri_batch = None
    tri_indices = []

    quad_color = (1,1,1,.25)
    quad_shader = None
    quad_batch = None
    quad_indices = []

    ngon_color = (1,0,0,.25)
    ngon_shader = None
    ngon_batch = None
    ngon_indices = []

    @classmethod
    def reset_data(cls):
        cls.obj = None
        cls.obj_name = ""
        cls.points = []

        cls.shader = gpu.shader.from_builtin('3D_UNIFORM_COLOR')

        cls.tri_batch = None
        cls.tri_indices = []

        cls.quad_batch = None
        cls.quad_indices = []

        cls.ngon_batch = None
        cls.ngon_indices = []

    @staticmethod
    def draw():
        glEnable(GL_LINE_SMOOTH)
        glEnable(GL_BLEND)
        glEnable(GL_DEPTH_TEST)
        glDepthFunc(GL_LESS)

        if DATA.tri_batch:
            DATA.shader.bind()
            DATA.shader.uniform_float('color', DATA.tri_color)
            DATA.tri_batch.draw(DATA.shader)
        if DATA.quad_batch:
            DATA.shader.bind()
            DATA.shader.uniform_float('color', DATA.quad_color)
            DATA.quad_batch.draw(DATA.shader)
        if DATA.ngon_batch:
            DATA.shader.bind()
            DATA.shader.uniform_float('color', DATA.ngon_color)
            DATA.ngon_batch.draw(DATA.shader)

def poly_draw_shader():
    DATA.draw()

def register_shader():
    global DRAW_HANDLER
    if not DRAW_HANDLER:
        DRAW_HANDLER = bpy.types.SpaceView3D.draw_handler_add(poly_draw_shader, tuple(), "WINDOW", "POST_VIEW")
    DATA.running = True

def unregister_shader():
    global DRAW_HANDLER
    if DRAW_HANDLER:
        bpy.types.SpaceView3D.draw_handler_remove(DRAW_HANDLER, "WINDOW")
        DRAW_HANDLER = None
    DATA.running = False

from bpy.app.handlers import persistent
@persistent
def remove_poly_debug_shader(dummy):
    unregister_shader()

# --- DEPSGRAPH --- #

@persistent
def poly_data_updater(scene):

    # Validate
    if DATA.obj_name not in scene.objects:
        unregister_shader()
        DATA.reset_data()
        unregister_deps()
        return

    # New ref of same object
    DATA.obj = scene.objects[DATA.obj_name]

    # Check for updates
    if DATA.obj.mode == 'EDIT':
        DATA.obj.update_from_editmode()

    insert_draw_data(DATA.obj)

def register_deps():
    bpy.app.handlers.depsgraph_update_post.append(poly_data_updater)

def unregister_deps():
    if poly_data_updater in bpy.app.handlers.depsgraph_update_post:
        bpy.app.handlers.depsgraph_update_post.remove(poly_data_updater)

# --- OPERATOR --- #

class HOPS_OT_Poly_Display_Debug(bpy.types.Operator):
    bl_idname = "hops.poly_debug_display"
    bl_label = "Poly Debug Display"
    bl_space_type = 'VIEW_3D'
    bl_region_type = 'WINDOW'
    bl_options = {"REGISTER", "UNDO"}
    bl_description = """Poly Debug Display

    Shows topological shader coloring for derived(actual) geometry.
    Recommended for topological management.

    """

    def invoke(self, context, event):

        # Validate
        if not context.active_object: return {"FINISHED"}
        if context.active_object.type != 'MESH': return {"FINISHED"}

        obj = context.active_object
        obj.hops.is_poly_debug_display = not obj.hops.is_poly_debug_display

        # Turn off drawing
        if not obj.hops.is_poly_debug_display:
            unregister_obj()
            bpy.ops.hops.display_notification(info='Disabled Poly Debug Display')
            return {"FINISHED"}

        register_obj(obj)
        bpy.ops.hops.display_notification(info='Enabled Poly Debug Display')
        return {"FINISHED"}

# --- SETUP / CLEAR --- #

def register_obj(obj):
    insert_draw_data(obj)
    register_shader()
    register_deps()

def unregister_obj():
    unregister_shader()
    unregister_deps()
    DATA.reset_data()

# --- UTILS --- #

def insert_draw_data(obj):

    mat = obj.matrix_world
    mesh = obj.data
    mesh.calc_loop_triangles()

    DATA.reset_data()
    DATA.obj_name = obj.name
    DATA.points = [mat @ (v.co + v.normal * .002) for v in mesh.vertices]

    # --- Tris --- #
    tris = [poly for poly in mesh.polygons if len(poly.vertices) == 3]
    indexes = [poly.index for poly in tris]
    DATA.tri_indices = []
    DATA.tri_batch = None

    for triangle in mesh.loop_triangles:
        if triangle.polygon_index in indexes:
            DATA.tri_indices.append([mesh.vertices[v].index for v in triangle.vertices])

    if DATA.tri_indices and DATA.points:
        DATA.tri_batch = batch_for_shader(DATA.shader, 'TRIS', {'pos': DATA.points}, indices=DATA.tri_indices)

    # --- Quads --- #
    quads = [poly for poly in mesh.polygons if len(poly.vertices) == 4]
    indexes = [poly.index for poly in quads]
    DATA.quad_indices = []
    DATA.quad_batch = None

    for triangle in mesh.loop_triangles:
        if triangle.polygon_index in indexes:
            DATA.quad_indices.append([mesh.vertices[v].index for v in triangle.vertices])

    if DATA.quad_indices and DATA.points:
        DATA.quad_batch = batch_for_shader(DATA.shader, 'TRIS', {'pos': DATA.points}, indices=DATA.quad_indices)

    # --- Ngons --- #
    ngons = [poly for poly in mesh.polygons if len(poly.vertices) > 4]
    indexes = [poly.index for poly in ngons]
    DATA.ngon_indices = []
    DATA.ngon_batch = None

    for triangle in mesh.loop_triangles:
        if triangle.polygon_index in indexes:
            DATA.ngon_indices.append([mesh.vertices[v].index for v in triangle.vertices])

    if DATA.ngon_indices and DATA.points:
        DATA.ngon_batch = batch_for_shader(DATA.shader, 'TRIS', {'pos': DATA.points}, indices=DATA.ngon_indices)


# coding: utf-8
import time
from time import strftime
from sqlalchemy import func
from pyQwSciLib import db
from pyQwSciLib import app
import datetime

# TODO default time
class EGOVProvinceCityInfo(db.Model):
    __bind_key__ = "mysql"
    __tablename__ = "egov_province_city_info"
    id = db.Column(db.INTEGER, primary_key=True)
    city = db.Column(db.String(20))
    province = db.Column(db.String(20))
    province_abbreviation = db.Column(db.String(10))  # 省份简称，地图用
    type = db.Column(db.INTEGER)  # 1 省/国务院， 2 市/部委
    remarks = db.Column(db.INTEGER)
    created_time = db.Column(db.TIMESTAMP)

    @staticmethod
    def query_province_info():
        """
        返回省级行政单位信息
        :return:
        """
        res = EGOVProvinceCityInfo.query.filter(
            EGOVProvinceCityInfo.type == 1
        ).all()
        ret = []
        for item in res:
            ret.append((item.id, item.city))
        return tuple(ret)

class EGOVPolicyPaper(db.Model):
    __bind_key__ = "mysql"
    __tablename__ = 'egov_policy_papers'
    id = db.Column(db.INTEGER, primary_key=True)
    province_name = db.Column(db.String(20))
    city_name = db.Column(db.String(20))
    title = db.Column(db.TEXT)
    time = db.Column(db.String(20))
    url = db.Column(db.TEXT)
    year = db.Column(db.INTEGER)
    province_id = db.Column(db.INTEGER)
    city_id = db.Column(
        db.INTEGER,
        db.ForeignKey('egov_province_city_info.id')
    )
    type = db.Column(db.INTEGER)
    have_file = db.Column(db.INTEGER)
    file_content = db.Column(db.BLOB)
    file_type = db.Column(db.String(10))
    created_time = db.Column(db.TIMESTAMP)

    def __init__(self, item):
        self.id = item[0]
        self.province_name = item[1]
        self.city_name = item[2]
        self.title = item[3]
        self.time = item[4]
        self.url = item[5]
        self.year = item[6]
        self.province_id = item[7]
        self.city_id = item[8]
        self.type = item[9]
        self.have_file = item[10]
        self.file_content = item[11]
        self.file_type = item[12]
        self.created_time = item[13]

    @staticmethod
    def group_by_year(province):
        """
        返回年份的统计信息
        :param province:
        :return:
        """
        query = db.session.query(
            EGOVPolicyPaper.year,
            func.count(EGOVPolicyPaper.id)
        )
        if not province == "all":
            query = query.filter(
                EGOVPolicyPaper.province_id == province
            )
        res = query.group_by(
                EGOVPolicyPaper.year
        ).all()
        return res

    @staticmethod
    def group_by_province(province, year):
        """
        按照省份分组
        :param province:
        :param year:
        :return:
        """
        if province == "all":  # 全部分组
            query = db.session.query(
                EGOVPolicyPaper.province_name,
                func.count(EGOVPolicyPaper.id)
            )
            if not year == "all":  # 查询具体年份
                query = query.filter(
                    EGOVPolicyPaper.year == year
                )
            query = query.group_by(
                EGOVPolicyPaper.province_name
            )
        else:  # 按照省份分组
            query = db.session.query(
                EGOVPolicyPaper.city_name,
                func.count(EGOVPolicyPaper.id)
            ).filter(
                EGOVPolicyPaper.province_id == province
            )
            if not year == "all":  # 查询具体年份
                query = query.filter(
                    EGOVPolicyPaper.year == year
                )
            query = query.group_by(
                EGOVPolicyPaper.city_name
            )
        res = query.order_by(
                func.count(EGOVPolicyPaper.id)
            ).all()
        return res

    @staticmethod
    def group_by_province_abbreviation():
        query = db.session.query(
            EGOVProvinceCityInfo.province_abbreviation,
            func.count(EGOVPolicyPaper.id)
        ).join(
            EGOVPolicyPaper
        ).filter(
            EGOVProvinceCityInfo.province_abbreviation != '0'
        ).group_by(
            EGOVProvinceCityInfo.province_abbreviation
        ).order_by(
            func.count(EGOVPolicyPaper.id)
        )
        res = query.all()
        return res

    @staticmethod
    def query_paper_items_by_province_and_year(province, year):
        """
        根据省份和年份 返回文件信息列表
        :param province:
        :param year
        :return:
        """
        query = db.session.query(
            EGOVPolicyPaper.id,
            EGOVPolicyPaper.title
        ).filter(
            EGOVPolicyPaper.have_file == 1
        )
        if province is None or province == u"all" or province == u"null":
            if year is None or year == u"all" or year == u"null":
                res = query.limit(100).all()
            else:  # year is specified
                res = query.filter(
                    EGOVPolicyPaper.year == year
                ).all()
        else:  # province is specified
            if year is None or year == u"all" or year == u"null":
                res = query.filter(
                    EGOVPolicyPaper.province_id == province
                ).all()
            else:
                res = query.filter(
                    EGOVPolicyPaper.province_id == province,
                    EGOVPolicyPaper.year == year
                ).all()
        return res

    @staticmethod
    def execute_raw_sql(sql):
        res = db.session.execute(
            sql, bind=db.get_engine(app, bind="mysql")
        ).fetchall()
        # res = db.session.execute(sql).fetchall()
        ret = []
        for item in res:
            paper_item = EGOVPolicyPaper(item)
            ret.append(paper_item)
        return ret

class EGOVPolicyPapersAnalysis(db.Model):
    __bind_key__ = "mysql"
    # __tablename__ = "egov_policy_papers_analysis"
    __tablename__ = "egov_policy_papers_analysis_copy"
    paper_id = db.Column(db.INTEGER,
                         db.ForeignKey('egov_policy_papers.id'),
                         primary_key=True)
    key_words = db.Column(db.TEXT)
    words_cloud = db.Column(db.TEXT)
    created_time = db.Column(
        db.TIMESTAMP,
        default=datetime.datetime.now()
    )

    def __init__(self, paper_id, key_words):
        self.paper_id = paper_id
        self.key_words = key_words

    def save(self):
        db.session.add(self)
        db.session.commit()

    @staticmethod
    def update():
        db.session.commit()

    @staticmethod
    def query_by_province_and_year(province, year):
        """
        根据省份和年份分组
        :param province:
        :param year:
        :return:
        """
        query = EGOVPolicyPapersAnalysis.query.join(EGOVPolicyPaper)
        if province == "all":  # province is not specified
            if year == "all":
                res = query.all()
            else:  # year is specified
                res = query.filter(
                    EGOVPolicyPaper.year == int(year)
                ).all()
        else:  # province is specified
            if year == "all":
                res = query.filter(
                    EGOVPolicyPaper.province_id == province
                ).all()
            else:  # year is specified
                res = query.filter(
                    EGOVPolicyPaper.province_id == province,
                    EGOVPolicyPaper.year == int(year)
                ).all()
        ret = []
        for item in res:
            ret.append((item.paper_id, item.key_words))
        return tuple(ret)

    @staticmethod
    def group_by_year(province):
        query = db.session.query(
            EGOVPolicyPaper.id,
            EGOVPolicyPapersAnalysis.key_words,
            EGOVPolicyPaper.year
        ).join(
            EGOVPolicyPapersAnalysis
        )
        if province == "all":  # province is not specified
            pass
        else:
            query = query.filter(
                EGOVPolicyPaper.province_id == province
            )
        res = query.order_by(EGOVPolicyPaper.year).all()
        return res

    @staticmethod
    def query_word_cloud_by_paper_id(paper_id):
        img_data = db.session.query(
            EGOVPolicyPapersAnalysis.words_cloud
        ).filter(
            EGOVPolicyPapersAnalysis.paper_id == paper_id
        ).first()
        return img_data[0]


# -*- coding:utf-8 -*-
__author__ = 'wgz'
__date__ = '2018/1/16 19:20'

import scrapy
from .. import items

class neihanSpider(scrapy.Spider):
    # 在命令行被调用的名称
    name = 'nhspider'
    # 限制只爬取这个域名下的数据
    allowed_domains = ['neihanshequ.com']
    # 爬取的url地址
    start_urls = (
        'http://neihanshequ.com/joke/?is_json=1&app_name=neihanshequ_web&max_time=1516101542.0',
    )

    def parse(self, response):
        print('********************************************')
        import json, time
        neihan_json = json.loads(response.text)['data']['data']
        # print(neihan_json)
        # print(json.loads(response.text)['data']['data'][0]['group']['text'])
        for i in range(20):
            username = neihan_json[i]['group']['user']['name']
            create_time = neihan_json[i]['group']['create_time']
            createtime = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(create_time))
            content = neihan_json[i]['group']['text']

            # print username
            # print create_time
            # print content

            item = items.neihanItem()
            item['username'] = username
            item['createtime'] = createtime
            item['content'] = content
            #
            # yield item
            return items