import numpy as np
import csv
import matplotlib.pyplot as plt
import os
from PIL import Image
from numpy import asarray
import copy
import math
#mwuahhahahahha it works! :)))

#l2 regularization
#dropout
#mini-batch
def main():
    showcost = 200
    #hyperparameters loading data:
    maxDataTest = 100
    imageSize = 120
    maxData = 600

    #hyperparameters network
    mini_batch_size = 200
    keepProb = 1
    lambd = 0.05 #between 0 - 0.1 the bigger lambda the bigger penalty on complicated weights
    layerDims = [imageSize * imageSize * 3, 400, 300, 200, 10,50,30,1]
    iterations = 2000
    learningRate = 0.001

    Xtrain, Ytrain = loadImages("train", imageSize, maxData)
    Xtrain, Ytrain = hussle(Xtrain, Ytrain) #for final training error
    mini_batches = random_mini_batches(Xtrain,Ytrain, mini_batch_size)
    Xtest, Ytest = loadImages("test", imageSize, maxDataTest)
    Xtest, Ytest = hussle(Xtest, Ytest)

    parameters = trainmodel(layerDims, mini_batches, iterations, learningRate, showcost, lambd, Xtest, Ytest, keepProb)
    trainingPrediction = makePrediction(Xtrain, parameters) #no dropout for test time
    testPrediction = makePrediction(Xtest, parameters) #no dropout for test time
    print("trainingAccuracy", accuracy(trainingPrediction, Ytrain))
    print("testAccuracy", accuracy(testPrediction, Ytest))

def random_mini_batches(X,Y, mini_batch_size):
    m = X.shape[1]
    mini_batches = []
    shuffled_X, shuffled_Y = hussle(X,Y)
    inc = mini_batch_size
    num_complete_minibatches = math.floor(m/mini_batch_size)
    for k in range(0, num_complete_minibatches):
        mini_batch_X = shuffled_X[:, k*inc:(k+1) * inc]
        mini_batch_Y = shuffled_Y[:, k*inc:(k+1) * inc]
        mini_batch = (mini_batch_X, mini_batch_Y)
        mini_batches.append(mini_batch)
    #handling non-complete mini_batch at the end:
    if m % mini_batch_size != 0:
        mini_batch_X = shuffled_X[:, num_complete_minibatches * mini_batch_size:]
        mini_batch_Y = shuffled_Y[:, num_complete_minibatches * mini_batch_size:]
        mini_batch = (mini_batch_X, mini_batch_Y)
        mini_batches.append(mini_batch)
    return mini_batches

def loadImages(folder, imageSize, maxData):
    catData = np.zeros((imageSize * imageSize * 3,1))
    counter = 0
    Y = np.full((1, maxData), int(1))
    for images in os.listdir(folder):
        #add 200 cat pictures
        if counter >= maxData:
            break
        counter +=1
        imageVector = processImage(images, folder, imageSize)
        catData = np.append(catData, imageVector, axis = 1)
    catData = catData[:,1:]
    catData = np.append(catData, Y, axis = 0)

    Y = np.full((1,maxData), int(0))
    dogData = np.zeros((imageSize * imageSize * 3,1))
    counter = 0
    for images in reversed(os.listdir(folder)):
        #adds 200 dogg pictures
        if counter >= maxData:
            break
        counter +=1
        imageVector = processImage(images, folder, imageSize)
        dogData = np.append(dogData, imageVector, axis = 1)
    dogData = dogData[:,1:]
    dogData = np.append(dogData, Y, axis = 0)

    Data = np.append(dogData, catData, axis = 1) #glue them together horizontally ofcourse

    #now randomize data:
    transposedData = Data.T
    np.random.shuffle(transposedData)
    Data = transposedData.T
    labels = np.array([Data[imageSize * imageSize * 3]])
    examples = np.delete(Data, imageSize * imageSize * 3, 0)
    examples = examples / 255
    return examples, labels

def processImage(images, folder, imageSize):
    #takes image returns one dimensional (vertical) image vector
        image = Image.open(os.path.join(folder, images)).resize((imageSize, imageSize))
        imageVector = asarray(image)
        imageVector = imageVector.reshape(imageVector.shape[0] * imageVector.shape[1] * imageVector.shape[2], 1)
        return imageVector

def hussle(X,Y):
    size = X.shape[0]
    Data = np.append(X, Y, axis = 0)
    transposedData = Data.T
    np.random.shuffle(transposedData)
    Data = transposedData.T
    labels = np.array([Data[size]])
    examples = np.delete(Data, size, 0)
    return examples, labels

def accuracy(prediction, Y):
    prediction[prediction > 0.5] = 1
    prediction[prediction <= 0.5] = 0

    difference = Y-prediction
    incorrect = np. count_nonzero(difference)
    false = incorrect / Y.shape[1] * 100
    accuracy = 100 - false
    return accuracy

def trainmodel(layerDims, mini_batches, iterations, learningRate, showcost, lambd, Xtest, Ytest, keepProb = 1):
    initialShowCost = showcost
    costs = []
    iterationlist = []
    #parameter initialization (xavier)
    initializeX, initializeY = mini_batches[0]
    parameters, Vdw, Vdb, Sdw, Sdb = parameterInitialization(initializeX, layerDims)
    counter = 0
    for i in range(iterations):
        print(i)
        for mini_batch in mini_batches:
            X,Y = mini_batch
            counter +=1
            iterationlist.append(counter)
            AL, forward_cache, D = L_model_forward(X, parameters, keepProb)

            #compute cost (maybe try out L2)
            thiscost = cost(AL, Y, parameters, lambd )
            costs.append(thiscost)
            #backward propagation (with gradient checking)
            grads = L_model_backward(AL, Y, forward_cache, lambd, parameters, keepProb, D)
            #gradCheck:

            #updating weights
            parameters = updateWeights(grads, parameters, layerDims, learningRate)
        if i == showcost:
            plt.title("cost function")
            plt.xlabel("Iterations")
            plt.ylabel("Cost")
            plt.plot(iterationlist, costs, color ="red")
            plt.show()
            showcost += initialShowCost
            trainingPrediction = makePrediction(X, parameters) #no dropout for test time
            testPrediction = makePrediction(Xtest, parameters) #no dropout for test time
            print("trainingAccuracy", accuracy(trainingPrediction, Y))
            print("testAccuracy", accuracy(testPrediction, Ytest))
    return parameters

def cost(A, Y, parameters, lambd):
    m = Y.shape[1]
    losses = -(np.multiply(np.log(A), Y) + np.multiply((1-Y), np.log(1-A)))
    cost = 1/m * np.sum(losses)
    frobeniusNorm = 0
    for l in range(1, int(len(parameters)/2) + 1):
        frobeniusNorm += np.sum(parameters["W" + str(l)] * parameters["W" + str(l)])
    L2_regularizationCost = lambd / (2*m) * frobeniusNorm
    cost += L2_regularizationCost
    return cost #checked

def makePrediction(X, parameters, keepProb = 1):
    AL, _, D = L_model_forward(X, parameters, keepProb)
    return AL

def updateWeights(grads, parameters, layerDims, learningRate):
    m = len(parameters) / 2
    for l in range(1, len(layerDims)):
        parameters["W" + str(l)] = parameters["W" + str(l)] - learningRate * grads["dW" + str(l)]
        parameters["b" + str(l)] = parameters["b" + str(l)] - learningRate * grads["db" + str(l)]
    return parameters

def parameterInitialization(X, layerDims):
    numPreviousNeurons = X.shape[0]
    parameters = {}
    for l in range(1, len(layerDims)):
        parameters["W" + str(l)] = np.random.randn(layerDims[l], layerDims[l-1]) * np.sqrt(1/layerDims[l-1])
        parameters["b" + str(l)] = np.zeros((layerDims[l], 1))
        numPreviousNeurons = layerDims[l]
    Vdw = 0
    Vdb = 0
    Sdw = 0
    Sdb = 0
    return parameters, Vdw, Vdb, Sdw, Sdb

def L_model_forward(X, parameters, keepProb):
    caches = []
    A = X
    #no dropout in input layer
    L = int(len(parameters) / 2)
    D = []
    for l in range(1, L):
        A_prev = A
        A, cache = linearActivationForward(A_prev, parameters["W" + str(l)], parameters["b" + str(l)], "relu")
        #store d one cache ahead for backward prop
        A,d = dropout(A, keepProb)
        D.append(d)
        #linear activation forward returns the next A for the next layer but also returns a cache containing
        #linear (W, A_PREV, b) and activation (Z, "activation" => "relu") cache
        caches.append(cache)
    AL, cache = linearActivationForward(A, parameters["W" + str(L)], parameters["b" + str(L)], "sigmoid")
    #no dropout in output layer
    caches.append(cache)
    return AL, caches, D

def dropout(A, keepProb):
    d = np.random.rand(A.shape[0], A.shape[1]) < keepProb
    A = np.multiply(A,d)
    A /= keepProb
    return A, d

def L_model_backward(AL, Y, caches, lambd, parameters, keepProb, D):
    grads = {}
    L = len(caches)
    m = Y.shape[1]
    dAl = -np.divide(Y, AL) + np.divide(1-Y, 1-AL)
    currentCache = caches[-1]
    dA_prev_temp, dW_temp, db_temp = linear_activation_backward(dAl, currentCache, "sigmoid")
    currentD = D[-1]
    dA_prev_temp = np.multiply(dA_prev_temp, currentD)/keepProb
    #grads["dA" + str(L-1)] = dA_prev_temp
    grads["dW" + str(L)] = dW_temp
    grads["db" + str(L)] = db_temp
    for l in reversed(range(L-1)):
        dA = dA_prev_temp
        currentCache = caches[l]
        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(dA, currentCache, "relu")
        if l != 0:
            #no dropout for input layer when l == 0
            currentD = D[l - 1]
            dA_prev_temp = np.multiply(dA_prev_temp, currentD) / keepProb
        #grads["dA" + str(l)] = dA_prev_temp
        grads["dW" + str(l+1)] = dW_temp + lambd/m * parameters["W" + str(l + 1)]
        grads["db" + str(l+1)] = db_temp
        #the l-1 makes sense, you give in the A of the l+1 in linearactivationbackward,
        #this function will return the previous a which is therefore l,
        #l-1 because you calculate current a with the next weights (see page 5 notes)
        #same logic goes for why L-1 when calculating dA[l-1]
    return grads

def linear_activation_backward(dA, cache, activation):
    linear_cache, activation_cache = cache
    W = linear_cache["W"]
    b = linear_cache["b"]
    A_prev = linear_cache["A_prev"]
    Z = activation_cache["Z"]
    m = Z.shape[1]

    if activation == "relu":
        A = relu(Z)
        dZ = reluBackward(Z, dA)
    elif activation == "tanh":
        A= tanh(Z)
        dZ = tanhBackward(Z, dA)
    elif activation == "sigmoid":
        A = sigmoid(Z)
        dZ = sigmoidBackward(Z, dA)
    dA_prev = np.dot(W.T, dZ)
    dW = 1/m * np.dot(dZ, A_prev.T)
    db = 1/m * np.sum(dZ, axis = 1, keepdims = True)
    return dA_prev, dW, db

def linearActivationForward(A_prev, W, b, activation):
    linear_cache = {
        "A_prev": A_prev,
        "W": W,
        "b": b
        }
    Z = np.dot(W, A_prev) + b
    activation_cache = {
        "Z": Z,
        "activation": activation
        }
    if activation == "tanh":
        A = tanh(Z)
    if activation == "sigmoid":
        A = sigmoid(Z)
    if activation == "relu":
        A = relu(Z)
    caches = (linear_cache, activation_cache)
    return A, caches

def tanh(Z):
    A = (np.exp(Z) - np.exp(-Z)) / (np.exp(Z) + np.exp(-Z))
    return A
def sigmoid(Z):
    A = 1 / (1+np.exp(-Z))
    return A
def relu(Z):
    Z[Z < 0] = 0 #each value beneath 0 will have activation 0, but Z otherwise
    A = Z
    return A

def tanhBackward(Z, dA):
    dZ = (1-np.power(tanh(Z),2)) * dA
    return dZ
def sigmoidBackward(Z, dA):
    dZ = (sigmoid(Z) * (1 - sigmoid(Z)))* dA
    return dZ
def reluBackward(Z, dA):
    Z[Z<0] = 0
    Z[Z>= 0] = 1
    dZ = Z * dA
    return dZ

def loadDataCSV(path):
    file = open(path)
    csvreader = csv.reader(file)
    dataset = []
    count = 0
    for row in csvreader:
           if count == 0:
               count +=1
               continue
           if count == 1:
               count = 2
               dataset = np.array([row])
               continue
           row = np.array([row])
           dataset = np.concatenate((dataset,row), axis = 0)

    trainSet = dataset[:int(len(dataset) * 0.7)].T
    testSet = dataset[int(len(dataset) * 0.7):].T
    Ytrain = trainSet[-1].astype(np.float).reshape(1, trainSet.shape[1])
    Ytest = testSet[-1].astype(np.float).reshape(1, testSet.shape[1])
    trainSet  = np.delete(trainSet, -1, 0)
    testSet = np.delete(testSet, -1, 0)
    Xtrain = trainSet.astype(np.float)
    Xtest = testSet.astype(np.float)
    means = 1 / Xtrain.shape[0] * np.sum(Xtrain, axis = 1, keepdims = True)
    Xtrain = Xtrain /means
    Xtrain = Xtrain / (np.sqrt(1/Xtrain.shape[0] * np.sum(Xtrain, axis =1, keepdims = True)))
    Xtest = Xtest / means
    Xtest = Xtest / (np.sqrt(1/Xtrain.shape[0] * np.sum(Xtrain, axis =1, keepdims = True)))

    return Xtrain, Ytrain, Xtest, Ytest

if __name__ == "__main__":
    main()


import evaluatoare.CardValue as cval
from queue import Queue
import os,sys
sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from backend.game_state import game_state as gs,piece
##############Things from the game code######################################################################################################################################################

##############Things from the game code######################################################################################################################################################

def Calculate_PositionValue(gamestate:gs,player,dezvoltari):
    ############### The variables
    resourceProduction=0
    developmentvalue=0
    LRaLA=0
    VP=0
    #1 Resource Production
    for tile in gamestate.tiles:
        if(tile.index!=gamestate.hottile):
            for piece in tile.pieces:
                if(piece.name=="asezare" and piece.player==player):
                    resourceProduction+=(7-abs(tile.value-7))/7*0.5
                if(piece.name=="oras" and piece.player==player):
                    resourceProduction+=(7-abs(tile.value-7))/7
                if(piece.name=="drum" and piece.player==player):
                    resourceProduction+=(7-abs(tile.value-7))/7*0.1

    #2 Expansion
    expansion=gamestate.constructi[player][0]*2+gamestate.constructi[player][1]+gamestate.constructi[player][0]/2
    #3 Development Cards
    developmentvalue=dezvoltari[player][1]*0.5+dezvoltari[player][2]*0.6+dezvoltari[player][3]*0.8+dezvoltari[player][4]
    #4. Longest Road and Largest Army
    if(gamestate.biggestArmy[0]==player):LRaLA+=2
    if(gamestate.biggestRoad[0]==player):LRaLA+=2
    #5. Victory Points
    VP=dezvoltari[player][0]
    #6. COMBINAREA VARIABILELOR
    rez=0
    rez=resourceProduction+expansion+developmentvalue+LRaLA+VP
    return rez


'''
International Morse Code defines a standard encoding where each letter is mapped to a series of dots and dashes, as follows:

    'a' maps to ".-",
    'b' maps to "-...",
    'c' maps to "-.-.", and so on.

For convenience, the full table for the 26 letters of the English alphabet is given below:

[".-","-...","-.-.","-..",".","..-.","--.","....","..",".---","-.-",".-..","--","-.","---",".--.","--.-",".-.","...","-","..-","...-",".--","-..-","-.--","--.."]

Given an array of strings words where each word can be written as a concatenation of the Morse code of each letter.

    For example, "cab" can be written as "-.-..--...", which is the concatenation of "-.-.", ".-", and "-...". We will call such a concatenation the transformation of a word.

Return the number of different transformations among all words we have.

Example 1:

Input: words = ["gin","zen","gig","msg"]
Output: 2
Explanation: The transformation of each word is:
"gin" -> "--...-."
"zen" -> "--...-."
"gig" -> "--...--."
"msg" -> "--...--."
There are 2 different transformations: "--...-." and "--...--.".

'''
from typing import List
def uniqueMorseRepresentations(words: List[str]) -> int:
    alpha = [chr(i) for i in range(97, 123)]
    code = [".-","-...","-.-.","-..",".","..-.","--.","....","..",".---","-.-",".-..","--","-.","---",".--.","--.-",".-.","...","-","..-","...-",".--","-..-","-.--","--.."]
    uni_code = dict(zip(alpha, code))

    def code(letter):
        result = ""
        for key, value in uni_code.items():
            if key == letter:
                result = result + value
        return result

    def word_to_code(word):
        return "".join([code(letter) for letter in word])

    result = [word_to_code(word) for word in words]
    return len(set(result))

words = ["gin","zen","gig","msg"]
print(uniqueMorseRepresentations(words))


import numpy
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
import pandas
from torch_geometric.data import DataLoader
from sklearn.preprocessing import scale
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from scipy.spatial.distance import correlation
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
from ane.util.data import normalize
from ane.geomagnetic_strom_forecasting.storm_detection import detect_storm

class FCNN(nn.Module):
    def __init__(self, dim_in, dim_out):
        super(FCNN, self).__init__()
        self.fc1 = nn.Linear(dim_in, 128)
        self.bn1 = nn.BatchNorm1d(128)
        self.fc2 = nn.Linear(128, 128)
        self.bn2 = nn.BatchNorm1d(128)
        self.fc3 = nn.Linear(128, dim_out)

    def forward(self, x):
        h = F.relu(self.bn1(self.fc1(x)))
        h = F.relu(self.bn2(self.fc2(h)))
        out = self.fc3(h)

        return out

    def fit(self, data_loader, optimizer, criterion):
        self.train()
        sum_train_losses = 0

        for data, targets in data_loader:
            data = data.cuda()
            targets = targets.cuda()

            preds = self(data)
            loss = criterion(preds, targets)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            sum_train_losses += loss.item()

        return sum_train_losses / len(data_loader)

    def predict(self, data_loader):
        self.eval()
        list_preds = list()

        with torch.no_grad():
            for data, _ in data_loader:
                data = data.cuda()

                preds = self(data)
                list_preds.append(preds)

        return torch.cat(list_preds, dim=0)

class RNNModel(nn.Module):
    def __init__(self, dim_in, dim_hidden, n_layers, len_seq, dim_emb, rnn_model):
        super(RNNModel, self).__init__()
        self.dim_hidden = dim_hidden
        self.n_layers = n_layers
        self.rnn_model = rnn_model

        if self.rnn_model == 'rnn':
            self.rnn = nn.RNN(dim_in, dim_hidden, n_layers, batch_first=True)
        elif self.rnn_model == 'lstm':
            self.rnn = nn.LSTM(dim_in, dim_hidden, n_layers, batch_first=True)
        elif self.rnn_model == 'gru':
            self.rnn = nn.GRU(dim_in, dim_hidden, n_layers, batch_first=True)

        self.bn1 = nn.BatchNorm1d(dim_hidden * len_seq)
        self.fc1 = nn.Linear(dim_hidden * len_seq, dim_emb)

    def forward(self, x):
        h_0 = torch.zeros(self.n_layers, x.shape[0], self.dim_hidden).cuda()

        if self.rnn_model == 'lstm':
            c_0 = torch.zeros(self.n_layers, x.shape[0], self.dim_hidden).cuda()
            z, _ = self.rnn(x, (h_0, c_0))
        else:
            z, _ = self.rnn(x, h_0)

        z = self.bn1(z.reshape(z.shape[0], -1))
        embs = self.fc1(z)

        return embs

class Transformer(nn.Module):
    def __init__(self, dim_in, dim_emb, n_layers=None, len_seq=None):
        super(Transformer, self).__init__()
        self.rnn_model = 'transformer'
        self.n_layers = n_layers
        self.rnn = nn.Transformer(dim_in, 7, 2, 2, 2, 0.05, batch_first=True)
        self.bn1 = nn.BatchNorm1d(dim_in * len_seq)
        self.fc1 = nn.Linear(dim_in * len_seq, dim_emb)

    def forward(self, src, tgt):
        z = self.rnn(src, tgt)
        z = self.bn1(z.reshape(z.shape[0], -1))
        embs = self.fc1(z)

        return embs

def get_seq_dataset(dataset, len_seq):
    seq_dataset = list()
    seq = list()

    for i in range(0, len_seq):
        seq.append(numpy.zeros(dataset.shape[1]))

    for i in range(0, dataset.shape[0]):
        seq.pop(0)
        seq.append(dataset[i, :])
        seq_dataset.append(numpy.hstack(seq))

    return numpy.vstack(seq_dataset)

def get_data_loader(*data, batch_size, shuffle=False, dtype=torch.float):
    tensors = [torch.tensor(d, dtype=dtype) for d in data]
    return DataLoader(TensorDataset(*tuple(tensors)), batch_size=batch_size, shuffle=shuffle)

def get_pos_neg_samples(seq, targets, tgt=None):
    pos_seq = list()
    pos_tgt = list()
    pos_targets = list()
    neg_seq = list()
    neg_tgt = list()
    neg_targets = list()

    for i in range(0, seq.shape[0]):
        idx = random.sample(range(0, seq.shape[0]), 2)

        if torch.norm(targets[i] - targets[idx[0]]) < torch.norm(targets[i] - targets[idx[1]]):
            pos_seq.append(seq[idx[0], :, :].view(1, seq.shape[1], seq.shape[2]))
            pos_targets.append(targets[idx[0]])
            neg_seq.append(seq[idx[1], :, :].view(1, seq.shape[1], seq.shape[2]))
            neg_targets.append(targets[idx[1]])

            if tgt is not None:
                pos_tgt.append(tgt[idx[0], :, :].view(1, tgt.shape[1], tgt.shape[2]))
                neg_tgt.append(tgt[idx[1], :, :].view(1, tgt.shape[1], tgt.shape[2]))
        else:
            pos_seq.append(seq[idx[1], :, :].view(1, seq.shape[1], seq.shape[2]))
            pos_targets.append(targets[idx[1]])
            neg_seq.append(seq[idx[0], :, :].view(1, seq.shape[1], seq.shape[2]))
            neg_targets.append(targets[idx[0]])

            if tgt is not None:
                pos_tgt.append(tgt[idx[1], :, :].view(1, tgt.shape[1], tgt.shape[2]))
                neg_tgt.append(tgt[idx[0], :, :].view(1, tgt.shape[1], tgt.shape[2]))
    if tgt is None:
        return torch.vstack(pos_seq), torch.vstack(pos_targets), torch.vstack(neg_seq), torch.vstack(neg_targets)
    else:
        return torch.vstack(pos_seq), torch.vstack(pos_tgt), torch.vstack(pos_targets),\
               torch.vstack(neg_seq), torch.vstack(neg_tgt), torch.vstack(neg_targets)

def loss_ane(seq, targets, emb_net, tgt=None):
    rand_idx = torch.randperm(seq.shape[0])
    seq_sample = seq[rand_idx, :, :]
    targets_sample = targets[rand_idx]

    if tgt is None:
        emb_data = F.normalize(emb_net(seq), p=2, dim=1)
        emb_sample = F.normalize(emb_net(seq_sample), p=2, dim=1)
    else:
        tgt_sample = tgt[rand_idx, :, :]
        emb_data = F.normalize(emb_net(seq, tgt), p=2, dim=1)
        emb_sample = F.normalize(emb_net(seq_sample, tgt_sample), p=2, dim=1)

    dist_emb = torch.sum((emb_data - emb_sample)**2, dim=1)
    dist_target = torch.sum((targets - targets_sample)**2, dim=1)

    return torch.mean((dist_emb - dist_target)**2)

def train(emb_net, data_loader, optimizer):
    if emb_net.rnn_model == 'transformer':
        return __train_transformer(emb_net, data_loader, optimizer)
    else:
        return __train_rnn(emb_net, data_loader, optimizer)

def __train_rnn(emb_net, data_loader, optimizer):
    emb_net.train()
    train_loss = 0

    for seq, targets in data_loader:
        seq = seq.cuda()
        targets = targets.cuda()
        loss = loss_ane(seq, targets, emb_net)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    return train_loss / len(data_loader)

def __train_transformer(emb_net, data_loader, optimizer):
    emb_net.train()
    train_loss = 0

    for batch in data_loader:
        src = batch[0].cuda()
        tgt = batch[1].cuda()
        targets = batch[2].cuda()
        loss = loss_ane(src, targets, emb_net, tgt=tgt)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    return train_loss / len(data_loader)

def test(emb_net, data_loader):
    if emb_net.rnn_model == 'transformer':
        return __test_transformer(emb_net, data_loader)
    else:
        return __test_rnn(emb_net, data_loader)

def __test_rnn(emb_net, data_loader):
    emb_net.eval()
    list_embs = list()

    with torch.no_grad():
        for seq, _ in data_loader:
            seq = seq.cuda()

            embs = F.normalize(emb_net(seq), p=2, dim=1)
            list_embs.append(embs)

    return torch.cat(list_embs, dim=0)

def __test_transformer(emb_net, data_loader):
    emb_net.eval()
    list_embs = list()

    with torch.no_grad():
        for batch in data_loader:
            src = batch[0].cuda()
            tgt = batch[1].cuda()

            embs = F.normalize(emb_net(src, tgt), p=2, dim=1)
            list_embs.append(embs)

    return torch.cat(list_embs, dim=0)

def get_seq_data(x, y, len_seq):
    x_seq = []
    y_seq = []

    for i in range(0, x.shape[0] - len_seq):
        x_seq.append(x[i:i+len_seq, :])
        y_seq.append(y[i+len_seq])

    return torch.FloatTensor(x_seq), torch.FloatTensor(y_seq).view([-1, 1])

def exec_dml(dataset_name, rnn_model, init_lr_emb, l2_emb, bs_emb, dim_emb,
             init_lr_pred, l2_pred, bs_pred):
    # Experiment settings.
    path_save_file = 'results/' + dataset_name
    len_seq = 8

    # Load dataset.
    data = numpy.array(pandas.read_excel('datasets/' + dataset_name + '.xlsx'))
    x = scale(data[:, 1:-1])
    y = data[:, -1]
    y_norm = normalize(y)
    n_train = int(0.8 * x.shape[0])

    if rnn_model == 'transformer':
        x_seq, y_pred = get_seq_data(x, y_norm, len_seq=len_seq)
        y_seq = torch.vstack([x_seq[1:, :], torch.zeros((1, len_seq, x.shape[1]))])
        loader_train = DataLoader(TensorDataset(x_seq[:n_train], y_seq[:n_train], y_pred[:n_train]), batch_size=bs_emb)
        loader_test = DataLoader(TensorDataset(x_seq[n_train:], y_seq[n_train:], y_pred[n_train:]), batch_size=bs_emb)
        emb_net = Transformer(dim_in=x.shape[1], dim_emb=dim_emb, len_seq=len_seq).cuda()
    else:
        x_seq, y_seq = get_seq_data(x, y_norm, len_seq=len_seq)
        loader_train = DataLoader(TensorDataset(x_seq[:n_train], y_seq[:n_train]), batch_size=bs_emb, shuffle=False)
        loader_test = DataLoader(TensorDataset(x_seq[n_train:], y_seq[n_train:]), batch_size=bs_emb, shuffle=False)
        emb_net = RNNModel(rnn_model=rnn_model, dim_emb=dim_emb, dim_in=x.shape[1],
                           dim_hidden=16, n_layers=2, len_seq=len_seq).cuda()

    optimizer = torch.optim.Adam(emb_net.parameters(), lr=init_lr_emb, weight_decay=l2_emb)

    # Train the embedding network.
    for epoch in range(0, 500):
        train_loss = train(emb_net, loader_train, optimizer)
        print('Epoch [{}/{}]\tTrain loss: {:.4f}'.format(epoch + 1, 500, train_loss))

    # Generate the latent embeddings of the data using the trained embedding network.
    emb_train = test(emb_net, loader_train).cpu().numpy()
    emb_test = test(emb_net, loader_test).cpu().numpy()

    # Save embedding results.
    df_train = pandas.DataFrame(numpy.hstack([emb_train, y[:n_train].reshape(-1, 1)]))
    df_test = pandas.DataFrame(numpy.hstack([emb_test, y[n_train:-len_seq].reshape(-1, 1)]))
    df_train.to_excel(path_save_file + '/embs_train.xlsx', header=None, index=None)
    df_test.to_excel(path_save_file + '/embs_test.xlsx', header=None, index=None)

    # Load embedding results.
    dataset_train = numpy.array(pandas.read_excel(path_save_file + '/embs_train.xlsx', header=None))
    dataset_test = numpy.array(pandas.read_excel(path_save_file + '/embs_test.xlsx', header=None))
    dataset_x = scale(numpy.vstack([dataset_train[:, :-1], dataset_test[:, :-1]]))
    dataset_train_x = dataset_x[:dataset_train.shape[0], :]
    dataset_test_x = dataset_x[dataset_train.shape[0]:, :]
    dataset_train_x_seq = get_seq_dataset(dataset_train_x, len_seq)
    dataset_test_x_seq = get_seq_dataset(dataset_test_x, len_seq)
    dataset_train_y = dataset_train[:, -1].reshape(-1, 1)
    dataset_test_y = dataset_test[:, -1].reshape(-1, 1)

    # Load the dataset on the embedding space.
    data_loader_train = get_data_loader(dataset_train_x_seq, dataset_train_y, batch_size=bs_pred, shuffle=True)
    data_loader_test = get_data_loader(dataset_test_x_seq, dataset_test_y, batch_size=bs_pred)

    # Define prediction model and optimizer.
    model = FCNN(dataset_train_x_seq.shape[1], dim_out=1).cuda()
    optimizer = torch.optim.Adam(model.parameters(), lr=init_lr_pred, weight_decay=l2_pred)
    criterion = torch.nn.MSELoss()

    # Train the prediction model.
    for epoch in range(0, 500):
        train_loss = model.fit(data_loader_train, optimizer, criterion)
        print(epoch, train_loss)

    # Evaluate the trained model.
    preds_test = model.predict(data_loader_test).cpu().numpy()
    mae = mean_absolute_error(dataset_test_y, preds_test)
    rmse = numpy.sqrt(mean_squared_error(dataset_test_y, preds_test))
    corr = correlation(dataset_test_y, preds_test)

    # Save the trained model and the extrapolation results.
    df = pandas.DataFrame(numpy.hstack([dataset_test_y, preds_test]))
    df.to_excel(path_save_file + '/preds.xlsx', index=None, header=['targets', 'preds'])
    precision, recall, f1 = detect_storm(path_save_file)

    return mae, rmse, corr, precision, recall, f1


from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
from unicodedata import name
from nltk import sent_tokenize
import sys
import copy
import kor_sentence

def remove_tags(data):
    p=re.compile(r'<.*?>')
    return p.sub('', data)

def remove_span(data):
    p=re.compile(r'.*?')
    return p.sub('',data)

def remove_comma(data):
    result=data.replace("
,
"," ")
    return result

def remove_semantic(data):
    p=re.compile(r'<semantics.*>.*?</semantics>',re.S)
    return p.sub('',data)

def remove_bracket(data):

    ck=-1
    st=0
    while 1:
        ck=data.find('#cite_note-',st)
        if ck==-1:
            break
        else:
            st=data.find('[',ck)
            fi=data.find(']',st)
            data=data.replace(data[st:fi+1],'')

    return data

def eng_sentence(final_header_eng):

    start=-1
    finish=-1
    cnt=0

    # '('로 시작해서 ')'까지 찾아서 제거

    x=0
    while 1:
        if x==len(final_header_eng):
            break

        start=final_header_eng[x].find('(')
        if start==-1:
            x=x+1
            continue

        finish=final_header_eng[x].find(')',start)
        if finish==-1:
            tmp=final_header_eng[x][start:len(final_header_eng[x])+1]
        else:
            tmp=final_header_eng[x][start:finish+1]

        final_header_eng[x]=final_header_eng[x].replace(tmp,'')
        start=final_header_eng[x].find('(')

        if start==-1:
            x=x+1

        finish=-1

    # ~~~')'까지 찾아서 제거
    for x in range(len(final_header_eng)):

        finish=final_header_eng[x].find(')')
        if finish==-1:
            continue

        tmp=final_header_eng[x][:finish+1]
        final_header_eng[x]=final_header_eng[x].replace(tmp,'')
        finish=-1

    # 공백이거나 '.'만 있는 라인제거

    for x in range(len(final_header_eng)):

        if final_header_eng[cnt]=='' or final_header_eng[cnt]=='.':
            del final_header_eng[cnt]
        else:
            cnt=cnt+1

        if len(final_header_eng)-1==cnt:
            break

    return final_header_eng

def check_table_index(sources):

    #쓸때 없는 table 위치 찾기(kor)
    table_st=[]
    table_fi=[]
    st=-1
    fi=-1
    while 1:
        st=str(sources).find('=len(table_fi):
            break
        ct=0
        while 1:
            if  j>=len(table_st) or table_fi[i]<table_st[j]:
                table_set[i].append(table_st[i])
                table_set[i].append(table_fi[j-1])
                break

            else:
                j=j+1
                ct=ct+1

        i=i+ct
    return table_set

def WeakTranslation(sourcesKOR_tmp,sourcesENG_tmp):

    start_k=str(sourcesKOR_tmp).find('
')
    start_e=str(sourcesENG_tmp).find('
')

    #문단이 없으면 다음 문서로 넘어가기
    if start_k==-1 or start_e==-1:
        return -1, -1

    finish_k=str(sourcesKOR_tmp).find('
')
    finish_e=str(sourcesENG_tmp).find('
')
    if finish_k==-1 or finish_e==-1:
        finish_k=0
        finish_e=0

    ########## KOREA ############
    tmp_k=str(sourcesKOR_tmp)[start_k:finish_k]
    tt=0
    while 1:

        idx=tmp_k.find('
',tt+1)
        if idx==-1:
            break
        else:
            tt=idx

    tmp_k=tmp_k[:tt]
    header_kor=remove_tags(tmp_k)
    header_kor=remove_comma(header_kor)
    header_kor=remove_bracket(header_kor)
    header_kor=remove_semantic(header_kor)
    header_kor = header_kor[:len(header_kor) - 4]
    final_header_kor = kor_sentence.kor_sentence(header_kor)

    ########## ENGLISH ############
    tmp_e=str(sourcesENG_tmp)[start_e:finish_e]
    tt=0
    while 1:

        idx=tmp_e.find('
',tt+1)
        if idx==-1:
            break
        else:
            tt=idx

    tmp_e=tmp_e[:tt]
    header_eng=remove_tags(tmp_e)
    header_eng=remove_comma(header_eng)
    header_eng=remove_bracket(header_eng)
    header_eng=remove_semantic(header_eng)
    final_header_eng = sent_tokenize(header_eng)
    final_header_eng = eng_sentence(final_header_eng)

    return final_header_kor, final_header_eng

def StrongTranslation(sourcesKOR_tmp, sourcesENG_tmp):

    #문단 부분만 추출
    para_kor = sourcesKOR_tmp.findAll('p')
    para_eng = sourcesENG_tmp.findAll('p')

    #문단이 없으면 다음 문서로 넘어가기
    if len(para_kor)==0 or len(para_eng)==0:
        return -1, -1

    ############# KOREA ############
    non_bmp_map=dict.fromkeys(range(0x10000,sys.maxunicode+1),0xfffd)
    header_kor=remove_comma(str(para_kor).translate(non_bmp_map))
    header_kor=remove_bracket(header_kor)
    header_kor=remove_semantic(header_kor)
    header_kor=remove_tags(header_kor)
    if header_kor[len(header_kor)-2]==',':
        header_kor=header_kor[1:len(header_kor)-2]
    else:
        header_kor=header_kor[1:len(header_kor)-1]
    #header_kor=remove_brachet(header_kor)
    final_header_kor=kor_sentence.kor_sentence(header_kor)

    ############# ENGLISH ############
    non_bmp_map2=dict.fromkeys(range(0x10000,sys.maxunicode+1),0xfffd)
    header_eng=remove_comma(str(para_eng).translate(non_bmp_map))
    header_eng=remove_bracket(header_eng)
    header_eng=remove_semantic(header_eng)
    header_eng=remove_tags(header_eng)
    if header_eng[len(header_eng)-2]==',':
        header_eng=header_eng[1:len(header_eng)-2]
    else:
        header_eng=header_eng[1:len(header_eng)-1]
    #header_eng=remove_brachet(header_eng)
    final_header_eng=sent_tokenize(header_eng)
    final_header_eng=eng_sentence(final_header_eng)

    return final_header_kor, final_header_eng

def header_for_link(sourcesKOR, sourcesENG, i, metric_result ):
    #print("[header_for_link] " + str(i))
    # 쓸때 없는 table 부분 삭제
    tmp_kor = str(sourcesKOR)
    tmp_eng = str(sourcesENG)
    table_set_kor = check_table_index(sourcesKOR)
    table_set_eng = check_table_index(sourcesENG)

    # 불필요한 table제거 (kor)
    kkk = []
    for j in range(len(table_set_kor)):
        if table_set_kor[j] == []:
            continue
        kkk.append(tmp_kor[table_set_kor[j][0]:table_set_kor[j][1] + 9])

    for j in range(len(kkk)):
        tmp_kor = tmp_kor.replace(str(kkk[j]), '')

    # 불필요한 table제거 (eng)
    eee = []
    for j in range(len(table_set_eng)):
        if table_set_eng[j] == []:
            continue
        eee.append(tmp_eng[table_set_eng[j][0]:table_set_eng[j][1] + 9])

    for j in range(len(eee)):
        tmp_eng = tmp_eng.replace(str(eee[j]), '')

    sourcesKOR_tmp = BeautifulSoup(tmp_kor, "html.parser")
    sourcesENG_tmp = BeautifulSoup(tmp_eng, "html.parser")

    # 문단 부분만 추출
    para_kor = sourcesKOR_tmp.findAll('p')
    para_eng = sourcesENG_tmp.findAll('p')

    '''
    #강한번역, 약한번역에 따른 처리
    if metric_result>=0.8:
        final_header_kor, final_header_eng = StrongTranslation(sourcesKOR_tmp, sourcesENG_tmp)

    else:
        final_header_kor, final_header_eng = WeakTranslation(sourcesKOR_tmp, sourcesENG_tmp)
    '''
    final_header_kor, final_header_eng = WeakTranslation(sourcesKOR_tmp, sourcesENG_tmp)
    if final_header_kor==-1 or final_header_eng==-1:
        return -1, -1

    ###########  KOREA   ###########
    tmp_kor_link = [[] for j in range(len(final_header_kor))]

    # <a ~ 부분 잘라내기
    for m in range(len(final_header_kor)):
        start = 0
        finish = 0
        while 1:
            start = final_header_kor[m].find('<a')
            if start != -1:
                finish = final_header_kor[m].find('')
                tmp_kor_link[m].append(final_header_kor[m][start:finish + 4])
            else:
                break

            while 1:
                start = final_header_kor[m].find('<a', start + 1)
                if start != -1:
                    finish = final_header_kor[m].find('', finish + 1)
                    tmp_kor_link[m].append(final_header_kor[m][start:finish + 4])
                else:
                    break
            break

    # <a ~ 부분에서 단어만 추출
    tmp = [[] for j in range(len(tmp_kor_link))]
    k_link_list = [[] for j in range(len(tmp_kor_link))]
    for k in range(len(tmp_kor_link)):
        if len(tmp_kor_link[k]) == 0:
            continue
        for j in range(len(tmp_kor_link[k])):
            # title 부분
            st = tmp_kor_link[k][j].find('title="')
            if st == -1:
                continue
            fi = tmp_kor_link[k][j].find('"', st + 7)
            tt = tmp_kor_link[k][j]

            if str(tt[fi - 1]) == ' ':
                tt = tt[st + 7:fi - 1]
            else:
                tt = tt[st + 7:fi]

            kk = tt.find(':')
            if kk >= 0:
                tt = tt[kk + 1:]
            k_link_list[k].append(str(tt))

    ###########  ENGLISH  ###########
    tmp_eng_link = [[] for j in range(len(final_header_eng))]
    # <a ~ 부분 잘라내기
    for m in range(len(final_header_eng)):
        start = 0
        finish = 0
        while 1:
            start = final_header_eng[m].find('<a')
            if start != -1:
                finish = final_header_eng[m].find('')
                tmp_eng_link[m].append(final_header_eng[m][start:finish + 4])
            else:
                break

            while 1:
                start = final_header_eng[m].find('<a', start + 1)
                if start != -1:
                    finish = final_header_eng[m].find('', finish + 1)
                    tmp_eng_link[m].append(final_header_eng[m][start:finish + 4])
                else:
                    break
            break

    # <a ~ 부분에서 단어만 추출
    tmp = [[] for j in range(len(tmp_eng_link))]
    e_link_list = [[] for j in range(len(tmp_eng_link))]
    for k in range(len(tmp_eng_link)):
        if len(tmp_eng_link[k]) == 0:
            continue
        for j in range(len(tmp_eng_link[k])):
            # title 부분
            st = tmp_eng_link[k][j].find('title="')
            if st == -1:
                continue
            fi = tmp_eng_link[k][j].find('"', st + 7)
            tt = tmp_eng_link[k][j]

            if str(tt[fi - 1]) == ' ':
                tt = tt[st + 7:fi - 1]
            else:
                tt = tt[st + 7:fi]

            kk = tt.find(':')
            if kk >= 0:
                tt = tt[kk + 1:]
            e_link_list[k].append(str(tt))

    return (k_link_list, e_link_list)


import json
import os
import pandas
import pyperclip
import selenium
import sys
import time

from bs4 import *

# 공통 로그 함수
def line_logging(*messages):
    import datetime
    log_time = datetime.datetime.today().strftime('[%Y/%m/%d %H:%M:%S]')
    log = list()
    for message in messages:
        log.append(str(message))
    print(log_time + ':[' + ' '.join(log) + ']', flush=True)

# crawling function
def get_post(p_url, p_param, p_sleep_time=2, p_flag_view_url=True):
    import time
    import urllib
    import requests

    url_full_path = p_url + '?' + urllib.parse.urlencode(p_param)
    if p_flag_view_url:
        line_logging(url_full_path)
    headers = {
        'content-type': 'application/json, text/javascript, */*; q=0.01',
        'User-Agent': 'Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15',
        'referer': 'http://finance.daum.net/domestic/exchange/COMMODITY-%2FCLc1'
    }
    try:
        results = requests.get(url_full_path, headers=headers)
        time.sleep(p_sleep_time)
        return results
    except:
        time.sleep(p_sleep_time * 2)
        results = requests.get(url_full_path, headers=headers)
        time.sleep(p_sleep_time)
        return results

# ====================================================================================================================================================================================================
# HTML Parsing
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# 국내 지수 (코스피/코스닥) 수집
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
def collect_korea(p_market, page_no=1, p_sleep_time=2):
    url = "https://finance.naver.com/sise/sise_index_day.nhn"

    list_price = list()
    param = {
        'code': p_market,
        'page': page_no
    }
    results = get_post(url, param, p_sleep_time=p_sleep_time)
    price_table = BeautifulSoup(results.content, "html.parser").find_all('table')[0]
    price_trs = BeautifulSoup(str(price_table), "html.parser").find_all('tr')
    for price_tr in price_trs:
        row = BeautifulSoup(str(price_tr), "html.parser").find_all('td')
        if len(row) > 3:
            list_price.append({
                'eod_date': int(row[0].text.strip().replace('.', '')),
                'item_code': p_market,
                'price_close': float(row[1].text.strip().replace(',', '')),
                'trade_amount': int(row[4].text.strip().replace(',', '')),
                'diff': float(row[2].text.strip().replace(',', '')),
                'rate': float(row[3].text.strip().replace(',', '').replace('%', '').replace('+', '')),
            })
    return pandas.DataFrame(list_price)

# 국내 지수 (코스피/코스닥) 수집 결과 저장
def save_kospi_and_kosdaq(p_market, page_to=10):
    page_from = 1
    df_index = pandas.DataFrame()
    count_of_date = 0

    for page_no in range(page_from, page_to):
        df_page = collect_korea(p_market, page_no=page_no)
        if df_index.shape[0] > 0:
            df = df_index[~df_index['eod_date'].isin(set(df_page['eod_date'].unique()))]
        else:
            df = df_index
        df_index = pandas.concat([df, df_page], sort=False)
        df_index = df_index.drop_duplicates()
        df_index = df_index.set_index(['eod_date'])
        df_index = df_index.sort_index(ascending=False)
        df_index = df_index.reset_index()

        line_logging('save_kospi_and_kosdaq', p_market, 'Start:', df_index.tail(1)['eod_date'].tolist()[0], ',Finish:', df_index.head(1)['eod_date'].tolist()[0], count_of_date, df_index.shape)
        if count_of_date == df_index.shape[0]:
            break
        count_of_date = df_index.shape[0]

    return df_index
# ====================================================================================================================================================================================================

# ====================================================================================================================================================================================================
# JSON loading
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# 한국은행 데이터 수집 함수
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
def get_ECOS_MM(p_flag):
    import urllib
    import urllib.request
    import urllib.parse
    import json

    base_url = 'https://ecos.bok.or.kr/jsp/vis/keystat/Key100Stat_n1.jsp'

    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Host': 'ecos.bok.or.kr',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15',
        'Accept-Language': 'en-US,en;q=0.9',
        'Referer': 'https://ecos.bok.or.kr/jsp/vis/keystat/',
        'Connection': 'keep-alive',
    }

    # 파라미터 세팅
    params = '?' + urllib.parse.urlencode({
        urllib.parse.quote_plus('languageFlg'): 'MM',
        urllib.parse.quote_plus('languageFlg2'): '1',
        urllib.parse.quote_plus('languageFlg3'): p_flag,
        urllib.parse.quote_plus('languageFlg4'): '%20',
        urllib.parse.quote_plus('languageFlg5'): 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15',
    })
    req = urllib.request.Request(base_url + urllib.parse.unquote(params), headers=headers)
    response_body = urllib.request.urlopen(req).read()
    json_data = json.loads(response_body)[0]

    list_row = list()
    for item_info in json_data:
        yyyymm = int(item_info['TIME'])
        if yyyymm < 201401:
            continue
        list_row.append({
            'yyyymm': yyyymm,
            'item_code': p_flag,
            'value': item_info['DATA_VALUE'],
        })

    time.sleep(1.5)
    return pandas.DataFrame(list_row)
# ====================================================================================================================================================================================================

# ====================================================================================================================================================================================================
# Selenium
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# 클립보드에 input을 복사한 뒤, 해당 내용을 actionChain을 이용해 로그인 폼에 붙여넣기
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
def copy_input(p_driver, xpath, input):
    pyperclip.copy(input)
    p_driver.find_element_by_xpath(xpath).click()
    selenium.webdriver.common.action_chains.ActionChains(p_driver).\
        key_down(selenium.webdriver.common.keys.Keys.COMMAND).send_keys('v').\
        key_up(selenium.webdriver.common.keys.Keys.COMMAND).perform()
# ====================================================================================================================================================================================================

# ====================================================================================================================================================================================================
# API
# ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
def get_ASOS(p_api_key, p_start_date, p_finish_date, stn_id, p_page_number=1):
    line_logging('get_ASOS is started.', p_start_date, p_finish_date, stn_id, p_page_number)
    import urllib
    import urllib.request
    import urllib.parse
    import json

    url = 'http://apis.data.go.kr/1360000/AsosDalyInfoService/getWthrDataList'
    # parameter for request
    params = '?' + urllib.parse.urlencode({
        urllib.parse.quote_plus('serviceKey'): p_api_key,
        urllib.parse.quote_plus('numOfRows'): '10',
        urllib.parse.quote_plus('pageNo'): p_page_number,
        urllib.parse.quote_plus('dataType'): 'json',
        urllib.parse.quote_plus('dataCd'): 'ASOS',
        urllib.parse.quote_plus('dateCd'): 'DAY',
        urllib.parse.quote_plus('startDt'): p_start_date,
        urllib.parse.quote_plus('endDt'): p_finish_date,
        urllib.parse.quote_plus('stnIds'): stn_id,
    })
    req = urllib.request.Request(url + urllib.parse.unquote(params))
    response_body = urllib.request.urlopen(req).read()
    json_data = json.loads(response_body)

    json_info = json_data['response']['body']
    list_items = json_info['items']['item']
    list_row = list()
    for item_info in list_items:
        list_row.append({
            'COL-01': str(item_info['stnId']),
            'COL-02': str(item_info['stnNm']),
            'COL-03': str(item_info['tm']),
            'COL-04': str(item_info['avgTa']),
            'COL-05': str(item_info['minTa']),
            'COL-06': str(item_info['minTaHrmt']),
            'COL-07': str(item_info['maxTa']),
            'COL-08': str(item_info['maxTaHrmt']),
            'COL-09': str(item_info['sumRnDur']),
            'COL-10': str(item_info['mi10MaxRn']),
            'COL-11': str(item_info['mi10MaxRnHrmt']),
            'COL-12': str(item_info['hr1MaxRn']),
            'COL-13': str(item_info['hr1MaxRnHrmt']),
            'COL-14': str(item_info['sumRn']),
            'COL-15': str(item_info['maxInsWs']),
            'COL-16': str(item_info['maxInsWsWd']),
            'COL-17': str(item_info['maxInsWsHrmt']),
            'COL-18': str(item_info['maxWs']),
            'COL-19': str(item_info['maxWsWd']),
            'COL-20': str(item_info['maxWsHrmt']),
            'COL-21': str(item_info['avgWs']),
            'COL-22': str(item_info['hr24SumRws']),
            'COL-23': str(item_info['maxWd']),
            'COL-24': str(item_info['avgTd']),
            'COL-25': str(item_info['minRhm']),
            'COL-26': str(item_info['minRhmHrmt']),
            'COL-27': str(item_info['avgRhm']),
            'COL-28': str(item_info['avgPv']),
            'COL-29': str(item_info['avgPa']),
            'COL-30': str(item_info['maxPs']),
            'COL-31': str(item_info['maxPsHrmt']),
            'COL-32': str(item_info['minPs']),
            'COL-33': str(item_info['minPsHrmt']),
            'COL-34': str(item_info['avgPs']),
            'COL-35': str(item_info['ssDur']),
            'COL-36': str(item_info['sumSsHr']),
            'COL-37': str(item_info['hr1MaxIcsrHrmt']),
            'COL-38': str(item_info['hr1MaxIcsr']),
            'COL-39': str(item_info['sumGsr']),
            'COL-40': str(item_info['ddMefs']),
            'COL-41': str(item_info['ddMefsHrmt']),
            'COL-42': str(item_info['ddMes']),
            'COL-43': str(item_info['ddMesHrmt']),
            'COL-44': str(item_info['sumDpthFhsc']),
            'COL-45': str(item_info['avgTca']),
            'COL-46': str(item_info['avgLmac']),
            'COL-47': str(item_info['avgTs']),
            'COL-48': str(item_info['minTg']),
            'COL-49': str(item_info['avgCm5Te']),
            'COL-50': str(item_info['avgCm10Te']),
            'COL-51': str(item_info['avgCm20Te']),
            'COL-52': str(item_info['avgCm30Te']),
            'COL-53': str(item_info['avgM05Te']),
            'COL-54': str(item_info['avgM10Te']),
            'COL-55': str(item_info['avgM15Te']),
            'COL-56': str(item_info['avgM30Te']),
            'COL-57': str(item_info['avgM50Te']),
            'COL-58': str(item_info['sumLrgEv']),
            'COL-59': str(item_info['sumSmlEv']),
            'COL-60': str(item_info['n99Rn']),
            'COL-61': str(item_info['iscs']),
            'COL-62': str(item_info['sumFogDur']),
        })

    line_logging('get_ASOS is finished.', p_start_date, p_finish_date, stn_id, p_page_number)

    time.sleep(1.5)
    return pandas.DataFrame(list_row), json_info['totalCount']


from art import logo
import random
cards = [11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]

#pregunta_1 = input("Quieres jugar Blackjack responde 'si' o 'no' ? \n ")
mano = []
mano_de1 = []

def activacionJuego(mano_de, final_deljuego):

    print(logo)
    mano_uno = random.sample(cards, 2)
    mano = mano_uno
    suma = 0
    for carta in mano:
        suma += carta
    mano_de1.append(mano_de)
    print(f"Tus cartas: {mano} Tu puntaje: {suma}\n Primera carta del dealer {mano_de1}")

    while True:
        pregunta_2 = input("Quieres otra carta responde 'si' o 'no'\n\n").lower()
        if pregunta_2 == "no":
            han = mano_de
            mano_de1.append(han)
            resul_dealer = sum(mano_de1)
            if sum(mano) > resul_dealer:
                 print(f"\n\n GANASTE !!!\n\nTus cartas: {mano} Tu puntaje: {suma}\n\nSegunda mano del dealer {han} Puntaje del dealer: {resul_dealer}\n\n")
            elif sum(mano) < resul_dealer:
                 print(f"\n\n Perdiste !!!\n\nTus cartas: {mano} Tu puntaje: {suma}\n\nSegunda mano del dealer {han} Puntaje del dealer: {resul_dealer}\n\n")
            else:
                print(f"\n\nEmpate !!!\n\nTus cartas: {mano} Tu puntaje: {suma}\nSegunda mano del dealer {han} Puntaje del dealer: {resul_dealer}\n")
            break

        elif pregunta_2 == "si":
               mano_ju = mano_de
               mano.append(mano_ju)
               suma = sum(mano)
               print(f"Tu carta es {mano_ju}, tu puntaje es {suma}")
               if suma > 21:
                    print(f"Tu carta es {mano_ju} Perdiste, te has pasado del limite, tu puntaje es {suma}")
                    break

mano_de = random.choice(cards)
final_juego = False
activacionJuego(mano_de, final_juego)


import requests
import json
import argparse

base_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/'

def mark_as_spam(id, api_key, verbose):
    print('Marking as spam: PR%d' % id)
    # 1) get bug info to find 'cc'
    u = base_url + 'bug/' + str(id)
    r = requests.get(u)
    response = json.loads(r.text)

    if 'error' in response and response['error']:
        print(response['message'])
        return

    # 2) mark the bug as spam
    bug = response['bugs'][0]
    creator = bug['creator']
    cc_list = bug['cc']
    data = {
        'status': 'RESOLVED',
        'resolution': 'INVALID',
        'summary': 'spam',
        'ids': [id],
        'api_key': api_key,
        'comment': { 'comment': 'spam'},
        'product': 'gcc',
        'component': 'spam',
        'version': 'unknown',
        'cc': {'remove': cc_list},
        'priority': 'P5',
        'severity': 'trivial',
        'url': '',
        'assigned_to': 'unassigned@gcc.gnu.org' }

    r = requests.put(u, json = data)
    if verbose:
        print(r)
        print(r.text)

    # 3) mark the first comment as spam
    r = requests.get(u + '/comment')
    response = json.loads(r.text)
    for c in response['bugs'][str(id)]['comments']:
        if c['creator'] == creator:
            comment_id = c['id']
            u2 = '%sbug/comment/%d/tags' % (base_url, comment_id)
            print(u2)
            r = requests.put(u2, json = {'comment_id': comment_id, 'add': ['spam'], 'api_key': api_key})
            if verbose:
                print(r)
                print(r.text)

    # 4) mark all attachments as spam
    r = requests.get(u + '/attachment')
    response = json.loads(r.text)
    attachments = response['bugs'][str(id)]
    for a in attachments:
        attachment_id = a['id']
        url = '%sbug/attachment/%d' % (base_url, attachment_id)
        r = requests.put(url, json = {'ids': [attachment_id],
            'summary': 'spam',
            'file_name': 'spam',
            'content_type': 'application/x-spam',
            'is_obsolete': True,
            'api_key': api_key})
        if verbose:
            print(r)
            print(r.text)

parser = argparse.ArgumentParser(description='Mark Bugzilla issues as spam.')
parser.add_argument('api_key', help = 'API key')
parser.add_argument('range', help = 'Range of IDs, e.g. 10-23,24,25,27')
parser.add_argument('--verbose', action = 'store_true', help = 'Verbose logging')

args = parser.parse_args()

chunks = args.range.split(',')
for c in chunks:
    parts = list(map(lambda x: int(x), c.split('-')))
    if len(parts) == 1:
        r = [parts[0]]
    else:
        r = range(parts[0], parts[1] + 1)

    for id in r:
        mark_as_spam(id, args.api_key, args.verbose)


from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.views.decorators.cache import cache_page
from django.views.decorators.csrf import csrf_exempt

from .forms import CommentForm, PostForm
from .models import Follow, Group, Post, User

@cache_page(20, key_prefix="index_page")
def index(request):
    post = Post.objects.select_related('author').all()
    paginator = Paginator(post, settings.POSTS_PER_PAGE)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, 'index.html', {'page': page})

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    post = group.posts.all()
    paginator = Paginator(post, settings.POSTS_PER_PAGE)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, 'group.html', {'group': group,
                  'post': post, 'page': page})

def profile(request, username):
    following = False
    user = get_object_or_404(User, username=username)
    post = user.posts.all()
    paginator = Paginator(post, settings.POSTS_PER_PAGE)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    if not request.user.is_anonymous and Follow.objects.filter(
            user=request.user, author=user).exists():
        following = True
    return render(request, 'profile.html', {
                  'author': user,
                  'following': following,
                  'post': post,
                  'page': page})

def post_view(request, username, post_id):
    following = False
    user = get_object_or_404(User, username=username)
    post = get_object_or_404(Post, author__username=username, id=post_id)
    form = CommentForm(request.POST or None)
    comments = post.comments.all()
    if not request.user.is_anonymous and Follow.objects.filter(
            user=request.user, author=user).exists():
        following = True
    return render(request, 'post.html', {
        'add_comment': True,
        'comments': comments,
        'form': form,
        'following': following,
        "author": user,
        'post': post
    })

@login_required
@csrf_exempt
def new_post(request):
    if request.method == 'POST':
        form = PostForm(request.POST or None, files=request.FILES or None)
        if form.is_valid():
            post = form.save(commit=False)
            post.author = request.user
            form.save()
            return redirect('index')
        return render(request, 'form.html', {'form': form})

    form = PostForm()
    return render(request, 'form.html', {'form': form})

@csrf_exempt
def post_edit(request, username, post_id):
    if username != request.user.username:
        return redirect(reverse('post', args=(username, post_id)))
    post = get_object_or_404(Post, author__username=username, id=post_id)
    form = PostForm(request.POST or None, files=request.FILES or None,
                    instance=post)
    if form.is_valid():
        form.save()
        return redirect('post', username, post_id)
    return render(request, 'form.html', {
        'edit': True,
        'post': post,
        'form': form
    })

@login_required
def add_comment(request, username, post_id):
    post = get_object_or_404(Post, id=post_id)
    if request.method == 'POST':
        form = CommentForm(request.POST or None)
        if form.is_valid():
            comment = form.save(commit=False)
            comment.post = post
            comment.author = request.user
            comment.save()
            return redirect('post', username, post_id)
        return render(request, 'includes/comments.html', {'form': form})

    form = CommentForm()
    return render(request, 'includes/comments.html', {
        'form': form})

@login_required
def follow_index(request):
    post = Post.objects.filter(author__following__user=request.user)
    paginator = Paginator(post, settings.POSTS_PER_PAGE)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, "follow.html", {
                  'post': post,
                  'page': page})

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username=username)
    if author != request.user:
        Follow.objects.get_or_create(user=request.user, author=author)
    return redirect('profile', username)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username=username)
    Follow.objects.filter(user=request.user, author=author).delete()
    return redirect('profile', username)

def page_not_found(request, exception):
    return render(
        request,
        "misc/404.html",
        {"path": request.path},
        status=404
    )

def server_error(request):
    return render(request, "misc/500.html", status=500)


import numpy as np
from matplotlib import pyplot as plt
import scipy.integrate
import scipy.special

fig = plt.figure(figsize=(3.5, 2.5))
ax = fig.add_subplot(111)

np.random.seed(0)
nruns = 20
neval = np.unique(np.round(np.logspace(0, 2, 50)).astype(int))

def integrand(x, a, b, c, d, e, f):
    I0arg = np.sqrt(np.square(c * np.cos(x) + d * np.sin(x)) + np.square(e * np.cos(x) + f * np.sin(x)))
    return np.exp(a * np.cos(x) + b * np.sin(x) + I0arg) * scipy.special.i0e(I0arg)

def periodic_quad(func, args, n):
    return 2*np.pi/n * np.sum(func(2*np.pi/n * np.arange(n), *args))

for irun in range(nruns):
    args = tuple(np.random.randn(6).tolist())
    exact, _ = scipy.integrate.quad(
        integrand, 0, 2*np.pi, args, epsabs=1e-14, epsrel=1e-14, limit=200)
    approx = np.asarray([
        periodic_quad(integrand, args, n) for n in neval])
    relerr = np.abs(approx - exact) / exact
    ax.plot(neval, relerr, '-k')

# TOA+SNR
# def integrand(x, a, b, c, d, e, f, g, h, i, j):
#     I0arg1 = np.sqrt(np.square(c * np.cos(x) + d * np.sin(x)) + np.square(e * np.cos(x) + f * np.sin(x)))
#     I0arg2 = np.sqrt(np.square(g * np.cos(x) + h * np.sin(x)) + np.square(i * np.cos(x) + j * np.sin(x)))
#     return np.exp(a * np.cos(x) + b * np.sin(x) + I0arg1 + I0arg2) * scipy.special.i0e(I0arg1) * scipy.special.i0e(I0arg2)
#
# for irun in range(nruns):
#     args = tuple(np.random.randn(10).tolist())
#     exact, _ = scipy.integrate.quad(
#         integrand, -1, 1, args, epsabs=1e-14, epsrel=1e-14, limit=200)
#     approx = np.asarray([
#         scipy.integrate.fixed_quad(integrand, -1, 1, args, n)[0] for n in neval])
#     relerr = np.abs(approx - exact) / exact
#     ax.plot(neval, relerr, '--k')

ax.set_xscale('log')
ax.set_yscale('log')
ax.set_ylim(1e-16, 1)
ax.set_xlabel('number of function evaluations')
ax.set_ylabel('relative error')

fig.subplots_adjust(left=0.175, bottom=0.15, right=0.95, top=0.95)

fig.savefig('polarization_angle_integral_convergence.pdf')


import datetime

import fastapi

import db
import mytypes

def search(state: str, district: str, mandal: str, from_time: datetime.datetime, to_time: datetime.datetime,
           resource_group_id: int):
    con = db.get_db()
    cur = con.cursor()
    print(state, district, mandal, from_time, to_time)
    cur.execute(
        "SELECT resource.*, availability.start_time, availability.end_time FROM \
        resource JOIN availability ON\
         availability.resource_id = resource.resourceid \
         WHERE availability.start_time <= ? AND availability.end_time >= ? \
          AND state = ? AND district = ? AND mandal = ? AND resource_group_id = ?",
        (from_time, to_time, state, district, mandal, resource_group_id,))
    rv = []
    for row in cur.fetchall():
        rv.append(
            mytypes.SearchResultsInfo(resource_id=row[0], owner=row[1], state=row[2], district=row[3], mandal=row[4],
                                      data=row[5], available_from=row[7], available_till=row[8]))
    return rv

def create(state: str, district: str, mandal: str, resource_group_id: int, data: str, owner: int):
    con = db.get_db()
    cur = con.cursor()
    cur.execute(
        "SELECT * FROM resource WHERE state = ? AND district = ? AND mandal = ? AND resource_group_id = ? AND owner = ?",
        (state, district, mandal, resource_group_id, owner,))
    if len(cur.fetchall()):
        raise fastapi.HTTPException(status_code=409, detail="already exists")
    cur.execute("INSERT INTO resource (owner, state, district, mandal, data, resource_group_id) VALUES (?,?,?,?,?,?)",
                (owner, state, district, mandal, data, resource_group_id,))
    rv = cur.lastrowid
    con.commit()
    return rv

def set_available_range(resource_id: int, start: datetime.datetime, end: datetime.datetime, user_id: int):
    con = db.get_db()
    cur = con.cursor()
    cur.execute("SELECT * FROM resource WHERE resourceid = ? AND owner = ?", (resource_id, user_id,))
    if not len(cur.fetchall()):
        raise fastapi.HTTPException(status_code=404, detail="not found")
    cur.execute("INSERT INTO availability(resource_id, start_time, end_time) VALUES (?,?,?)",
                (resource_id, start, end,))
    con.commit()

def order(resource_id: int, client: int, from_time: datetime.datetime, to_time: datetime.datetime, qty: float):
    con = db.get_db()
    cur = con.cursor()
    cur.execute("SELECT owner FROM resource WHERE resourceid = ?", (resource_id,))
    lessor = cur.fetchone()[0]
    cur.execute(
        "INSERT INTO orders(resourceid, lessor, lessee, start_time, end_time, quantity, order_status)\
         VALUES (?,?,?,?,?, ?, ?)",
        (resource_id, lessor, client, from_time, to_time, qty, 1,))
    con.commit()

def get_orders(user_id: int):
    con = db.get_db()
    cur = con.cursor()

    cur.execute(
        "SELECT orderid, r.resourceid, lessor, username, \
        start_time, end_time, quantity, order_status, r.resource_group_id\
          FROM orders JOIN user ON userid = lessee JOIN resource r on orders.resourceid = r.resourceid WHERE lessor = ?",
        (user_id,))
    rows = cur.fetchall()
    rv = []
    for row in rows:
        rv.append(mytypes.OrderInfo(orderid=row[0], resource=row[1], lessor=row[2], lessee=row[3], start_time=row[4],
                                    end_time=row[5], quantity=row[6], order_status=row[7], resource_group=row[8]))
    return rv

def update_order(user_id: int, order_id: int, new_status: int):
    con = db.get_db()
    cur = con.cursor()
    cur.execute("UPDATE orders SET order_status = ? WHERE orderid = ? AND lessor = ?", (new_status, order_id, user_id))
    con.commit()


import os
from flask import Flask, request, jsonify
from werkzeug.utils import secure_filename
from loader import Loader

UPLOAD_FOLDER = os.getcwd() + '/data'
ALLOWED_EXTENSIONS = {'csv'}

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.secret_key = "secret_key"

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def validate_file(req_file):
    if 'file' not in req_file:
        return jsonify({"response": "missing file key in request"})
    else:
        file = req_file['file']
        if file.filename == '':
            return jsonify({"response": "missing file in request"})
        else:
            if file and allowed_file(file.filename):
                filename = secure_filename(file.filename)
                path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
                file.save(path)
                try:
                    loader = Loader(path)
                except AssertionError:
                    return jsonify({"response": "filename was incorrect, please use correct filenames"})
                else:
                    loader.process_data()
                    status = loader.status
                    return jsonify({"response": f"successfully saved the file and processed it with status: {status}"})
            else:
                return jsonify({"response": "file extension not allowed"})

@app.route('/')
def index():
   return ''

@app.route('/upload', methods=['POST'])
def upload():
    return validate_file(request.files)

if __name__== '__main__':
    port = int(os.environ.get('PORT', 5000))
    app.run(debug=True, host='0.0.0.0', port=port)


from test.support import open_urlresource
import unittest

from http.client import HTTPException
import sys
from unicodedata import normalize, unidata_version

TESTDATAFILE = "NormalizationTest.txt"
TESTDATAURL = "http://www.pythontest.net/unicode/" + unidata_version + "/" + TESTDATAFILE

def check_version(testfile):
    hdr = testfile.readline()
    return unidata_version in hdr

class RangeError(Exception):
    pass

def NFC(str):
    return normalize("NFC", str)

def NFKC(str):
    return normalize("NFKC", str)

def NFD(str):
    return normalize("NFD", str)

def NFKD(str):
    return normalize("NFKD", str)

def unistr(data):
    data = [int(x, 16) for x in data.split(" ")]
    for x in data:
        if x > sys.maxunicode:
            raise RangeError
    return "".join([chr(x) for x in data])

class NormalizationTest(unittest.TestCase):
    def test_main(self):
        # Hit the exception early
        try:
            testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
                                        check=check_version)
        except PermissionError:
            self.skipTest(f"Permission error when downloading {TESTDATAURL} "
                          f"into the test data directory")
        except (OSError, HTTPException):
            self.fail(f"Could not retrieve {TESTDATAURL}")

        with testdata:
            self.run_normalization_tests(testdata)

    def run_normalization_tests(self, testdata):
        part = None
        part1_data = {}

        for line in testdata:
            if '#' in line:
                line = line.split('#')[0]
            line = line.strip()
            if not line:
                continue
            if line.startswith("@Part"):
                part = line.split()[0]
                continue
            try:
                c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
            except RangeError:
                # Skip unsupported characters;
                # try at least adding c1 if we are in part1
                if part == "@Part1":
                    try:
                        c1 = unistr(line.split(';')[0])
                    except RangeError:
                        pass
                    else:
                        part1_data[c1] = 1
                continue

            # Perform tests
            self.assertTrue(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
            self.assertTrue(c4 ==  NFC(c4) ==  NFC(c5), line)
            self.assertTrue(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
            self.assertTrue(c5 ==  NFD(c4) ==  NFD(c5), line)
            self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
                            NFKC(c3) == NFKC(c4) == NFKC(c5),
                            line)
            self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
                            NFKD(c3) == NFKD(c4) == NFKD(c5),
                            line)

            # Record part 1 data
            if part == "@Part1":
                part1_data[c1] = 1

        # Perform tests for all other data
        for c in range(sys.maxunicode+1):
            X = chr(c)
            if X in part1_data:
                continue
            self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c)

    def test_bug_834676(self):
        # Check for bug 834676
        normalize('NFC', '\ud55c\uae00')

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
"""
This class provides read and write functions of VASP POSCAR file.
"""

import numpy as np
import copy

class POSCAR(object):
    """
    POSCAR class enables POSCAR read/write functions.
    """

    def __init__(self,fname="POSCAR"):
        self.fname = fname
        self.h = np.zeros((3,3),dtype=float)
        self.num_atoms= []
        self.pos= []
        self.flags= []
        self.species= []

#     def to_dict(self):
#         """
#         Returns a dictionary-type variable of the object.
#         """
#         dict= {}
#         dict.update({'comment':self.c1})
#         dict.update({'afac':self.afac})
#         dict.update({})

    def read(self,fname = 'POSCAR'):
        self.fname = fname
        f= open(fname,'r')
        #.....1st line: comment
        self.c1= f.readline()
        #.....2nd line: multiplying factor
        self.afac= float(f.readline().split()[0])
        #.....3rd-5th lines: lattice vectors
        # NOTE (180409): The definition of the hmat here is different from hmat in NAPSystem.
        # The hmat in NAPSystem is transpose of this.
        data= f.readline().split()
        self.h[0]= [ float(x) for x in data ]
        data= f.readline().split()
        self.h[1]= [ float(x) for x in data ]
        data= f.readline().split()
        self.h[2]= [ float(x) for x in data ]
        #.....6th line: num of atoms
        data= f.readline().split()
        if not data[0].isdigit(): # if it is not digit, read next line
            self.species = copy.copy(data)
            data = f.readline().split()
        self.num_atoms= np.array([ int(n) for n in data ])
        #.....7th line: comment
        self.c7= f.readline()
        if self.c7[0] in ('s','S'):
            self.c8= f.readline()
        #.....following lines: atom positions
        for ni in self.num_atoms:
            for j in range(ni):
                data= f.readline().split()
                self.pos.append(np.array([ float(x) for x in data[0:3] ]))
                if len(data) > 3:
                    if len(data) == 6:
                        self.flags.append([ x for x in data[3:6] ])
                    elif len(data) == 4:
                        self.flags.append([ data[3],data[3],data[3] ])
                else:
                    self.flags.append([ 'T', 'T', 'T' ])
        f.close()

    def write(self,fname='POSCAR'):
        f= open(fname,'w')
        f.write(self.c1)
        f.write(' {0:12.7f}\n'.format(self.afac))
        f.write(' {0:12.7f} {1:12.7f} {2:12.7f}\n'.format(self.h[0,0],
                                                          self.h[0,1],
                                                          self.h[0,2]))
        f.write(' {0:12.7f} {1:12.7f} {2:12.7f}\n'.format(self.h[1,0],
                                                          self.h[1,1],
                                                          self.h[1,2]))
        f.write(' {0:12.7f} {1:12.7f} {2:12.7f}\n'.format(self.h[2,0],
                                                          self.h[2,1],
                                                          self.h[2,2]))
        for n in self.num_atoms:
            f.write(' {0:3d}'.format(n))
        f.write('\n')

        f.write(self.c7)
        if hasattr(self, 'c8'):
            f.write(self.c8)

        for i in range(len(self.pos)):
            f.write(' {0:12.7f} {1:12.7f} {2:12.7f}'.format(self.pos[i][0],
                                                            self.pos[i][1],
                                                            self.pos[i][2]))
            f.write(' {0} {1} {2}\n'.format(self.flags[i][0],
                                            self.flags[i][1],
                                            self.flags[i][2]))
        f.close()

if __name__ == '__main__':
    poscar= POSCAR()
    poscar.read()
    poscar.write(fname='TEST-POSCAR')