from keras.datasets import mnist
from keras.layers import Dense, LSTM
from keras.layers.core import Dropout
from keras.utils import to_categorical
from keras.models import Sequential
import get_data as gd
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

def data_pre():
    '''
    Get the data and stack all the data together with input data and labels
    '''
    print('extracting gunshot&ngunshot features...')
    x_Gunshot,s_Gunshot,x_nGunshot,s_nGunshot,dataset,As_Gunshot= gd.extract_features()
    print('gunshot&ngunshot features extracted!')
    X = np.vstack((x_Gunshot, x_nGunshot)).astype(np.float64)
    length = len(X)
    X = np.reshape(X,(length,100,-1))
    # Fit a per-column scaler
    y = np.hstack((np.ones(len(x_Gunshot)), np.zeros(len(x_nGunshot))))
    return X, y

def build_model(lstm_layers,dense_layers):
    model = Sequential()
    model.add(LSTM(units=64, input_shape=(nb_time_steps, nb_input_vector),return_sequences=True))
    for i in range(lstm_layers - 1):
            model.add(LSTM(output_dim=32 * (lstm_layers-i),
                        activation='relu',
                        return_sequences=True))
    model.add(LSTM(output_dim=32,
                        activation='relu',
                        return_sequences=False))
    for i in range(dense_layers - 1):
            model.add(Dense(output_dim=256,
                            activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(2, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

def acu_curve(y,prob):
    fpr,tpr,threshold = roc_curve(y,prob) ###计算真正率和假正率
    roc_auc = auc(fpr,tpr) ###计算auc的值

    plt.figure()
    lw = 2
    plt.figure(figsize=(10,10))
    plt.plot(fpr, tpr, color='darkorange',
             lw=lw, label='ROC curve (area = %0.3f)' % roc_auc) ###假正率为横坐标，真正率为纵坐标做曲线
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])

    font1 = {'family' : 'Times New Roman',
    'weight' : 'normal',
    'size'   : 40,
    }
    font2 = {'family' : 'Times New Roman',
    'weight' : 'normal',
    'size'   : 50,
    }
    plt.xlabel('False Positive Rate',font1)
    plt.ylabel('True Positive Rate',font1)
    plt.title('Receiver operating characteristic curve',font2)
    plt.legend(loc="lower right")

    plt.show()

# parameters for LSTM
nb_lstm_outputs = 30  #Number of output
nb_time_steps = 100  #time steps
nb_input_vector = 1728 #input vector
lstm_layers = 3
dense_layers = 1

#data preprocessing: tofloat32, normalization, one_hot encoding
X, y = data_pre()
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=rand_state)

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

y_train = to_categorical(y_train, num_classes=2)
y_test = to_categorical(y_test, num_classes=2)

model = build_model(lstm_layers,dense_layers)
#compile:loss, optimizer, metrics

#train: epcoch, batch_size
model.fit(X_train, y_train, epochs=30, batch_size=64, verbose=1)
model.summary()
score = model.evaluate(X_test, y_test,batch_size=64, verbose=1)

y_score = model.predict(X_test)
yt = []
yc = []
for i in range(len(y_test)):
    yt.append(np.where(y_test[i]==1)[0].tolist()[0])
    yc.append(np.where(y_score[i]==1)[0].tolist()[0])
acu_curve(yt,yc)
target_names = ['not_gunshot','gunshot']
print(classification_report(yt, yc, target_names=target_names))
print(confusion_matrix(yt, yc))

print(score)


from bs4 import BeautifulSoup
from requests import get
import requests
import random
import os
import time
import nltk
import nltk.data
import telegram
from telegram import Updater

class IdealListener(object):
    start_response = [ 'Привет)', \
                       'Как твой день?', \
                       'Как дела?']
    common_response = ['Да, я тоже так думаю',\
                 'какая глубокая мысль',\
                 'я никогда не думал об этом с этой точки зрения',\
                 'как интересно, расскажи подробнее',\
                 'это точно']
    question_response = ['посмотрим', \
                         'не так плохо' \
                         'нормально']
    exclaim_response = ['Вот это да!',\
                        'Никогда бы не подумал',\
                        'Ничего себе...',\
                        'Случится же']
    end_response = [ 'С тобой интересно общаться, но мне пора идти(', \
                     'see around)', \
                     'Пока)', \
                     'До встречи)',
                     'Приятно было поболтать',
                     'see you later)']

    def __init__(self, token):
        IdealListener.__init__(self)
        self.updater = Updater(token=token)
        self.dispatcher = self.updater.dispatcher
        self.tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

        self.dispatcher.addTelegramCommandHandler('start', self.start)
        self.dispatcher.addTelegramCommandHandler('echo', self.echo)
        self.dispatcher.addTelegramCommandHandler('die', self.die)
        self.dispatcher.addTelegramMessageHandler(self.echo)

    def send_msg(self, bot, update, msg):
        time_start = time.time()
        while time.time() - time_start < 0.1 * len(msg):
            bot.sendChatAction(chat_id=update.message.chat_id, action=telegram.ChatAction.TYPING)
            time.sleep(1.0)

        bot.sendMessage(chat_id=update.message.chat_id, text=msg)

    def start(self, bot, update):
        self.send_msg(bot, update, self.start_response[random.randint(0, len(self.start_response)-1)])

    def echo(self, bot, update):
        msg = update.message.text
        ending = msg.strip()[-1]
        response = ''

        if ending == '?':
            response = self.question_response[random.randint(0, len(self.question_response)-1)]
        elif ending == '!':
            response = self.exclaim_response[random.randint(0, len(self.exclaim_response)-1)]
        else:
            response = self.common_response[random.randint(0, len(self.common_response)-1)]

        for line in self.tokenizer.tokenize(response):
            self.send_msg(bot, update, line)

    def die(self, bot, update):
        bot.sendMessage(chat_id=update.message.chat_id,
                        text=self.end_response[random.randint(0, len(self.end_response))-1])

    def launch(self):
        self.updater.start_polling()

    def stop(self):
        self.updater.stop()

if __name__ == '__main__':
    token = "bot_secret"
    bot = IdealListener(token)
    bot.launch()


from os import path
from PIL import Image, ExifTags
import tkinter as tk
from tkinter.filedialog import askopenfilename
import datetime

#Сбор метадаты
def get_MetaData(image):
    img = Image.open(image)
    exif = { ExifTags.TAGS[k]: v for k, v in img._getexif().items() if k in ExifTags.TAGS }
    return exif

#ПОХУЙ
def main():
    #Получение файла
    image = str(get_file())
    #Проверка расширения файла
    full_name = path.basename(image)
    name = path.splitext(full_name)[1]
    if name == ".jpg":
        #Получение даты
        data = get_MetaData(image = image)
        write_file(data)
    else:
        print("Только JPG")
        exit

#Сборка файла
def get_file():
    filename = askopenfilename()
    return filename

#Запись файла
def write_file(data):
    print("Записать Данные в отдельный файл? 1/0")
    otv = int(input())
    if otv == 1:
        date = str(datetime.datetime.now())
        file_data = open(date + ".txt", "w")
        for i in data:
            file_data.write(i + ": "+ str(data[i]))
            file_data.write("\n")

main()


#!/usr/bin/python

import glob
import shutil
import sys
import os

tran_type=''
active_lang=''

if len(sys.argv) == 2 and sys.argv[1] == 'localize':
	tran_type=sys.argv[1]
elif len(sys.argv) == 3 and sys.argv[1] == 'internationalize':
	tran_type=sys.argv[1]
	active_lang=sys.argv[2]
else:
	sys.stderr.write('Usage: %s localize\n' % sys.argv[0])
	sys.stderr.write('  example: %s internationalize English-EN\n' % sys.argv[0])
	sys.exit(1)

#if localize, ensure hidden .config file is devoid of i18n
#if internationalize, ensure the hidden .config file contains plugin-gargoyle-i18n and the target language package
#if os.path.exists('./package/plugin-gargoyle-i18n/files/etc/uci-defaults/zzz-plugin-gargoyle-i18n'):
for config_file in glob.glob('./*-src/.config'):
	#there should be only one
	if tran_type=='internationalize':
		print ('Editing config file to build in %s translation\n' % (active_lang,))

	cfg_fileFO = open(config_file, 'rb')
	cfg_doc=cfg_fileFO.readlines()
	cfg_fileFO.close()

	newcfg_doc=[]
	for cline in cfg_doc:
		anewline=''

		if (cline.startswith('CONFIG_PACKAGE_plugin-gargoyle-i18n') or cline.startswith('CONFIG_PACKAGE_gargoyle-i18n')) and tran_type=='localize' :
			anewline='# '+cline
		if (cline.startswith('CONFIG_PACKAGE_plugin-gargoyle-i18n') or cline.startswith('CONFIG_PACKAGE_gargoyle-i18n')) and tran_type=='internationalize' :
			#sorry, but I'm sure your some slick brotha and you've got mad dope skillz, but...
			anewline='# '+cline
		if cline.startswith('CONFIG_PACKAGE_haserl-i18n') and tran_type=='localize' :
			anewline='CONFIG_PACKAGE_haserl=y\n'

		if anewline != '':
			newcfg_doc.append(anewline)
		else:
			newcfg_doc.append(cline)

	if tran_type=='internationalize':
		newcfg_doc.append('#\n')
		newcfg_doc.append('# Gargoyle I18N\n')
		newcfg_doc.append('#\n')
		newcfg_doc.append('\n')
		newcfg_doc.append('CONFIG_PACKAGE_gargoyle-i18n=y\n')
		found_lang=False
		for langpack in glob.glob('./package/plugin-gargoyle-i18n-*/files/www/i18n/*'):
			lang=os.path.basename(langpack)
			if lang==active_lang:
				found_lang=True
				newcfg_doc.append('CONFIG_PACKAGE_plugin-gargoyle-i18n-%s=y\n' % (lang,))
			else:
				newcfg_doc.append('CONFIG_PACKAGE_plugin-gargoyle-i18n-%s=m\n' % (lang,))

		newcfg_doc.append('\n')

		if found_lang == False:
			sys.stderr.write('finalize was unable to find the target language\n')
			sys.exit(1)

	cfg_fileFO = open(config_file, 'wb')
	cfg_fileFO.seek(0)
	cfg_fileFO.writelines(newcfg_doc)
	cfg_fileFO.close()


class Solution:
	# @return a boolean
	def isInterleave(self, s1, s2, s3):
		if len(s3) != len(s1) + len(s2):
			return False
		dp = [False for i in xrange(len(s2) + 1)]
		dp[0] = True
		for i in xrange(1, len(s2) + 1):
			dp[i] = dp[i - 1] and s2[i - 1] == ss[i - 1]
		for i in xrange(1, len(s1)):
			dp[0] = s1[i - 1] == s3[i - 1]
			for j in xrange(1,len(s2) + 1):
				dp[j] = s1[i - 1] == s3[i + j - 1] and dp[j] \
					or s2[j - 1] == s3[i + j - 1] and dp[j - 1]
		return dp[len(s2)]


import csv

"""Enable to produce sample ttl"""
sample = False

"""Define usefull predicates"""
predicate_name = "rdfs:label"
predicate_long = "geo-pos:long"
predicate_lat = "geo-pos:lat"
predicate_purpose = "org:purpose"
predicate_located = "gn:locatedIn"
predicate_eff = "dim-etab:effectif"

"""Define csv headers"""
SIREN = "SIREN"
NIC = "NIC"
APEN = "APEN700"
NOMEN_LONG = "NOMEN_LONG"
LONGITUDE = "longitude"
LATITUDE = "latitude"
DEP = "DEPET"
COM = "COMET"
EFF = "EFETCENT"

""""Define prefixes"""
BASE_URI_SIRET = "http://id.insee.fr/sirene/siret/"
BASE_URI_POINT = "http://id.insee.fr/geo/point/"
BASE_URI_API = "http://id.insee.fr/codes/nafr2/sousClasse/";
BASE_URI_DEP = "http://id.insee.fr/geo/departement/";
BASE_URI_COM = "http://id.insee.fr/geo/commune/";

""""Usefull methods"""
def addChevrons( string ):
   return "<" + string + ">"

def create_line( predicate, object ):
   return "    " + predicate + " " + object.replace('"','')  + " ; \n"

def create_line_string( predicate, object ):
   return "    " + predicate + " \"" + object.replace('"','')  + "\" ; \n"

def create_line_uri( predicate, object ):
   return "    " + predicate + " " + addChevrons(object.replace('"',''))  + " ; \n"

def create_line_float( predicate, object ):
   return "    " + predicate + " \"" + object.replace('"','')  + "\"^^xsd:float ; \n"

def create_point(siret, lat, long):
    point_uri = addChevrons(BASE_URI_POINT+siret)
    point ="    geo:hasPointGeometry %s .\n"%(point_uri)
    point += "%s geo:asWKT \"POINT(%s %s)\"^^geo:wktLiteral .\n"%(point_uri, long, lat)
    return point

def create_effectif_uri(eff):
    if (eff == 'NN'):
        return "tranche-eff:0"
    eff = int(eff)
    if (eff >= 500):
        return "tranche-eff:500"
    if (eff >= 200):
        return "tranche-eff:200"
    if (eff >= 100):
        return "tranche-eff:100"
    if (eff >= 50):
        return "tranche-eff:50"
    if (eff >= 20):
        return "tranche-eff:20"
    if (eff >= 10):
        return "tranche-eff:10"
    if (eff >= 1):
        return "tranche-eff:1"
    return "tranche-eff:0"

def create_code_list_item(code, label):
    item = "\n"
    item += "tranche-eff:%s a skos:Concept ; \n"%(code)
    item += "    skos:inScheme <http://id.insee.fr/codes/tranche-effectif> ; \n"
    item += "    skos:prefLabel \"%s\"@fr . \n"%(label)
    return item

def create_code_list():
   cl = "<http://id.insee.fr/codes/tranche-effectif> a skos:ConceptScheme . \n"
   cl += create_code_list_item("0", "0 salarié ou effectif inconnu")
   cl += create_code_list_item("1", "1 à 9 salariés")
   cl += create_code_list_item("10", "10 à 19 salariés")
   cl += create_code_list_item("20", "20 à 49 salariés")
   cl += create_code_list_item("50", "50 à 99 salariés")
   cl += create_code_list_item("100", "100 à 199 salariés")
   cl += create_code_list_item("200", "200 à 499 salariés")
   cl += create_code_list_item("500", "500 salariés ou plus")
   return cl

"""Build ttl file from csv"""
with open('geo-sirene.ttl', 'w', encoding="utf8") as output:
    """Write prefixes"""
    output.write("@prefix  rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n")
    output.write("@prefix  geo-pos: <http://www.w3.org/2003/01/geo/wgs84_pos#> .\n")
    output.write("@prefix  xsd: <http://www.w3.org/2001/XMLSchema#> .\n")
    output.write("@prefix  rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .\n")
    output.write("@prefix  geo: <http://www.opengis.net/ont/geosparql#> .\n")
    output.write("@prefix  org: <http://www.w3.org/ns/org#> .\n")
    output.write("@prefix  gn: <http://www.geonames.org/ontology#> .\n")
    output.write("@prefix  tranche-eff: <http://id.insee.fr/codes/tranche-effectif/> . \n")
    output.write("@prefix  dim-etab: <http://id.insee.fr/sirene/etablissement/> .\n")
    output.write("@prefix  skos: <http://www.w3.org/2004/02/skos/core#> . \n")
    output.write("\n")

    """Write code list"""
    output.write(create_code_list())
    output.write("\n")

    with open('geo_sirene.csv', 'r', encoding="utf8") as csvfile:
        csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')

        """Get usefull header indexes"""
        headers = next(csvreader)

        indice_SIREN =  headers.index(SIREN)
        indice_NIC =  headers.index(NIC)
        indice_NOMEN_LONG =  headers.index(NOMEN_LONG)
        indice_LONGITUDE =  headers.index(LONGITUDE)
        indice_LATITUDE =  headers.index(LATITUDE)
        indice_APEN =  headers.index(APEN)
        indice_DEP =  headers.index(DEP)
        indice_COM =  headers.index(COM)
        indice_EFF =  headers.index(EFF)

        i=0
        """Get lines to build resources writing triples"""
        for row in csvreader:
            siret = row[indice_SIREN]+row[indice_NIC]
            better_apen = row[indice_APEN][:2]+"."+row[indice_APEN][2:]
            dep = row[indice_DEP]
            com = dep+ row[indice_COM]

            output.write(addChevrons(BASE_URI_SIRET+siret)+"\n")
            output.write("    rdf:type org:OrganizationalUnit ;\n ")
            output.write(create_line_string(predicate_name,row[indice_NOMEN_LONG]))
            output.write(create_line_uri(predicate_purpose,BASE_URI_API+better_apen))
            output.write(create_line_uri(predicate_located,BASE_URI_DEP+dep))
            output.write(create_line_uri(predicate_located,BASE_URI_COM+com))
            output.write(create_line_float(predicate_long,row[indice_LONGITUDE]))
            output.write(create_line_float(predicate_lat,row[indice_LATITUDE]))
            output.write(create_line(predicate_eff,create_effectif_uri(row[indice_EFF])))
            output.write(create_point(siret, row[indice_LATITUDE], row[indice_LONGITUDE] ))
            output.write("\n")

            i+=1

            if (i%100000==0):
                print(i)
                if(sample):
                    break


import os

os.environ['FI_EFA_FORK_SAFE'] = '1'

import reframe as rfm
import reframe.utility.sanity as sn
import hackathon as hack
from reframe.core.launchers import LauncherWrapper
import os
import csv

@rfm.simple_test
class PICSARliteTest(hack.HackathonBase):

    # Where to run the binaries 'aws:c6gn' on Arm or 'aws:c5n' on Intel
    valid_systems = ['aws:c5n' if os.uname().machine == 'x86_64' else 'aws:c6gn']

    # Logging Variables
    log_team_name = 'TeamEPCC'
    log_app_name = 'PICSARlite'
    log_test_name = 'test'
    #log_test_type = 'profile'

    # Define test case
    # In this case we download the file from GitHub and write as picsarlite.in - the expected input file
    prerun_cmds = [
        'wget --quiet -O picsarlite.in https://raw.githubusercontent.com/ECP-WarpX/picsar/348830b444c65ca305aa3f89cd72fef7c65abd7d/examples/example_decks_fortran/test.pixr',
        'touch prof.csv',
    ]

    # Define Execution
    # Binary to run
    executable = 'picsar'
    # Command line options to pass
    executable_opts = ["picsarlite.in"]
    # Where the output is written to
    logfile = 'picsarlite.out'
    # Store the output file (used for validation later)
    keep_files = [logfile]

    # Parameters - Compilers - Defined as their Spack specs (use spec or hash)
    spec = parameter([
            'picsarlite/x4t2nz4', #'picsarlite@0.1%gcc@10.3.0',
            'picsarlite@0.1%arm@21.0.0.879',
            'picsarlite@0.1%nvhpc@21.2'
    ])

    # Parameters - MPI / Threads - Used for scaling studies
    my_parallelism = [
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 64},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 32},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 16},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 8},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 4},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 2},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 1},
            #
            { 'nodes' : 1, 'mpi' : 2, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 4, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 8, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 16, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 32, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 64, 'omp' : 1},
            #
            { 'nodes' : 1, 'mpi' : 2, 'omp' : 32},
            { 'nodes' : 1, 'mpi' : 4, 'omp' : 16},
            { 'nodes' : 1, 'mpi' : 8, 'omp' : 8},
            { 'nodes' : 1, 'mpi' : 16, 'omp' : 4},
            { 'nodes' : 1, 'mpi' : 32, 'omp' : 2},
            #
            { 'nodes' : 2, 'mpi' : 32, 'omp' : 4},
            { 'nodes' : 4, 'mpi' : 64, 'omp' : 4},
            { 'nodes' : 8, 'mpi' : 128, 'omp' : 4},
    ] if 'aws:c6gn' in valid_systems else [
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 72},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 36},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 18},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 9},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 3},
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 1},
            #
            { 'nodes' : 1, 'mpi' : 1, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 3, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 9, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 18, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 36, 'omp' : 1},
            { 'nodes' : 1, 'mpi' : 72, 'omp' : 1},
            #
            { 'nodes' : 1, 'mpi' : 2, 'omp' : 36},
            { 'nodes' : 1, 'mpi' : 4, 'omp' : 18},
            { 'nodes' : 1, 'mpi' : 8, 'omp' : 9},
            { 'nodes' : 1, 'mpi' : 9, 'omp' : 8},
            { 'nodes' : 1, 'mpi' : 18, 'omp' : 4},
            { 'nodes' : 1, 'mpi' : 36, 'omp' : 2},
            #
            { 'nodes' : 2, 'mpi' : 16, 'omp' : 9},
            { 'nodes' : 4, 'mpi' : 32, 'omp' : 9},
            { 'nodes' : 8, 'mpi' : 64, 'omp' : 9},
    ]

    parallelism = parameter(my_parallelism)

    @run_before('run')
    def prepare_job(self):
        self.job.options += ['--exclusive']

    @run_before('sanity')
    def set_sanity_patterns(self):
        # simple check: make sure to find 21x "Fields dump in", indicating 21 iterations
        num_messages = sn.len(sn.findall(r'\s+Fields\s+dump\s+in\s+', self.stdout))
        self.sanity_patterns = sn.assert_eq(num_messages, 21)

        # Performance Testing
        # We dont set an expected value
        self.reference = {
           '*': {'Total runtime': (0, None, None, 's'),}
        }

        # Total runtime
        perf_regex = r'Total\s+runtime\s+on\s+\d+\s+CPUS\s+=\s+(\S+)'
        self.perf_patterns = {
                'Total runtime': sn.extractsingle(perf_regex, self.stderr, 1, float, item=-1)
        }

    # Before we run, we want to add MAP to the launcher
    # But we want to export the functions list to a csv file
    @run_before('run')
    def set_profiler(self):
      self.proffile = 'prof.csv'
      self.keep_files.append(self.proffile)
      self.modules.append('arm-forge@21.0')
      self.job.launcher = LauncherWrapper(self.job.launcher, 'map',
                   ['--profile', '--export-functions='+self.proffile])

    # Now we can retrieve the csv file and parse it for the top 10 functions
    # We log them to the `hot_spots` variable
    # This then gets logged by ReFrame
    @run_before('sanity')
    def get_prof_data(self):
        self.hot_spots = []
        file_path = os.path.join(self.stagedir, self.proffile)
        with open(file_path, newline='') as csvfile:
            reader = csv.reader(csvfile, delimiter=',', quotechar='"')
            next(reader, None) # Skip Header
            for index, row in zip(range(10), reader):
               self.hot_spots.append(
                       {
                           "position" : index,
                           "name" : row[7],
                           "percent" : row[2],
                           "mpi_percent" : row[4]
                        })


import os
import sys
import argparse
import math

import time
import numpy as np

import log as logging
from core import MyModel, confusion_matrix
import trading_data as tdata
import constants
import colors
import utils
import tensorflow as tf

LOG = logging.getLogger(__name__)
epochs = constants.EPOCHS
EPOCHS = constants.EPOCHS

def fit(inputs,
        outputs,
        mu,
        sigma,
        units=1,
        activation='tanh',
        nb_plays=1,
        learning_rate=0.001,
        loss_file_name="./tmp/my_model_loss_history.csv",
        weights_name='model.h5',
        loss_name='mse',
        batch_size=10,
        ensemble=1,
        force_train=False,
        learnable_mu=False):

    # epochs = 20000
    # epochs = 10000
    epochs = 10000
    start = time.time()
    input_dim = batch_size

    timestep = 1
    input_dim = inputs.shape[0]
    # timestep = inputs.shape[0] // input_dim
    # steps_per_epoch = inputs.shape[0] // input_dim
    steps_per_epoch = 1

    mymodel = MyModel(input_dim=input_dim,
                      timestep=timestep,
                      units=units,
                      activation=activation,
                      nb_plays=nb_plays,
                      learning_rate=learning_rate,
                      ensemble=ensemble,
                      learnable_mu=learnable_mu)

    LOG.debug("Learning rate is {}".format(learning_rate))

    preload_weights = False

    if force_train or \
      not os.path.isfile("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays)):
        mymodel.fit2(inputs=inputs,
                     mu=mu,
                     sigma=sigma,
                     outputs=outputs,
                     epochs=epochs,
                     verbose=1,
                     steps_per_epoch=steps_per_epoch,
                     loss_file_name=loss_file_name,
                     preload_weights=preload_weights,
                     weights_fname=weights_fname)
        end = time.time()
        LOG.debug("time cost: {}s".format(end-start))
        LOG.debug("saving weights info")
        mymodel.save_weights(weights_fname)
        LOG.debug("finished saving weights")
    else:
        LOG.debug("already trained, ignore. If you still want to re-train , you can pass flag `force_train`")

def predict(inputs,
            outputs,
            units=1,
            activation='tanh',
            nb_plays=1,
            weights_name='model.h5'):
    with open("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays), 'r') as f:
        line = f.read()

    shape = list(map(int, line.split(":")))

    assert len(shape) == 3, "shape must be 3 dimensions"
    start = time.time()

    input_dim = shape[2]
    timestep = shape[1]

    # num_samples = inputs.shape[0] // (input_dim * timestep)
    if input_dim * timestep > inputs.shape[0]:
        # we need to append extra value to make test_inputs and train_outputs have the same
        # keep test_outputs unchanged
        inputs = np.hstack([inputs, np.zeros(input_dim*timestep-test_inputs.shape[0])])

    start = time.time()
    parallel_prediction = True
    mymodel = MyModel(input_dim=input_dim,
                      timestep=timestep,
                      units=units,
                      activation=activation,
                      nb_plays=nb_plays,
                      parallel_prediction=parallel_prediction)

    mymodel.load_weights(weights_fname)
    predictions = mymodel.predict_parallel(inputs)

    end = time.time()
    LOG.debug("time cost: {}s".format(end-start))
    predictions = predictions[:outputs.shape[0]]
    loss = ((predictions - outputs) ** 2).mean()
    loss = float(loss)
    LOG.debug("loss: {}".format(loss))
    return inputs[:outputs.shape[0]], predictions

def trend(prices,
          B,
          mu,
          sigma,
          units=1,
          activation='tanh',
          nb_plays=1,
          weights_name='model.h5',
          trends_list_fname=None,
          ensemble=1):

    # best_epoch = None
    # try:
    #     with open("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays), 'r') as f:
    #         line = f.read()
    # except FileNotFoundError:
    # # if True:
    #     epochs = []
    #     base = '/'.join(weights_fname.split('/')[:-1])
    #     for _dir in os.listdir(base):
    #         if os.path.isdir('{}/{}'.format(base, _dir)):
    #             try:
    #                 epochs.append(int(_dir.split('-')[-1]))
    #             except ValueError:
    #                 pass

    #     if not epochs:
    #         raise Exception("no trained parameters found")

    #     best_epoch = max(epochs)
    #     best_epoch = 15000
    #     LOG.debug("Best epoch is {}".format(best_epoch))
    #     dirname = '{}-epochs-{}/{}plays'.format(weights_fname[:-3], best_epoch, nb_plays)
    #     if not os.path.isdir(dirname):
    #         # sanity checking
    #         raise Exception("Bugs inside *save_wegihts* or *fit2*")
    #     with open("{}/input_shape.txt".format(dirname), 'r') as f:
    #         line = f.read()

    with open("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays), 'r') as f:
        line = f.read()

    shape = list(map(int, line.split(":")))

    assert len(shape) == 3, "shape must be 3 dimensions"
    input_dim = shape[2]

    timestep = 1
    shape[1] = timestep

    mymodel = MyModel(input_dim=input_dim,
                      timestep=timestep,
                      units=units,
                      activation=activation,
                      nb_plays=nb_plays,
                      parallel_prediction=True,
                      ensemble=ensemble)

    mymodel.load_weights(weights_fname, extra={'shape': shape})
    guess_trend = mymodel.trend(prices=prices, B=B, mu=mu, sigma=sigma)

    loss = float(-1.0)
    return guess_trend, loss

def plot_graphs_together(price_list, noise_list, mu, sigma,
                         units=1,
                         activation='tanh',
                         nb_plays=1,
                         weights_name='model.h5',
                         trends_list_fname=None, ensemble=1):
    best_epoch = None

    # try:
    #     with open("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays), 'r') as f:
    #         line = f.read()
    # except FileNotFoundError:
    if True:
        epochs = []
        base = '/'.join(weights_fname.split('/')[:-1])
        for _dir in os.listdir(base):
            if os.path.isdir('{}/{}'.format(base, _dir)):
                try:
                    epochs.append(int(_dir.split('-')[-1]))
                except ValueError:
                    pass

        if not epochs:
            raise Exception("no trained parameters found")

        best_epoch = max(epochs)
        best_epoch = 15000
        LOG.debug("Best epoch is {}".format(best_epoch))
        dirname = '{}-epochs-{}/{}plays'.format(weights_fname[:-3], best_epoch, nb_plays)
        if not os.path.isdir(dirname):
            # sanity checking
            raise Exception("Bugs inside *save_wegihts* or *fit2*")
        with open("{}/input_shape.txt".format(dirname), 'r') as f:
            line = f.read()

    shape = list(map(int, line.split(":")))
    assert len(shape) == 3, "shape must be 3 dimensions"
    input_dim = shape[2]

    timestep = 1
    shape[1] = timestep
    parallelism = True
    mymodel = MyModel(input_dim=input_dim,
                      timestep=timestep,
                      units=units,
                      activation=activation,
                      nb_plays=nb_plays,
                      parallel_prediction=parallelism,
                      ensemble=ensemble)

    mymodel.load_weights(weights_fname, extra={'shape': shape, 'parallelism': parallelism, 'best_epoch': best_epoch, 'use_epochs': True})
    mymodel.plot_graphs_together(prices=price_list, noises=noise_list, mu=mu, sigma=sigma)

def visualize(inputs,
              mu=0,
              sigma=1,
              units=1,
              activation='tanh',
              nb_plays=1,
              weights_name='model.h5'):

    with open("{}/{}plays/input_shape.txt".format(weights_name[:-3], nb_plays), 'r') as f:
        line = f.read()
    shape = list(map(int, line.split(":")))

    assert len(shape) == 3, "shape must be 3 dimensions"
    input_dim = shape[2]
    # timestep = inputs.shape[0] // input_dim
    timestep = 1
    shape[1] = timestep
    start = time.time()

    mymodel = MyModel(input_dim=input_dim,
                      timestep=timestep,
                      units=units,
                      activation=activation,
                      nb_plays=nb_plays)

    mymodel.load_weights(weights_fname, extra={'shape': shape})
    mymodel.visualize_activated_plays(inputs=inputs)

def plot(a, b, trend_list):
    from matplotlib import pyplot as plt
    x = range(1, a.shape[0]+1)
    diff1 = ((a[1:] - a[:-1]) >= 0).tolist()
    diff2 = ((b[1:] - a[:-1]) >= 0).tolist()
    fig, (ax1, ax2, ax3) = plt.subplots(3, 1, sharex=True)
    ax1.plot(x, a, color='blue')
    ax1.plot(x, b, color='black')

    for index, d1, d2 in zip(x[1:], diff1, diff2):
        if d1 is True and d2 is True:
            ax1.scatter([index], [b[index-1]], marker='^', color='green')
        elif d1 is False and d2 is False:
            ax1.scatter([index], [b[index-1]], marker='^', color='green')
        elif d1 is False and d2 is True:
            ax1.scatter([index], [b[index-1]], marker='s', color='black')
        elif d1 is True and d2 is False:
            ax1.scatter([index], [b[index-1]], marker='s', color='black')

    ax2.plot(x, a, color='blue')
    min_trend_list = trend_list.min(axis=1)
    max_trend_list = trend_list.max(axis=1)
    ax2.fill_between(x, min_trend_list, max_trend_list, facecolor='gray', alpha=0.5, interpolate=True)
    ax3.plot(x, a, color='blue')
    trend_list_ = [trend for trend in  trend_list]
    ax3.boxplot(trend_list_)

    plt.show()
    # fname = "/Users/baymax_testios/Desktop/1.png"
    fname = "./1.png"
    fig.savefig(fname, dpi=400)

def ttest_rel(method1, method2):
    # outputs = np.array(outputs).reshape(-1)
    # guess_prices = np.array(guess_prices).reshape(-1)

    # loss1 =  ((guess_prices - prices[start_pos:end_pos]) ** 2)
    # loss2 = np.abs(guess_prices - prices[start_pos:end_pos])
    # loss3 = (prices[start_pos:end_pos] - prices[start_pos-1:end_pos-1]) ** 2
    # loss4 = np.abs(prices[start_pos:end_pos] - prices[start_pos-1:end_pos-1])

    # LOG.debug("root sum square loss1: {}".format((loss1.sum()/(end_pos-start_pos))**(0.5)))
    # LOG.debug("root sum square loss2: {}".format((loss3.sum()/(end_pos-start_pos))**(0.5)))
    # LOG.debug("total abs loss1: {}".format((loss2.sum()/(end_pos-start_pos))))
    # LOG.debug("total abs loss2: {}".format((loss4.sum()/(end_pos-start_pos))))

    # guess_prices_list = np.array(guess_prices_list)
    pass

def rmse_bucket(ground_truth_noise, ground_truth_price, predict_price, price_steps=10, noise_steps=10):
    diff_price_list = np.abs(ground_truth_price[1:] - ground_truth_price[:-1])
    diff_noise_list = np.abs(ground_truth_noise[1:] - ground_truth_noise[:-1])

    delta_price_step = (np.max(diff_price_list) - np.min(diff_price_list)) / price_steps
    delta_noise_step = (np.max(diff_noise_list) - np.min(diff_noise_list)) / noise_steps

    delta_price_list = [delta_price_step * i + np.min(diff_price_list) for i in range(price_steps+1)]
    delta_noise_list = [delta_noise_step * i + np.min(diff_noise_list) for i in range(noise_steps+1)]

    diff_ground_truth_predict_of_price_list = ground_truth_price[1:] - predict_price

    bucket = { (p, n) : [] for p in range(price_steps+1) for n in range(noise_steps+1)}

    max_val = -1
    i = 0
    for dp, dn, diff in zip(diff_price_list, diff_noise_list, diff_ground_truth_predict_of_price_list):
        _p_idx = (dp - np.min(diff_price_list)) / delta_price_step
        _n_idx = (dn - np.min(diff_noise_list)) / delta_noise_step

        p_idx = math.floor(_p_idx)
        n_idx = math.ceil(_n_idx)

        # import ipdb; ipdb.set_trace()

        p_idx1 = p_idx
        n_idx1 = n_idx

        val = diff * diff
        # val = abs(diff)
        if val > max_val:
            max_val = val

        # single constraints >= p, <= n
        # for _n_idx in range(n_idx, noise_steps+1):
        #     for _p_idx in range(0, p_idx+1):
        #         bucket[(_p_idx, _n_idx)].append((val, dp, dn, i, predict_price[i], ground_truth_price[i+1], ground_truth_price[i]))

        bucket[(p_idx, n_idx)].append((val, dp, dn, i, predict_price[i], ground_truth_price[i+1], ground_truth_price[i]))
        i += 1

    return bucket, delta_price_step, delta_noise_step, delta_price_list, delta_noise_list, max_val

def rmse3d():
    # https://stackoverflow.com/questions/23670178/matplotlib-3d-bar-chart-axis-issue
    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt
    from matplotlib import gridspec
    import numpy as np
    from matplotlib import colors as mcolors
    from matplotlib import cm

    base_file = "./new-dataset/models/diff_weights/method-sin/activation-None/state-0/markov_chain/mu-0/sigma-110/units-10000/nb_plays-20/points-1000/input_dim-1/mu-0-sigma-110-points-1000.csv"

    trend_file = "./new-dataset/models/diff_weights/method-sin/activation-None/state-0/markov_chain/mu-0/sigma-110/units-20/nb_plays-20/points-1000/input_dim-1/predictions-mu-0-sigma-110-points-1000/activation#-elu/state#-0/units#-100/nb_plays#-100/ensemble-11/loss-mle/trends-batch_size-1500.csv"
    _, ground_truth_noise = tdata.DatasetLoader.load_data(base_file)
    ground_truth_price, predict_price = tdata.DatasetLoader.load_data(trend_file)

    ground_truth_price = ground_truth_price[:100]
    predict_price = predict_price[:100]
    # ground_truth_noise =
    baseline_price = ground_truth_price[:-1]
    # ground_truth_price = ground_truth_price[1:]
    predict_price = predict_price[1:]
    # ground_truth_noise = ground_truth_noise[1001:1010]
    ground_truth_noise = ground_truth_noise[1000:1100]

    price_steps = 5
    noise_steps = 5

    assert price_steps == noise_steps, "price_steps and noise_steps must be the same"
    _baseline_bucket, delta_price_step, delta_noise_step, delta_price_list, delta_noise_list, baseline_max_rmse = rmse_bucket(ground_truth_noise, ground_truth_price, baseline_price, price_steps, noise_steps)
    _predict_bucket, _, _, _, _, predict_max_rmse  = rmse_bucket(ground_truth_noise, ground_truth_price, predict_price, price_steps, noise_steps)

    max_rmse = predict_max_rmse if predict_max_rmse > baseline_max_rmse else baseline_max_rmse
    rmse_steps = 5
    delta_rmse = max_rmse / rmse_steps
    delta_rmse_list = [i*delta_rmse for i in range(rmse_steps+1)]

    def _helper(ax, _bucket, x, y, zlabel='rmse', color='cyan', func=lambda v: v):
        _zz = np.zeros((price_steps+1, noise_steps+1), dtype=np.float32)

        for k, v in _bucket.items():
            if len(v) != 0:
                _p, _n = k
                # ipdb;ipdb.set_trace()
                # _p * delta_price_step + np.min(diff_price_list)
                _v = [vv[0] for vv in v]
                _pv = [vv[1] for vv in v]
                _nv = [vv[2] for vv in v]
                _zz[k] = func(_v)
                print("{}, ({}, {}), {}, {}, {}, {}".format(k, delta_price_list[_p], delta_noise_list[_n], v, _zz[k],  min(_v), max(_nv)))

        z = _zz.T.ravel()
        print("zz: ", _zz)
        # import ipdb; ipdb.set_trace()

        bottom = np.zeros_like(z)
        ax.bar3d(x, y, bottom, width, depth, z, shade=True, color=color)

        ax.w_xaxis.set_ticks(_x)
        xticks = np.array(["{:.3f}".format(p) for p in delta_price_list])
        ax.w_xaxis.set_ticklabels(xticks)

        ax.w_yaxis.set_ticks(_y + 0.5)
        yticks = np.array(["{:.1f}".format(n) for n in delta_noise_list])
        ax.w_yaxis.set_ticklabels(yticks)

        ax.set_xlabel('$\Delta p$')
        ax.set_ylabel('$\Delta n$')
        ax.set_zlabel(zlabel)
        return z

    fig = plt.figure(constrained_layout=True)

    spec = gridspec.GridSpec(ncols=2, nrows=2, figure=fig)

    ax1 = fig.add_subplot(spec[0, 0], projection='3d')
    ax2 = fig.add_subplot(spec[0, 1], projection='3d', sharez=ax1)

    ax3 = fig.add_subplot(spec[1, 0], projection='3d')
    ax4 = fig.add_subplot(spec[1, 1], projection='3d', sharez=ax3)

    _x = np.arange(len(delta_price_list))
    _y = np.arange(len(delta_noise_list))

    _xx, _yy = np.meshgrid(_x, _y)
    print("_xx: ", _xx)
    print("_yy: ", _yy)
    # import ipdb; ipdb.set_trace()
    x, y = _xx.ravel(), _yy.ravel()

    width = 0.5
    depth = 0.5

    print("width: ", width, ", depth: ", depth)

    values = np.linspace(0.2, 1., x.shape[0])
    colors = cm.rainbow(values)

    if baseline_max_rmse > predict_max_rmse:
        _helper(ax2, _predict_bucket, x, y, zlabel='PREDICT-RMSE',
                # color=mcolors.CSS4_COLORS['darkorange'],
                color=colors,
                func=lambda v: (sum(v)/len(v))**0.5)
                # func=lambda v: (sum(v)/len(v)))
        _helper(ax1, _baseline_bucket, x, y, zlabel='BASELINE-RMSE',
                # color=mcolors.CSS4_COLORS['dodgerblue'],
                color=colors,
                func=lambda v: (sum(v)/len(v))**0.5)
                # func=lambda v: (sum(v)/len(v)))
    else:
        _helper(ax1, _baseline_bucket, x, y, zlabel='BASELINE-RMSE',
                # color=mcolors.CSS4_COLORS['dodgerblue'],
                color=colors,
                func=lambda v: (sum(v)/len(v))**0.5)
                # func=lambda v: (sum(v)/len(v)))
        _helper(ax2, _predict_bucket, x, y, zlabel='PREDICT-RMSE',
                # color=mcolors.CSS4_COLORS['darkorange'],
                color=colors,
                func=lambda v: (sum(v)/len(v))**0.5)
                # func=lambda v: (sum(v)/len(v)))

    print("================================================================================")
    _helper(ax3, _baseline_bucket, x, y, zlabel='BASELINE-COUNTS',
            # color=mcolors.CSS4_COLORS['dodgerblue'],
            color=colors,
            func=lambda v: len(v))
    _helper(ax4, _predict_bucket, x, y, zlabel='PREDICT-COUNTS',
            # color=mcolors.CSS4_COLORS['darkorange'],
            color=colors,
            func=lambda v: len(v))

    plt.show()
    import ipdb; ipdb.set_trace()

if __name__ == "__main__":
    # print("Plot RMSE 3D")
    # rmse3d()
    # import ipdb; ipdb.set_trace()

    LOG.debug(colors.red("Test multiple plays"))

    parser = argparse.ArgumentParser()
    parser.add_argument("--batch_size", dest="batch_size",
                        default=1000,
                        type=int)
    parser.add_argument("--__nb_plays__", dest="__nb_plays__",
                        default=2,
                        type=int)
    parser.add_argument("--__units__", dest="__units__",
                        default=5,
                        type=int)
    parser.add_argument("--__activation__", dest="__activation__",
                        default=None,
                        type=str)

    parser.add_argument('--trend', dest='trend',
                        default=False, action='store_true')
    parser.add_argument('--predict', dest='predict',
                        default=False, action='store_true')
    parser.add_argument('--plot', dest='plot',
                        default=False, action='store_true')
    parser.add_argument('--visualize_activated_plays', dest='visualize_activated_plays',
                        default=False, action='store_true')
    parser.add_argument('--__mu__', dest='__mu__',
                        default=0,
                        type=float)
    parser.add_argument('--__sigma__', dest='__sigma__',
                        default=110,
                        type=float)
    parser.add_argument('--ensemble', dest='ensemble',
                        default=2,  # start from 1
                        type=int)
    parser.add_argument('--force-train', dest='force_train',
                        default=False, action='store_true')

    parser.add_argument('--learnable-mu', dest='learnable_mu',
                        default=False, action='store_true')

    argv = parser.parse_args(sys.argv[1:])
    # Hyper Parameters
    # learning_rate = 0.003
    learning_rate = 0.07

    batch_size = argv.batch_size

    # loss_name = 'mse'
    loss_name = 'mle'

    method = 'sin'
    # method = 'mixed'
    # method = 'noise'
    interp = 1
    # do_prediction = False
    do_prediction = argv.predict
    do_confusion_matrix = False
    mc_mode = True
    do_trend = argv.trend
    do_plot = argv.plot
    do_visualize_activated_plays = argv.visualize_activated_plays
    ensemble = argv.ensemble
    with_noise = True

    diff_weights = True

    run_test = False

    mu = 0
    sigma = 110

    points = 1000
    input_dim = 1
    ############################## ground truth #############################
    nb_plays = 20
    # units is 10000 special for dataset comes from simulation
    units = 20
    state = 0
    activation = 'tanh'
    activation = None
    ############################## predicitons #############################
    __nb_plays__ = argv.__nb_plays__
    __units__ = argv.__units__
    # __nb_plays__ = 2
    # __units__ = 2

    __state__ = 0
    __activation__ = argv.__activation__
    # __activation__ = 'relu'
    # __activation__ = None
    # __activation__ = 'tanh'
    # __mu__ = 2.60
    __mu__ = 0
    __sigma__ = 1
    # __mu__ = argv.__mu__
    # __sigma__ = argv.__sigma__

    if method == 'noise':
        with_noise = True

    if with_noise is False:
        mu = 0
        sigma = 0

    if diff_weights is True:
        # input_file_key = 'models_diff_weights'
        # loss_file_key = 'models_diff_weights_loss_history'
        if mc_mode is True:
            weights_file_key = 'models_diff_weights_mc_saved_weights'
        else:
            weights_file_key = 'models_diff_weights_saved_weights'
        # predictions_file_key = 'models_diff_weights_predictions'
        weights_file_key = 'models_diff_weights_mc_saved_weights'
    else:
        # input_file_key = 'models'
        # loss_file_key = 'models_loss_history'
        # weights_file_key = 'models_saved_weights'
        # predictions_file_key = 'models_predictions'
        raise

    weights_file_key = 'models_diff_weights_mc_stock_model_saved_weights'

    # XXXX: place weights_fname before run_test
    weights_fname = constants.DATASET_PATH[weights_file_key].format(method=method,
                                                                    activation=activation,
                                                                    state=state,
                                                                    mu=mu,
                                                                    sigma=sigma,
                                                                    units=units,
                                                                    nb_plays=nb_plays,
                                                                    points=points,
                                                                    input_dim=input_dim,
                                                                    __activation__=__activation__,
                                                                    __state__=__state__,
                                                                    __units__=__units__,
                                                                    __nb_plays__=__nb_plays__,
                                                                    loss=loss_name,
                                                                    ensemble=ensemble,
                                                                    batch_size=batch_size)
    if interp != 1:
        if do_prediction is False:
            raise
        if run_test is True:
            raise
        elif run_test is False:
            raise
    elif interp == 1:
        if run_test is True:
            raise
        elif run_test is False:
            if diff_weights is True:
                input_file_key = 'models_diff_weights'
                loss_file_key = 'models_diff_weights_loss_history'
                predictions_file_key = 'models_diff_weights_predictions'
            else:
                raise

    # if do_trend is True:
    ################### markov chain #############################
    if mc_mode is True:
        input_file_key = 'models_diff_weights_mc_stock_model'
        loss_file_key = 'models_diff_weights_mc_stock_model_loss_history'
        predictions_file_key = 'models_diff_weights_mc_stock_model_predictions'
        if do_trend is True:
            predictions_file_key = 'models_diff_weights_mc_stock_model_trends'
            trends_list_file_key = 'models_diff_weights_mc_stock_model_trends_list'
    else:
        input_file_key = 'models_diff_weights_mc'
        loss_file_key = 'models_diff_weights_mc_loss_history'
        predictions_file_key = 'models_diff_weights_mc_predictions'

    fname = constants.DATASET_PATH[input_file_key].format(interp=interp,
                                                          method=method,
                                                          activation=activation,
                                                          state=state,
                                                          mu=mu,
                                                          sigma=sigma,
                                                          units=units,
                                                          nb_plays=nb_plays,
                                                          points=1000,
                                                          input_dim=input_dim)

    LOG.debug("Load data from file: {}".format(colors.cyan(fname)))
    if do_prediction is True and do_trend is True:
        raise Exception("both do predictions and do_trend are True")

    # Debug Dima hysteresis behaviours
    # fname = 'new-dataset/models/diff_weights/method-sin/activation-None/state-0/markov_chain/mu-0/sigma-110/units-10000/nb_plays-20/points-1000/input_dim-1/mu-0-sigma-110-points-1000-debug-5.csv'
    inputs, outputs = tdata.DatasetLoader.load_data(fname)
    if do_trend is False:
        # inputs, outputs = inputs[:points], outputs[:points]
        pass
    if mc_mode is True:
        # inputs, outputs = outputs, inputs
        pass
    else:
        inputs, outputs = outputs, inputs
        # gap = 5
        # inputs, outputs = inputs[::gap], outputs[::gap]
        # # inputs = np.arange(800)[::4].astype(np.float32)
        # # inputs = np.zeros(800)[::4].astype(np.float32)
        # # mu = 0
        # # sigma = 0.5
        # # points = 200
        # # noise = np.random.normal(loc=mu, scale=sigma, size=points).astype(np.float32)
        # # inputs += noise
        # mu1 = 4
        # sigma1 = 2.5
        # inputs = tdata.DatasetGenerator.systhesis_markov_chain_generator(200, mu1, sigma1)

        pass

    # inputs, outputs = outputs, inputs
    # inputs, outputs = inputs[:2000], outputs[:2000]
    # It's for debug variables
    # inputs, outputs = inputs[:1500*20], outputs[:1500*20]
    # inputs, outputs = inputs[::20], outputs[::20]

    loss_history_file = constants.DATASET_PATH[loss_file_key].format(interp=interp,
                                                                     method=method,
                                                                     activation=activation,
                                                                     state=state,
                                                                     mu=mu,
                                                                     sigma=sigma,
                                                                     units=units,
                                                                     nb_plays=nb_plays,
                                                                     points=points,
                                                                     input_dim=input_dim,
                                                                     __activation__=__activation__,
                                                                     __state__=__state__,
                                                                     __units__=__units__,
                                                                     __nb_plays__=__nb_plays__,
                                                                     loss=loss_name,
                                                                     ensemble=ensemble,
                                                                     batch_size=batch_size)

    predicted_fname = constants.DATASET_PATH[predictions_file_key].format(interp=interp,
                                                                          method=method,
                                                                          activation=activation,
                                                                          state=state,
                                                                          mu=mu,
                                                                          sigma=sigma,
                                                                          units=units,
                                                                          nb_plays=nb_plays,
                                                                          points=points,
                                                                          input_dim=input_dim,
                                                                          __activation__=__activation__,
                                                                          __state__=__state__,
                                                                          __units__=__units__,
                                                                          __nb_plays__=__nb_plays__,
                                                                          loss=loss_name,
                                                                          ensemble=ensemble,
                                                                          batch_size=batch_size)

    if mc_mode is True and do_trend is True:
        trends_list_fname = constants.DATASET_PATH[trends_list_file_key].format(interp=interp,
                                                                                method=method,
                                                                                activation=activation,
                                                                                state=state,
                                                                                mu=mu,
                                                                                sigma=sigma,
                                                                                units=units,
                                                                                nb_plays=nb_plays,
                                                                                points=points,
                                                                                input_dim=input_dim,
                                                                                __activation__=__activation__,
                                                                                __state__=__state__,
                                                                                __units__=__units__,
                                                                                __nb_plays__=__nb_plays__,
                                                                                loss=loss_name,
                                                                                ensemble=ensemble,
                                                                                batch_size=batch_size)

    LOG.debug('############################  SETTINGS #########################################')
    LOG.debug('# Learning Rate: {}'.format(learning_rate))
    LOG.debug('# points: {}'.format(points))
    LOG.debug('# nb_plays: {}'.format(nb_plays))
    LOG.debug('# units: {}'.format(units))
    LOG.debug('# activation: {}'.format(activation))
    LOG.debug("# mu: {}".format(mu))
    LOG.debug("# sigma: {}".format(sigma))
    LOG.debug("# state: {}".format(state))
    LOG.debug('# __nb_plays__: {}'.format(__nb_plays__))
    LOG.debug('# __units__: {}'.format(__units__))
    LOG.debug('# __activation__: {}'.format(__activation__))
    LOG.debug("# __mu__: {}".format(__mu__))
    LOG.debug("# __sigma__: {}".format(__sigma__))
    LOG.debug("# __state__: {}".format(__state__))

    LOG.debug("# do_prediction: {}".format(do_prediction))
    LOG.debug("# do_trend: {}".format(do_trend))
    LOG.debug("# do_fit: {}".format(not (do_prediction and do_trend)))
    LOG.debug("# mc_mode: {}".format(mc_mode))

    LOG.debug('# train_fname: {}'.format(colors.cyan(fname)))
    LOG.debug('# predicted_fname: {}'.format(colors.cyan(predicted_fname)))
    LOG.debug('# weights_fname: {}'.format(colors.cyan(weights_fname)))

    LOG.debug('################################################################################')

    # input(colors.red("Press Enter to continue..."))

    # try:
    #     predicted_fname = 'new-dataset/models/diff_weights/method-sin/activation-None/state-0/markov_chain/mu-0/sigma-110/units-20/nb_plays-20/points-1000/input_dim-1/predictions-mu-0-sigma-110-points-1000/activation#-elu/state#-0/units#-100/nb_plays#-100/ensemble/loss-mle/trends-batch_size-1500.csv'
    #     a, b = tdata.DatasetLoader.load_data(predicted_fname)
    #     # inp, trend_list = tdata.DatasetLoader.load_data(trends_list_fname)
    #     # assert np.allclose(a, inp, atol=1e-5)
    #     confusion = confusion_matrix(a, b)
    #     LOG.debug(colors.purple("confusion matrix is: {}".format(confusion)))

    #     # plot(a, b, trend_list)

    #     hnn_rsme = (((b[:-1] - a[:-1]) ** 2).mean())**(0.5)
    #     baseline_rsme =  (((a[1:] - a[:-1]) ** 2).mean())**(0.5)
    #     # loss2 = np.abs(guess_prices - prices[start_pos:end_pos])
    #     # loss3 = (prices[start_pos:end_pos] - prices[start_pos-1:end_pos-1]) ** 2
    #     # loss4 = np.abs(prices[start_pos:end_pos] - prices[start_pos-1:end_pos-1])
    #     LOG.debug("hnn-RMSE: {}".format(hnn_rsme))
    #     LOG.debug("baseline-RMSE: {}".format(baseline_rsme))

    #     sys.exit(0)
    # except FileNotFoundError:
    #     LOG.warning("Not found prediction file, no way to create confusion matrix")

    if mc_mode is True and do_trend is True:
        predictions, loss = trend(prices=inputs[:batch_size*2],
                                  B=outputs[:batch_size*2],
                                  mu=__mu__,
                                  sigma=__sigma__,
                                  units=__units__,
                                  activation=__activation__,
                                  nb_plays=__nb_plays__,
                                  weights_name=weights_fname,
                                  trends_list_fname=trends_list_fname,
                                  ensemble=ensemble)
        # inputs = inputs[batch_size:batch_size+predictions.shape[-1]]
        # inputs = inputs[batch_size:batch_size+predictions.shape[-1]]
        # inputs = inputs[1000:1100]
        import ipdb; ipdb.set_trace()
        inputs = inputs[1510:1610]
        # inputs = inputs[1510:1515]
    elif do_visualize_activated_plays is True:
        LOG.debug(colors.red("Load weights from {}, DO VISUALIZE ACTIVATED PLAYS".format(weights_fname)))
        visualize(inputs=inputs[:batch_size],
                  mu=__mu__,
                  sigma=__sigma__,
                  units=__units__,
                  activation=__activation__,
                  nb_plays=__nb_plays__,
                  weights_name=weights_fname)
        sys.exit(0)
    elif do_prediction is True:

        LOG.debug(colors.red("Load weights from {}".format(weights_fname)))
        # import ipdb; ipdb.set_trace()
        # inputs, outputs = inputs[:batch_size], outputs[:batch_size]
        predictions, best_epoch = predict(inputs=inputs,
                                           outputs=outputs,
                                           units=__units__,
                                           activation=__activation__,
                                           nb_plays=__nb_plays__,
                                           weights_name=weights_fname)
        if best_epoch is not None:
            predicted_fname = "{}-epochs-{}.csv".format(predicted_fname[:-4], best_epoch)

    elif do_plot is True:
        inputs, outputs = inputs[:batch_size], outputs[:batch_size]
        plot_graphs_together(price_list=inputs, noise_list=outputs, mu=__mu__, sigma=__sigma__,
                             weights_name=weights_fname,
                             units=__units__,
                             activation=__activation__,
                             nb_plays=__nb_plays__,
                             ensemble=ensemble)
        sys.exit(0)
    else:
        LOG.debug("START to FIT via {}".format(colors.red(loss_name.upper())))
        _inputs, _outputs = inputs[:2000], outputs[:2000]
        train_inputs, train_outputs = _inputs[:1500], _outputs[:1500]
        test_inputs, test_outputs = _inputs[1500:], _outputs[1500:]

        fit(inputs=train_inputs,
            outputs=train_outputs,
            mu=__mu__,
            sigma=__sigma__,
            units=__units__,
            activation=__activation__,
            nb_plays=__nb_plays__,
            learning_rate=learning_rate,
            loss_file_name=loss_history_file,
            weights_name=weights_fname,
            loss_name=loss_name,
            batch_size=batch_size,
            ensemble=ensemble,
            force_train=argv.force_train,
            learnable_mu=argv.learnable_mu)

        inputs, predictions = predict(inputs=test_inputs,
                                      outputs=test_outputs,
                                      units=__units__,
                                      activation=__activation__,
                                      nb_plays=__nb_plays__,
                                      weights_name=weights_fname)

    LOG.debug("Write data into predicted_fname: {}".format(predicted_fname))
    tdata.DatasetSaver.save_data(inputs, predictions, predicted_fname)


import sys, os, re
import json

from kafka import KafkaConsumer, TopicPartition
consumer = KafkaConsumer()
consumer.assign([TopicPartition('test', 0)])
consumer.seek_to_beginning()

for message in consumer:
  message_bytes = message.value
  message_string = message_bytes.decode()
  message_object = json.loads(message_string)
  print(message_object)


# Copyright (c) 2021: Zhiyuan Nan (namjw@hanyang.ac.kr).
#
# This work is licensed under the terms of the MIT license.
# For a copy, see <https://opensource.org/licenses/MIT>.

import torch as T
import random
import numpy as np
from torch.utils.tensorboard import SummaryWriter

import sys
import os
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))

from icsl_rl.agent import DQNAgent, DoubleDQNAgent, DuelingDQNAgent, D3QNAgent, NoisyDQNAgent
from icsl_rl.agent import DDPGAgent, TD3Agent, SACAgent, PPOAgent, A2CAgent, BC_SACAgent
from icsl_rl.arguments import get_args
from icsl_rl.runner import Runner

# Random Seed Settings
def _random_seed(seed):
    if T.backends.cudnn.enabled:
        T.backends.cudnn.benchmark = False
        T.backends.cudnn.deterministic = True

    T.manual_seed(seed)
    T.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)
    print('Using GPU : ', T.cuda.is_available() , ' |  Seed : ', seed)

if __name__ == '__main__':
    args = get_args()
    writer = SummaryWriter('./logs/' + args.algorithm)
    _random_seed(args.seed)

    if args.algorithm == 'DQN':
        agent = DQNAgent(args)

    if args.algorithm == 'Double_DQN':
        agent = DoubleDQNAgent(args)

    if args.algorithm == 'Dueling_DQN':
        agent = DuelingDQNAgent(args)

    if args.algorithm == 'D3QN':
        agent = D3QNAgent(args)

    if args.algorithm == 'Noisy_DQN':
        agent = NoisyDQNAgent(args)

    if args.algorithm == 'DDPG':
        agent = DDPGAgent(args)

    if args.algorithm == 'TD3':
        agent = TD3Agent(args)

    if args.algorithm == 'SAC':
        agent = SACAgent(args)

    if args.algorithm == 'PPO':
        agent = PPOAgent(args)

    if args.algorithm == 'A2C':
        agent = A2CAgent(args)

    if args.algorithm == 'BC_SAC':
        agent = BC_SACAgent(args)

    runner = Runner(agent, args, agent.env, writer)

    if args.evaluate:
        returns = runner.evaluate()
        print('Average returns is', returns)
        runner.gif(agent, agent.env)
    else:
        if args.algorithm == 'PPO':
            runner.ppo_run()
        else:
            runner.run()


"""
Author: Andrew Stark (andrew.stark@optimumpps.co.uk)

Initialisation for Web App

"""
from flask import Flask, render_template
from flask_mail import Mail
from flask_oauthlib.client import OAuth
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_moment import Moment
from config import config
from flask_sslify import SSLify
from flask_debugtoolbar import DebugToolbarExtension
from flask_heroku import Heroku

mail = Mail()
oauth = OAuth()
db = SQLAlchemy()

login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'

toolbar = DebugToolbarExtension()
heroku = Heroku()
moment = Moment()

def create_app(config_name):
    """
    Create the instance of the Flask Web App
    """
    app = Flask(__name__)
    app.config.from_object(config[config_name])
    config[config_name].init_app(app)

    mail.init_app(app)
    oauth.init_app(app)
    # db.init_app(app)
    login_manager.init_app(app)
    heroku.init_app(app)
    moment.init_app(app)

    if app.config['DEBUG'] == True:
        toolbar.init_app(app)

    if not app.debug and not app.testing and not app.config['SSL_DISABLE']:
        sslify = SSLify(app)

    @app.errorhandler(404)
    def page_not_found(err):
        """
        Return the page not found HTTP error
        """
        return render_template("error/404.html", err=err)

    @app.errorhandler(403)
    def page_not_accessible(err):
        """
        Return the page not accessible HTTP error
        """
        return render_template("error/403.html", err=err)

    @app.errorhandler(500)
    def internal_server_error(err):
        """
        Return the internal server error HTTP error
        """
        return render_template("error/500.html", err=err)

    from . views.auth import auth as auth_blueprint
    app.register_blueprint(auth_blueprint, url_prefix='/auth')

    from . views.main import main as main_blueprint
    app.register_blueprint(main_blueprint)

    return app


import pandas as pd
import argparse
import pickle
import warnings
warnings.filterwarnings('ignore')

parser = argparse.ArgumentParser()
parser.add_argument("-input_data", required=True, type=str)
parser.add_argument("-output_data", required=True, type=str)
parser.add_argument("-model", required=True, type=str)
args = parser.parse_args()

def fetch_model(model_path):
	with open(model_path, "rb") as f:
		rule_model = pickle.loads(f.read())
		return rule_model

def predict(input_data, output_data, model):
	if 'rules' in model.keys():
		data = pd.read_csv(input_data).query(model['rules']).fillna(0)
		ID = data['phone'].copy(deep=True)
		del data['phone']
		clf = model['model']
		pred = clf.predict(data)
		pred_proba = clf.predict_proba(data)
		output = pd.DataFrame({"ID": ID, "pred": pred, "pred_proba": pred_proba[:, 1]})
		output.query("pred == 1")[['ID', "pred_proba"]].to_csv(output_data, index=False)
		return output

if __name__ == "__main__":
	model = fetch_model(args.model)
	input_data = args.input_data
	output_data = args.output_data
	predict(input_data, output_data, model)


#===============================================================================
#
# EHOSTDL Tools build script
#
# GENERAL DESCRIPTION
#    EHOSTDL image build script
#
# Copyright (c) 2009-2012 by QUALCOMM, Incorporated.
# All Rights Reserved.
# QUALCOMM Confidential and Proprietary
#
#-------------------------------------------------------------------------------
#
#  $Header: //source/qcom/qct/core/pkg/bootloaders/rel/1.2.2/boot_images/core/bsp/tools/flash/ehostdl/build/SConscript#2 $
#  $DateTime: 2012/08/30 10:00:04 $
#  $Author: coresvc $
#  $Change: 2751359 $
#                      EDIT HISTORY FOR FILE
#
#  This section contains comments describing changes made to the module.
#  Notice that changes are listed in reverse chronological order.
#
# when       who     what, where, why
# --------   ---     ---------------------------------------------------------
# 08/23/12   sb      Add required libs to call pmic related functions in ehostdl
# 03/08/12   vm      Updated proc value for 8K targets
# 03/06/12   vm      Updated to use new image template and functions
# 12/29/11   sv      Disable ehostdl compilation for 8960, 8064, 8930
# 10/31/11   eo      Restore code start address to include header bytes.
# 10/31/11   eo      Update code start address for ehostdl hex image
# 10/26/11   sy      Calling buildProduct library to add compiled hex into BuildProduct.txt
# 09/16/11   sv      Update compiler options, run Ehostdl on Sparrow for 9x15
# 08/16/11   jz      Support ehostdl on 9x15
# 08/16/11   jz      Support ehostdl on 9x15
# 06/22/11   eo      Disable ehostdl compilation 9x15 apps
# 01/31/11   jz      Added aliases to support modem build
# 12/08/10   bn      Enable ehostdl compilation for 6615's corebsp alias
# 12/01/10   bn      Enable ehostdl compilation for 6615
# 11/04/10   sv      Disable ehostdl compilation for 6615
# 10/26/10   bn      Removed ehostdl from 8660 build
# 10/21/10   sb      Adding changes to support latest boot code updates for SCMM
# 09/20/10   sb      Remove referring to a1std.lib of rvct flavor
# 08/18/10   bb      .scl file cleanup for e-hostdl/hostdl
# 08/12/10   nr      Replaced nand_log.h with flash_log.h
# 08/08/10   bb      Fix for RVCT 4.0 build failure
# 08/03/10   jz      With LPDDR1/2 auto-detect, FEATURE_LPDDR2 is no longer needed
# 07/19/10   bb      Added support RVCT 4.0
# 07/13/10   bb      Added support for 6615
# 07/12/10   jz      Fixed for interleaved support
# 06/24/10   sb      Define FEATURE_512BYTES_NAND to support 512B on SCMM
# 06/18/10   sb      Fixing compilation warnings on MDM9K
# 05/14/10   jz      Increased BOOT_HEAP_SIZE for 7x30
# 05/07/10   sb      MBN builder changes for secure/nonsecure build
# 03/11/10   jz      Added alias amss_secure for secure signing
# 03/02/10   sb      ehostdl support on Voyager
# 02/18/10   op      Use hostdl_7x30.scl for Halcyon
# 01/15/10   op      Change by Halcyon boot, reduce BOOT_HEAP_SIZE
# 01/14/10   eo      Add support to reduce build code size
# 12/18/09   jz      Updated for LPDDR2 support
# 11/24/09   jz      Fixed compiler warnings
# 11/20/09   jz      Enable ehostdl support on Halcyon
# 11/01/09   ng      Added support for secure boot authentication
# 10/16/09   eo      Updates to support eHostdl on Voyager
# 09/22/09   wd      Updates to sync with Halcyon and Voyager
# 09/21/09   jz      Fixed for ehostdl support and turned on ehostdl build on scmm
# 09/08/09   eo      Update MDM9K SCONS scripts to new SCONS framework
# 08/12/09   mh      Branched and updated to generate image only
#===============================================================================
import os
Import('env')
env = env.Clone()

#------------------------------------------------------------------------------
# Add new options/arguments to build system
# Needs to be done before check aliases otherwise help won't be updated
#------------------------------------------------------------------------------
vars = Variables(None)

# Defining this to enable PRINTF through command line
vars.Add(
   EnumVariable('EHOSTDL_USES_PRINTF',
      'Overide USES_PRINTF flag used in EHOSTDL', 'no',
      allowed_values=('yes', 'no'), map={}, ignorecase=0
   )
)

# Update environment and read command line variables
vars.Update(env)

Help(vars.GenerateHelpText(env))

#------------------------------------------------------------------------------
# Aliases: Determines when  to build ehostdl
#------------------------------------------------------------------------------
# aliases:  first alias is always the target then the other possibles aliases
# amss      : Alias for AMSS; anything that needs to be build for amss
#             as a whole target (componenets from modem + apps)
# modem     : Only the modem (not applicable to single processor targets)
# amss_secure: Alias for building images with secure signing process
# amss_arm11: amss for arm11 processor
# amss_arm9 : amss for arm9 processor
# arm11     : Build components that is for ARM11 only
# arm9      : Build components that is for ARM9 only
# all       : Buidl all components

if env['MSM_ID'] in ['8660', '8x60', '8960', '8064', '8930']:
   aliases = [
      'not_supported',
   ]
else:
   aliases = [
      'ehostdl', 'enandprg', 'tools', 'flashtools',
      'modem', 'amss', 'amss_arm9', 'amss_arm11', 'amss_secure',
      'arm9', 'arm11', 'cbsp', 'all',
      'coreimg_arm11', 'coreimage_modem',
      'cbsp_arm11', 'cbsp_modem',
   ]

if env['MSM_ID'] in ['8660', '8x60', '8960','8064', '8930']:
   proc = 'not_supported'

if env['MSM_ID'] in ['9x00']:
   proc = 'arm9'

if env['MSM_ID'] in ['9x15']:
   proc = 'sparrow'

if env['MSM_ID'] in ['6195', '6295', '6695', '6615', '7x30']:
   proc = 'arm11'

env.InitImageVars(
   alias_list = aliases,                           # list of aliases, unique name index [0]
   proc = proc,                                    # proc settings
   config = 'modem',                               # config settings
   build_tags = ['MODEM_PROC', 'EHOSTDL_IMAGE',
   'SSBI_MINI_BOOT_DRIVER', 'PMIC_MINI_BOOT_DRIVER',
   'XPU_HAL_BOOT_DRIVER', 'M2VMT_HAL_BOOT_DRIVER',
   'SECCFG_SBL1_LIB'],                             # list of build tags for sub lib scripts
)

if not env.CheckAlias():
   Return()

#------------------------------------------------------------------------------
# Init default values for this PROC/Image
#------------------------------------------------------------------------------
env.Replace(BUILD_TOOL_CHAIN = 'yes')
env.Replace(USES_FLASH_REDUCED_BUILD = 'yes')

#------------------------------------------------------------------------------
# Configure and load in CBSP uses and path variables
#------------------------------------------------------------------------------
env.InitBuildConfig()

#---------------------------------------------------------------------------
# Load in the tools scripts
#---------------------------------------------------------------------------
env.LoadToolScript('rvct', toolpath = ['${BUILD_SCRIPTS_ROOT}'])
env.LoadToolScript('hostdl_gen_hex', toolpath = ['${BUILD_ROOT}/core/bsp/tools/flash/build'])

if env['MSM_ID'] in ['7x30', '9x00', '9x15']:
   env.LoadToolScript('modem_defs', toolpath = ['${BUILD_SCRIPTS_ROOT}'])

if env['MSM_ID'] in ['6195', '6295', '6695', '6615']:
   env.LoadToolScript('single_defs', toolpath = ['${BUILD_SCRIPTS_ROOT}'])

env.Append(ARMCC_CODE = " ${ARM_SPLIT_SECTIONS}")
env.Replace(ARMLD_NOREMOVE_CMD = "")

# Force debug and symbols for flash tools, required by T32 swbp interface
env.Replace(ARM_DBG = "-g --dwarf2")            # Enable DWARF2 format debug tables

# Compile on CORETEX A5
if 'RVCT4' in env.get('ARMTOOLS', 'not found'):
   env.Replace(ARMCC_OPT = "${ARM_SPACE_TIME} ${ARM_OPT_2}")
   env.Append(ASFLAGS = " --keep ${ARM_NO_UNALIGNED_ACCESS}")
   env.Replace(ARMCC_APCS = "--apcs /${ARM_INTRWORK}")
   env.Replace(ARMCC_CODE = "${ARM_END} ${ARM_NO_UNALIGNED_ACCESS}")
   env.Append(ARMCXX_CODE = " ${ARM_NO_UNALIGNED_ACCESS}")

#----------------------------------------------------------------------------
# Define the PRINT SOURCES.PRINT LOGS will be enabled only if USES_PRINTF is
# defined on the scons environment, to disable it DON'T define it. As the
# scripts check to see if the key is present and not the value!!
#----------------------------------------------------------------------------
env.Replace(USES_PRINTF = "no")
env.Replace(TNOISYLEVEL = "3")

#----------------------------------------------------------------------------
# Define macro NAND_PAGE_SIZE used to pass a value to macro 'MAX_PAGE_SIZE',
# which is used by nand tools and flash drivers (when compiled with tools)
# to size the static buffers.
#----------------------------------------------------------------------------
NAND_PAGE_SIZE = "4096"

# ---------------------------------------------------------------------------
# Determine the platform.
# Please see //source/qcom/qct/core/storage/flash/main/latest/src/README.txt
# for details on flash configurations.
# ---------------------------------------------------------------------------
FLASH_CFG_ID = "UNKNOWN"

# The following for FLASH_7K_CFG is not used but is place holder for
# future targets that shoudl use this configurations for hal/dal/tools
if env['MSM_ID'] in ['7627']:
   FLASH_CFG_ID = "FLASH_7K_CFG"
   FLASH_DAL_CFG = "config_v1"
   FLASH_HAL_CFG = "config_v1"
   FLASH_NAND_TOOLS_CFG = "config_v1"
   FLASH_NOR_TOOLS_CFG = "config_v1"
   FLASH_HOSTDL_CFG = "config_v1"
   SRCROOT = "../../../../.."

if env['MSM_ID'] in ['6195', '6295', '6695', '6615']:
   FLASH_CFG_ID = "FLASH_6K_CFG"
   FLASH_DAL_CFG = "config_v2"
   FLASH_HAL_CFG = "config_v1"
   FLASH_NAND_TOOLS_CFG = "config_v2"
   FLASH_NOR_TOOLS_CFG = "config_v1"
   FLASH_HOSTDL_CFG = "config_v2"
   SRCROOT = "../../../../../.."

if env['MSM_ID'] in ['7630', '7x30']:
   FLASH_CFG_ID = "FLASH_7K_DUAL_NAND_CFG"
   FLASH_DAL_CFG = "config_v1"
   FLASH_HAL_CFG = "config_v2"
   FLASH_NAND_TOOLS_CFG = "config_v2"
   FLASH_NOR_TOOLS_CFG = "config_v1"
   FLASH_HOSTDL_CFG = "config_v1"
   SRCROOT = "../../../../../.."

   if env.has_key('USES_DUAL_NAND_INTERLEAVE') and env['USES_DUAL_NAND_INTERLEAVE'] == "yes" :
      NAND_PAGE_SIZE = "8192"

if env['MSM_ID'] in ['9x00', '9x15']:
   FLASH_CFG_ID = "FLASH_9K_CFG"
   FLASH_DAL_CFG = "config_v1"
   FLASH_HAL_CFG = "config_v1"
   FLASH_NAND_TOOLS_CFG = "config_v2"
   FLASH_NOR_TOOLS_CFG = "config_v1"
   FLASH_HOSTDL_CFG = "config_v2"
   SRCROOT = "../../../../../.."

if env['MSM_ID'] in ['9x00']:
   env.Append(CPPDEFINES = [
      "FLASH_WATCHDOG_H=\\\"msm.h\\\"",
   ])

#----------------------------------------------------------------------------
# Overide uses_print setting from cmd line
#----------------------------------------------------------------------------
if env['EHOSTDL_USES_PRINTF'] == "yes":
   env.Replace(USES_PRINTF = "yes")

#----------------------------------------------------------------------------
# Define the PRINT SOURCES.PRINT LOGS will be enabled only if USES_PRINTF is
# set to YES in the make file
#----------------------------------------------------------------------------
if env.has_key('USES_PRINTF') and env['USES_PRINTF'] == "yes":
   env.Append(CPPDEFINES = [
      "FLASH_LOG_H=\\\"flash_log.h\\\"",
      "NOISY",
      "TNOISY=" + str(env['TNOISYLEVEL']),
   ])

#print env.Dump()

# NOTE: the following addresses should come from the .builds file
# TBD: remove the following hard-coded addresses when scons can pass them down
if env['MSM_ID'] in ['6195', '6295', '6695', '6615']:
   EHOSTDL_DATA_ADDR = "0x0"
   EHOSTDL_CODE_START_ADDR = "0x20012028"
   EHOSTDL_CODE_ADDR_HDR = "0x20012000"
   EHOSTDL_IMEM_STACK_BASE = "0x20012000"
   EHOSTDL_RAM_STACK_BASE = "0x20012000"
elif env['MSM_ID'] in ['9x00']:
   EHOSTDL_DATA_ADDR = "0x0"
   EHOSTDL_CODE_START_ADDR = "0x40012028"
   EHOSTDL_CODE_ADDR_HDR = "0x40012000"
   EHOSTDL_IMEM_STACK_BASE = "0x40012000"
   EHOSTDL_RAM_STACK_BASE = "0x40012000"
elif env['MSM_ID'] in ['7x30', '7630']:
   EHOSTDL_DATA_ADDR = "0x0"
   EHOSTDL_CODE_START_ADDR = "0x80000028"
   EHOSTDL_CODE_ADDR_HDR = "0x80000000"
   EHOSTDL_IMEM_STACK_BASE = "0x80070000"
   EHOSTDL_RAM_STACK_BASE = "0x80070000"
elif env['MSM_ID'] in ['9x15']:
   EHOSTDL_DATA_ADDR = "+0x0"
   EHOSTDL_CODE_START_ADDR = "0x2A000050"
   EHOSTDL_CODE_ADDR_HDR = "0x2A000000"
   EHOSTDL_IMEM_STACK_BASE = "0x2A040000"
   EHOSTDL_RAM_STACK_BASE = "0x2A000000"
   EHOSTDL_RPM_JUMP_BASE = "0x00043D00"

EHOSTDL_APP_RAM_ZI_BASE = "0x4000"

# add extension flags for EHOSTDL
env.Append(CPPDEFINES = [
   "BUILD_TOOL_CHAIN",
   "MAX_PAGE_SIZE=" + NAND_PAGE_SIZE,
   "FLASH_SINGLE_THREADED",
   "ENDIAN_LITTLE",
   "BUILD_HOSTDL",
   "BUILD_NANDPRG",
   "HOSTDL_PROCESS_USB_BUFFERS",
   "EXTENDED_NAND_CMDS",
   "DAL_CONFIG_MMU_SCHEME=DAL_CONFIG_MMU_SCHEME_NONE",
   "DAL_IMAGE_FLASH_TOOLS",
   "CODE_START_ADDR=" + EHOSTDL_CODE_START_ADDR,
   "DATA_ADDR=" + EHOSTDL_DATA_ADDR,
   "APP_RAM_ZI_REGION_OFFSET=" + EHOSTDL_APP_RAM_ZI_BASE,
   "FEATURE_HS_USB_BASIC",
   "BUILD_EHOSTDL",
   "EHOSTDL_CODE_ADDR_HDR=" + EHOSTDL_CODE_ADDR_HDR,
   "EHOSTDL_IMEM_STACK_BASE=" + EHOSTDL_IMEM_STACK_BASE,
   "EHOSTDL_RAM_STACK_BASE=" + EHOSTDL_RAM_STACK_BASE,
])

# For 7x30 and 9x00, FEATURE_STANDALONE_MODEM will be defined in .builds file.
# So no need to define it in Sconscript file
if env['MSM_ID'] in ['6195', '6295', '6695', '6615']:
   env.Append(CPPDEFINES = [
      "FEATURE_STANDALONE_MODEM",
      "BUILD_BOOT_CHAIN_DBL",
   ])

   if env.has_key('USES_512NAND'):
      env.Append(CPPDEFINES = [
        "FEATURE_512BYTES_NAND",
     ])

if env['MSM_ID'] in ['9x15']:
   env.Append(CPPDEFINES = [
   "EHOSTDL_RPM_JUMP_BASE=" + EHOSTDL_RPM_JUMP_BASE,
   "EHOSTDL_STARTUP_SOURCE=\\\"hostdl_sparrow_startup.o\\\"",
   "FLASH_EHOSTDL_INCLUDE_CLK_INIT",
   "FLASH_ENABLE_WDOG_CFG",
   ])
else:
   env.Append(CPPDEFINES = [
   "EHOSTDL_STARTUP_SOURCE=\\\"hostdl_startup.o\\\"",
   "BUILD_EHOSTDL_DBL",
   ])

if env['MSM_ID'] in ['9x00', '9x15']:
   env.Append(CPPDEFINES = [
      "TARGET_ID_9K",
      "USE_USB_ONLY",
   ])

if env['MSM_ID'] in ['7x30', '7630']:
   env.Append(CPPDEFINES = [
      "BOOT_HEAP_SIZE=60000",
      "FLASH_INCLUDE_EHOSTDL_H=1",
   ])
else:
   env.Append(CPPDEFINES = [
      "BOOT_HEAP_SIZE=20000",
   ])

#------------------------------------------------------------------------------
# Decide which build steps to perform
#------------------------------------------------------------------------------
# Regular build steps (no filter) is build everything, once a user starts
# using filters we have to make decisions depending on user input.
#
# The LoadAUSoftwareUnits function will take care of filtering subsystem, units,
# etc.  This is to take care of what steps to enable disable from the top level
# script, such as building files specify in this script i.e. quartz, stubs, etc.

do_local_files = True
do_link = True
do_post_link = True

# Get user input from command line
filter_opt = env.get('FILTER_OPT')

# Limit build processing based on filter option
if filter_opt is not None:
   do_link = False
   do_post_link = False

   if not env.FilterMatch(os.getcwd()):
      do_local_files = False

#-------------------------------------------------------------------------------
# Libraries Section
#-------------------------------------------------------------------------------
core_libs, core_objs = env.LoadAUSoftwareUnits('core')
modem_libs, modem_objs = env.LoadAUSoftwareUnits('modem', 'rfa/pmic')
ehostdl_units = [core_libs, core_objs,]

if do_local_files:
#-------------------------------------------------------------------------------
# EHOSTDL Environment
#-------------------------------------------------------------------------------
   env.RequireRestrictedApi('BOOT')

#-------------------------------------------------------------------------------
# Begin building EHOSTDL
#-------------------------------------------------------------------------------
   env.Replace(TARGET_NAME = 'enandprg_${BUILD_ID}')
#-------------------------------------------------------------------------------
# Generate Scatter Load File (SCL)
#-------------------------------------------------------------------------------
   if env['MSM_ID'] in ['7x30', '7630']:
      target_scl = env.SclBuilder('${SHORT_BUILDPATH}/${TARGET_NAME}',
      '${BUILD_ROOT}/core/storage/flash/tools/src/hostdl/ehostdl_in.scl')
   elif env['MSM_ID'] in ['9x15']:
      target_scl = env.SclBuilder('${SHORT_BUILDPATH}/${TARGET_NAME}',
      '${BUILD_ROOT}/core/storage/flash/tools/src/hostdl/hostdl_in.scl')
   else:
      target_scl = env.SclBuilder('${SHORT_BUILDPATH}/${TARGET_NAME}',
      '${BUILD_ROOT}/core/storage/flash/tools/src/hostdl/ehostdl_in_legacy.scl')

if do_link:
#-------------------------------------------------------------------------------
# Generate enandprg.elf
#-------------------------------------------------------------------------------
   libs_path = env['INSTALL_LIBPATH']

   enandprg_elf = env.Program('${SHORT_BUILDPATH}/${TARGET_NAME}',
      source=[core_objs, modem_objs], LIBS=[core_libs, modem_libs], LIBPATH=libs_path)

   env.Depends(enandprg_elf, target_scl)

   Clean(enandprg_elf, env.subst('${SHORT_BUILDPATH}/${TARGET_NAME}.map'))
   Clean(enandprg_elf, env.subst('${SHORT_BUILDPATH}/${TARGET_NAME}.sym'))

# Copy elf to needed locations for AMSS tools to load on target
   install_enandprg_elf = env.Install("${BUILD_ROOT}/core/storage/flash/tools/src/hostdl",
      enandprg_elf)

if do_post_link:
#----------------------------------------------------------------------------
# Generate EHOSTDL BIN
#----------------------------------------------------------------------------
   target_bin = env.BinBuilder('${SHORT_BUILDPATH}/${TARGET_NAME}', enandprg_elf)

#----------------------------------------------------------------------------
# Generate EHOSTDL mbn
#----------------------------------------------------------------------------
   target_mbn = env.MbnBuilder('${SHORT_BUILDPATH}/${TARGET_NAME}', target_bin,
      IMAGE_TYPE="ehostdl", FLASH_TYPE="nand")

#-------------------------------------------------------------------------------
# Generate  ENANDPRG.HEX
#-------------------------------------------------------------------------------
   enprg_hex = env.NPRGHexBuilder('ENPRG${MSM_ID}_${BUILD_ID}', target_mbn,
      HOSTDL_ADDR=EHOSTDL_CODE_ADDR_HDR, CODE_START_OFFSET=0x0)

   install_enandprg_hex = env.InstallAs('${MBN_ROOT}/ENPRG${MSM_ID}.hex', enprg_hex)

   enandprg_product =  env.BuildProductsFile(
      '${SHORT_BUILDPATH}/${TARGET_NAME}',install_enandprg_hex)

#===============================================================================
# Define units that will be built
#===============================================================================
   ehostdl_units = [enandprg_product,]

# Add aliases
for alias in aliases:
   env.Alias(alias, ehostdl_units)


""" various methods for interacting with psi4. i.e. getting gradients, hessians, options etc """
import logging
from psi4.driver import json_wrapper
from .printTools import (print_array_string, print_mat_string)

# TODO delete this method once psi4 writes a method that does this (just better)
def collect_psi4_options(options):
    """Is meant to look through the dictionary of psi4 options being passed in and
    pick out the basis set and QM method used (Calcname)
    which are appened to the list of psi4 options
    """
    keywords = {}
    for opt in options['PSI4']:
        keywords[opt] = options['PSI4'][opt]

    basis = keywords['BASIS']
    del keywords['BASIS']
    QM_method = keywords['CALCNAME']
    del keywords['CALCNAME']

    return QM_method, basis, keywords

def get_optking_options_psi4(all_options):
    optking_user_options = {}
    optking_options = all_options['OPTKING']
    optking_user_options['OPTKING'] = {}
    for opt, optval in optking_options.items():
        if optval['has_changed']:
            optking_user_options[opt] = optval['value']

    return optking_user_options

def psi4_calculation(new_geom, o_json, driver='gradient'):
    """ Call psi4 to perform a calculation"""

    # is there something broken about dummy atoms here?
    logger = logging.getLogger(__name__)
    geom = o_json.to_JSON_geom(new_geom)
    json_input = o_json.update_geom_and_driver(geom, driver)
    logger.debug("Getting %s from Psi4 through JSON interface\n" % (driver))

    qc_input = "Input to run_json\n"
    for k,v in json_input.items():
        if k != 'raw_output':
            qc_input += "%s,%s\n" % (k, v)
    logger.debug(qc_input)

    rval = json_wrapper.run_json(json_input, True)

    qc_output = "Return from run_json\n"
    for k,v in rval.items():
        if k != 'raw_output':
            qc_output += "%s,%s\n" % (k, v)
    logger.debug(qc_output)

    return rval

#################
## Thses are all methods that are not longer used. Keeping them around for reference if needed. Will remove completely later
#################
#def set_geometry(newGeom):
#    """Define a function for optking that takes a Cartesian, numpy geometry, and
#    puts it in place for subsequent gradient computations.  This function
#    may move COM or reorient; will return possible changed cartesian geometry.
#    """
#
#    mol = psi4.core.get_active_molecule()
#    psi_geom =  psi4.core.Matrix.from_array(newGeom)
#    mol.set_geometry(psi_geom)
#    mol.update_geometry()
#    return np.array( mol.geometry() )
#    newGeom[:] = np.array( mol.geometry() )

#def energy_func(xyz, printResults=True):
#    """This function only matters for special purposes like 1D line searching.
#    The energy is usually obtained from the gradient function.
#    """
#
#    mol = psi4.core.get_active_molecule()
#    xyz[:] = setGeometry_func(xyz) #id like for this to be in my get_energy methods
#    E, wfn = psi4.driver.energy(calcName, molecule=mol, return_wfn=True)
#    if printResults:
#        psi4.core.print_out('\tEnergy: %15.10f\n' % E)
#    return E
#
#def gradient_func(xyz, printResults=True):
#    """Define a function for optking that returns an energy and cartesian
#    gradient in numpy array format."""
#
#    mol = psi4.core.get_active_molecule()
#    #xyz[:] = setGeometry_func(xyz)
#    psi4gradientMatrix, wfn = psi4.driver.gradient(calcName, molecule=mol, return_wfn=True)
#    gradientMatrix = np.array( psi4gradientMatrix )
#    E = wfn.energy()
#    if printResults:
#       psi4.core.print_out( '\tEnergy: %15.10f\n' % E)
#       psi4.core.print_out( '\tGradient\n')
#       psi4.core.print_out( str(gradientMatrix) )
#       psi4.core.print_out( "\n")
#    return E, np.reshape(gradientMatrix, (gradientMatrix.size))
#
#
#def hessian_func(xyz, printResults=False):
#    """Gets a hessian from psi4 (cartesian)"""
#    H = psi4.driver.hessian(calcName, molecule=mol)
#    if printResults:
#        psi4.core.print_out( 'Hessian\n')
#        H.print_out()
#        psi4.core.print_out( "\n")
#    return np.array( H )
#


# File of the class Utils, containing various helper functions
# File: utils.py
# Author: Atharva Kulkarni

import pandas as pd
import numpy as np
from gensim.models import KeyedVectors
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import nltk
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer

class Utils():
    """" Class containing various helper functions """

    # -------------------------------------------- Class Constructor --------------------------------------------
    def __init__(self):
        """ Class Constructor """
        self.tokenizer = Tokenizer(oov_token='[oov]')

    # -------------------------------------------- Function to read data --------------------------------------------

    def read_data(self, path, columns=[]):
        """ Function to read data
        @param path (str): path to the data to be read.
        @columns (list): List of columns to be read.
        return data (pandas.DataFrame): a dataframe of the data read.
        """
        delimiter = path.split(".")[-1]

        if delimiter == "csv":
            if not columns:
                data = pd.read_csv(path, sep=",")
                return data
            else:
                data = pd.read_csv(path, sep=",", usecols=columns)
                return data

        elif delimiter == "tsv":
            if not columns:
                data = pd.read_csv(path, sep="\t")
                return data
            else:
                data = pd.read_csv(path, sep="\t", usecols=columns)
                return data

        else:
            return None

    # -------------------------------------------- Function read dictionary --------------------------------------------

    def get_dict(self, path, key_column, value_column):
        """ Function to read a file into dictionary.
        @param path (str): path to file.
        return dict: created dictionary.
        """
        data = pd.read_csv(path)
        return dict(zip(data[key_column].values.tolist(), data[value_column].values.tolist()))

    # -------------------------------------------- Function to tokenize and pad input text --------------------------------------------

    def tokenize_and_pad(self, corpus, maxlen=100, padding_type='post', truncating_type='post', mode="train"):
        """ Function to prepare text for model input (tokenize and pad).
        @param corpus (list): the corpus to prepare.
        @param maxlen (int): max allowed length of input texts.
        @param padding_type (str): padding type (post or pre).
        @param truncating_type (str): truncating type (post or pre).
        @mode (str): specify train or test mode.
        """
        if mode == "train":
            self.tokenizer.fit_on_texts(corpus)

        corpus = self.tokenizer.texts_to_sequences(corpus)
        corpus = np.asarray(pad_sequences(corpus,
                                          padding=padding_type,
                                          truncating=truncating_type,
                                          maxlen=maxlen))
        return corpus, self.tokenizer

    # -------------------------------------------- Function to read embeddings --------------------------------------------

    def get_embedding_matrix(self, path, vocab, top=50000):
        """ Function to get the word embedding matrix.
        @param path (str): path to the word embeddings file.
        @param vocab (list): list of corpus vocab.
        @return embedding_matrix (np.array): embedding matrix.
        """
        if path.split(".")[-1] == "bin":
            embedding_model = KeyedVectors.load_word2vec_format(path, binary=True, limit=top)
        else:
            embedding_model = KeyedVectors.load_word2vec_format(path, binary=False, limit=top)

        embeddin_model_vocab =[word for word in embedding_model.vocab.keys()]
        final_vocab = list(set(vocab) | set(embeddin_model_vocab))

        embedding_matrix = np.zeros((len(final_vocab), 300))
        cnt = 0
        for index in range(len(final_vocab)):
            if final_vocab[index] in embeddin_model_vocab:
                vec = embedding_model.wv[final_vocab[index]]
                if vec is not None:
                    embedding_matrix[index] = vec
            else:
                cnt = cnt + 1
                continue
        print("zero embeddings: ", cnt)
        return embedding_matrix

     # -------------------------------------------- Function to create age bins --------------------------------------------

    def categorize_age(self, age):
        if age >0 and age <=25:
            return 0
        elif age >25 and age <=40:
            return 1
        elif age >40 and age <=60:
            return 2
        elif age >60:
            return 3

    # -------------------------------------------- Function to create age bins --------------------------------------------

    def categorize_income(self, income):
        if income >0 and income <=25000:
            return 0
        elif income >25000 and income <=75000:
            return 1
        elif income >75000:
            return 2

    # ----------------------------------- Function to calculate weight of each word -----------------------------------

    def get_word_weights(self, essay, feature):
        ''' Function to calculate weight of each word according to the specified feature
        @param essay (pd.Series) list of essays.
        @param feature (pd.Series) The target variable (gold_empthy_bin or gold_distres_bin).
        @return weight (dict) The dictionary contribution of each word to the feature passed.
        '''
        dictionary1 = {}   #to count the number of times a word contributes to feature=1
        dictionary0 = {}   #to count the number of times a word contributes to feature=0
        words=[]
        for i in range(0,len(essay)):   #loop to find both counts(feature=1, feature=0) for each word
            words = essay[i].split()
            for word in words:
                if word.isalpha():
                    if word in dictionary1.keys():
                        if feature[i] == 1:
                            dictionary1[word] = dictionary1[word] + 1
                        else :
                            dictionary0[word] = dictionary0[word] + 1
                    else:
                        if feature[i] == 1:
                            dictionary1[word] = 1
                            dictionary0[word] = 0
                        else:
                            dictionary0[word] = 1
                            dictionary1[word] = 0

        weight = {}  #to store weight of each word
        for i in dictionary0:
            weight[i]=((dictionary1[i]-dictionary0[i])/(dictionary1[i]+dictionary0[i]))

        return weight

    # ----------------------------------- Function to calculate score for each Essay -----------------------------------

    def get_essay_empathy_distress_scores(self, essay, word_weights, transform='original'):
        ''' Function to calculate score for each Essay.
        @param essay (list) list of essays.
        @param word_weights (dict) Tthe dictionary obtained from the wordWeight function
        @return essay_weight (list): list of weights for each essay.
        '''
        essay_weight = []
        for i in range(0, len(essay)):
            weight = 0
            for word in essay[i].split():
                if word in word_weights.keys():
                    weight = weight + float(word_weights[word])
            essay_weight.append(weight)
        if transform == "original":
            return np.reshape(essay_weight, (len(essay_weight), 1))
        elif transform == "tan-inverse":
            np.arctan(np.reshape(essay_weight, (len(essay_weight), 1)))

    # ----------------------------------- Function to get word emotion and vad scores -----------------------------------

    def get_word_scores(self, sentiment='anger'):
        path = "/content/gdrive/My Drive/WASSA-2021-Shared-Task/resources/NRC-resources/"+sentiment+"-scores.txt"
        word_scores = {}
        with open(path) as f:
            data = f.readlines()
            for row in data:
                row = row.split("\t")
                row[1] = row[1].split("\n")
                word_scores[row[0]] = row[1][0]
        f.close()
        return word_scores

    # -------------------------------------------- Function to get essay emotion and vad scores --------------------------------------------

    def get_essay_nrc_scores(self, essay, nrc_features, normalize=False):
        word_scores_list = []
        for element in nrc_features:
            word_scores = self.get_word_scores(element)
            word_scores_list.append(word_scores)
        essay_scores = np.zeros((len(essay), len(word_scores_list)))

        for i in range(len(essay)):
            for j in range(len(word_scores_list)):
                score = 0
                cnt = 0
                for word in essay[i].split():
                    if word in word_scores_list[j].keys():
                        cnt = cnt = 1
                        score = score + float(word_scores_list[j].get(word))
                essay_scores[i][j] = score

        return essay_scores