# transaction.py
# HMC E85 8 December 2017
# hfang@hmc.edu, mjenrungrot@hmc.edu

import hashlib
import json
import ed25519
import binascii

FEE_PER_TRANSACTION = 1

class Transaction:
    def __init__(self):
        """
        Constructor for a transaction.
        """
        self.id = None
        self.hash = None
        self.type = None
        self.data = {
            "inputs": [],
            "outputs": []
        }

    def __repr__(self):
        """
        String representation for a transaction
        """
        # jsonDumper is used for recursively converting an object to correct JSON output format
        def jsonDumper(obj):
            return obj.__dict__
        return str(json.dumps(self, default=jsonDumper))

    def toHash(self):
        """
        Compute hash for the transaction
        """
        strInput = str(self.id) + str(self.type) + str(self.data)
        return hashlib.sha256(strInput.encode('utf-8')).hexdigest()

    def check(self):
        """
        Check that the transaction is valid, e.g. signed by the correct
        individual.
        """

        # Check if the computed hash matches the transaction's hash.
        transactionHash = self.toHash()
        if(transactionHash != self.hash):
            raise ValueError("hash value of transaction error")

        # Check if the signature of all input transactions are correct.
        for inputTransaction in self.data["inputs"]:
            transactionHash = inputTransaction["transaction"]
            index = inputTransaction["index"]
            amount = inputTransaction["amount"]
            address = inputTransaction["address"]
            signature = inputTransaction["signature"]
            publicKey = address

            # Generate the Message Hash from the transaction hash, index, and address
            messageHash = str(transactionHash) + str(index) + str(address)
            messageHashed = hashlib.sha256(messageHash.encode('utf-8')).digest()

            # Generate the verifying key from the public key
            verifying_key = ed25519.VerifyingKey(binascii.a2b_hex(publicKey))

            # Check if the the signature of the transaction is valid by checking if we can verify the messageHash
            # by the signature.
            verification = None
            try:
                verifying_key.verify(binascii.a2b_hex(signature), messageHashed)
                verification = True
                print("signature is good!")
            except ed25519.BadSignatureError:
                verification = False
                print("signature is bad!")

            # Verify the signed transaction
            if not verification:
                raise ValueError("Signed transaction is invalid by verification process")

        # Check if the sum of input transactions are enough for the sum of output transactions
        # plus fee.
        if self.type == "regular":
            sumOfInputsAmount = 0
            sumOfOutputsAmount = 0

            for inputTransaction in self.data["inputs"]:
                sumOfInputsAmount += int(inputTransaction["amount"])
            for outputTransaction in self.data["outputs"]:
                sumOfOutputsAmount += int(outputTransaction["amount"])

            if(sumOfInputsAmount < sumOfOutputsAmount):
                raise ValueError("Input amount is less than output amount")

            if((sumOfInputsAmount - sumOfOutputsAmount) < FEE_PER_TRANSACTION):
                raise ValueError("Not enough fee")

        return True

def createTransaction(data):
    """
    Create a transaction from dictionary.
    """
    transaction = Transaction()
    transaction.id = data["id"]
    transaction.type = data["type"]
    transaction.data = data["data"]
    transaction.hash = transaction.toHash()
    return transaction

def createTransactionObject(data):
    """
    Create a transaction from transaction.
    """
    transaction = Transaction()
    transaction.id = data.id
    transaction.type = data.type
    transaction.data = data.data
    transaction.hash = transaction.toHash()
    return transaction


from os import listdir
from os.path import join, splitext

from utils.ploting.heat_map import HeatMap
from numpy import asarray
from PIL import Image

def plot_heatmap_data(path_to_orig_image):
    for image_file in listdir(path_to_orig_image):
        image_name, _ = splitext(image_file)

        path_to_mask_image = join(path_to_orig_image, '../..', 'pred_mask')
        mask_filename = \
            [join(path_to_mask_image, image) for image in listdir(path_to_mask_image) if image_name in image][0]

        with Image.open(mask_filename) as mask:
            with Image.open(join(path_to_orig_image, image_file)) as orig_image:
                hm = HeatMap(asarray(orig_image), asarray(mask))

                hm.save(image_name,
                        transparency=0.5,
                        format='jpg',
                        save_path='../../scripts/heatmap_data',
                        color_map='jet')


import pygame

from src import constants
from src.character import Character
from src.blob import Blob

import random
from math import sin, cos

class BlobSpawner:
    def __init__(self):
        self.last_spawn = pygame.time.get_ticks()

    def spawn_blob(self, app):
        x = random.uniform(0, constants.FIELD_WIDTH)
        y = random.uniform(0, constants.FIELD_HEIGHT)
        app.objects.append(Blob((x, y)))
        app.blob_count += 1

    def update(self, app):
        current_time = pygame.time.get_ticks()

        if self.last_spawn + constants.BLOB_SPAWNER_COOLDOWN <= current_time:
            for _ in range(0, constants.BLOB_SPAWNER_MAX_PER_TICK):
                if app.blob_count >= constants.BLOB_SPAWNER_UPPER_LIMIT:
                    break

                self.spawn_blob(app)

            self.last_spawn = current_time


import pandas as pd
import os
import re
from sklearn.model_selection import train_test_split
import shutil
import requests
import time
#-- This File is Step 2 of the LaArt Pipeline --
# 1 Input: (1) latinamerican_art.csv
# 2 Output: (1) latinamerican_art.csv (2) la_image_fpaths.csv (3) validLa_image_fpaths.csv
#import the metadata for La Art (this is a result of a MySql Database query script) with minor modifications
la_image_metadata = pd.read_csv('../../../data_samples/LaArt/latinamerican_art.csv')
shape_initial = la_image_metadata.shape
#dropping unneccessary rows (1 removed)
la_image_metadata = la_image_metadata.where(la_image_metadata.title.apply(pd.notna)).dropna(how='all')
la_image_metadata = la_image_metadata.where(la_image_metadata.forwarddisplayname.apply(pd.notna)).dropna(how='all')
la_image_metadata = la_image_metadata.where(la_image_metadata.objectid.apply(pd.notna)).dropna(how='all')
# adding expected file name (limiting title to 100 characters and concatenating it with forwardisplayname and objectid to create UUID filepath)
la_image_metadata['file_name'] = la_image_metadata['title'].apply(lambda x: x.replace(' ','_').replace('/', '&')).apply(lambda x: x[:100]) + '_' + la_image_metadata['forwarddisplayname'].apply(lambda x: x.replace(' ', '_').replace('/', '&')) + '_' + la_image_metadata['objectid'].apply(lambda x: str(int(x)) + '.jpg')
#adding the expected root directory of the image files
la_image_directory = '../latinamerican-2-imagefolder-split/'
la_image_metadata['directory'] = [la_image_directory] * len(la_image_metadata)
#subfolder split (train 70% /test 30%) need to be identified randomly
train_data, test_data = train_test_split(la_image_metadata, test_size=0.3)
train_data['subfolder'] = ['train'] * len(train_data)
test_data['subfolder'] = ['test'] * len(test_data)
#adding new split (subfolder) column to la_image_metadata
la_image_metadata = pd.concat([train_data, test_data]).reset_index(drop=True)
# adding expected filepath (directory + filename)
la_image_metadata['image_fp'] = la_image_metadata.directory + la_image_metadata.subfolder + '/' + la_image_metadata.file_name

#To verify data shape during process
shape_change_1 = la_image_metadata.shape
print('Shape starting: ', shape_initial)
print('Shape after edit 1: ', shape_change_1)

# selects the features relevant to run the download script
la_image_fpaths = la_image_metadata.loc[:, ['objectid', 'directory','subfolder','file_name', 'image_fp']]

## for next code section in notebook, images will be downloaded
## I assume no images are downloaded & image_fp/directory not created
def download_image(url, path, name, headers):
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        with open(name, "wb") as f:
            f.write(response.content)
        shutil.move(name, path)
    else:
        print(response.status_code)
# Define HTTP Headers
ua_header = {
    "User-Agent": "Chrome/51.0.2704.103",
}

# this block now uses the la_image_fpaths table
print("Download starting... please wait :)")
# This block iterates over all the images
for i in range(0, len(la_image_fpaths)):
    # Define URL of an image
    expanded_url = la_image_fpaths.expanded_url[i]
    # Define image file name, file path to place
    file_name = la_image_fpaths.file_name[i]
    fp = la_image_fpaths.image_fp[i]
    # Download image
    # timer delay (15 seconds)
    time.sleep(15)
    download_image(expanded_url, fp, file_name, ua_header)

#checking that the filepath / naming conventions I used are consistent
from os.path import exists
file_exists = []
for i in range(len(la_image_fpaths)):
    file_exists.append(exists(la_image_fpaths.image_fp[i]))
la_image_fpaths['file_downloaded'] = file_exists

presentLa_image_fpaths = la_image_fpaths.where(la_image_fpaths.file_downloaded == True).dropna(how='all')

missingLa_image_fpaths = la_image_fpaths.where(la_image_fpaths.file_downloaded == False).dropna(how='all')

#saving the image fp data for use with the model
presentLa_image_fpaths.to_csv('../../../data_samples/LaArt/presentLa_image_fpaths.csv', index=False)
missingLa_image_fpaths.to_csv('../../../data_samples/LaArt/missingLa_image_fpaths.csv', index=False)
print('CSV Created: ../../../data_samples/LaArt/presentLa_image_fpaths.csv')
print('CSV Created: ../../../data_samples/LaArt/missingLa_image_fpaths.csv')
la_image_metadata.to_csv('../../../data_samples/LaArt/latinamerican_art.csv', index=False)
print('CSV Edited: ../../../data_samples/LaArt/latinamerican_art.csv')

perc_exists = la_image_fpaths.file_downloaded.sum()/len(file_exists)
total = la_image_fpaths.shape[0]
whole_num_exists = perc_exists * total
text = 'The amount of images downloaded is {} percent. Which means {} is amount downloaded, out of {} in latinamerican_art.csv'
print(text.format(perc_exists, whole_num_exists, total))
print("download_LaArt.py finished.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 11 16:58:06 2019

@author: l.kate
"""
# Load pakages
import pandas as pd
import numpy as np
import string
from datetime import datetime
import tensorflow as tf
import os
import re
import json
from nltk.corpus import stopwords
from sklearn.model_selection import train_test_split
from sklearn.metrics import median_absolute_error as mae
from sklearn.metrics import mean_squared_error as mse
from sklearn.metrics import accuracy_score as acc
import matplotlib.pyplot as plt
# Preprocess functions in keras
from keras.preprocessing.text import one_hot, Tokenizer
from keras.preprocessing.sequence import pad_sequences
# Models in keras
from keras.models import Sequential
from keras import initializers
from keras.layers import Dropout, Activation,Embedding,CuDNNGRU,Bidirectional,Convolution1D, MaxPooling1D, Input, Dense, BatchNormalization
from keras.layers.recurrent import LSTM, GRU
from keras.layers import concatenate, Conv1D, CuDNNLSTM
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, SpatialDropout1D
from keras.callbacks import Callback, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from keras.models import Model, load_model
from keras.optimizers import Adam, SGD, RMSprop
from keras import regularizers
from keras.layers.core import Reshape, Flatten
from keras.utils import to_categorical
#Word2Vec
import gensim
from gensim.models import Word2Vec
from gensim.utils import simple_preprocess
from gensim.models.keyedvectors import KeyedVectors

Corpus = pd.read_csv('corpus.csv')[['datetime','text_final']]
Corpus_date = pd.read_csv('corpus_date.csv')
t = Tokenizer()
t.fit_on_texts(Corpus_date['docs'])

# Take a look at our tokens
print(list(t.word_index.items())[-1])
print(len(list(t.word_index.items())))

vocab_size = len(t.word_index) + 1
'''
# integer encode the documents, run once
corpus_date = pd.DataFrame(sp500.Date,columns = ['Date'])
docs = Corpus['text_final'].tolist()
corpus_date['docs'] = [" ".join(docs[sp500.iloc[i-1].ind:sp500.iloc[i].ind]) if i >0 else " ".join(docs[:sp500.iloc[i].ind]) for i in range(len(sp500))]
#corpus_date.to_csv('corpus_date.csv',index = None)
'''

encoded_docs = t.texts_to_sequences(Corpus_date['docs'])
# pad documents to a max length
max_length = max(len(x) for x in encoded_docs)
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')

'''
# load the whole GloVe embedding into memory
embeddings_index = {}
with open('glove.840B.300d.txt', encoding='utf-8') as f:
    for line in f:
        values = line.split(' ')
        word = values[0]
        embedding = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = embedding
print('Loaded %s word vectors.' % len(embeddings_index))
'''
# Import Word2Vec
embeddings_index = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)

# create a weight matrix for words in training docs
embedding_dim = 300
embedding_matrix = np.zeros((vocab_size, embedding_dim))

for word, i in t.word_index.items():
    if word in embeddings_index:
        embedding_matrix[i] = embeddings_index[word]
    else:
        embedding_matrix[i] = np.random.normal(0,np.sqrt(0.25),embedding_dim)
    if i%5000 ==0:
        print(i)

'''
Split the data into training set and test set
cutoff year: 2012 (train:2006-2011 [1309 observations], test:2012-2013)
Params:
    x_train/x_test: financial news headlines
    tech_train/tech_test: technical indicators
    y_train/y_test: labels
'''
x_train, x_test = padded_docs[:1309],padded_docs[1309:]

from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder(sparse=False)
label = sp500['label']
y_ohe = ohe.fit_transform(label.values.reshape(-1, 1))
y_train, y_test = y_ohe[:1309],y_ohe[1309:]

tech_indicators1 = tech_indicators[tech_indicators.columns[~tech_indicators.columns.isin(['Date'])]]
tech_train = tech_indicators1.iloc[:1309,]
tech_test = tech_indicators1.iloc[1309:,]
tech_test = tech_test.reset_index(drop = True)

x_train = np.array(x_train)
x_test = np.array(x_test)
tech_train = np.array(tech_train)
tech_test = np.array(tech_test)
# Reshape tech_train and tech_test to fit the LSTM layer
tech_train = np.reshape(tech_train,(tech_train.shape[0],1,tech_train.shape[1]))
tech_test = np.reshape(tech_test,(tech_test.shape[0],1,tech_test.shape[1]))

y_train = np.array(y_train)
y_test = np.array(y_test)
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

print(len(x_train)) #1309
print(len(x_test))  #472
print(len(tech_train)) #1309
print(len(tech_test))  #472
print(x_train.shape,tech_train.shape)
# Number of tech indicators
tech_num = len(tech_indicators1.columns)

file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
                              save_best_only = True, mode = "min")
early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)

def build_model(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
    inp = Input(shape = (max_length,))
    x = Embedding(vocab_size, embedding_dim, weights = [embedding_matrix], trainable = False)(inp)
    x1 = SpatialDropout1D(dr)(x)

    x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
    x1 = Conv1D(32, kernel_size=3, padding='valid', kernel_initializer='he_uniform')(x_gru)
    avg_pool1_gru = GlobalAveragePooling1D()(x1)
    max_pool1_gru = GlobalMaxPooling1D()(x1)

    x3 = Conv1D(32, kernel_size=2, padding='valid', kernel_initializer='he_uniform')(x_gru)
    avg_pool3_gru = GlobalAveragePooling1D()(x3)
    max_pool3_gru = GlobalMaxPooling1D()(x3)

    x_lstm = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
    x1 = Conv1D(32, kernel_size=3, padding='valid', kernel_initializer='he_uniform')(x_lstm)
    avg_pool1_lstm = GlobalAveragePooling1D()(x1)
    max_pool1_lstm = GlobalMaxPooling1D()(x1)

    x3 = Conv1D(32, kernel_size=2, padding='valid', kernel_initializer='he_uniform')(x_lstm)
    avg_pool3_lstm = GlobalAveragePooling1D()(x3)
    max_pool3_lstm = GlobalMaxPooling1D()(x3)

    x = concatenate([avg_pool1_gru, max_pool1_gru, avg_pool3_gru, max_pool3_gru,
                    avg_pool1_lstm, max_pool1_lstm, avg_pool3_lstm, max_pool3_lstm])
    x = BatchNormalization()(x)
    x = Dropout(0.2)(Dense(128,activation='relu') (x))
    x = BatchNormalization()(x)
    x = Dropout(0.2)(Dense(100,activation='relu') (x))
    x = Dense(5, activation = "sigmoid")(x)
    model = Model(inputs = inp, outputs = x)
    model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
    history = model.fit(x_train, y_train, batch_size = 128, epochs = 15, validation_split=0.1,
                        verbose = 1, callbacks = [check_point, early_stop])
    model = load_model(file_path)
    return model

model = build_model(lr = 1e-4, lr_d = 0, units = 128, dr = 0.5)
pred = model.predict(x_test, batch_size = 1024)
(eval_loss, eval_accuracy) = model.evaluate( x_test, y_test)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

# author: Thomas Kaergel kaergel at b1-systems.de>
# last-modified:: 2019-01-17
# purpose: backupscript for use with cron and borg
# name: borgcron

import os
import datetime
import argparse
import sys
import yaml
import logging
import re
import socket
import struct
import subprocess
import sys
import time

defaultlogfile = 'borgcron.log'
defaultloglevel = logging.WARNING

LOG = logging.getLogger(__name__)

class options(object):
    def __init__(self):
        self._init_parser()

    def _init_parser(self):
        self.parser = argparse.ArgumentParser()
        self.parser.add_argument('-s', "--stdout",
                                 help='print log to stdout too',
                                 action='store_true')
        self.group = self.parser.add_mutually_exclusive_group(required=False)
        self.group.add_argument('-d', '--debug',
                                help="print lots of debugging statements",
                                action="store_const",
                                default=defaultloglevel,
                                dest="loglevel", const=logging.DEBUG)
        self.group.add_argument('-v', '--verbose',
                                help="be verbose",
                                action="store_const",
                                default=defaultloglevel,
                                dest="loglevel", const=logging.INFO)
        self.parser.add_argument('-p', "--pretend",
                                 help='nothing is really done! '
                                      'borgcron just generates the borg '
                                      'command-lines, prints them to stdout'
                                      'and exits',
                                 action='store_true')
        self.parser.add_argument('configfile',
                                 nargs='?',
                                 type=argparse.FileType('r'),
                                 default=sys.stdin,
                                 help='a config file in YAML format.'
                                      ' Can also be read from stdin.')
        self.parser.add_argument('-l', "--logfile",
                                 metavar='logfile',
                                 # nargs='*',
                                 type=argparse.FileType('a'),
                                 default=defaultlogfile,
                                 help='default logfile is borgcron.log')

    def parse(self, args=None):
        return self.parser.parse_args(args)

class clicommand(object):
    def __init__(self, cmdline, logfile):
        self._cmdline = cmdline
        self._logfile = logfile

    def execute(self):
        LOG.info("executing cli-command: \n")
        proc = subprocess.Popen(self._cmdline,
                                stdout = self._logfile,
                                stderr = subprocess.STDOUT,
                                universal_newlines = True,
                                shell = False)
        returncode = proc.wait()
        LOG.info("command returned: %s" % returncode)
        return returncode

class backupdir(object):
    def __init__(self, directory):
        self._name = directory["name"]
        self._days = directory["days"]
        self._month = directory["month"]
        self._years = directory["years"]
        self._namedepth = directory["namedepth"]

    def _get_title(self):
        LOG.debug("_get_title says self._name is %s" % self._name)
        title = ""
        path = self._name
        depth = self._namedepth
        while path is not "/" and depth > -1:
            path, folder = os.path.split(path)
            LOG.debug("path prefix is %s. foldername is %s" % (path, folder))
            if depth > 0:
                title = "°" + folder + title
            else:
                title = folder + title
            depth -= 1
        LOG.debug("title is: %s" % title)
        return title

    def generate_cmds(self, args, flags, comp, repo):
        LOG.info("Starting backup of directroy %s" % self._name)
        LOG.debug("pretend is: %s" % args.pretend)
        now = datetime.datetime.today().strftime('%Y-%m-%d_%H%M%S')
        create = ['/usr/local/bin/borg', 'create', '-C']
        create += [comp]
        create += flags.split()
        create += [repo + '::' + self._get_title() + '-' + now]
        create += [self._name]
        LOG.info("generated borg create command is:")
        LOG.info(create)
        prune = ['/usr/local/bin/borg', 'prune']
        prune += flags.split()
        prune += '-d', str(self._days)
        prune += '-m', str(self._month)
        prune += '-y', str(self._years)
        prune += repo,
        prune += '-P', self._get_title()
        LOG.info("generated borg prune command is:")
        LOG.info(prune)
        return create, prune

def wake_on_lan(cfg):

    LOG.info("Sending WakeOnLAN magic paket.")
    mac_str = cfg["config"]["wol_mac_address"]
    net_bcast = cfg["config"]["remote_bcast_address"]
    addr_byte = mac_str.split(':')
    hw_addr = struct.pack('BBBBBB', int(addr_byte[0], 16),
                                    int(addr_byte[1], 16),
                                    int(addr_byte[2], 16),
                                    int(addr_byte[3], 16),
                                    int(addr_byte[4], 16),
                                    int(addr_byte[5], 16))
    msg = b'\xff' * 6 + hw_addr * 16
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    sock.sendto(msg, (net_bcast, 7))
    sock.sendto(msg, (net_bcast, 9))
    sock.close()
    return

def check_server(cfg):

    LOG.info("Checking if ssh on remote host is listening... plz wait")
    wol_ip = cfg["config"]["remote_host_address"]
    sock = socket.socket()
    count = 0
    res = sock.connect_ex((wol_ip, 22))
    while res != 0 and count < 300:
        time.sleep(1)
        count += 1
        sock.close()
        sock = socket.socket()
        res = sock.connect_ex((wol_ip, 22))
    if res != 0:
        LOG.critical("Remote host did not come up!")
        exit(1)
    LOG.info("Remote host is now up!")
    time.sleep(3)
    return

def configlogger(args):
    LOG.setLevel(defaultloglevel)
    logfile = open(args.logfile.name, 'a')
    fh = logging.StreamHandler(logfile)
    fh.setFormatter(logging.Formatter(
                    '%(asctime)s - %(levelname)s - %(message)s'))
    LOG.addHandler(fh)
    if args.stdout:
        LOG.warn('duplicating log to stdout')
        ch = logging.StreamHandler()
        ch.setFormatter(logging.Formatter(
                        '%(asctime)s - '
                        '%(levelname)s - %(message)s'))
        LOG.addHandler(ch)
    LOG.warn('-' * 80)
    LOG.warn("borgcron by Thomas Kärgel - starting new backup run")
    LOG.warn('Switching to loglevel %s' % args.loglevel)
    LOG.setLevel(args.loglevel)
    return logfile

def parseconfig(args):
    if os.path.isfile(args.configfile.name):
        LOG.info("Reading YAML config from file")
        stream = open(args.configfile.name, 'r')
    else:
        LOG.warn("Awaiting YAML config from stdin")
        stream = sys.stdin
    try:
        config = yaml.safe_load(stream)
        LOG.debug("YAML config is:\n" + yaml.dump(config,
                                                  explicit_start=True,
                                                  default_flow_style=False))
    except yaml.YAMLError as exc:
        logging.critical(exc)
        exit(1)
    return config

def pretendoutput(createcmd, prunecmd):
    createstr = ''
    for i in createcmd:
        createstr = createstr + str(i) + ' '
    print("-" * 80)
    print("create command is:")
    print(createstr)
    prunestr = ''
    for i in prunecmd:
        prunestr = prunestr + str(i) + ' '
    print("-" * 80)
    print("prune command is:")
    print(prunestr)

def main():
    args = options().parse()
    logfile = configlogger(args)
    LOG.debug("args:\n" + str(args))
    cfg = parseconfig(args)
    repo = cfg["config"]["target_repository"]
    comp = cfg["config"]["compression"]
    flags = cfg["config"]["borg_options"]
    prescript = cfg["config"]["prerun_script"]
    if cfg["config"]["wake_remote_host"] and not args.pretend:
        wake_on_lan(cfg)
        check_server(cfg)
    LOG.debug("prescript is: %s" % prescript.split())
    if prescript != "":
        returncode = clicommand(prescript.split(), logfile).execute()
        if returncode == 1:
            LOG.critical("execution of prerun_script failed! -> bailing out.")
            exit(1)
    if "directories" in cfg["config"]:
        for directory in cfg["config"]["directories"]:
            LOG.debug("item config is:\n" +
                      yaml.dump(directory,
                                explicit_start=True,
                                default_flow_style=False))
            dir_item = backupdir(directory)
            LOG.debug("dir_item.name is %s" % dir_item._name)
            createcmd, prunecmd = dir_item.generate_cmds(args, flags, comp, repo)
            if args.pretend:
                pretendoutput(createcmd, prunecmd)
            else:
                clicommand(createcmd, logfile).execute()
                if directory["prune"]:
                    clicommand(prunecmd, logfile).execute()
    if cfg["config"]["shutdown_remote_host"]:
        sshtarget = repo.split(':')
        shutdowncmd = ("ssh %s %s" % (sshtarget[0],
                       cfg["config"]["shutdown_command"]))
        LOG.debug("shutdowncmd is: %s" % shutdowncmd)
        if not args.pretend:
            LOG.info("Shuting down remote host.")
            returncode = clicommand(shutdowncmd.split(),
                                    logfile).execute()
            if returncode == 1:
                LOG.critical("Shutdown of remote failed!")
                exit(1)
    postscript = cfg["config"]["postrun_script"]
    LOG.debug("postscript is: %s" % postscript.split())
    if postscript != "":
        returncode = clicommand(postscript.split(), logfile).execute()
        if returncode == 1:
            LOG.critical("execution of postrun_script failed! -> bailing out.")
            exit(1)

if __name__ == "__main__":
    main()
    exit(0)


import logging
from composite_field import CompositeField
from dateutil.relativedelta import relativedelta
from django.db import models
from django.utils.translation import ugettext_lazy as _

logger = logging.getLogger('debits')
"""The logger used by Debits."""

class Period(CompositeField):
    """Period (for example of recurring payment or of a trial subscription).

    It may be expressed in days, weeks, months, or years. Several units cannot be mixed:
    for example, it cannot be 2 months and 13 days."""
    UNIT_DAYS = 1
    UNIT_WEEKS = 2
    UNIT_MONTHS = 3
    UNIT_YEARS = 4

    period_choices = ((UNIT_DAYS, _("days")),  # different processors may support a part of it
                      (UNIT_WEEKS, _("weeks")),
                      (UNIT_MONTHS, _("months")),
                      (UNIT_YEARS, _("years")))
    """For Django :class:`~django.forms.ChoiceField`."""

    unit = models.SmallIntegerField()
    """days, weeks, months, or years."""

    count = models.SmallIntegerField()
    """The number of the units"""

    def __init__(self, unit=None, count=None):
        super().__init__()
        if unit is not None:
            self['unit'].default = unit
        if count is not None:
            self['count'].default = count

# The following functions do not work as a method, because
# CompositeField is replaced with composite_field.base.CompositeField.Proxy:

def period_to_string(period):
    """Human readable description of a period.

    Args:
        period: `Period` field.

    Returns:
        A human readable string."""
    hash = {e[0]: e[1] for e in Period.period_choices}
    return "%d %s" % (period.count, hash[period.unit])

def period_to_delta(period):
    """Convert :class:`Period` to :class:`relativedelta`."""
    return {
        Period.UNIT_DAYS: lambda: relativedelta(days=period.count),
        Period.UNIT_WEEKS: lambda: relativedelta(weeks=period.count),
        Period.UNIT_MONTHS: lambda: relativedelta(months=period.count),
        Period.UNIT_YEARS: lambda: relativedelta(years=period.count),
    }[period.unit]()


from flask import Flask

from redis import Redis

app = Flask('Gaze Capture Server')
conn = Redis()

@app.route("/")
def ping():

    rep = conn.get("updated")

    if rep is not None:
        return rep
    return ''

# get last frame gaze coords
#
@app.route("/gaze")
def gaze_track():

    rep = conn.get("gaze_coord")

    if rep is not None:
        return rep
    return ''

# get last blinks count
#
@app.route("/blink")
def blink_count():

    rep = conn.get("blinks")

    if rep is not None:
        return rep
    return ''

@app.route("/msg")
def msg():

    rep = conn.get("msg")

    if rep is not None:
        return rep
    return ''


# coding: utf-8

# # Imports

# In[1]:

import pandas as pd
import os
import numpy as np
from rdkit.Chem import AllChem as Chem
import sys
sys.path.append('..')

# In[2]:

WORK_DIR = os.getcwd()

DATA_DIR = os.path.join(WORK_DIR,'original')

data_processed_dir = os.path.join(WORK_DIR,'processed')
if not os.path.exists(data_processed_dir):
    os.makedirs(data_processed_dir)

# # Read Raw Data

# Read the raw data files, rename columns, drop NaN and extraneous information

# ### ChemFluor

# In[3]:

data_location = os.path.join(DATA_DIR, 'chem_fluor/Alldata_SMILES.xlsx')
chemfluor_df = pd.read_excel(data_location)
chemfluor_df.rename(columns={'SMILES':'smiles',"Absorption/nm":'peakwavs_max'}, inplace=True)
chemfluor_df = chemfluor_df[['smiles','solvent','peakwavs_max']].copy()
chemfluor_df.dropna(inplace=True)
chemfluor_df['source'] = 'chemfluor'
chemfluor_df

# ### DSSCDB

# In[4]:

data_location = os.path.join(DATA_DIR, 'dsscdb')
xlsx_files = [x for x in os.listdir(data_location) if x.endswith('.xlsx')]
dsscdb_df = pd.DataFrame()
for file in xlsx_files:
    file_location = os.path.join(data_location, file)
    dsscdb_df = dsscdb_df.append(pd.read_excel(file_location), ignore_index=True, sort=True)

dsscdb_df.rename(columns={'SMILES':'smiles','SOLVENT':'solvent','ABSORPTION_MAXIMA':'peakwavs_max'}, inplace=True)
dsscdb_df = dsscdb_df[['smiles','solvent','peakwavs_max']].copy()
dsscdb_df.dropna(inplace=True)
dsscdb_df['source'] = 'dsscdb'
dsscdb_df

# ### DyeAgg

# In[5]:

data_location = os.path.join(DATA_DIR, 'dye_agg/new_dssc_Search_results.csv')
dyeagg_df = pd.read_csv(data_location, sep=';')
dyeagg_df.rename(columns={'STRUCTURE':'smiles','SOLVENT':'solvent','PEAK_ABSORPTION_SOLUTION':'peakwavs_max'},
                 inplace=True)
dyeagg_df = dyeagg_df[['smiles','solvent','peakwavs_max']].copy()
dyeagg_df.dropna(inplace=True)
dyeagg_df['source'] = 'dyeagg'
dyeagg_df

# ### CDEx

# In[6]:

data_location = os.path.join(DATA_DIR, 'jcole/paper_allDB.csv')
jcole_df = pd.read_csv(data_location)
jcole_df.rename(columns={'SMI':'smiles', "lambda_max (Exp,  nm)": 'peakwavs_max'}, inplace=True)
jcole_df = jcole_df[['smiles','solvent','peakwavs_max']].copy()
jcole_df.dropna(inplace=True)
jcole_df['source'] = 'cdex'
jcole_df

# ### Deep4Chem

# In[7]:

data_location = os.path.join(DATA_DIR, 'joung/DB_for_chromophore_Sci_Data_rev02.csv')
joung_df = pd.read_csv(data_location)

joung_df.rename(columns={'Chromophore':'smiles','Solvent':'solvent', 'Absorption max (nm)':'peakwavs_max'},
                inplace=True)

# Drop measurements where solvent == 'gas'
gas_idx = joung_df[joung_df['solvent']=='gas'].index
joung_df.drop(index=gas_idx, inplace=True)

# Drop measurements taken in solid state (original authors wrote file as chromophore == solvent for these cases)
ss_idx = joung_df[joung_df['smiles']==joung_df['solvent']].index
joung_df.drop(index=ss_idx, inplace=True)

joung_df = joung_df[['smiles','solvent','peakwavs_max']].copy()
joung_df.dropna(inplace=True)
joung_df['source'] = 'deep4chem'

joung_df

# # Convert Solvent Names to SMILES

# In[8]:

no_solvent_smiles_df = pd.concat([chemfluor_df, dsscdb_df, dyeagg_df, jcole_df])
no_solvent_smiles_df.reset_index(drop=True, inplace=True)
no_solvent_smiles_df['solvent'] = no_solvent_smiles_df['solvent'].apply(lambda x: x.lower().strip())
no_solvent_smiles_df

# In[9]:

# Exclude all solvents that are ionic, mixtures, polymers, or not clear what they are

bad_solvents = ['-','n','10 − 4–10 − 6 m','quartz','silica','pbs','pvc','ch3cn-ch2cl2','acetonitrile + tert-butanol',
                'cl','poly(ethylene glycol)s','chcl','methanol, tert-butyl alcohol-acetonitrile',
                'methanol-chloroform','tbaf','chcl3-ch3oh','ch 2 cl 2 + 0.2 mmol l − 1 bu 4 nbf','ch3oh–h2o','pmma',
                'ch3oh+chcl3','t-buoh-an','ni','ch 2 cl','acetonitrile-dmso','acetonitrile and tert-butyl alcohol',
                'sds','tert-butanol–acetonitrile','chcl3–ch3oh','barium sulfate','mecn-dcm','chloroform + methanol',
                'ch3oh-chcl3','kbr','2-methyl-2-propanol-acetonitrile']
no_solvent_smiles_df = no_solvent_smiles_df.loc[~no_solvent_smiles_df['solvent'].isin(bad_solvents)]

for bad_symbol in ['%','/',':']:
    no_solvent_smiles_df = no_solvent_smiles_df.loc[~no_solvent_smiles_df['solvent'].str.contains(bad_symbol)]

no_solvent_smiles_df

# In[10]:

# Dictionary for standardizing abbreviations and different names for the same solvents
replace_solvents_master_dict = {'acn':'acetonitrile', 'c6h6':'benzene', 'c6h12':'cyclohexane',
                                'ccl4':'carbon tetrachloride', 'ch2cl2':'dichloromethane',
                                'ch 2 cl 2':'dichloromethane','ch3cn':'acetonitrile',
                                'ch 3 cn':'acetonitrile','ch 3 oh':'methanol', 'chcl3':'chloroform',
                                'chroroform':'chloroform',
                                'chx':'chlorohexidine',
                                'ctc':'carbon tetrachloride',
                                'dichloromethane.':'dichloromethane','dcb':'1,2-dichlorobenzene',
                                'dcm':'dichloromethane',
                                'dee': 'diethyl ether',
                                'dimethylsulfoxide':'dmso', 'dimethylsufoxide':'dmso', 'dimethyl sulfoxide':'dmso',
                                'dma':'dimethylacetamide', 'dmf':'dimethylformamide',
                                'ethylacetate': 'ethyl acetate', 'etoh':'ethanol',
                                'etoac':'ethyl acetate', 'h 2 o':'water', 'h2o':'water', 'hcl':'hydrochloric acid',
                                'hex':'hexane',
                                'mch': 'methylcyclohexane',
                                'mecn':'acetonitrile', 'meoh':'methanol',
                                'me-thf':'2-methyltetrahydrofuran', 'mthf':'2-methyltetrahydrofuran',
                                'nmp':'n-methyl-2-pyrrolidone','o-c6h4cl2': 'orthodichlorobenzene',
                                'pgmea':'propylene glycol methyl ether acetate',
                                'phcl':'chlorobenzene',
                                'phcn':'benzonitrile',
                                'tfa':'trifluoroacetic acid', 'tfe':'2,2,2-trifluoroethanol',
                                'thf':'tetrahydrofuran', 'toluol':'toluene'}

# Dictionary for replacing all full solvent names with solvent SMILES
solvent_smiles_dict = {'cyclohexane': 'C1CCCCC1', 'dmso': 'CS(=O)C','ethylene glycol': 'C(CO)O',
                        'ethanol': 'CCO','methanol': 'CO','dioxane': 'C1COCCO1','pyridine': 'c1ccncc1',
                        'chlorobenzene': 'Clc1ccccc1','water': 'O','glycerin': 'OCC(O)CO',
                        'dichloromethane': 'ClCCl','carbon tetrachloride': 'ClC(Cl)(Cl)Cl',
                        'toluene': 'Cc1ccccc1','chloronaphthalene': 'Clc1cccc2ccccc12','hexane': 'CCCCCC',
                        'acetonitrile': 'CC#N','chloroform': 'ClC(Cl)Cl','benzene': 'c1ccccc1',
                        'dimethylformamide': 'CN(C)C=O','tetrahydrofuran': 'C1CCOC1',
                        'triethylamine': 'CCN(CC)CC','bromobenzene': 'Brc1ccccc1',
                        '2-methylbutane': 'CCC(C)C','2-pentanone': 'CCCC(C)=O',
                        'di-n-butyl ether': 'CCCCOCCCC','glycerol': 'OCC(O)CO','methyl formate': 'COC=O',
                        '2-propanol': 'CC(C)O','diisopropyl ether': 'CC(C)OC(C)C','1-octanol': 'CCCCCCCCO',
                        '2,2,2-trifluoroethanol': 'OCC(F)(F)F','2-methyltetrahydrofuran': 'CC1CCCO1',
                        'methylcyclohexane': 'CC1CCCCC1','1-butanol': 'CCCCO','heptane': 'CCCCCCC',
                        'ethyl acetate': 'CCOC(C)=O','1,2-dichlorobenzene': 'Clc1ccccc1Cl',
                        '1-decanol': 'CCCCCCCCCCO','formamide': 'NC=O','acetone': 'CC(C)=O',
                        'dimethylacetamide': 'CN(C)C(C)=O','o-dimethoxybenzene': 'COc1ccccc1OC',
                        'n-methylformamide': 'CNC=O','diethyl ether': 'CCOCC','1-hexanol': 'CCCCCCO',
                        '1-methyl-2-pyrrolidinone': 'CN1CCCC1=O','butyl acetate': 'CCCCOC(C)=O',
                        'tert-pentanol': 'CCC(C)(C)O','2-methyl-2-propanol': 'CC(C)(C)O',
                        '1,2-propanediol': 'CC(O)CO','1-propanol': 'CCCO','methyl acetate': 'COC(C)=O',
                        'trifluoroacetic acid': 'OC(=O)C(F)(F)F',
                        'n-methyl-2-pyrrolidone': 'CN1CCCC1=O', '1,4-dioxane': 'C1COCCO1',
                        'hydrochloric acid': 'Cl', 'benzonitrile': 'N#Cc1ccccc1',
                        'propanol': 'CCCO', 'propylene carbonate': 'CC1COC(=O)O1',
                        'benzyl alcohol': 'OCc1ccccc1', 'butanol': 'CCCCO',
                        'propylene glycol methyl ether acetate': 'COCC(C)OC(C)=O', 'isopropanol': 'CC(C)O',
                        'methylene chloride': 'ClCCl', 'n-hexane': 'CCCCCC',
                        'octene': 'CCCCCCC=C', 'mops': 'O[S](=O)(=O)CCCN1CCOCC1',
                        'tetrachloromethane': 'ClC(Cl)(Cl)Cl', 'pentane': 'CCCCC',
                        'chlorobenzene': 'Clc1ccccc1', 'acetic acid': 'CC(O)=O', '1,2-dichloroethane': 'ClCCCl',
                       'chlorohexidine':'Clc1ccc(NC(=N)NC(=N)NCCCCCCNC(=N)NC(=N)Nc2ccc(Cl)cc2)cc1',
                       'orthodichlorobenzene':'C1=CC=C(C(=C1)Cl)Cl', 'tert-butyl alcohol':'CC(C)(C)O',
               }

solvents_set = set(no_solvent_smiles_df['solvent'])
#print(len(solvents_set))

solvents_for_df_dict = {}
for solvent in solvents_set:
    if solvent in replace_solvents_master_dict.keys():
        solvent_ = replace_solvents_master_dict[solvent]
    else:
        solvent_ = solvent
    solvents_for_df_dict[solvent] = solvent_smiles_dict[solvent_]

# In[11]:

# Replace names with SMILES
no_solvent_smiles_df['solvent'] = no_solvent_smiles_df['solvent'].map(solvents_for_df_dict)
no_solvent_smiles_df

# # Combine All Sources

# In[12]:

df = pd.concat([no_solvent_smiles_df, joung_df])
df = df.loc[df['peakwavs_max']!='-'].copy()
df['peakwavs_max'] = df['peakwavs_max'].apply(lambda x: float(x))
df

# # Filtering

# ### Remove Molecules that Cannot be Sanitized by RDKit

# In[13]:

def sanitize_smiles(smiles):
    try:
        smiles = Chem.MolToSmiles(Chem.MolFromSmiles(smiles))
    except:
        smiles = np.nan
    return smiles

df['smiles'] = df['smiles'].apply(lambda x: sanitize_smiles(x))
df['solvent'] = df['solvent'].apply(lambda x: sanitize_smiles(x))
df.dropna(inplace=True)
df

# ### Remove Clusters (SMILES containing ".")

# In[14]:

cluster_idx = df[df['smiles'].str.contains('\.')].index
#print('Removing {} rows'.format(len(cluster_idx)))
df.drop(index=cluster_idx, inplace=True)
df

# # Export DataFrame to CSV File

# In[15]:

df[['smiles','solvent','peakwavs_max','source']].to_csv(f'{data_processed_dir}/all_lambda_max_abs_including_duplicates.csv',
                                                        index=False)

# # Stats

# ### Table 2

# In[16]:

df['source'].value_counts()

# In[17]:

df.drop_duplicates(subset=['smiles','solvent'])

# In[18]:

df.groupby(['smiles','solvent']).count().query('source > 1')

# In[19]:

len(set(df['smiles']))

# In[20]:

len(set(df['solvent']))

# ### Table S1

# In[21]:

dict(df['solvent'].value_counts())


import redis
import time
import sys
import ast

r = redis.Redis(host='localhost', port=6379, db=0)

def subscribeChannel(channels):
    p = r.pubsub()
    p.subscribe(channels)

    while True:
        message = p.get_message()
        if message:
            print(message)
        time.sleep(0.01)

def publishChannel(channel,msg):
    r.publish(channel, msg)

if __name__ == "__main__":
    args = sys.argv

    if args[1]=='subscribeChannel':

        args[2]=bytes(args[2],"utf-8")
        args[2] = args[2].decode("UTF-8")
        args[2]= ast.literal_eval(args[2])
        for channel in args[2]:
            print(f"subscribed to : {channel}\n")
        globals()[args[1]](args[2])

    elif args[1]=='publishChannel':
        print(f"Published Message : {args[3]}")
        globals()[args[1]](args[2],args[3])


from PyQt5.QtWidgets import QWidget
from PyQt5 import QtWidgets

class Chose_worker(QWidget):
    def __init__(self, list_workers):
        super(Chose_worker, self).__init__()

        widget = QtWidgets.QHBoxLayout()
        self.chose = QtWidgets.QComboBox()
        self.chose.addItem("")
        for worker in list_workers:
            self.chose.addItem(worker[0])
        widget.addWidget(self.chose)

        self.setLayout(widget)


import unittest

# from SNDG.Structure.StructureVariant import StructureVariant
from SNDG import execute
from io import StringIO
import tempfile
import json

class TestVcfSnpeffIO(unittest.TestCase):

    def test_residues_mapping(self):
        stdout = tempfile.NamedTemporaryFile()
        execute("python3 -m SNDG.Structure.StructureVariant residues -i ./test/prot2.fasta --pdb_data /tmp/data -s 2VYI"
                , stdout=stdout)

        with open(stdout.name) as output:
            contents = output.read()
        stdout.close()
        expected = ' '.join("""pdb     chain   resid   alt     ref     pos     pdb_pos
2vyi    A       81      P       G       23      1
2vyi    A       160     K       A       160     77
2vyi    B       160     K       A       160     74""".split())
        self.assertEqual(
            expected, ' '.join(contents.strip().split())
        )

    def test_residues_ann(self):
        stdout = tempfile.NamedTemporaryFile()
        execute("cat test/test.tbl | python -m SNDG.Structure.StructureVariant ann --pdb_data /tmp/data "
                , stdout=stdout)
        with open(stdout.name) as output:
            contents = output.read()
        stdout.close()
        anns = [json.loads(x) for x in contents.split("###") if x.strip()]
        self.assertEqual(4, len(anns))
        r2vyi_B_160_K = [x["ann"] for x in anns if x["residue"] == "2vyi_B_160_K"][0]

        self.assertEqual(0.705, r2vyi_B_160_K["pockets"][0]["druggabilitty"])

        r1azm_A_91_P = [x["ann"] for x in anns if x["residue"] == "1azm_A_91_P"][0]
        self.assertTrue("BINDING SITE FOR RESIDUE AZM A 262" in
                        [x["details"] for x in r1azm_A_91_P["binding"]])

if __name__ == '__main__':
    unittest.main()


from notes import containNotes, noteToNumber, notesToChord, chordList, chordToNote, chordListAlt, notesToScale, scaleList, scaleListAlt, scaleToNote
from transitions.extensions import GraphMachine
from utils import send_text_message, send_menu_carousel, send_chord, send_not_found, send_chord_note, send_scale, send_scale_note, send_fsm, send_demo

class TocMachine():
    def __init__(self):
        self.notes = []
        self.chord = -1
        self.scale = -1
        self.machine = GraphMachine(
            model=self,
            **{
                "states":["start", "menu", "chord", "chordResult", "chordNote", "chordNoteRootnote", "chordNoteType"
                , "scale", "scaleResult", "scaleNote", "scaleNoteRootnote", "scaleNoteType", "fsm", "demo"],
                "transitions":[
                    {
                        "trigger": "advance",
                        "source": "*",
                        "dest": "fsm",
                        "conditions": "is_going_to_fsm",
                    },
                    {
                        "trigger": "advance",
                        "source": "*",
                        "dest": "menu",
                        "conditions": "is_going_to_menu",
                    },
                    {
                        "trigger": "advance",
                        "source": "menu",
                        "dest": "chord",
                        "conditions": "is_going_to_chord",
                    },
                    {
                        "trigger": "advance",
                        "source": "menu",
                        "dest": "chordNote",
                        "conditions": "is_going_to_chordNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "chord",
                        "dest": "chordResult",
                        "conditions": "is_going_to_chordResult",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordResult",
                        "dest": "chordResult",
                        "conditions": "is_going_to_chordResult",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNote",
                        "dest": "chordNoteRootnote",
                        "conditions": "is_going_to_chordNoteRootnote",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteRootnote",
                        "dest": "chordNoteType",
                        "conditions": "is_going_to_chordNoteType",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteType",
                        "dest": "chordNote",
                        "conditions": "is_going_back_to_chordNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteType",
                        "dest": "chordNoteType",
                        "conditions": "is_going_to_chordNoteType",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteType",
                        "dest": "chordNoteRootnote",
                        "conditions": "is_going_to_chordNoteRootnote",
                    },
                    {
                        "trigger": "advance",
                        "source": "menu",
                        "dest": "scale",
                        "conditions": "is_going_to_scale",
                    },
                    {
                        "trigger": "advance",
                        "source": "menu",
                        "dest": "scaleNote",
                        "conditions": "is_going_to_scaleNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "scale",
                        "dest": "scaleResult",
                        "conditions": "is_going_to_scaleResult",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleResult",
                        "dest": "scaleResult",
                        "conditions": "is_going_to_scaleResult",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNote",
                        "dest": "scaleNoteRootnote",
                        "conditions": "is_going_to_scaleNoteRootnote",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteRootnote",
                        "dest": "scaleNoteType",
                        "conditions": "is_going_to_scaleNoteType",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteType",
                        "dest": "scaleNote",
                        "conditions": "is_going_back_to_scaleNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteType",
                        "dest": "scaleNoteType",
                        "conditions": "is_going_to_scaleNoteType",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteType",
                        "dest": "scaleNoteRootnote",
                        "conditions": "is_going_to_scaleNoteRootnote",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteType",
                        "dest": "chordNote",
                        "conditions": "is_going_to_change_rootNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "chordNoteType",
                        "dest": "chordNoteRootnote",
                        "conditions": "is_going_to_change_type",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteType",
                        "dest": "scaleNote",
                        "conditions": "is_going_to_change_rootNote",
                    },
                    {
                        "trigger": "advance",
                        "source": "scaleNoteType",
                        "dest": "scaleNoteRootnote",
                        "conditions": "is_going_to_change_type",
                    },
                    {
                        "trigger": "advance",
                        "source": "fsm",
                        "dest": "menu",
                        "conditions": "is_going_to_menu",
                    },
                    {
                        "trigger": "advance",
                        "source": "menu",
                        "dest": "demo",
                        "conditions": "is_going_to_demo",
                    },
                    {
                        "trigger": "advance",
                        "source": "demo",
                        "dest": "menu",
                        "conditions": "is_going_to_menu",
                    },
                ],
                "initial":"start",
                "auto_transitions":False,
                "show_conditions":True,
            },
        )

    #condition
    def is_going_to_menu(self, event):
        text = event.message.text
        return "選單" in text

    def is_going_to_chord(self, event):
        text = event.message.text
        return "和弦" in text and "組成音" not in text

    def is_going_to_chordNote(self, event):
        text = event.message.text
        return "和弦" in text and "組成音" in text

    def is_going_to_chordResult(self, event):
        print("!" + str(self.notes))
        text = event.message.text
        if(len(containNotes(text)) != 0): self.notes = containNotes(text)
        return len(containNotes(text)) != 0

    def is_going_to_chordNoteRootnote(self, event):
        text = event.message.text
        if(len(containNotes(text)) != 0): self.notes = containNotes(text)
        return len(containNotes(text)) != 0

    def is_going_to_chordNoteType(self, event):
        self.chord = -1
        text = event.message.text
        text = text.strip()
        for i in range(len(chordList)):
            for j in range(len(chordList[i])):
                if text == chordList[i][j]:
                    self.chord = i
                    break
            else:
                continue
            break
        if self.chord == -1:
            for i in range(len(chordListAlt)):
                for j in range(len(chordListAlt[i])):
                    if str(text).lower() == chordListAlt[i][j]:
                        self.chord = i
                        break
                else:
                    continue
                break
        return self.chord != -1

    def is_going_back_to_chordNote(self, event):
        text = event.message.text
        return "重來" in text or "再來" in text or "重查" in text

    def is_going_to_scale(self, event):
        text = event.message.text
        return "音階" in text and "組成音" not in text

    def is_going_to_scaleNote(self, event):
        text = event.message.text
        return "音階" in text and "組成音" in text

    def is_going_to_scaleResult(self, event):
        text = event.message.text
        if(len(containNotes(text)) != 0): self.notes = containNotes(text)
        return len(containNotes(text)) != 0

    def is_going_to_scaleNoteRootnote(self, event):
        text = event.message.text
        if(len(containNotes(text)) != 0): self.notes = containNotes(text)
        return len(containNotes(text)) != 0

    def is_going_to_scaleNoteType(self, event):
        self.scale = -1
        text = event.message.text
        text = text.strip()
        for i in range(len(scaleList)):
            for j in range(len(scaleList[i])):
                if text == scaleList[i][j]:
                    self.scale = i
                    break
            else:
                continue
            break
        if self.scale == -1:
            for i in range(len(scaleListAlt)):
                for j in range(len(scaleListAlt[i])):
                    if str(text).lower() == scaleListAlt[i][j]:
                        self.scale = i
                        break
                else:
                    continue
                break
        return self.scale != -1

    def is_going_back_to_scaleNote(self, event):
        text = event.message.text
        return "重來" in text or "再來" in text or "重查" in text

    def is_going_to_change_rootNote(self, event):
        text = event.message.text
        return "更改" in text and "根音" in text

    def is_going_to_change_type(self, event):
        text = event.message.text
        return "更改" in text and "種類" in text

    def is_going_to_fsm(self, event):
        text = event.message.text
        return "fsm" in str(text).lower()

    def is_going_to_demo(self, event):
        text = event.message.text
        return "demo" in str(text).lower()

    #on enter
    def on_enter_menu(self, event):
        print("I'm entering menu")
        reply_token = event.reply_token
        send_menu_carousel(reply_token)

    def on_enter_chord(self, event):
        print("I'm entering chord")
        reply_token = event.reply_token
        text = "請輸入數個音符（A~G，可搭配升降記號）。我會想辦法告訴你他們可以組成的和弦。"
        send_text_message(reply_token, text)

    def on_enter_chordNote(self, event):
        print("I'm entering chordNote")
        reply_token = event.reply_token
        text = "請先輸入和弦的「根音」，或點選下方常用音符。"
        send_text_message(reply_token, text, "note")

    def on_enter_chordResult(self, event):
        print("I'm entering chordResult")
        reply_token = event.reply_token
        print(self.notes)
        root_note, whichChord = notesToChord(self.notes)
        if whichChord > -1:
            send_chord(reply_token, root_note, whichChord)
        else:
            send_not_found(reply_token)

    def on_enter_chordNoteRootnote(self, event):
        print("I'm entering chordNoteRootnote")
        reply_token = event.reply_token
        text = "請輸入和弦「種類」，或點選下方常用和弦。"
        send_text_message(reply_token, text, "chord")

    def on_enter_chordNoteType(self, event):
        print("I'm entering chordNoteType")
        reply_token = event.reply_token
        print(self.notes)
        text = chordToNote(self.notes, self.chord)
        send_chord_note(reply_token, self.notes[0], self.chord, text)

    def on_enter_scale(self, event):
        print("I'm entering scale")
        reply_token = event.reply_token
        text = "請輸入數個音符（A~G，可搭配升降記號）。我會想辦法告訴你他們可以組成的音階。"
        send_text_message(reply_token, text)

    def on_enter_scaleNote(self, event):
        print("I'm entering scaleNote")
        reply_token = event.reply_token
        text = "請先輸入音階的「根音」，或點選下方常用音符。"
        send_text_message(reply_token, text, "note")

    def on_enter_scaleResult(self, event):
        print("I'm entering scaleResult")
        reply_token = event.reply_token
        print(self.notes)
        root_note, whichScale = notesToScale(self.notes)
        if whichScale > -1:
            send_scale(reply_token, root_note, whichScale)
        else:
            send_not_found(reply_token)

    def on_enter_scaleNoteRootnote(self, event):
        print("I'm entering scaleNoteRootnote")
        reply_token = event.reply_token
        text = "請輸入音階「種類」，或點選下方常用音階。"
        send_text_message(reply_token, text, "scale")

    def on_enter_scaleNoteType(self, event):
        print("I'm entering chordNoteType")
        reply_token = event.reply_token
        print(self.notes)
        text = scaleToNote(self.notes, self.scale)
        send_scale_note(reply_token, self.notes[0], self.scale, text)

    def on_enter_fsm(self, event):
        print("I'm entering fsm")
        reply_token = event.reply_token
        send_fsm(reply_token)

    def on_enter_demo(self, event):
        print("I'm entering demo")
        reply_token = event.reply_token
        send_demo(reply_token)


import urllib3
from bs4 import BeautifulSoup
import os
import subprocess
import sys

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
http = urllib3.PoolManager()

global_num_issues = 0

group_name = sys.argv[1]
base_url = 'https://github.com/' + group_name

def get_parsed_page(url_requested):
	page_http = http.request('GET', url_requested)
	page_html = str(page_http.data.decode('utf-8'))
	return BeautifulSoup(page_html, 'html.parser')

def get_repo_security_labels(repo_url):
	parsed_labels_html = get_parsed_page(repo_url + '/labels?&q=security')
	return [element.text.strip() for element in parsed_labels_html.find_all("span", attrs={"class": "label-name"})]

def download_issue(repo_name, issue_id, security_related, gt_file, label=None):
	global group_name
	global global_num_issues

	issue_page = get_parsed_page(base_url + '/' + repo_name + '/issues/' + issue_id)

	issue_title = issue_page.find("h1", attrs={"class": "discussion-title"})
	if issue_title is None:
		issue_title = issue_page.find("span", attrs={"class": "js-issue-title"})
	if label is None:
		print("Downloading {} {}".format(repo_name, issue_id))
	else:
		print("Downloading {} {} <{}>".format(repo_name, issue_id, label))
	issue_title = issue_title.text.strip()

	issue_description_element = issue_page.find("div", attrs={"class": "discussion-starting-comment"})
	if issue_description_element is None:
		issue_description_element = issue_page.find("td", attrs={"class": "d-block comment-body markdown-body js-comment-body"})
	issue_description = issue_description_element.text

	global_issue_identifier = '{}_{}_{}'.format(group_name, repo_name, issue_id)
	with open('organizations/{}/{}/{}.txt'.format(group_name, repo_name, global_issue_identifier), 'w+') as issue_file:
		issue_file.write(issue_title + '\n\n')
		issue_file.write(issue_description)
	gt_file.write("{} {}\n".format(global_issue_identifier, '(1,0)' if security_related else '(0,1)'))

	global_num_issues += 1

def get_num_pages(parsed_html):
	pages = parsed_html.find("div", attrs={"class": "pagination"})
	if pages is None:
		return 1
	n_pages = int(pages.findChildren("a")[-2].text)
	return n_pages

def download_issues(repo_name, security_labels):
	global group_name

	if not os.path.exists('organizations/{}/{}'.format(group_name, repo_name)):
		os.makedirs('organizations/{}/{}'.format(group_name, repo_name))

	gt_file = open('organizations/{}/{}/ground_truth.txt'.format(group_name, repo_name), 'w+')
	# First download all issues with security tags
	num_security_issues = 0
	for label in security_labels:
		if any(ord(c) >= 128 for c in label):
			print("{} includes non-ascii characters, ignoring".format(label))
			continue

		needs_quotes = False
		if ' ' in label:
			needs_quotes = True
		url_label = label.replace(' ', '+').replace(':', '%3A')
		if needs_quotes:
			url_label = "\"" + url_label + "\""
		url_label = '%3A' + url_label
		security_issues_html = get_parsed_page(base_url + '/' + repo_name + '/issues?page=1&q=label' + url_label)
		n_pages = get_num_pages(security_issues_html)
		for page in range(1, n_pages + 1):
			issues_page_url = base_url + '/' + repo_name + '/issues?page={}&q=label{}'.format(page, url_label)
			security_issues_html = get_parsed_page(issues_page_url)
			issue_table = security_issues_html.find("div", attrs={"class": "js-navigation-container js-active-navigation-container"})
			if issue_table is None:
				print("No issues found for label: {}".format(label))
				continue
			issue_rows = issue_table.findChildren(recursive=False)
			for issue in issue_rows:
				issue_id = issue.attrs["id"][6:]
				download_issue(repo_name, issue_id, True, gt_file, label)
				num_security_issues += 1
	print("Found {} security-related issues".format(num_security_issues))
	if num_security_issues == 0:
		subprocess.call(['rm', '-rf', '{}/{}'.format(group_name, repo_name)])
		return

	security_labels = set(security_labels)
	# Then download an equal number of issues issues which do not have security tags
	num_non_security = 0
	parsed_issues_html = get_parsed_page(base_url + '/' + repo_name + '/issues?page=1&q=')
	n_pages = get_num_pages(parsed_issues_html)
	for page in range(1, n_pages + 1):

		issues_page_url = base_url + '/' + repo_name + '/issues?page={}&q='.format(page)
		parsed_issues_html = get_parsed_page(issues_page_url)
		issue_table = parsed_issues_html.find("div", attrs={"class": "js-navigation-container js-active-navigation-container"})
		if issue_table is None:
			issues_page_url = base_url + '/' + repo_name + '/pulls?page={}&q='.format(page)
			parsed_issues_html = get_parsed_page(issues_page_url)
			issue_table = parsed_issues_html.find("div", attrs={"class": "js-navigation-container js-active-navigation-container"})

		issue_rows = issue_table.findChildren(recursive=False)

		for issue in issue_rows:

			issue_labels_element = issue.find("span", attrs={"class": "labels"})
			if issue_labels_element is not None:
				has_security_label = False
				for label in issue_labels_element.findChildren():
					label = label.text.strip()
					if label in security_labels:
						has_security_label = True
						break
				if has_security_label:
					continue

			issue_id = issue.attrs["id"][6:]
			if num_non_security < num_security_issues:
				download_issue(repo_name, issue_id, False, gt_file)
				num_non_security += 1
			else:
				print("Finished downloading {} security issues and {} non-security issues for repo: {}".format(num_security_issues, num_non_security, repo_name))
				gt_file.close()
				return
	print("Finished downloading {} security issues and {} non-security issues for repo: {}".format(num_security_issues, num_non_security, repo_name))
	gt_file.close()

n_pages = get_num_pages(get_parsed_page(base_url))
for page in range(1, n_pages + 1):
	page_url = base_url + '?page=' + str(page)
	parsed_page_html = get_parsed_page(page_url)

	repos = [element.text.strip() for element in parsed_page_html.find_all("a", attrs={"itemprop":"name codeRepository"})]
	for repo in repos:
		print("Checking repo [{}] for security labels".format(repo))
		labels = get_repo_security_labels(base_url + '/' + repo)
		if len(labels) > 0:
			print("[{}] has labels: {}".format(repo, str(labels)))
			download_issues(repo, labels)
			print("Total issues downloaded: {}\n".format(global_num_issues))
		else:
			print("[{}] has no security labels\n".format(repo))

print("Final issue download count for ({}): {}".format(group_name, global_num_issues))


# -*- coding: utf-8 -*-
"""
Created on Sun Jun  2 14:36:52 2019

@author: Darin
"""

import numpy as np
import scipy.sparse as sparse
import matplotlib.pyplot as plt
from FEM import FEM
import Material
import Interpolation
import Update
from Optimization import PyOpt

#def run():
fem = FEM()
Nelx = 100
Nely = 50
Dimensions = [0, 10, 0, 5]
fem.Create2DMesh(Dimensions, Nelx, Nely, maxLevels=10)
fem.SolveSystem = fem.SolveSystemDirect
ofst = 1e-10

lower = np.array([10-ofst, 2.5-ofst])
upper = np.array([10+ofst, 2.5+ofst])
loads = [0, -1e4]
loadSpecs = [{'lower':lower, 'upper':upper, 'force':loads}]
fem.AddLoad(loadSpecs)

lower = np.array([0-ofst, 0-ofst])
upper = np.array([0+ofst, 5+ofst])
bc = [0, 0]
bcSpecs = [{'lower':lower, 'upper':upper, 'disp':bc}]
fem.AddBC(bcSpecs)

#    springpoly = np.array([[-1, 0.5-ofst], [2, 0.5-ofst], [2, 0.5+ofst], [-1, 0.5+ofst]])
#    stiff = [1, 1]
#    springSpecs = [{'poly':springpoly, 'stiff':stiff}]
#    fem.AddSprings(springSpecs)

#fem.Plot()

fem.SetMaterial(Material.PlaneStressElastic(15e9, 0.3))

#update = Update.OCUpdateScheme(0.2, 0.5, np.linspace(0, 1, fem.nElem, endpoint=False),#0.5 * np.ones(fem.nElem),
#                               np.zeros(fem.nElem), np.ones(fem.nElem))
update = Update.MMA(0.5 * np.ones(fem.nElem), 1, np.zeros(fem.nElem), np.ones(fem.nElem))

opt = PyOpt(fem, update)

# Minimum feature size filter
radius = 1.5 * (Dimensions[1] - Dimensions[0]) / Nelx
opt.Filter = opt.ConstructDensityFilter(radius=radius, nElx=[Nelx, Nely])

# Maximum feature size filter
radius = 0.5 * (Dimensions[1] - Dimensions[0]) / Nelx
opt.R = opt.ConstructDensityFilter(radius=radius, nElx=[Nelx, Nely])
rowsum = opt.R.indptr[1:] - opt.R.indptr[:-1]
edge = rowsum.max() - rowsum
opt.R = opt.R.tocoo()
opt.R = sparse.csr_matrix((np.concatenate([1+0*opt.R.data, edge]),
                           (np.concatenate([opt.R.row, np.arange(opt.R.shape[0])]),
                            np.concatenate([opt.R.col, opt.R.shape[0]*np.ones(opt.R.shape[0], dtype=int)]))))

fem.Initialize()

from Functions import Compliance, Volume, Stability
opt.AddFunction(Compliance, 1., 0, 1, 'objective')
opt.AddFunction(Volume, 0.4, 0, 1, 'constraint')

for penal in np.linspace(1,4,7):
    print("Penalty set to %1.2f" % penal)
    opt.SetInterpolation(Interpolation.SIMP_CUT(opt.Filter, opt.R, 50, penal,
                                                0, 0.01, minStiff=1e-10))
    opt.Optimize(maxit=30, plt_freq=10)


from wsgiref.simple_server import make_server
import json
import urllib.request
import conf

# The application interface is a callable object
def application(environ, start_response):

    # Build the response body possibly
    # using the supplied environ dictionary
    response_body = 'Request method: %s' % environ['REQUEST_METHOD']

    # HTTP response code and message
    status = '200 OK'

    # HTTP headers expected by the client
    # They must be wrapped as a list of tupled pairs:
    # [(Header name, Header value)].
    response_headers = [
        ('Content-Type', 'text/plain'),
        ('Content-Length', str(len(response_body)))
    ]

    # Send them to the server using the supplied function
    start_response(status, response_headers)

    # Return the response body. Notice it is wrapped
    # in a list although it could be any iterable.
    return [response_body.encode()]

def public_folder(environ, start_response):

    # Construct the API request headers
    headers = {
        "Content-type": "application/json",
        "Authorization": "Bearer " + conf.oauth_token
    }

    # Get the filename from the request
    path = environ.get("PATH_INFO", "")

    # First see if the file has been shared before
    data = json.dumps( { "path": path } ).encode("utf-8")

    # Do the request
    req = urllib.request.Request("https://api.dropboxapi.com/2/sharing/list_shared_links",
                                data=data, headers=headers)

    try:
        # If we get here the request was successful

        # Get the reponse, decode it and tranform the json in a dictionary
        response = urllib.request.urlopen(req)
        data = json.loads(response.read().decode('utf8'))

        # get the url from the dictionary
        url = data["links"][0]["url"]

        # Return the HTTP response code and location
        status = "302 Found"
        start_response(status, [("Location", url)])

        return [b'']

    except urllib.error.URLError as e:
        # error code 409 means the file is not shared yet
        if e.code != 409:

            # We have hit an unexpected error
            print(e.reason)
            print(e.read().decode("utf8", 'ignore'))
            start_response("500 Internal server error")
            return [b'']

    # New file. Not shared yet, share it now
    data = json.dumps( { "path": path, "settings": { "requested_visibility": "public" } } ).encode("utf-8")
    req = urllib.request.Request("https://api.dropboxapi.com/2/sharing/create_shared_link_with_settings",
                                data=data, headers=headers)

    try:
        response = urllib.request.urlopen(req)
        data = json.loads(reponse.read().decode('utf8'))

        #print(response.read().decode('utf8'))
        print(data)
        print(data[0]["url"])

        # Return the HTTP response code and location
        status = "302 Found"
        start_response(status, [("Location", url)])

        return [b'']

    except urllib.error.URLError as e:

        if e.code == 409:

            # The requested file is not there
            status = "404 Not Found"
            response_body = 'File Not Found: %s' % path

            # HTTP headers expected by the client
            # They must be wrapped as a list of tupled pairs:
            # [(Header name, Header value)].
            response_headers = [
                ('Content-Type', 'text/plain'),
                ('Content-Length', str(len(response_body)))
            ]

            start_response(status, response_headers)
            return [response_body.encode()]

        print(e.reason)
        print(e.read().decode("utf8", 'ignore'))

        start_response("500 Internal server error")
        return [b'']

#
# https://docs.python.org/3.6/library/wsgiref.html
#

def main():

    if conf.oauth_token == "" or conf.public_folder == "":
        print("Missing configuration!")
        print("Make sure the values in 'conf.py' are set.")
        return

    # Instantiate the server
    httpd = make_server('localhost', 8080, public_folder)

    # Wait for a single request, serve it and quit
    httpd.handle_request()

    # Start the webserver
    #httpd.serve_forever()

if __name__ == "__main__":
    main()


from backend import constants, logger
from backend.api import radarr, sonarr
from backend.database.statement import select
from backend.scheduler.tasks import catalogue

import telegram
import datetime

def notifyImmediately(bot, job):
    # Pulls the watch_id, and decodes the bytes back to the data
    try:
        watch_id = job.context.decode('utf-8')
        data = watch_id.split(";")
    except:
        logger.error(__name__, "notifyImmediately() failed to extract and process watch_id")
        return True
    # If the media isn't in the database, refreshes the databases first
    if(int(data[1]) == constants.NOTIFIER_MEDIA_TYPE_MOVIE and select.getMovie(data[0]) == None and radarr.enabled):
        catalogue.updateMovies(None, None)
    elif(int(data[1]) == constants.NOTIFIER_MEDIA_TYPE_TELEVISION and select.getShow(data[0]) == None and sonarr.enabled):
        catalogue.updateTelevision(None, None)
    # Gets the metadata for the movie and the list of users who need notifications
    metadata = select.getMetadata(watch_id, data[1])
    users = select.getUsersImmediateUpdate(data[0], data[1])
    # If there are no users watching this show or movie, print a log and return
    if(len(users) == 0):
        logger.info(__name__, "New content ({}): Notified no users".format(metadata[2]), "INFO_GREEN")
        return True
    # Build the messages
    if(int(data[1]) == constants.NOTIFIER_MEDIA_TYPE_TELEVISION):
        msg_simple = constants.NOTIFIER_IMMEDIATELY_HEADER + buildSimpleTelevisionMessage(metadata)
        msg_complex = constants.NOTIFIER_IMMEDIATELY_HEADER + buildComplexTelevisionMessage(metadata)
        is_upgrade = metadata[9]
    elif(int(data[1]) == constants.NOTIFIER_MEDIA_TYPE_MOVIE):
        msg_simple = constants.NOTIFIER_IMMEDIATELY_HEADER + buildSimpleMovieMessage(metadata)
        msg_complex = constants.NOTIFIER_IMMEDIATELY_HEADER + buildComplexMovieMessage(metadata)
        is_upgrade = metadata[5]
    # Process user messages
    notifications_counter = 0
    for user in users:
        user_data = select.getUser(user[0])
        # Checks if the user is banned or restricted
        if((not constants.RESTRICTED_NOTIFICATIONS or user_data[1] != constants.ACCOUNT_STATUS_RESTRICTED) and user_data[1] != constants.ACCOUNT_STATUS_BANNED):
            # Checks if the media was an upgrade
            if(is_upgrade == 1):
                if(user_data[3] == constants.ACCOUNT_UPGRADE_YES):
                    # Gets the complexity and sends the appropriate message
                    if(user_data[2] == constants.ACCOUNT_DETAIL_SIMPLE):
                        bot.send_message(chat_id=user_data[0], text=msg_simple, parse_mode=telegram.ParseMode.MARKDOWN)
                    elif(user_data[2]== constants.ACCOUNT_DETAIL_COMPLEX):
                        bot.send_message(chat_id=user_data[0], text=msg_complex, parse_mode=telegram.ParseMode.MARKDOWN)
                    notifications_counter += 1
            else:
                # Gets the complexity and sends the appropriate message
                if(user_data[2] == constants.ACCOUNT_DETAIL_SIMPLE):
                    bot.send_message(chat_id=user_data[0], text=msg_simple, parse_mode=telegram.ParseMode.MARKDOWN)
                elif(user_data[2]== constants.ACCOUNT_DETAIL_COMPLEX):
                    bot.send_message(chat_id=user_data[0], text=msg_complex, parse_mode=telegram.ParseMode.MARKDOWN)
                notifications_counter += 1
    logger.info(__name__, "New content ({}): notified {} user(s)".format(metadata[2], notifications_counter), "INFO_GREEN")

def notifyDaily(bot, job):
    users = select.getUsers()
    for user in users:
        # Gets the notifiers
        notifiers = select.getNotifiersForUserDaily(user[0], constants.NOTIFIER_MEDIA_TYPE_TELEVISION)
        # Makes sure the user has at least one TV notifier
        if(len(notifiers) != 0):
            # Builds the message
            msg = buildReportMessage(notifiers, constants.NOTIFIER_DAILY_HEADER, constants.NOTIFIER_FREQUENCY_DAILY)
            # Makes sure there is new content to notify the user about and send it if there is
            if(msg != constants.NOTIFIER_DAILY_HEADER):
                bot.send_message(chat_id=user[0], text=msg, parse_mode=telegram.ParseMode.MARKDOWN)
    logger.info(__name__, "Daily content report sent to all applicable users", "INFO_GREEN")

def notifyWeekly(bot, job):
    users = select.getUsers()
    for user in users:
        # Gets the notifiers
        notifiers = select.getNotifiersForUserWeekly(user[0], constants.NOTIFIER_MEDIA_TYPE_TELEVISION)
        # Makes sure the user has at least one TV notifier
        if(len(notifiers) != 0):
            # Builds the message
            msg = buildReportMessage(notifiers, constants.NOTIFIER_WEEKLY_HEADER, constants.NOTIFIER_FREQUENCY_WEEKLY)
            # Makes sure there is new content to notify the user about and send it if there is
            if(msg != constants.NOTIFIER_WEEKLY_HEADER):
                bot.send_message(chat_id=user[0], text=msg, parse_mode=telegram.ParseMode.MARKDOWN)
    logger.info(__name__, "Weekly content report sent to all applicable users", "INFO_GREEN")

# Returns a formatted string for some television metadata (simple format)
def buildSimpleTelevisionMessage(metadata):
    return "*{}*\nSeason {} Episode {}\n\n".format(metadata[2], metadata[5], metadata[6])

#Returns a formatted string for some television metadata (complex format)
def buildComplexTelevisionMessage(metadata):
    return "*{}*\nSeason {} Episode {}\n\"_{}_\"\n\n{} | {} | {}\n\n".format(metadata[2], metadata[5], metadata[6], metadata[4], metadata[3], metadata[7], constants.NOTIFIER_QUALITY_VERSIONS[int(metadata[8])])

# Returns a formatted string for some movie metadata (simple format)
def buildSimpleMovieMessage(metadata):
    return "*{}*".format(metadata[2])

# Returns a formatted string for some movie metadata (complex format)
def buildComplexMovieMessage(metadata):
    return "*{}*\n\n{} | {}".format(metadata[2], metadata[3], constants.NOTIFIER_QUALITY_VERSIONS[int(metadata[4])])

# Returns a formatted string for all new television for that day for a user's notifiers
def buildReportMessage(notifiers, header, timeframe):
    msg = header
    for notifier in notifiers:
        if(timeframe == constants.NOTIFIER_FREQUENCY_DAILY):
            metadata = select.getMetadataPastDay(constants.NOTIFIER_MEDIA_TYPE_TELEVISION, notifier[2], constants.ACCOUNT_UPGRADE_NO)
        elif(timeframe == constants.NOTIFIER_FREQUENCY_WEEKLY):
            metadata = select.getMetadataPastWeek(constants.NOTIFIER_MEDIA_TYPE_TELEVISION, notifier[2], constants.ACCOUNT_UPGRADE_NO)
        if(len(metadata) != 0):
            msg += "\n*{}*: {} new episode(s)".format(str(metadata[0][2]), len(metadata))
    return msg

# Calculates the amount of seconds until the time to send the daily notification
# Will be off by -1 to -5 seconds, but it's close enough that it's negligible
def secondsToDaily():
    now = datetime.datetime.now()
    try:
        time = constants.NOTIFICATION_TIME.split(":")
        # If it's still the same day but past the notification time, adds a day
        if(now.hour >= int(time[0]) and now.minute >= int(time[1])):
            notification_time = datetime.datetime(now.year, now.month, now.day+1, int(time[0]), int(time[1]))
        else:
            notification_time = datetime.datetime(now.year, now.month, now.day, int(time[0]), int(time[1]))
    except:
        logger.error(__name__, "Invalid NOTIFICATION_TIME in backend/constants.py")
        exit()
    # Returns the total amount of seconds until the notification time
    return int(abs((now-notification_time).total_seconds()))

# Calculates the amount of seconds until the time to send the weekly notification
# Will be off by -1 to -5 seconds, but it's close enough that it's negligible
def secondsToWeekly():
    now = datetime.datetime.now()
    try:
        time = constants.NOTIFICATION_TIME.split(":")
        day = constants.WEEKDAYS.index(constants.NOTIFICATION_DAY)
        if(now.hour >= int(time[0]) and now.minute >= int(time[1])):
            # If it's still the same day but past the notification time, adds a day
            notification_time = datetime.datetime(now.year, now.month, now.day+1, int(time[0]), int(time[1]))
        else:
            notification_time = datetime.datetime(now.year, now.month, now.day, int(time[0]), int(time[1]))
        # Adds days to the notification time until it reaches the correct day
        while notification_time.weekday() != day:
            notification_time += datetime.timedelta(days=1)
    except:
        logger.error(__name__, "Invalid NOTIFICATION_TIME or NOTIFICATION_DAY in backend/constants.py")
        exit()
    # Returns the total amount of seconds until the notification time
    return int(abs((now-notification_time).total_seconds()))


from flask import Blueprint, render_template, redirect, url_for, request, \
        flash, jsonify
from flask.ext.login import login_required, login_user, logout_user, \
        current_user
from datetime import datetime, timedelta
from sklearn.metrics import silhouette_score
from sklearn.cluster import MiniBatchKMeans as KMeans
import numpy as np

from .. import db
from ..models.user import User
from ..models.device import Device
from ..models.measure import MeasurePoint, MeasureValue, SENSOR_NAMES

bp = Blueprint("analysis", __name__)

@bp.route('/')
def index():
    return render_template('analysis/index.html')

@bp.route('/status/')
def status():
    context = {}
    context['total_users'] = User.query.count()
    context['total_devices'] = Device.query.count()
    context['total_measures'] = MeasurePoint.query.count()

    return render_template('analysis/status.html', **context)

@bp.route('/maps/')
def maps():
    return render_template('analysis/maps.html')

def _point_to_json_dict(point):
    item = dict(timestamp=point.timestamp.timestamp(),
                deviceId=point.device_id,
                latitude=point.latitude, longitude=point.longitude,
                accuracy=point.accuracy, values=[])
    for value in point.values.all():
        item['values'].append(dict(type=value.type_name, value=value.value))
    return item

@bp.route('/maps/area/')
def area():
    args = request.args.get
    try:
        date_from = datetime.strptime(args('from'), '%Y-%m-%d')
    except (ValueError, TypeError):
        date_from = None
    try:
        date_to = datetime.strptime(args('to'), '%Y-%m-%d')
        date_to += timedelta(days=1)
    except (ValueError, TypeError):
        date_to = None

    points = MeasurePoint.query
    points = points.filter(MeasurePoint.longitude > args('west')
                  ).filter(MeasurePoint.longitude < args('east')
                  ).filter(MeasurePoint.latitude < args('north')
                  ).filter(MeasurePoint.latitude > args('south'))
    if date_from is not None:
        points = points.filter(MeasurePoint.timestamp >= date_from)
    if date_to is not None:
        points = points.filter(MeasurePoint.timestamp < date_to)

    measures = [_point_to_json_dict(p) for p in points]
    return jsonify(count=len(measures), points=measures)

CLUSTER_SENSOR_TYPES = ['Temperature', 'Humidity',
                        'OxidizingGas', 'ReducingGas']

@bp.route('/maps/cluster/', methods=['POST'])
def do_clustering():
    keys = request.get_json()

    points = []
    values = []
    types = set(CLUSTER_SENSOR_TYPES)
    for id, timestamp in keys:
        time = datetime.fromtimestamp(timestamp)
        point = MeasurePoint.query.get((id, time))
        value = {v.type_name: v.value for v in point.values.all()}
        if not set(value.keys()) >= types:
            continue
        # Normalization:
        value['OxidizingGas'] /= 1000
        value['ReducingGas'] /= 10000
        points.append(point)
        values.append(value)

    X = [[v[t] for t in CLUSTER_SENSOR_TYPES]
         for v in values]
    X = np.array(X)
    k = KMeans(n_clusters=2, init='k-means++')
    k.fit(X)
    score = silhouette_score(X, k.labels_)

    groups = [[_point_to_json_dict(p) for p, l in
              zip(points, k.labels_) if l == i] for i in
              range(max(k.labels_) + 1)]

    return jsonify(score=score, groups=groups)


import pytest

import os
import time
import glob
import pathlib

import numpy as np

from deltametrics import utils
from deltametrics import mobility as mob
from deltametrics import sample_data

class TestNoStratigraphyError:

    def test_needs_obj_argument(self):
        with pytest.raises(TypeError):
            raise utils.NoStratigraphyError()

    def test_only_obj_argument(self):
        _mtch = "'str' object has no*."
        with pytest.raises(utils.NoStratigraphyError, match=_mtch):
            raise utils.NoStratigraphyError('someobj')

    def test_obj_and_var(self):
        _mtch = "'str' object has no attribute 'somevar'."
        with pytest.raises(utils.NoStratigraphyError, match=_mtch):
            raise utils.NoStratigraphyError('someobj', 'somevar')

class TestLineToCells:

    def test_flat_inputs(self):
        x0, y0, x1, y1 = 10, 40, 50, 40
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 41

    def test_vert_inputs(self):
        x0, y0, x1, y1 = 40, 10, 40, 70
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 61

    def test_quadrantI_angle_inputs(self):
        x0, y0, x1, y1 = 10, 10, 60, 92
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 83
        # check line is sorted correctly: p0 --> p1
        assert np.all(ret1[:, 0] == np.array([x0, y0]))
        assert np.all(ret1[:, -1] == np.array([x1, y1]))

    def test_quadrantII_angle_inputs(self):
        x0, y0, x1, y1 = 80, 20, 40, 50
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 41
        # check line is sorted correctly: p0 --> p1
        assert np.all(ret1[:, 0] == np.array([x0, y0]))
        assert np.all(ret1[:, -1] == np.array([x1, y1]))

    def test_quadrantIII_angle_inputs(self):
        x0, y0, x1, y1 = 80, 70, 40, 40
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 41
        # check line is sorted correctly: p0 --> p1
        assert np.all(ret1[:, 0] == np.array([x0, y0]))
        assert np.all(ret1[:, -1] == np.array([x1, y1]))

    def test_quadrantIV_angle_inputs(self):
        x0, y0, x1, y1 = 10, 80, 60, 30
        ret1 = utils.line_to_cells(np.array([[x0, y0], [x1, y1]]))
        ret2 = utils.line_to_cells((x0, y0), (x1, y1))
        ret3 = utils.line_to_cells(x0, y0, x1, y1)
        ret1, ret2, ret3 = np.vstack(ret1), np.vstack(ret2), np.vstack(ret3)
        assert (np.all(ret1 == ret2) and np.all(ret1 == ret3))
        assert ret1.shape[1] == 51
        assert np.all(ret1[:, 0] == np.array([x0, y0]))
        assert np.all(ret1[:, -1] == np.array([x1, y1]))

    def test_bad_inputs(self):
        x0, y0, x1, y1 = 10, 10, 60, 92
        with pytest.raises(TypeError, match=r'Length of input .*'):
            _ = utils.line_to_cells(x0, y0, x1, y1, x0, y1)
        with pytest.raises(ValueError):
            _ = utils.line_to_cells(
                np.array([[x0, y0], [x1, y1], [x0, y1]]))

chmap = np.zeros((5, 4, 4))
# define time = 0
chmap[0, :, 1] = 1
# every time step one cell of the channel will migrate one pixel to the right
for i in range(1, 5):
    chmap[i, :, :] = chmap[i - 1, :, :].copy()
    chmap[i, -1 * i, 1] = 0
    chmap[i, -1 * i, 2] = 1
# define the fluvial surface - entire 4x4 area
fsurf = np.ones((4, 4))
# define the index corresponding to the basemap at time 0
basevalue = [0]
# define the size of the time window to use
time_window = 5

def test_linear_fit():
    """Test linear curve fitting."""
    ch_abandon = mob.calculate_channel_abandonment(
        chmap, basevalues_idx=basevalue, window_idx=time_window)
    yfit, popts, cov, err = utils.curve_fit(ch_abandon, fit='linear')
    assert pytest.approx(yfit) == np.array([4.76315477e-24, 2.50000000e-01,
                                            5.00000000e-01, 7.50000000e-01,
                                            1.00000000e+00])
    assert pytest.approx(popts) == np.array([2.50000000e-01, 4.76315477e-24])
    assert pytest.approx(cov) == np.array([[1.76300984e-25, -0.00000000e+00],
                                           [0.00000000e+00,  5.28902953e-24]])
    assert pytest.approx(err) == np.array([4.19882108e-13, 2.29978902e-12])

def test_harmonic_fit():
    """Test harmonic curve fitting."""
    ch_abandon = mob.calculate_channel_abandonment(
        chmap, basevalues_idx=basevalue, window_idx=time_window)
    yfit, popts, cov, err = utils.curve_fit(ch_abandon, fit='harmonic')
    assert pytest.approx(yfit) == np.array([-0.25986438, 0.41294455,
                                            0.11505591, 0.06683947,
                                            0.04710091])
    assert pytest.approx(popts) == np.array([-0.25986438, -1.62929608])
    assert pytest.approx(cov) == np.array([[0.50676407, 1.26155952],
                                           [1.26155952, 4.3523343]])
    assert pytest.approx(err) == np.array([0.71187364, 2.08622489])

def test_invalid_fit():
    """Test invalid fit parameter."""
    ch_abandon = mob.calculate_channel_abandonment(
        chmap, basevalues_idx=basevalue, window_idx=time_window)
    with pytest.raises(ValueError):
        utils.curve_fit(ch_abandon, fit='invalid')

def test_exponential_fit():
    """Test exponential fitting."""
    ydata = np.array([10, 5, 2, 1])
    yfit, popts, cov, err = utils.curve_fit(ydata, fit='exponential')
    assert pytest.approx(yfit) == np.array([10.02900253, 4.85696353,
                                            2.22612537, 0.88790858])
    assert pytest.approx(popts) == np.array([10.02900253, -0.49751195,
                                             0.67596451])
    assert pytest.approx(cov) == np.array([[0.0841566, 0.04554967, 0.01139969],
                                           [0.04554967, 0.59895713, 0.08422946],
                                           [0.01139969, 0.08422946,
                                            0.01327807]])
    assert pytest.approx(err) == np.array([0.29009757, 0.77392321, 0.11523053])

def test_format_number_float():
    _val = float(5.2)
    _fnum = utils.format_number(_val)
    assert _fnum == '10'

    _val = float(50.2)
    _fnum = utils.format_number(_val)
    assert _fnum == '50'

    _val = float(15.0)
    _fnum = utils.format_number(_val)
    assert _fnum == '20'

def test_format_number_int():
    _val = int(5)
    _fnum = utils.format_number(_val)
    assert _fnum == '0'

    _val = int(6)
    _fnum = utils.format_number(_val)
    assert _fnum == '10'

    _val = int(52)
    _fnum = utils.format_number(_val)
    assert _fnum == '50'

    _val = int(15)
    _fnum = utils.format_number(_val)
    assert _fnum == '20'

def test_format_table_float():
    _val = float(5.2)
    _fnum = utils.format_table(_val)
    assert _fnum == '5.2'

    _val = float(50.2)
    _fnum = utils.format_table(_val)
    assert _fnum == '50.2'

    _val = float(15.0)
    _fnum = utils.format_table(_val)
    assert _fnum == '15.0'

    _val = float(15.03689)
    _fnum = utils.format_table(_val)
    assert _fnum == '15.0'

    _val = float(15.0689)
    _fnum = utils.format_table(_val)
    assert _fnum == '15.1'

def test_format_table_int():
    _val = int(5)
    _fnum = utils.format_table(_val)
    assert _fnum == '5'

    _val = int(5.8)
    _fnum = utils.format_table(_val)
    assert _fnum == '5'

    _val = int(5.2)
    _fnum = utils.format_table(_val)
    assert _fnum == '5'

@pytest.mark.xfail(raises=ImportError,
                   reason='pyDeltaRCM is not a required dependency')
def test_time_from_log_new(tmp_path):
    """Generate run+logfile and then read runtime from it."""
    from pyDeltaRCM.model import DeltaModel
    delta = DeltaModel(out_dir=str(tmp_path))  # init delta to make log file
    time.sleep(1)
    delta.finalize()  # finalize and end log file
    log_path = os.path.join(tmp_path, os.listdir(tmp_path)[0])  # path to log
    elapsed_time = utils.runtime_from_log(log_path)
    # elapsed time should exceed 0, but exact time will vary
    assert isinstance(elapsed_time, float)
    assert elapsed_time > 0

def test_time_from_log_sampledataset(tmp_path):
    golfpath = pathlib.PurePath(sample_data._get_golf_path())
    # find the log file there
    found = glob.glob(os.path.join(golfpath.parent, '*.log'))
    assert len(found) == 1
    elapsed_time = utils.runtime_from_log(found[0])
    # elapsed time should exceed 0, but exact time will vary
    assert isinstance(elapsed_time, float)
    assert elapsed_time > 0