#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import logging
import optparse
import os
import sys
import time

ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
ACTIVATE_SCRIPT = os.path.join(ROOT_PATH, 'jenkins-env', 'bin', 'activate_this.py')
TREEHERDER_CONFIG_FILE = os.path.join(ROOT_PATH, '.jenkins.properties')

def main():
    parser = optparse.OptionParser()
    parser.add_option('--debug',
                      dest='debug',
                      action='store_true',
                      default=False)
    parser.add_option('--log-folder',
                      dest='log_folder',
                      default='log',
                      help='Folder to write notification log files into')
    parser.add_option('--log-level',
                      dest='log_level',
                      default='INFO',
                      help='Logging level, default: %default')
    parser.add_option('--authfile',
                      dest='authfile',
                      default='.authentication.ini',
                      help='Path to the authentiation file for Pulse Guardian and Jenkins')
    parser.add_option('--push-message',
                      dest='message',
                      help='Log file of a Pulse message to process for Jenkins')
    parser.add_option('--display-only',
                      dest='display_only',
                      action='store_true',
                      default=False,
                      help='Only display build properties and don\'t trigger jobs.')
    options, args = parser.parse_args()

    if not len(args):
        parser.error('A configuration file has to be passed in as first argument.')

    logging.Formatter.converter = time.gmtime
    logging.basicConfig(level=options.log_level,
                        format='%(asctime)s %(levelname)5s %(name)s: %(message)s',
                        datefmt='%Y-%m-%dT%H:%M:%SZ')
    logger = logging.getLogger('mozmill-ci')

    # Configure logging levels for sub modules. Set to ERROR by default.
    sub_log_level = logging.ERROR
    if options.log_level == logging.getLevelName(logging.DEBUG):
        sub_log_level = logging.DEBUG
    logging.getLogger('mozdownload').setLevel(sub_log_level)
    logging.getLogger('redo').setLevel(sub_log_level)
    logging.getLogger('requests').setLevel(sub_log_level)
    logging.getLogger('thclient').setLevel(sub_log_level)

    # Auto import the virtual environment so the script can directly
    # be called without having to source into first.
    try:
        execfile(ACTIVATE_SCRIPT, dict(__file__=ACTIVATE_SCRIPT))
        logger.info('Virtual environment activated successfully.')
    except IOError:
        logger.exception('Could not activate virtual environment at "{}"'.format(ACTIVATE_SCRIPT))
        sys.exit(1)

    from lib.automation import FirefoxAutomation

    FirefoxAutomation(configfile=args[0],
                      authfile=options.authfile,
                      treeherder_configfile=TREEHERDER_CONFIG_FILE,
                      debug=options.debug,
                      log_folder=options.log_folder,
                      logger=logger,
                      message=options.message,
                      display_only=options.display_only)

if __name__ == "__main__":
    main()


import numpy as np
import pandas as pd
from os import path
import argparse
from interval import interval
from itertools import chain
from mir_parser import MirParser

# Pass tuples of peak ranges (.mir) and full dataframes(.h5)
def find_consensus_peaks(t, *args):
	# Notice current problems
	print("Chaining problems are not resolved yet!")
	print("Multiple overlaps are resolved by revoking any kind of merge, this is a temporary solution!")

	# Allow decimal and percent values
	if t > 1:
		t = t / 100

	# Parse Peak Ranges:
	parsed_mir = []
	for mir_path, _ in args:
		parsed_mir.append(MirParser(mir_path).parse_peaks())

	# Parse Dataframes:
	parsed_frames = []
	for _, frame_path in args:
		parsed_frames.append(pd.read_hdf(frame_path))

	# Dict for future peak lists
	peak_dict = dict((key,[]) for key in range(len(args)))

	# Dict to determine dataset membership
	peak_ranges_dict = dict((idx, peak_ranges) for idx, peak_ranges in enumerate(parsed_mir))

	joint_peak_ranges = list(chain(*parsed_mir))
	joint_peak_ranges.sort(key= lambda x: x[0])

	# Use reduces joint peak ranges list to reduce search time in find_max_overlap()
	reduced_joint_peak_ranges = joint_peak_ranges[:] # ?: not necessary because of index usage?

	# Initialize candidates as dict to ensure correkt if statement at the beginning
	# Candidates will change between dict and list a few times
	candidates = dict()

	# Calculate the overlap between ranges and join them if they exceed a defined threshold
	for idx, peak_ref in enumerate(joint_peak_ranges):
		if peak_ref in chain(*candidates.values()): # skip peaks that were already merged durin the maximum overlap detection
			continue # ?: should work because of ordered
		else:
			candidates = []
			ref = interval[peak_ref.min_mass, peak_ref.max_mass]
			for peak_compare in joint_peak_ranges[idx:]: # include self peak
				compare = interval[peak_compare.min_mass, peak_compare.max_mass]
				pct_ref, pct_compare = calc_percent_overlap(ref, compare)
				if pct_ref > t or pct_compare > t:
					candidates.append(peak_compare)
				else:
					# check if current candidate list can be extendet by spanning overlaps
					candidates = find_max_overlap(candidates, reduced_joint_peak_ranges[idx:], t) # ?: should work because of ordered # candidates is list
					candidates = check_for_split(candidates, t) # candidates is dict
					break
			else:
				# if the last iteration of the for loop is an overlap, candidates will remain as list but needs to be a dict
				# ?: Changing candidates list into simple dict may be enough, check if find_max_overlap() and check_for_split() is needed!
				if type(candidates) != dict:
					candidates = find_max_overlap(candidates, reduced_joint_peak_ranges[idx:], t) # ?: is this needed? # candidates is list
					#candidates = check_for_split(candidates, t) # candidates is dict
					candidates = check_for_revoke(candidates, t) # revokes any merge in case of chaining or multiple overlap, e.g. D1: x & y overlap with D2: z
		# ?: check if double entries will occur -> should not because of starting if statement
		# Define new peak based on midpoint of all candidates
		# ?: weight by intensity could improve this
		for key, value in candidates.items():
			peak = define_peak(value)
			# add new peak to each data set that added at least one candidate
			for peak_ranges in value:
				for k, v in peak_ranges_dict.items():
					if peak_ranges in v:
						peak_dict[k].extend([peak])

	for key, value in peak_dict.items(): # ?: should work because of ordered
		parsed_frames[key].columns = value

	for idx, frame in enumerate(parsed_frames):
		key = ""
		frame_path = args[idx][1]
		frame_path = frame_path.replace(path.sep, "/")
		with pd.HDFStore(frame_path) as f:
			key = f.keys()[0][1:]
		with pd.HDFStore(path.splitext(frame_path)[0] + "_aligned_" + str(t*100) + "pct.h5", complib='blosc', complevel=9) as store:
			store[key] = frame

# Calculate how many percent the overlap spans in each interval
def calc_percent_overlap(interval_a, interval_b):
	if interval_a & interval_b:
		span_a = interval_a[0].sup - interval_a[0].inf
		span_b = interval_b[0].sup - interval_b[0].inf
		overlap = interval_a & interval_b
		span_o = overlap[0].sup - overlap[0].inf
		pct_a = (span_o/span_a)
		pct_b = (span_o/span_b)
		return pct_a, pct_b
	else:
		return 0, 0

# Define peak based on a overlapping peak ranges
def define_peak(peak_ranges):
	min_peak = min([peak.min_mass for peak in peak_ranges])
	max_peak = max([peak.max_mass for peak in peak_ranges])
	return round(interval[min_peak, max_peak].midpoint[0][0], 3)

# check for each candidate in candidates if a candidate can extend the overlap by spanning across multiple disconnected peak ranges
def find_max_overlap(candidates, joint_peak_ranges, t):
	to_iter = candidates
	for candidate in to_iter:
		ref = interval[candidate.min_mass, candidate.max_mass]
		for peak_compare in joint_peak_ranges: # reduce compare by reducing to iter
			compare = interval[peak_compare.min_mass, peak_compare.max_mass]
			pct_ref, pct_compare = calc_percent_overlap(ref, compare)
			if pct_ref > t or pct_compare > t:
				if peak_compare not in candidates:
					candidates.append(peak_compare)
	if candidates != to_iter:
		candidates = find_max_overlap(candidates, joint_peak_ranges, t)
	return candidates

def check_for_split(candidates, t):
	split_dict = dict()
	key_counter = 0
	for idx, candidate in enumerate(candidates):
		ref = interval[candidate.min_mass, candidate.max_mass]
		for candidate_compare in candidates[idx:]: # ?: should work because of ordered
			compare = interval[candidate_compare.min_mass, candidate_compare.max_mass]
			pct_ref, pct_compare = calc_percent_overlap(ref, compare)
			if pct_ref < t and pct_compare < t:
				split_dict[key_counter] = [candidate]
				key_counter += 1
				split_dict[key_counter] = [candidate_compare]
				key_counter += 1
	if len(split_dict): # If no split happend, put the whole candidate array in one split dict entry
		for key, value in split_dict.items():
			ref = interval[value[0].min_mass, value[0].max_mass]
			for candidate_compare in candidates: # ?: should work because of ordered
				compare = interval[candidate_compare.min_mass, candidate_compare.max_mass]
				if compare != ref:
					pct_ref, pct_compare = calc_percent_overlap(ref, compare)
					if pct_ref > t or pct_compare > t:
						split_dict[key].extend([candidate_compare])
	else:
		split_dict[key_counter] = candidates
	return split_dict

def check_for_revoke(candidates, t):
	split_dict = dict()
	key_counter = 0
	for idx, candidate in enumerate(candidates):
		ref = interval[candidate.min_mass, candidate.max_mass]
		for candidate_compare in candidates[idx:]: # ?: should work because of ordered
			compare = interval[candidate_compare.min_mass, candidate_compare.max_mass]
			pct_ref, pct_compare = calc_percent_overlap(ref, compare)
			if pct_ref < t and pct_compare < t:
				split_dict[key_counter] = [candidate]
				key_counter += 1
				split_dict[key_counter] = [candidate_compare]
				key_counter += 1
	if len(split_dict): # If no split happend, put the whole candidate array in one split dict entry
		for key, value in split_dict.items(): # If split happend, do not merge anything and use mean for each individual value
			split_dict[key] = [round(np.mean([value[0].min_mass, value[0].max_mass]),3)]
	else:
		split_dict[key_counter] = candidates
	return split_dict

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description="Align selected peak ranges of multiple data sets.")
	parser.add_argument("-t", type=float, action="store", dest="t", help="Percentual overlap between two selected ranges.")
	parser.add_argument("-p", nargs='+', action="store", dest="paths", help="Tuples of paths of form: file1.mir file1.h5 file2.mir file2.h5, ...")
	args = parser.parse_args()
	try:
		path_tuples = list(zip(args.paths[::2], args.paths[1::2]))
		find_consensus_peaks(args.t, *path_tuples)
	except:
		parser.print_help()


# ref from official zeroQ
# https://github.com/amirgholami/ZeroQ/blob/ba37f793dbcb9f966b58f6b8d1e9de3c34a11b8c/classification/classification/distill_data.py
# @file Different utility functions
# Copyright (c) Yaohui Cai, Zhewei Yao, Zhen Dong, Amir Gholami
# All rights reserved.
# This file is part of ZeroQ repository.
#
# ZeroQ is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ZeroQ is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ZeroQ repository.  If not, see <http://www.gnu.org/licenses/>.
# *

import os
import torch
import torch.nn as nn
import copy
import tqdm
import torch.optim as optim
from train_script.utils import *
from utils.val import validation
from train_script.get_zeroQ_data import *
from utils.quantize_model import quantize_model, freeze_act, freeze_bn, un_freeze_act

def getDistilData(teacher_model,
                  dataset,
                  batch_size,
                  num_batch=1,
                  for_inception=False):
    """
	Generate distilled data according to the BatchNorm statistics in the pretrained single-precision model.
	Currently only support a single GPU.
	teacher_model: pretrained single-precision model
	dataset: the name of the dataset
	batch_size: the batch size of generated distilled data
	num_batch: the number of batch of generated distilled data
	for_inception: whether the data is for Inception because inception has input size 299 rather than 224
	"""

    # initialize distilled data with random noise according to the dataset
    dataloader = getRandomData(dataset=dataset,
                               batch_size=batch_size,
                               for_inception=for_inception)

    print('****** Data loaded ******')

    eps = 1e-6
    # initialize hooks and single-precision model
    hooks, hook_handles, bn_stats, refined_gaussian = [], [], [], []
    teacher_model = teacher_model.cuda()
    teacher_model = teacher_model.eval()

    # get number of BatchNorm layers in the model
    layers = sum([
        1 if isinstance(layer, nn.BatchNorm2d) else 0
        for layer in teacher_model.modules()
    ])

    for n, m in teacher_model.named_modules():
        if isinstance(m, nn.Conv2d) and len(hook_handles) < layers:
            # register hooks on the convolutional layers to get the intermediate output after convolution and before BatchNorm.
            hook = output_hook()
            hooks.append(hook)
            hook_handles.append(m.register_forward_hook(hook.hook))
        if isinstance(m, nn.BatchNorm2d):
            # get the statistics in the BatchNorm layers
            bn_stats.append(
                (m.running_mean.detach().clone().flatten().cuda(),
                 torch.sqrt(m.running_var +
                            eps).detach().clone().flatten().cuda()))
    assert len(hooks) == len(bn_stats)

    print("Training ZeroQ Dataset with {} batch...".format(num_batch))
    for i, gaussian_data in enumerate(dataloader):

        if i == num_batch:
            break

        # initialize the criterion, optimizer, and scheduler
        gaussian_data = gaussian_data.cuda()
        gaussian_data.requires_grad = True
        crit = nn.CrossEntropyLoss().cuda()
        optimizer = optim.Adam([gaussian_data], lr=0.5)
        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
                                                         min_lr=1e-4,
                                                         verbose=False,
                                                         patience=100)

        input_mean = torch.zeros(1, 3).cuda()
        input_std = torch.ones(1, 3).cuda()

        for it in tqdm.tqdm(range(500)):
            teacher_model.zero_grad()
            optimizer.zero_grad()
            for hook in hooks:
                hook.clear()
            output = teacher_model(gaussian_data)
            mean_loss = 0
            std_loss = 0

            # compute the loss according to the BatchNorm statistics and the statistics of intermediate output
            for cnt, (bn_stat, hook) in enumerate(zip(bn_stats, hooks)):
                tmp_output = hook.outputs
                bn_mean, bn_std = bn_stat[0], bn_stat[1]
                tmp_mean = torch.mean(tmp_output.view(tmp_output.size(0),
                                                      tmp_output.size(1), -1),
                                      dim=2)
                tmp_std = torch.sqrt(
                    torch.var(tmp_output.view(tmp_output.size(0),
                                              tmp_output.size(1), -1),
                              dim=2) + eps)
                mean_loss += own_loss(bn_mean, tmp_mean)
                std_loss += own_loss(bn_std, tmp_std)
            tmp_mean = torch.mean(gaussian_data.view(gaussian_data.size(0), 3,
                                                     -1),
                                  dim=2)
            tmp_std = torch.sqrt(
                torch.var(gaussian_data.view(gaussian_data.size(0), 3, -1),
                          dim=2) + eps)
            mean_loss += own_loss(input_mean, tmp_mean)
            std_loss += own_loss(input_std, tmp_std)
            total_loss = mean_loss + std_loss

            # update the distilled data
            total_loss.backward()
            optimizer.step()
            scheduler.step(total_loss.item())

        refined_gaussian.append(gaussian_data.detach().clone())

    for handle in hook_handles:
        handle.remove()

    print("Training ZeroQ Dataset Done ...........")
    return refined_gaussian

# training

def train_zeroQ(fp_model, q_model, val_dataloder, batch_size=32, for_incep=False):

    dataloader = getDistilData(
        fp_model.cuda(),
        "imagenet",
        batch_size=batch_size,
        for_inception=for_incep)

    # Freeze BatchNorm statistics
    q_model.eval()
    q_model = q_model.cuda()
    q_model = un_freeze_act(q_model)

    # update activation
    with torch.no_grad():
        for _, inputs in enumerate(dataloader):
            if isinstance(inputs, list):
                inputs = inputs[0]
                inputs = inputs.cuda()
                _ = quantize_model(inputs)

    print("Zero Shot Quantization Finished .....")

    q_model = freeze_act(q_model)
    print("Eval after zeroQ update")
    q_top_1, q_top_5 = validation(val_dataloder, q_model)

    return q_model


import sys
import os
import subprocess
import time
import serial

skList = ['7', '2', '3', '14', '5', '0', '16', '11', '8', '12', '1', '6']
#skList = ['7']
dac10List = ['400', '400', '677', '384', '400', '695', '760', '730', '753', '770', '716', '742']
#dac10List = ['800']
#skList = ['6']
#dac10List = ['742']

for Sk in skList :
	inputF = open('Core/EASI_Probe.txt', 'r')
	outputF = open('Conf/EASI_Probe_' + Sk + '.txt', 'w')
	s = inputF.readline()
	y = int(Sk)<<10
	w = "{:04x}".format(256+y)
	outputF.write(w + '\n')
	while s:
		s = inputF.readline()
		outputF.write(s)
	inputF.close()
	outputF.close()

	inputF = open('Core/EASI_Hold_Pot_46ns.txt', 'r')
	outputF = open('Conf/EASI_Hold_Pot_46ns_' + Sk + '.txt', 'w')
	s = inputF.readline()
	y = int(Sk)<<10
	w = "{:04x}".format(736+y)
	outputF.write(w + '\n')
	while s:
		s = inputF.readline()
		outputF.write(s)
	inputF.close()
	outputF.close()

	inputF = open('Core/EASI_TimeOut_Pot_300ns.txt', 'r')
	outputF = open('Conf/EASI_TimeOut_Pot_300ns_' + Sk + '.txt', 'w')
	s = inputF.readline()
	y = int(Sk)<<10
	w = "{:04x}".format(704+y)
	outputF.write(w + '\n')
	while s:
		s = inputF.readline()
		outputF.write(s)
	inputF.close()
	outputF.close()

subprocess.call("./Reset")

skCounter = 0

for Sk in skList :
	arg = ["./ResetSlave", Sk]
	subprocess.call(arg)

subprocess.call("./Init")

for dac10 in dac10List:
	inputF = open('EASIprog.c', 'r')
	outputF = open('EASIprog.out', 'w')
	s = inputF.readline()
	while s:
		outputF.write(s)
		#if ('//DAC8' in s):
		#	outputF.write('for (i=0; i<32; i++) error = DACbiasSC_EASI(SC_EASI, i, ' + str(dac8) + ');')
		#	outputF.write('\n')
		#	s = inputF.readline()
		if ('//DAC10' in s) :
			outputF.write('\tDAC10thrsSC_EASI(SC_EASI,' + str(dac10) +');\n')
			s = inputF.readline()
		s = inputF.readline()
	inputF.close()
	outputF.close()
	arg = ["mv", "EASIprog.out", "EASIprog.c"]
	subprocess.call(arg)
	arg = ["gcc", "-O2", "EASIprog.c", "libreriaSC_EASI.c", "-o", "EASIprog"]
	subprocess.call(arg)
	subprocess.call("./EASIprog")

	inputF = open('Core/EASI_Slow_Control.txt', 'r')
	outputF = open('Conf/EASI_Slow_Control_' + skList[skCounter] + '.txt', 'w')
	s = inputF.readline()
	y = int(Sk)<<10
	w = "{:04x}".format(224 + y)
	outputF.write(w + '\n')
	while s:
		s = inputF.readline()
		outputF.write(s)
	inputF.close()
	outputF.close()
	print(dac10 + '\t' + skList[skCounter] + '\n')
	skCounter = skCounter + 1

for Sk in skList :
	arg = ["./SendFSlaves", "Conf/EASI_Probe_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendFSlaves", "Conf/EASI_Hold_Pot_46ns_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendFSlaves", "Conf/EASI_TimeOut_Pot_300ns_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendFSlaves", "Conf/EASI_Slow_Control_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendHV_NORUMP", "Conf_HV/EASI_HV-OUT_ShutDown_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendFSlaves", "Conf_HV/EASI_SwHV_ON_" + Sk + ".txt"]
	subprocess.call(arg)
	arg = ["./SendHV", "Conf_HV/EASI_HV-OUT_Vbias_" + Sk + ".txt"]
	subprocess.call(arg)
	time.sleep(1)
	print('---' + Sk + '\n')

arg = ["./SendFMaster", "MasterCMD/EN_MUX0.txt"]
subprocess.call(arg)
time.sleep(1)
arg = ["./SendFMaster", "MasterCMD/EN_MUX1.txt"]
subprocess.call(arg)
time.sleep(1)
arg = ["./SendFMaster", "MasterCMD/EN_MUX2.txt"]
subprocess.call(arg)
time.sleep(1)
arg = ["./SendFMaster", "MasterCMD/EN_MUX3.txt"]
subprocess.call(arg)
time.sleep(1)
arg = ["./SendFMaster", "MasterCMD/EN_MUX4.txt"]
subprocess.call(arg)

print ("---> System Ready!\n")
count = 0
evts = 100

while(count<1):
	count = count + 1
	print("Run " + str(count) + " :: Now reading")
	outputF = open('slaveDatas1', 'w')
	outputF.write(str(int(round(time.time()*1000))) + '\n')
	outputF.close()
	arg = ['./ReadSlave', str(evts) , '2', '3', '7', '14', '5', '0', '16', '11', '8', '12', '1', '6']
	#arg = ['./ReadSlave', str(evts) , skList[0]]
	subprocess.call(arg)
	outputF = open('slaveDatas1', 'a')
	outputF.write(str(int(round(time.time()*1000))))
	outputF.close()
	arg = ['mv', '/home/muBot/slaveDatas1', '/home/muray/TestRun_New/slaveData_evts' + str(evts) + '_run' + str(count)]
	#arg = ['mv', '/home/muBot/slaveDatas1', '/home/muray/TestRun_New/slaveData_evts' + str(evts) + '_sk' + skList[0]]
	subprocess.call(arg)

print ('\n--- Shutting Down System\n')
for Sk in skList :
	#argv = ["./SendHV_rDown", "Conf_HV/EASI_HV-OUT_Vbias_" + Sk + ".txt"]
	#subprocess.call(argv)
	argv = ["./SendHV_NORUMP", "Conf_HV/EASI_HV-OUT_ShutDown_" + Sk + ".txt"]
	subprocess.call(argv)
	argv = ["./SendFSlaves", "Conf_HV/EASI_SwHV_OFF_" + Sk + ".txt"]
	subprocess.call(argv)


# Copyright 2020 Lorna Authors. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import re
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint as cp
from collections import OrderedDict

from .utils import densenet_params
from .utils import get_model_params
from .utils import bn_function_factory
from .utils import load_pretrained_weights

class DenseLayer(nn.Sequential):
    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False):
        super(DenseLayer, self).__init__()
        self.add_module("norm1", nn.BatchNorm2d(num_input_features)),
        self.add_module("relu1", nn.ReLU(inplace=True)),
        self.add_module("conv1", nn.Conv2d(num_input_features, bn_size *
                                           growth_rate, kernel_size=1, stride=1,
                                           bias=False)),
        self.add_module("norm2", nn.BatchNorm2d(bn_size * growth_rate)),
        self.add_module("relu2", nn.ReLU(inplace=True)),
        self.add_module("conv2", nn.Conv2d(bn_size * growth_rate, growth_rate,
                                           kernel_size=3, stride=1, padding=1,
                                           bias=False)),
        self.drop_rate = drop_rate
        self.memory_efficient = memory_efficient

    def forward(self, *prev_features):
        bn_function = bn_function_factory(self.norm1, self.relu1, self.conv1)
        if self.memory_efficient and any(prev_feature.requires_grad for prev_feature in prev_features):
            bottleneck_output = cp.checkpoint(bn_function, *prev_features)
        else:
            bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        if self.drop_rate > 0:
            new_features = F.dropout(new_features, p=self.drop_rate,
                                     training=self.training)
        return new_features

class DenseBlock(nn.Module):
    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False):
        super(DenseBlock, self).__init__()
        for i in range(num_layers):
            layer = DenseLayer(
                num_input_features + i * growth_rate,
                growth_rate=growth_rate,
                bn_size=bn_size,
                drop_rate=drop_rate,
                memory_efficient=memory_efficient,
            )
            self.add_module(f"denselayer{i + 1}", layer)

    def forward(self, init_features):
        features = [init_features]
        for name, layer in self.named_children():
            new_features = layer(*features)
            features.append(new_features)
        return torch.cat(features, 1)

class Transition(nn.Sequential):
    def __init__(self, num_input_features, num_output_features):
        super(Transition, self).__init__()
        self.add_module("norm", nn.BatchNorm2d(num_input_features))
        self.add_module("relu", nn.ReLU(inplace=True))
        self.add_module("conv", nn.Conv2d(num_input_features, num_output_features,
                                          kernel_size=1, stride=1, bias=False))
        self.add_module("pool", nn.AvgPool2d(kernel_size=2, stride=2))

class DenseNet(nn.Module):
    """Densenet-BC model class, based on
    `"Densely Connected Convolutional Networks" <https://arxiv.org/pdf/1608.06993.pdf>`_
    """

    def __init__(self, blocks_args=None, global_params=None):
        # growth_rate=32, block_config=(6, 12, 24, 16),
     #            num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False):

        super().__init__()
        assert isinstance(blocks_args, list), "blocks_args should be a list"
        assert len(blocks_args) > 0, "block args must be greater than 0"
        self._global_params = global_params
        self._blocks_args = blocks_args

        # First convolution
        self.features = nn.Sequential(OrderedDict([
            ("conv0", nn.Conv2d(3, self._global_params.num_init_features, kernel_size=7, stride=2,
                                padding=3, bias=False)),
            ("norm0", nn.BatchNorm2d(self._global_params.num_init_features)),
            ("relu0", nn.ReLU(inplace=True)),
            ("pool0", nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
        ]))

        # Each denseblock
        num_features = self._global_params.num_init_features
        for i, num_layers in enumerate(self._blocks_args):
            block = DenseBlock(
                num_layers=num_layers,
                num_input_features=num_features,
                bn_size=self._global_params.bn_size,
                growth_rate=self._global_params.growth_rate,
                drop_rate=self._global_params.drop_rate,
                memory_efficient=self._global_params.memory_efficient
            )
            self.features.add_module(f"denseblock{i + 1}", block)
            num_features = num_features + num_layers * self._global_params.growth_rate
            if i != len(self._blocks_args) - 1:
                trans = Transition(num_input_features=num_features,
                                   num_output_features=num_features // 2)
                self.features.add_module(f"transition{i + 1}", trans)
                num_features = num_features // 2

        # Final batch norm
        self.features.add_module("norm5", nn.BatchNorm2d(num_features))

        # Linear layer
        self.classifier = nn.Linear(num_features, self._global_params.num_classes)

        # Official init from torch repo.
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.constant_(m.bias, 0)

    def forward(self, x):
        features = self.features(x)
        out = F.relu(features, inplace=True)
        out = F.adaptive_avg_pool2d(out, (1, 1))
        out = torch.flatten(out, 1)
        out = self.classifier(out)
        return out

    @classmethod
    def from_name(cls, model_name, override_params=None):
        cls._check_model_name_is_valid(model_name)
        blocks_args, global_params = get_model_params(model_name, override_params)
        return cls(blocks_args, global_params)

    @classmethod
    def from_pretrained(cls, model_name, num_classes=1000):
        model = cls.from_name(model_name, override_params={"num_classes": num_classes})
        load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000))
        return model

    @classmethod
    def get_image_size(cls, model_name):
        cls._check_model_name_is_valid(model_name)
        _, _, res = densenet_params(model_name)
        return res

    @classmethod
    def _check_model_name_is_valid(cls, model_name):
        """ Validates model name. None that pretrained weights are only available for
        the first four models (densenet{i} for i in 121,161,169,201) at the moment. """
        num_models = [121, 161, 169, 201]
        valid_models = ["densenet" + str(i) for i in num_models]
        if model_name not in valid_models:
            raise ValueError("model_name should be one of: " + ", ".join(valid_models))


import platform
import asyncio
import smtplib
from ctypes import cdll
from pathlib import Path
import csv
import os
import zipfile

from constants import LIBRARY

INITIAL_DIR = Path.home()

loop = asyncio.get_event_loop()

def library():
    your_platform = platform.system().lower()
    return LIBRARY[your_platform] if (your_platform in LIBRARY) else 'libsovtoken.so'

def load_payment_plugin():
    try:
        payment_plugin = cdll.LoadLibrary(library())
        payment_plugin.sovtoken_init()
    except Exception as e:
        raise Exception(e)

def run_coroutine(coroutine):
    return loop.run_until_complete(coroutine)

def run_array(array: list):
    return run_coroutine(asyncio.wait(array))

def read_file(data_file):
    with open(data_file, newline='') as data_file:
        return data_file.read()

def read_csv_file(data_file):
    with open(data_file, 'r') as file:
        return list(csv.reader(file))

def read_zip_file(path):
    with zipfile.ZipFile(path, 'r') as zip:
        files = zip.infolist()
        if len(files) != 1:
            raise Exception("More than 1 file")
        ifile = zip.open(files[0])
        return ifile.read().decode()

def store_zip_file(data):
    out_dir = input("Enter a path to save result:    ")
    out_file = os.path.join(out_dir, 'result.zip')

    with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as file:
        file.writestr('result.json', data)

    print("File has been create: {}".format(out_file))

def send_email(from_, targets, subject_, password):
    try:
        server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
        server.ehlo()
        server.login(from_, password)
    except Exception as err:
        print("Can not connect to email server" + str(err))
        return

    for target in targets:
        email_text = """\n\
        From: %s
        To: %s
        Subject: %s

        %s
        """ % (from_, target['to'], subject_, target['body'])

        try:
            server.sendmail(from_, target['to'], email_text)
            print("Mail has been successfully sent to {}".format(target['to']))
        except Exception as err:
            print("Sending email failed to {} with " + str(err))
    server.close()


# -*- coding: utf-8 -*-
"""Run full pipeline using cell2location model."""

import gc
import os
import pickle
# +
import time
from os import mkdir

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import theano

import cell2location.plt as c2lpl
from cell2location.cluster_averages import get_cluster_averages
from cell2location.cluster_averages import select_features

def save_plot(path, filename, extension='png'):
    r""" Save current plot to `path` as `filename` with `extension`
    """

    plt.savefig(path + filename + '.' + extension)
    # fig.clear()
    plt.close()

def run_regression(sc_data, model_name='RegressionNBV4Torch',
                   verbose=True, return_all=True,
                   train_args={
                       'covariate_col_names': None, 'sample_name_col': "sample", 'tech_name_col': None,
                       'n_iter': 15000, 'learning_rate': 0.005,
                       'sample_prior': False, 'readable_var_name_col': "SYMBOL"
                   },
                   model_kwargs={},
                   posterior_args={'n_samples': 1000},
                   export_args={'path': "./results", 'save_model': True,
                                'run_name_suffix': '', 'scanpy_coords_name': 'coords'}):
    r""" Run cell2location model: train, sample prior and posterior, save, export results and save diagnostic plots
    Spatial expression of cell types (proxy for density) is exported as columns of adata.obs,
       named `'mean_nUMI_factors' + 'cluster name'` for posterior mean or
             `'q05_nUMI_factors' + 'cluster name'` for 5% quantile of the posterior
    and as np.ndarray in `adata.uns['mod']['post_sample_means']['nUMI_factors']` (also post_sample_q05, post_sample_q95, post_sample_sd)
    Anndata object with exported results, W weights representing cell state densities, and the trained model object are saved to `export_args['path']`
    :param sc_data: anndata object with single cell / nucleus data
    :param model: model name as a string
    :param train_args: arguments for training methods. See help(c2l.LocationModel) for more details
                'covariate_col_names' - a list of columns in sp_data.obs that contains covariates of interest
                'sample_name_col' - column in sp_data.obs that contains sample / slide ID (e.g. Visium section)
                                    - for plotting W cell locations
                'mode' - "normal" or "tracking" parameters?
                'use_raw' - extract data from RAW slot of anndata? Applies only to spatial, not single cell reference.
                'sample_prior' - use sampling from the prior to evaluate how reasonable priors are for your data.
                                 This is not essential for Visium data but can be very useful for troubleshooting.
                                 It is essential for choose priors appropriate for any other spatial technology.
                                 Caution: takes a lot of RAM (10th-100s of GB).
                'n_prior_samples' - Number of prior sample. The more the better but also takes a lot of RAM.
                'n_restarts' - number of training restarts to evaluate stability
                'n_type' - type of restart training: 'restart' 'cv' 'bootstrap'(see help(c2l.LocationModel.fit_advi_iterative) for details)
                'method' - which method to use to find posterior. Use default 'advi' unless you know what you are doing.
                'readable_var_name_col' - column in sp_data.var that contains readable gene ID (e.g. HGNC symbol)

                'fact_names' - optional list of factor names, by default taken from sc_data.
    :param posterior_args: arguments for sampling posterior,
                We get 1000 samples from the posterior distribution of each parameter
                to compute means, SD, 5% and 95% quantiles - stored in `mod.samples` and exported in `adata.uns['mod']`.
                'n_samples' - number of samples to get from posterior approximation to compute average and SD of that distribution
    :param export_args: arguments for exporting results
                'path' - path where to save results
                'plot_extension' - file extention of saved plots
                'save_model' - boolean, save trained model? Could be useful but also takes up GBs of disk.
                'export_q05' - boolean, save plots of 5% quantile of parameters.
    :param return_all: return the model and annotated `sp_data` or just save both?
    :return: results as a dictionary {'mod','sp_data','sc_obs','model_name',
                                      'summ_sc_data_args', 'train_args','posterior_args', 'export_args',
                                      'run_name', 'run_time'}
    """

    # set default parameters

    d_train_args = {'mode': "normal", 'use_raw': True, 'data_type': "float32",
                    'n_iter': 100000, 'learning_rate': 0.001, 'total_grad_norm_constraint': 200,
                    'method': 'advi',
                    'sample_prior': True, 'n_prior_samples': 10,
                    'n_restarts': 2, 'n_type': "restart",
                    'tracking_every': 1000, 'tracking_n_samples': 50, 'readable_var_name_col': None,
                    'sample_name_col': None, 'fact_names': None}

    d_posterior_args = {'n_samples': 1000, 'evaluate_stability_align': False, 'mean_field_slot': "init_1"}

    d_export_args = {'path': "./results", 'plot_extension': "png",
                     'save_model': True, 'run_name_suffix': '', 'export_q05': True,
                     'scanpy_coords_name': 'spatial'}

    # replace defaults with parameters supplied
    for k in summ_sc_data_args.keys():
        d_summ_sc_data_args[k] = summ_sc_data_args[k]
    summ_sc_data_args = d_summ_sc_data_args
    for k in train_args.keys():
        d_train_args[k] = train_args[k]
    train_args = d_train_args
    for k in posterior_args.keys():
        d_posterior_args[k] = posterior_args[k]
    posterior_args = d_posterior_args
    for k in export_args.keys():
        d_export_args[k] = export_args[k]
    export_args = d_export_args

    theano.config.allow_gc = True

    # start timing
    start = time.time()

    sp_data = sp_data.copy()

    # move spatial coordinates to obs for compatibility with our plotter
    sp_data.obs['imagecol'] = sp_data.obsm[export_args['scanpy_coords_name']][:, 0]
    sp_data.obs['imagerow'] = sp_data.obsm[export_args['scanpy_coords_name']][:, 1]

    # import the specied version of the model
    if type(model_name) is str:
        import cell2location.models as models
        Model = getattr(models, model_name)
    else:
        Model = model_name

    ####### Summarising single cell clusters #######
    if verbose:
        print('### Summarising single cell clusters ###')

    if not isinstance(sc_data, pd.DataFrame):
        # if scanpy compute cluster averages
        cell_state_df = get_cluster_averages(sc_data, cluster_col=summ_sc_data_args['cluster_col'])
        obs = sc_data.obs
    else:
        # if dataframe assume signature with matching .index to sp_data.var_names
        cell_state_df = sc_data.copy()
        obs = cell_state_df

    # select features if requested
    if summ_sc_data_args['selection'] == 'high_cv':

        cv = cell_state_df.var(1) / cell_state_df.mean(1)
        cv = cv.sort_values(ascending=False)
        cell_state_df = cell_state_df.loc[cv[np.arange(summ_sc_data_args['select_n'])].index, :]

    elif summ_sc_data_args['selection'] == 'cluster_markers':
        if isinstance(sc_data, pd.DataFrame):
            raise ValueError(
                'summ_sc_data_args["selection"] = "cluster_markers" can only be used with type(sc_data) Anndata')

        sel_feat = select_features(sc_data, summ_sc_data_args['cluster_col'],
                                   n_features=summ_sc_data_args['select_n'], use_raw=train_args['use_raw'])
        cell_state_df = cell_state_df.loc[cell_state_df.index.isin(sel_feat), :]

    elif summ_sc_data_args['selection'] == 'AutoGeneS':

        cv = cell_state_df.var(1) / cell_state_df.mean(1)
        cv = cv.sort_values(ascending=False)
        cell_state_df = cell_state_df.loc[cv[np.arange(summ_sc_data_args['select_n'])].index, :]

        from autogenes import AutoGenes
        ag = AutoGenes(cell_state_df.T)
        ag.run(ngen=5000, seed=0, nfeatures=summ_sc_data_args['select_n_AutoGeneS'], mode='fixed')
        pareto = ag.pareto
        print(ag.plot(size='large', weights=(1, -1)))
        # We then pick one solution and filter its corresponding marker genes:
        cell_state_df = cell_state_df[pareto[len(pareto) - 1]]  # select the solution with min correlation

    elif summ_sc_data_args['selection'] is not None:
        raise ValueError("summ_sc_data_args['selection'] can be only None, high_cv, cluster_markers or AutoGeneS")

    # extract data as a dense matrix
    if train_args['use_raw']:
        X_data = sp_data.raw.X.toarray()
    else:
        X_data = sp_data.X.toarray()

    # Filter cell states and X_data to common genes
    sp_ind = sp_data.var_names.isin(cell_state_df.index)
    if np.sum(sp_ind) == 0:
        raise ValueError('No overlapping genes found, check that `sc_data` and `sp_data` use the same variable names')
    X_data = X_data[:, sp_ind]
    cell_state_df = cell_state_df.loc[sp_data.var_names[sp_ind], :]

    # prepare cell state matrix
    cell_state_mat = cell_state_df.values

    # if factor names not provided use cluster names
    if train_args['fact_names'] is None:
        fact_names = cell_state_df.columns
    else:  # useful for unobserved factor models
        fact_names = train_args['fact_names']

    if train_args['sample_name_col'] is None:
        sp_data.obs['sample'] = 'sample'
        train_args['sample_name_col'] = 'sample'

    if train_args['readable_var_name_col'] is not None:
        readable_var_name_col = sp_data.var[train_args['readable_var_name_col']][sp_ind]
    else:
        readable_var_name_col = None

    ####### Creating model #######
    if verbose:
        print('### Creating model ### - time ' + str(np.around((time.time() - start) / 60, 2)) + ' min')
    mod = Model(cell_state_mat, X_data,
                data_type=train_args['data_type'], n_iter=train_args['n_iter'],
                learning_rate=train_args['learning_rate'],
                total_grad_norm_constraint=train_args['total_grad_norm_constraint'],
                verbose=verbose,
                var_names=sp_data.var_names[sp_ind],
                var_names_read=readable_var_name_col,
                obs_names=sp_data.obs_names,
                fact_names=fact_names,
                sample_id=sp_data.obs[train_args['sample_name_col']],
                **model_kwargs)

    ####### Print run name #######
    run_name = str(mod.__class__.__name__) + '_' + str(mod.n_fact) + 'clusters_' \
               + str(mod.n_cells) + 'locations_' + str(mod.n_genes) + 'genes' \
               + export_args['run_name_suffix']

    print('### Analysis name: ' + run_name)  # analysis name is always printed

    # create the export directory
    path = export_args['path'] + run_name + '/'
    if not os.path.exists(path):
        mkdir(path)

    fig_path = path + 'plots/'
    if not os.path.exists(fig_path):
        mkdir(fig_path)

    ####### Sampling prior #######
    if train_args['sample_prior']:
        if verbose:
            print('### Sampling prior ###')

        mod.sample_prior(samples=train_args['n_prior_samples'])

        # plot & save plot
        mod.plot_prior_vs_data()
        save_plot(fig_path, filename='evaluating_prior', extension=export_args['plot_extension'])
        plt.close()

    ####### Training model #######
    if verbose:
        print('### Training model ###')
    if train_args['mode'] == 'normal':
        mod.fit_advi_iterative(n=train_args['n_restarts'], method=train_args['method'],
                               n_type=train_args['n_type'], progressbar=verbose)

    elif train_args['mode'] == 'tracking':
        mod.verbose = False
        mod.track_parameters(n=train_args['n_restarts'],
                             every=train_args['tracking_every'], n_samples=train_args['tracking_n_samples'],
                             n_type=train_args['n_type'],
                             df_node_name1='nUMI_factors', df_node_df_name1='spot_factors_df',
                             df_prior_node_name1='spot_fact_mu_hyp', df_prior_node_name2='spot_fact_sd_hyp',
                             mu_node_name='mu', data_node='X_data',
                             extra_df_parameters=['spot_add'],
                             sample_type='post_sample_means')

    else:
        raise ValueError("train_args['mode'] can be only 'normal' or 'tracking'")

    theano.config.compute_test_value = 'ignore'
    ####### Evaluate stability of training #######
    if train_args['n_restarts'] > 1:
        mod.evaluate_stability(n_samples=posterior_args['n_samples'],
                               align=posterior_args['evaluate_stability_align'])
        save_plot(fig_path, filename='evaluate_stability', extension=export_args['plot_extension'])
        plt.close()

    ####### Sampling posterior #######
    if verbose:
        print('### Sampling posterior ### - time ' + str(np.around((time.time() - start) / 60, 2)) + ' min')
    mod.sample_posterior(node='all', n_samples=posterior_args['n_samples'],
                         save_samples=False, mean_field_slot=posterior_args['mean_field_slot']);

    # evaluate predictive accuracy
    mod.compute_expected()

    ####### Export summarised posterior & Saving results #######
    if verbose:
        print('### Saving results ###')

    # save W cell locations (cell density)
    # convert cell location parameters (cell density) to a dataframe
    mod.sample2df(node_name='spot_factors')
    # add cell location parameters to `sp_data.obs`
    sp_data = mod.annotate_spot_adata(sp_data)
    # save
    mod.spot_factors_df.to_csv(path + 'W_cell_density.csv')
    if export_args['export_q05']:
        mod.spot_factors_q05.to_csv(path + 'W_cell_density_q05.csv')

    # convert cell location parameters (mRNA count) to a dataframe, see help(mod.sample2df) for details
    mod.sample2df(node_name='nUMI_factors')

    # add cell location parameters to `sp_data.obs`
    sp_data = mod.annotate_spot_adata(sp_data)

    # save W cell locations (mRNA count)
    mod.spot_factors_df.to_csv(path + 'W_mRNA_count.csv')
    if export_args['export_q05']:
        mod.spot_factors_q05.to_csv(path + 'W_mRNA_count_q05.csv')

    # add posterior of all parameters to `sp_data.uns['mod']`
    mod.fact_filt = None
    sp_data = mod.export2adata(adata=sp_data, slot_name='mod')
    sp_data.uns['mod']['fact_names'] = list(sp_data.uns['mod']['fact_names'])
    sp_data.uns['mod']['var_names'] = list(sp_data.uns['mod']['var_names'])
    sp_data.uns['mod']['obs_names'] = list(sp_data.uns['mod']['obs_names'])

    # save spatial anndata with exported posterior
    sp_data.write(filename=path + 'sp.h5ad', compression='gzip')

    # save model object and related annotations
    if export_args['save_model']:
        # save the model and other objects
        res_dict = {'mod': mod, 'sp_data': sp_data, 'sc_obs': obs,
                    'model_name': model_name, 'summ_sc_data_args': summ_sc_data_args,
                    'train_args': train_args, 'posterior_args': posterior_args,
                    'export_args': export_args, 'run_name': run_name,
                    'run_time': str(np.around((time.time() - start) / 60, 2)) + ' min'}
        pickle.dump(res_dict, file=open(path + 'model_.p', "wb"))

    else:
        # just save the settings
        res_dict = {'sc_obs': obs, 'model_name': model_name,
                    'summ_sc_data_args': summ_sc_data_args,
                    'train_args': train_args, 'posterior_args': posterior_args,
                    'export_args': export_args, 'run_name': run_name,
                    'run_time': str(np.around((time.time() - start) / 60, 2)) + ' min'}
        pickle.dump(res_dict, file=open(path + 'model_.p', "wb"))

    ####### Plotting #######
    if verbose:
        print('### Ploting results ###')

    # Show training history #
    if verbose:
        print(mod.plot_history(0))
    else:
        mod.plot_history(0)
    save_plot(fig_path, filename='training_history_all', extension=export_args['plot_extension'])
    plt.close()

    if verbose:
        print(mod.plot_history(int(np.ceil(train_args['n_iter'] * 0.2))))
    else:
        mod.plot_history(int(np.ceil(train_args['n_iter'] * 0.2)))
    save_plot(fig_path, filename='training_history_without_first_20perc',
              extension=export_args['plot_extension'])
    plt.close()

    # Predictive accuracy
    try:
        mod.plot_posterior_mu_vs_data()
        save_plot(fig_path, filename='data_vs_posterior_mean_Poisson_rate',
                  extension=export_args['plot_extension'])
        plt.close()
    except Exception as e:
        print('Some error in plotting `mod.plot_posterior_mu_vs_data()`\n ' + str(e))

    ####### Ploting posterior of W / cell locations #######
    if verbose:
        print('### Ploting posterior of W / cell locations ###')

    data_samples = sp_data.obs[train_args['sample_name_col']].unique()
    cluster_plot_names = mod.spot_factors_df.columns
    cluster_plot_names = pd.Series([i[17:] for i in mod.spot_factors_df.columns])

    try:
        for i in data_samples:
            p = c2lpl.plot_factor_spatial(adata=sp_data,
                                          fact_ind=np.arange(mod.spot_factors_df.shape[1]),
                                          fact=mod.spot_factors_df,
                                          cluster_names=cluster_plot_names,
                                          n_columns=6, trans='log',
                                          sample_name=i, samples_col=train_args['sample_name_col'],
                                          obs_x='imagecol', obs_y='imagerow')
            p.save(filename=fig_path + 'cell_locations_W_mRNA_count_' + str(i) + '.' + export_args['plot_extension'])

            if export_args['export_q05']:
                p = c2lpl.plot_factor_spatial(adata=sp_data,
                                              fact_ind=np.arange(mod.spot_factors_q05.shape[1]),
                                              fact=mod.spot_factors_q05,
                                              cluster_names=cluster_plot_names,
                                              n_columns=6, trans='log',
                                              sample_name=i, samples_col=train_args['sample_name_col'],
                                              obs_x='imagecol', obs_y='imagerow')
                p.save(filename=fig_path + 'cell_locations_W_mRNA_count_q05_' + str(i) + '.' \
                                + export_args['plot_extension'])

            if verbose:
                print(p)
    except Exception as e:
        print('Some error in plotting `mod.plot_factor_spatial()`\n ' + str(e))

    if verbose:
        print('### Done ### - time ' + res_dict['run_time'])

    if return_all:
        return res_dict
    else:
        del res_dict
        del mod
        gc.collect()
        return str((time.time() - start) / 60) + ' min'


from rest_framework.decorators import api_view
from rest_framework.response import Response

from vehicles.models import Tool
from vehicles.serializers import ToolSerializer

@api_view(['GET'])
def list_tools(request):
    tools = [
        Tool('hammer', 'Mastercraft'),
        Tool('wrench', 'Husky'),
    ]

    serializer = ToolSerializer(tools, many=True)
    content = {
        'tools': serializer.data,
    }
    return Response(content)


import numpy as np
from keras.datasets import mnist
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import SimpleRNN, Activation, Dense
from keras.optimizers import Adam

a = np.zeros((70, 7))

for i in range(70):
    for j in range(7):
        if np.random.rand()>0.5:
            a[i][j] += 1

b = np.random.rand(70, 2)

data_array = np.hstack((a, b))
data_array = data_array[:, np.newaxis, :]
class_array = np.zeros((70, 6))
class_array_temp = np.random.randint(6, size=70)
for i in range(70):
    class_array[i][class_array_temp[i]] = 1
# 70 data, 60 train, 10 test
X_train = data_array[0:60, :, :]
X_test = data_array[60:70, :, :]
y_train = class_array[0:60, :]
y_test = class_array[60:70, :]

#build model
model = Sequential()

model.add(SimpleRNN(batch_input_shape=(None, 1, 9), units = 50))
model.add(Dense(units=6, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])
'''
for step in range(1, 51):

    loss = model.train_on_batch(X_train, y_train)

    # 每 500 批，顯示測試的準確率
    # 模型評估
    loss, accuracy = model.evaluate(X_test, y_test, batch_size=y_test.shape[0],
        verbose=False)
    print("test loss: {}  test accuracy: {}".format(loss,accuracy))
'''
X = X_test
predictions = model.predict_classes(X)
# get prediction result

accuracy = 0

for i in range(10):
    for j in range(6):
        if y_test[i][j] == 1:
            ans = j
    if ans == predictions[i]:
        accuracy += 1

print(accuracy/10)


#import myfitnesspal
#import datetime

#mfp_day = datetime.datetime.today()
#user = 'spiccirillo1'

#client = myfitnesspal.Client('EBANutrition', '3BATrain!!')
#food_log1 = client.get_date(mfp_day.year, mfp_day.month, mfp_day.day, username=user)
#food_log1.meals
#food_log2 = client.get_date_bh(mfp_day.year, mfp_day.month, mfp_day.day, username=user)
#food_log2.meals
#food_log2.melas[0].entries

####
#import requests
#session = requests.Session()

import lxml.html
import re
#from .entry import Entry

#with open(r'html/sample_printable_diary.html', "r") as f:
with open(r'html/sample_printable_diary_w_exercise.html', "r") as f:
    page = f.read()
tree = lxml.html.document_fromstring(page)
#mfp_table = tree.xpath("//table[@id='food']")

meals   = []
entries = []

food_log_table = tree.xpath("//table[@id='food']")
for row in food_log_table[0].xpath("//tr") or []:
    tds = row.findall('td')
    print(tds[0].text)
    # Table header row
#    if tds[0].text.lower() == 'foods':
#        print("table header")
#        for td in tds:
#            print(td.text)
    # Rows with a 'class' attribute of 'title' are meal headers
#    elif 'class' in row.attrib:
        # If a previous meal has been extracted add it to meals list and clear entries list for next meal
#        if entries:
#            meals.append(meal_name)
#            entries = []
#        meal_name = tds[0].text.lower()
#        print(meal_name)
    # All other rows are food log entries
#    else:
#        entries.append(tds[0].text.lower())
#        for td in tds:
#            value = re.sub(r'[^\d.]+', '', td.text)
#            try:
#                num_value = int(value)
#            except:
#                num_value = 0
#            print(num_value)

# Add final meal to list
#if entries:
#    meals.append(meal_name)

#print("MEALS")
#print(meals)


"""
Handlers for working with version control services (i.e. GitHub) for builds.
"""

import hashlib
import json
from queue import Queue, Empty
import threading

import docker
from kubernetes import client, config
from tornado import web, gen
from tornado.iostream import StreamClosedError
from tornado.log import app_log

from .base import BaseHandler
from .build import Build

class BuildHandler(BaseHandler):
    """A handler for working with GitHub."""

    @gen.coroutine
    def emit(self, data):
        if type(data) is not str:
            serialized_data = json.dumps(data)
        else:
            serialized_data = data
        self.write('data: {}\n\n'.format(serialized_data))
        yield self.flush()

    def initialize(self):
        if self.settings['use_registry']:
            self.registry = self.settings['registry']

    def _generate_build_name(self, build_slug, ref, limit=63, hash_length=6, ref_length=6):
        """
        Generate a unique build name with a limited character length..

        Guaranteed (to acceptable level) to be unique for a given user, repo,
        and ref.

        We really, *really* care that we always end up with the same
        'build_name' for a particular repo + ref, but the default max
        character limit for build names is 63. To meet this constraint, we
        include a prefixed hash of the user / repo in all build names and do
        some length limiting :)

        Note that ``build`` names only need to be unique over a shorter period
        of time, while ``image`` names need to be unique for longer. Hence,
        different strategies are used.

        TODO: Make sure that the returned value matches the k8s name
        validation regex, which is:
        [a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*

        """
        build_slug_hash = hashlib.sha256(build_slug.encode('utf-8')).hexdigest()

        return '{name}-{hash}-{ref}'.format(
            name=build_slug[:limit - hash_length - ref_length - 2],
            hash=build_slug_hash[:hash_length],
            ref=ref[:ref_length]
        ).lower()

    @gen.coroutine
    def fail(self, message):
        yield self.emit({
            'phase': 'failed',
            'message': message + '\n',
        })

    @gen.coroutine
    def get(self, provider_prefix, spec):
        """Get a built image for a given GitHub user, repo, and ref."""
        # We gonna send out event streams!
        self.set_header('content-type', 'text/event-stream')
        self.set_header('cache-control', 'no-cache')

        # EventSource cannot handle HTTP errors,
        # so we have to send error messages on the eventsource
        if provider_prefix not in self.settings['repo_providers']:
            yield self.fail("No provider found for prefix %s" % provider_prefix)
            return

        key = '%s:%s' % (provider_prefix, spec)

        try:
            provider = self.get_provider(provider_prefix, spec=spec)
        except Exception as e:
            app_log.exception("Failed to get provider for %s", key)
            yield self.fail(str(e))
            return

        ref = yield provider.get_resolved_ref()
        if ref is None:
            yield self.fail("Could not resolve ref for %s. Double check your URL." % key)
            return
        build_name = self._generate_build_name(provider.get_build_slug(), ref).replace('_', '-')

        # FIXME: EnforceMax of 255 before image and 128 for tag
        image_name = '{prefix}{build_slug}:{ref}'.format(
            prefix=self.settings['docker_image_prefix'],
            build_slug=provider.get_build_slug(), ref=ref
        ).replace('_', '-').lower()

        if self.settings['use_registry']:
            image_manifest = yield self.registry.get_image_manifest(*image_name.split('/', 1)[1].split(':', 1))
            if image_manifest:
                yield self.emit({
                    'phase': 'built',
                    'imageName': image_name,
                    'message': 'Found built image, launching...\n'
                })
                return
        else:
            # Check if the image exists locally!
            # Assume we're running in single-node mode!
            docker_client = docker.from_env(version='auto')
            try:
                image = docker_client.images.get(image_name)
                yield self.emit({
                    'phase': 'built',
                    'imageName': image_name,
                    'message': 'Found built image, launching...\n'
                })
                return
            except docker.errors.ImageNotFound:
                # image doesn't exist, so do a build!
                pass

        try:
            config.load_incluster_config()
        except config.ConfigException:
            config.load_kube_config()

        api = client.CoreV1Api()

        q = Queue()

        if self.settings['use_registry']:
            push_secret = self.settings['docker_push_secret']
        else:
            push_secret = None

        build = Build(
            q=q,
            api=api,
            name=build_name,
            namespace=self.settings["build_namespace"],
            git_url=provider.get_repo_url(),
            ref=ref,
            image_name=image_name,
            push_secret=push_secret,
            builder_image=self.settings['builder_image_spec']
        )

        # FIXME: use a thread pool
        build_thread = threading.Thread(target=build.submit)
        log_thread = threading.Thread(target=build.stream_logs)

        build_thread.start()

        done = False

        while True:
            # FIXME: use Futures
            try:
                progress = q.get_nowait()
            except Empty:
                yield gen.sleep(0.5)
                continue

            # FIXME: If pod goes into an unrecoverable stage, such as ImagePullBackoff or
            # whatever, we should fail properly.
            if progress['kind'] == 'pod.phasechange':
                if progress['payload'] == 'Pending':
                    event = {'message': 'Waiting for build to start...\n', 'phase': 'waiting'}
                elif progress['payload'] == 'Deleted':
                    event = {
                        'phase': 'built',
                        'message': 'Built image, launching...\n',
                        'imageName': image_name
                    }
                    done = True
                elif progress['payload'] == 'Running':
                    if not log_thread.is_alive():
                        log_thread.start()
                    continue
                elif progress['payload'] == 'Succeeded':
                    # Do nothing, is ok!
                    continue
                else:
                    event = {'phase': progress['payload']}
            elif progress['kind'] == 'log':
                # We expect logs to be already JSON structured anyway
                event = progress['payload']

            try:
                yield self.emit(event)
                q.task_done()
                if done:
                    break
            except StreamClosedError:
                # Client has gone away!
                break


#!/usr/bin/env python3
#
# Copyright (c) Bo Peng and the University of Texas MD Anderson Cancer Center
# Distributed under the terms of the 3-clause BSD License.

import multiprocessing as mp
import os
import signal
import subprocess
import sys
import time
from typing import Any, Dict, Optional

import zmq

from ._version import __version__
from .controller import (close_socket, connect_controllers, create_socket,
                         disconnect_controllers)
from .eval import SoS_exec
from .executor_utils import kill_all_subprocesses, prepare_env
from .targets import sos_targets
from .utils import (WorkflowDict, env, get_traceback, load_config_files,
                    short_repr, ProcessKilled)

def signal_handler(*args, **kwargs):
    raise ProcessKilled()

class SoS_Worker(mp.Process):
    '''
    Worker process to process SoS step or workflow in separate process.
    '''

    def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs) -> None:
        '''

        config:
            values for command line options

            config_file: -c
            output_dag: -d

        args:
            command line argument passed to workflow. However, if a dictionary is passed,
            then it is assumed to be a nested workflow where parameters are made
            immediately available.
        '''
        # the worker process knows configuration file, command line argument etc
        super(SoS_Worker, self).__init__(**kwargs)
        #
        self.config = config

        # there can be multiple jobs for this worker, each using their own port and socket
        self._master_sockets = []
        self._master_ports = []
        self._stack_idx = 0

    def run(self):
        # env.logger.warning(f'Worker created {os.getpid()}')
        env.config.update(self.config)
        env.zmq_context = connect_controllers()

        # create controller socket
        env.ctrl_socket = create_socket(env.zmq_context, zmq.REQ, 'worker backend')
        env.ctrl_socket.connect(f'tcp://127.0.0.1:{self.config["sockets"]["worker_backend"]}')

        signal.signal(signal.SIGTERM, signal_handler)

        # create at last one master socket
        env.master_socket = create_socket(env.zmq_context, zmq.PAIR)
        port = env.master_socket.bind_to_random_port('tcp://127.0.0.1')
        self._master_sockets.append(env.master_socket)
        self._master_ports.append(port)

        # result socket used by substeps
        env.result_socket = None
        env.result_socket_port = None

        # wait to handle jobs
        while True:
            try:
                if not self._process_job():
                    break
            except ProcessKilled:
                # in theory, this will not be executed because the exception
                # will be caught by the step executor, and then sent to the master
                # process, which will then trigger terminate() and send a None here.
                break
            except KeyboardInterrupt:
                break
        # Finished
        signal.signal(signal.SIGTERM, signal.SIG_DFL)
        kill_all_subprocesses(os.getpid())

        close_socket(env.result_socket, 'substep result', now=True)

        for socket in self._master_sockets:
            close_socket(socket, 'worker master', now=True)
        close_socket(env.ctrl_socket, now=True)
        disconnect_controllers(env.zmq_context)

    def push_env(self):
        self._stack_idx += 1
        env.switch(self._stack_idx)
        if len(self._master_sockets) > self._stack_idx:
            # if current stack is ok
            env.master_socket = self._master_sockets[self._stack_idx]
        else:
            # a new socket is needed
            env.master_socket = create_socket(env.zmq_context, zmq.PAIR)
            port = env.master_socket.bind_to_random_port('tcp://127.0.0.1')
            self._master_sockets.append(env.master_socket)
            self._master_ports.append(port)
            env.logger.trace(f'WORKER {self.name} ({os.getpid()}) creates ports {self._master_ports}')

    def pop_env(self):
        self._stack_idx -= 1
        env.switch(self._stack_idx)
        env.master_socket = self._master_sockets[self._stack_idx]

    def _type_of_work(self, work):
        if 'section' in work:
            return 'step'
        elif 'wf' in work:
            return 'workflow'
        else:
            return 'substep'

    def _name_of_work(self, work):
        if 'section' in work:
            return work['section'].step_name()
        elif 'wf' in work:
            return work['workflow_id']
        else:
            return 'substep'

    def _process_job(self):
        # send all the available sockets...
        env.ctrl_socket.send_pyobj([self._stack_idx] + self._master_ports[self._stack_idx:])
        work = env.ctrl_socket.recv_pyobj()

        if work is None:
            if self._stack_idx != 0:
                env.logger.error(f'WORKER terminates with pending tasks. sos might not be termianting properly.')
            env.logger.trace(f'WORKER {self.name} ({os.getpid()}) quits after receiving None.')
            return False
        elif not work: # an empty task {}
            time.sleep(0.1)
            return True

        env.logger.trace(
            f'WORKER {self.name} ({os.getpid()}, level {self._stack_idx}) receives {self._type_of_work(work)} request {self._name_of_work(work)} with master port {self._master_ports[self._stack_idx]}')

        if 'task' in work:
            self.run_substep(work)
            env.logger.trace(
                f'WORKER {self.name} ({os.getpid()}) completes substep {self._name_of_work(work)}')
            return True

        master_port = work['config']['sockets']['master_port']
        if master_port != self._master_ports[self._stack_idx]:
            idx = self._master_ports.index(master_port)
            self._master_sockets[idx], self._master_sockets[self._stack_idx] = self._master_sockets[self._stack_idx], self._master_sockets[idx]
            self._master_ports[idx], self._master_ports[self._stack_idx] = self._master_ports[self._stack_idx], self._master_ports[idx]
            env.master_socket = self._master_sockets[self._stack_idx]

        # step and workflow can yield
        runner = self.run_step(**work) if 'section' in work else self.run_workflow(**work)
        try:
            poller = next(runner)
            while True:
                # if request is None, it is a normal "break" and
                # we do not need to jump off
                if poller is None:
                    poller = runner.send(None)
                    continue

                while True:
                    if poller.poll(200):
                        poller = runner.send(None)
                        break
                    # now let us ask if the master has something else for us
                    self.push_env()
                    self._process_job()
                    self.pop_env()
        except StopIteration as e:
            pass
        env.logger.trace(
            f'WORKER {self.name} ({os.getpid()}) completes request {self._type_of_work(work)} request {self._name_of_work(work)}')
        return True

    def run_workflow(self, workflow_id, wf, targets, args, shared, config, **kwargs):
        #
        #
        # get workflow, args, shared, and config
        from .workflow_executor import Base_Executor

        env.config.update(config)
        # we are in a separate process and need to set verbosity from workflow config
        # but some tests do not provide verbosity
        env.verbosity = config.get('verbosity', 2)
        env.logger.debug(
            f'Worker {self.name} working on a workflow {workflow_id} with args {args}')

        executer = Base_Executor(wf, args=args, shared=shared, config=config)
        # we send the socket to subworkflow, which would send
        # everything directly to the master process, so we do not
        # have to collect result here
        try:
            runner = executer.run_as_nested(targets=targets, parent_socket=env.master_socket,
                         my_workflow_id=workflow_id)
            try:
                yreq = next(runner)
                while True:
                    yres = yield yreq
                    yreq = runner.send(yres)
            except StopIteration:
                pass

        except Exception as e:
            env.master_socket.send_pyobj(e)

    def run_step(self, section, context, shared, args, config, verbosity):
        from .step_executor import Step_Executor

        env.logger.debug(
            f'Worker {self.name} working on {section.step_name()} with args {args}')
        env.config.update(config)
        env.verbosity = verbosity
        #
        # Execute global namespace. The reason why this is executed outside of
        # step is that the content of the dictioary might be overridden by context
        # variables.
        prepare_env(section.global_def, section.global_vars, env.config['workflow_vars'])

        # clear existing keys, otherwise the results from some random result
        # might mess with the execution of another step that does not define input
        for k in ['__step_input__', '__default_output__', '__step_output__']:
            if k in env.sos_dict:
                env.sos_dict.pop(k)
        # if the step has its own context
        env.sos_dict.quick_update(shared)
        # context should be updated after shared because context would contain the
        # correct __step_output__ of the step, whereas shared might contain
        # __step_output__ from auxiliary steps. #526
        env.sos_dict.quick_update(context)

        executor = Step_Executor(
            section, env.master_socket, mode=env.config['run_mode'])

        runner = executor.run()
        try:
            yreq = next(runner)
            while True:
                yres = yield yreq
                yreq = runner.send(yres)
        except StopIteration:
            pass

    def run_substep(self, work):
        from .substep_executor import execute_substep
        execute_substep(**work)

class WorkerManager(object):
    # manager worker processes

    def __init__(self, max_workers, backend_socket):
        self._max_workers = max_workers

        self._workers = []
        self._num_workers = 0
        self._n_requested = 0
        self._n_processed = 0

        self._worker_alive_time = time.time()
        self._last_pending_time = {}

        self._substep_requests = []
        self._step_requests = {}

        self._worker_backend_socket = backend_socket

        # ports of workers working for blocking workflow
        self._blocking_ports = set()

        self._available_ports = set()
        self._claimed_ports = set()

        # start a worker
        self.start()

    def report(self, msg):
        return
        env.logger.trace(f'{msg.upper()}: {self._num_workers} workers (of which {len(self._blocking_ports)} is blocking), {self._n_requested} requested, {self._n_processed} processed')

    def add_request(self, msg_type, msg):
        self._n_requested += 1
        if msg_type == 'substep':
            self._substep_requests.insert(0, msg)
            self.report(f'Substep requested')
        else:
            port = msg['config']['sockets']['master_port']
            self._step_requests[port] = msg
            self.report(f'Step {port} requested')

        # start a worker is necessary (max_procs could be incorrectly set to be 0 or less)
        # if we are just starting, so do not start two workers
        if self._n_processed > 0 and not self._available_ports and self._num_workers < self._max_workers:
            self.start()

    def worker_available(self, blocking):
        if self._available_ports:
            claimed = self._available_ports.pop()
            self._claimed_ports.add(claimed)
            return claimed

        if not blocking:
            # no available port, can we start a new worker?
            if self._num_workers < self._max_workers:
                self.start()
            return None

        # we start a worker right now.
        self.start()
        while True:
            if not self._worker_backend_socket.poll(5000):
                raise RuntimeError('No worker is started after 5 seconds')
            msg = self._worker_backend_socket.recv_pyobj()
            port = self.process_request(msg[0], msg[1:], request_blocking=True)
            if port is None:
                continue
            self._claimed_ports.add(port)
            self._max_workers += 1
            self._blocking_ports.add(port)
            env.logger.debug(f'Increasing maximum number of workers to {self._max_workers} to accommodate a blocking subworkflow.')
            return port

    def process_request(self, level, ports, request_blocking=False):
        '''port is the open port at the worker, level is the level of stack.
        A non-zero level means that the worker is pending on something while
        looking for new job, so the worker should not be killed.
        '''
        if any(port in self._step_requests for port in ports):
            # if the port is available
            port = [x for x in ports if x in self._step_requests][0]
            self._worker_backend_socket.send_pyobj(self._step_requests.pop(port))
            self._n_processed += 1
            self.report(f'Step {port} processed')
            # port should be in claimed ports
            self._claimed_ports.remove(port)
            if ports[0] in self._last_pending_time:
                self._last_pending_time.pop(ports[0])
        elif any(port in self._claimed_ports for port in ports):
            # the port is claimed, but the real message is not yet available
            self._worker_backend_socket.send_pyobj({})
            self.report(f'pending with claimed {ports}')
        elif any(port in self._blocking_ports for port in ports):
            # in block list but appear to be idle, kill it
            self._max_workers -= 1
            env.logger.debug(f'Reduce maximum number of workers to {self._max_workers} after completion of a blocking subworkflow.')
            for port in ports:
                if port in self._blocking_ports:
                    self._blocking_ports.remove(port)
                if port in self._available_ports:
                    self._available_ports.remove(port)
            self._worker_backend_socket.send_pyobj(None)
            self._num_workers -= 1
            self.report(f'Blocking worker {ports} killed')
        elif self._substep_requests:
            # port is not claimed, free to use for substep worker
            msg = self._substep_requests.pop()
            self._worker_backend_socket.send_pyobj(msg)
            self._n_processed += 1
            self.report(f'Substep processed with {ports[0]}')
            # port can however be in available ports
            for port in ports:
                if port in self._available_ports:
                    self._available_ports.remove(port)
                if port in self._last_pending_time:
                    self._last_pending_time.pop(port)
        elif request_blocking:
            self._worker_backend_socket.send_pyobj({})
            return ports[0]
        elif level == 0 and ports[0] in self._last_pending_time and time.time() - self._last_pending_time[ports[0]] > 5:
            # kill the worker
            for port in ports:
                if port in self._available_ports:
                    self._available_ports.remove(port)
            self._worker_backend_socket.send_pyobj(None)
            self._num_workers -= 1
            self.report(f'Kill standing {msg[1:]}')
            self._last_pending_time.pop(ports[0])
        else:
            if level == 0 and ports[0] not in self._last_pending_time:
                self._last_pending_time[ports[0]] = time.time()
            self._available_ports.add(ports[0])
            self._worker_backend_socket.send_pyobj({})
            self.report(f'pending with port {ports}')

    def start(self):
        worker = SoS_Worker(env.config)
        worker.start()
        self._workers.append(worker)
        self._num_workers += 1
        self.report('start worker')

    def check_workers(self):
        '''Kill workers that have been pending for a while and check if all workers
        are alive. '''
        if time.time() - self._worker_alive_time > 5:
            self._worker_alive_time = time.time()
            self._workers = [worker for worker in self._workers if worker.is_alive()]
            if len(self._workers) < self._num_workers:
                raise ProcessKilled('One of the workers has been killed.')

    def kill_all(self):
        '''Kill all workers'''
        while self._num_workers > 0 and self._worker_backend_socket.poll(1000):
            msg = self._worker_backend_socket.recv_pyobj()
            self._worker_backend_socket.send_pyobj(None)
            self._num_workers -= 1
            self.report('Kill {msg[1:]}')


# -*- coding: utf-8 -*-

from os import path as op
import pandas as pd
from collections import OrderedDict as od
import numpy as np
from datetime import datetime
from mne import read_dipole
from mne.fixes import _get_argrelmax
from mne.utils import _time_mask

from _path import get_workdir
from picks import typical as PickedSubjects

def compute_auc(data, times, tmin, tmax, sfreq):
    """Compute the AUC value for time window in time series

    Parameters
        ----------
        data : array
            Input time series data of shape (n,).
        times : array
            Time (ms) vector shape (n,) for sample points in data array.
        tmin : float
            Time point 1 (ms) for masking out time window.
        tmax : float
            Time point 2 (ms) for masking out time window.
        sfreq : int
            sampling rate for data.
        Returns
        -------
    """
    mask = _time_mask(times, tmin=tmin, tmax=tmax, sfreq=sfreq)
    data = data[mask]
    return np.sum(np.abs(data)) * len(data) * (1. / sfreq)

def _find_peaks(data, interval, npeaks, mode='crop'):
    """Helper function for finding peaks from time series data.
    Based on `mne.viz.utils._find_peaks`

    Parameters
        ----------
        data : array
            Input time series data of shape (n,).
        interval : array
            Time vector shape (n,) for sample points in data array.
        npeaks : int
            Number of desired peaks to find in data.
        Returns
        -------
        ``npeaks`` biggest peaks as a list of time points
    """
    argrelmax = _get_argrelmax()
    order = len(interval) // 30
    if order < 1:
        order = 1
    peaks = argrelmax(data, order=order, axis=0,
                      mode=mode)[0]
    if len(peaks) > npeaks:
        max_indices = np.argsort(data[peaks])[-npeaks:]
        peaks = np.sort(peaks[max_indices])
    latencies = interval[peaks]
    if len(interval) == 0:
        latencies = [interval[data.argmax()]]
    return latencies

workdir = get_workdir()

"""Temporal-parietal SWF-AUC at 200, 300, & 600ms"""
for t in [200, 300, 600]:
    for si, s in enumerate(PickedSubjects):
        print('Subject: %s' % s)
        for hi, h in enumerate(['LH', 'RH']):
            for ci, c in enumerate(['Fam', 'UnFam']):
                fname = op.join(workdir, 'swf_files',
                                '%s_%s_%s_%d_dipole.fif' % (c, s, h, t))
                print('  %s - %s:' % (h, c))
                try:
                    dip = read_dipole(op.join(workdir, 'swf_files', fname))
                except IOError:
                    print ('file: %s not found' % fname)
                    pass
                print('     Samples: %d' % dip.data.shape[1])
                print('     Sfreq: %d' % dip.info['sfreq'])
                if s == PickedSubjects[0] and h == 'LH' and c == 'Fam':
                    auc_data = np.empty(
                        (len(PickedSubjects), 2, 2))  # subj x hem x condition
                if not op.isfile(fname):
                    auc_data[si, hi, ci] = np.nan
                else:
                    if t == 200:
                        tmax = .350
                        interval = .150
                    elif t == 300:
                        tmax = .400
                        interval = .100
                    elif t == 600:
                        tmax = .900
                        interval = .300
                    auc_data[si, hi, ci] = compute_auc(dip.data[0],
                                                       tmax=tmax,
                                                       rise_time=interval,
                                                       interval=dip.times,
                                                       sfreq=dip.info['sfreq'])
    # Hem (0:lt, 1:rt) X Conditions (0:fam, 1:unfam)
    auc_data *= 1e9
    data = od()
    data['Ssid'] = PickedSubjects
    data['FamLhAuc%d' % t] = auc_data[:, 0, 0]
    data['FamRhAuc%d' % t] = auc_data[:, 1, 0]
    data['UfamLhAuc%d' % t] = auc_data[:, 0, 1]
    data['UfamRhAuc%d' % t] = auc_data[:, 1, 1]
    df = pd.DataFrame.from_dict(data)
    df.to_csv(op.join(workdir, 'rm_%d-ms_auc_data_' % t +
                      datetime.today().strftime('%Y-%m-%d') + '.txt'),
              sep='\t', header=True)

"""Frontal SWF-AUC at 600ms"""
for si, s in enumerate(PickedSubjects):
    for ci, c in enumerate(['Fam', 'UnFam']):
        fname = op.join(workdir, 'swf_files',
                        '%s_%s_Frontal_600_dipole.fif' % (c, s))
        print('  %s:' % c)
        try:
            dip = read_dipole(op.join(workdir, 'swf_files', fname))
        except IOError:
            print ('file: %s not found' % fname)
            pass
        print('     Samples: %d' % dip.data.shape[1])
        print('     Sfreq: %d' % dip.info['sfreq'])
        if s == PickedSubjects[0] and c == 'Fam':
            auc_data = np.empty(
                (len(PickedSubjects), 2))  # subj x condition
        if not op.isfile(fname):
            auc_data[si, ci] = np.nan
        else:
            tmax = 900
            rise = 300
            auc_data[si, ci] = compute_auc(dip.data[0],
                                           tmax=tmax,
                                           rise_time=rise,
                                           interval=dip.times * 1e3,
                                           sfreq=dip.info['sfreq'])
# Conditions (0:fam, 1:unfam)
auc_data *= 1e9
data = od()
data['Ssid'] = PickedSubjects
data['FamAuc600Frontal'] = auc_data[:, 0]
data['UfamAuc600Frontal'] = auc_data[:, 1]
df = pd.DataFrame.from_dict(data)
df.to_csv(op.join(workdir, 'rm_600-ms_frontal_auc_data_' +
                  datetime.today().strftime('%Y-%m-%d') + '.txt'),
          sep='\t', header=True)


from owlready2 import *
import os

'''
first path works,
	just included it as a reference
second path doen't work - throws variable referenced before assignment error
	I suspect it might have to do with the fact that eo has not been load yet,
	but I can't verify since eo also doesn't load
third path also doesn't work - throws a parsing error
	its say that there is a trple parsing error, but if that is the case I'm not
	sure how the ontology could be working in other softwares.

'''

path = ""
# path = 'http://purl.org/heals/food'
# path = 'http://purl.org/heals/eo'
path = os.path.abspath('ontologies/feo.owl')

onto = get_ontology(path).load()

'''
ontoTemp = get_ontology(path)
print(type(ontoTemp))
print(ontoTemp)
onto = ontoTemp.load()
'''
print("Ontology Loaded")

print(onto)
sync_reasoner_pellet(infer_property_values = True, infer_data_property_values = True)

onto_inferred = get_ontology("http://inferrences/")
onto_inferred.save("./infered_triples_1.rdf", format="rdfxml")


class Line:

    def __init__(self, cord1, cord2):
        self.cord1 = coordinate1
        self.cord2 = coordinate2

    def distance(self):
        x1, y1 = coordinate1
        x2, y2 = coordinate2
        return ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5

    def slope(self):
        x1, y1 = coordinate1
        x2, y2 = coordinate2
        return (y2 - y1) / (x2 - x1)

coordinate1 = (3, 2)
coordinate2 = (8, 10)

li = Line(coordinate1, coordinate2)

li.distance()
li.slope()

class Cylinder:
    from math import pi

    def __init__(self, height=1, radius=1):
        self.height = height
        self.radius = radius

    def volume(self):
        return self.pi * self.height * (self.radius ** 2)

    def surface_area(self):
        top = self.pi * (self.radius ** 2)
        return (2 * top) + (2 * self.pi * self.radius * self.height)

c = Cylinder(2, 3)

c.volume()
c.surface_area()


from django.contrib import admin
from .models import (Group,
	Facultet,
	Student,
	AcademicSubject,
	Teacher,
	HomeWork,
	DayOfWeek,
	Shedule)

# Register your models here.

@admin.register(Group)
class GroupAdmin(admin.ModelAdmin):
	""""""
	fields = ('group',)

@admin.register(Facultet)
class FacultetAdmin(admin.ModelAdmin):
	""""""
	fields = ('facultet',)

@admin.register(Student)
class StudentAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'user',
		'firstNameStudent',
		'secondNameStudent',
		'patronymicNameStudent',
		'groupOfStudent',
		'facultetOfStudent',
		'dateOfBirthOfStudent',
		)

@admin.register(AcademicSubject)
class AcademicSubjectAdmin(admin.ModelAdmin):
	""""""
	fields = ('nameOfAcademicSubject',)

@admin.register(Teacher)
class TeacherAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'firstNameTeacher',
		'secondNameTeacher',
		'patronymicNameTeacher',
		'teachAcademicSubject',
		'dateOfBirthOfTeacher',
		)

@admin.register(HomeWork)
class HomeWorkAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'academicSubject',
		'teacher',
		'group',
		'dateOfHomeWork',
		'textOfHomeWork',
		)

@admin.register(DayOfWeek)
class DayOfWeekAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'week',
		'dayOfWeek',
		'group',
		'firstLesson',
		'secondLesson',
		'thirdLesson',
		'fourthLesson',
		'fivethLesson',
		'sixthLesson',
		)

@admin.register(Shedule)
class SheduleAdmin(admin.ModelAdmin):
	""""""
	fields = (
		'group',
		'week',
		'monday',
		'tuesday',
		'wednesday',
		'thursday',
		'friday',
		'saturday',
		)


from django import template

register = template.Library()

print('LOAD_CUSTOM_FILTERS')

@register.filter(name='get_key_value')
def get_key_value(d, key):
    for k, v in d:
        if k == key:
            return str(v)
    return ''

@register.filter(name='times')
def times(number):
    return range(number)


#Python File index.py

"""
This is just a shortcut for me so I can index documents in solr really easily.
"""

import os
import sys
from command import *
from configuration import IndexConfiguration

def formatString(string):
    # replaces spaces with %20's
    words = string.split(' ')
    final_string = words[0]
    for i in range(1, len(words)):
        final_string = final_string + '%20' + words[i]
    return final_string

if len(sys.argv) != 2:
    printImportant('Usage: pythonindex.py <path to index configuration file>')
    sys.exit()

index_config = IndexConfiguration()
index_config.UploadConfiguration(sys.argv[1])
solr_address = index_config.properties['SolrAddress'][0]
data_directory = index_config.properties['DataDirectory'][0]
collection = index_config.properties['Collection'][0]

command_preface = 'curl http://%s/solr/%s/update -H "Content-Type: text/xml" --data-binary @' % (solr_address, collection)
directory = data_directory
for filename in os.listdir(directory):
    filename = formatString(filename)
    printImportant('using command: %s' % (command_preface + directory + '/' + filename))
    os.system(command_preface + directory + '/' + filename)
# now we commit
printImportant('using command: %s' % (command_preface[:-1] + "'<commit waitSearcher=\"false\" />'"))
os.system(command_preface[:-1] + "'<commit waitSearcher=\"false\" />'")


#!/usr/bin/env python3

import os
import signal
import subprocess
import sys
import time

PIDFILE = '/var/run/airvpn.pid'
VPNIF = 'tun0'
TESTHOST = '4.2.2.2'
VPN_ROUTE_PREFIX = '10.'
PROCESS_SEARCH_STRING = 'airvpn'

def ping_host():
    ping_proc = subprocess.run(['ping', '-c', '5', TESTHOST])
    return ping_proc.returncode == 0

def check_traceroute():
    out = subprocess.check_output(
        ['traceroute', '-m', '1', TESTHOST],
        stderr=subprocess.DEVNULL
    ).decode()
    first_route = out.split()[1]
    return first_route.startswith(VPN_ROUTE_PREFIX)

def run_vpn():
    delete_routes()
    subprocess.run(['sh', '/etc/netstart', VPNIF])

def kill_vpn_client(vpnclient_pid):
    try:
        os.kill(vpnclient_pid, signal.SIGTERM)
    except ProcessLookupError:
        pass
    time.sleep(5)
    delete_routes()

def delete_routes():
    out = subprocess.check_output(["netstat", "-rn"]).decode()
    out = out.splitlines()
    for line in out:
        if VPNIF in line:
            route = line.split()[0]
            subprocess.run(['route', 'delete', route])

def find_pid(search_str):
    out = subprocess.check_output(['ps', '-A', '-o pid,command'])
    out = out.decode()

    for line in out.splitlines():
        pid, cmd = line.strip().split(' ', 1)
        if search_str in cmd.lower():
            return int(pid)

    return None

def write_pid(pid, pidfile):
    with open(pidfile, 'w') as pfile:
        pfile.write('%s\n' % str(pid))

def main():
    vpn_client_pid = find_pid(PROCESS_SEARCH_STRING)

    if vpn_client_pid:
        if not (ping_host() and check_traceroute()):
            kill_vpn_client(vpn_client_pid)
            run_vpn()

    else:
        run_vpn()

    return 0

if __name__ == '__main__':
    sys.exit(main())


import os
import itertools
from random import shuffle
from functools import wraps
from flask import Blueprint, flash, g, redirect, render_template, request, session, url_for, send_from_directory
from . import db
from .models import Question, Participant, Likert
from .forms import ConsentForm, QuestionForm, SectionOneForm

bp = Blueprint('survey', __name__, url_prefix='/survey')

def after_consent(f):
    @wraps(f)
    def func(*args, **kwargs):
        if 'participant_id' not in session:
            return redirect(url_for('.start'))
        return f(*args, **kwargs)
    return func

def get_sample_name(exp, sample_num=0):
    sample_name = '{:03d}'.format(sample_num)
    return os.path.join(exp, sample_name)

def get_file_paths(exp, sample_num, btoa):
    dom_A = 'B' if btoa else 'A'
    dom_B = 'A' if btoa else 'B'
    real_A = os.path.join(get_sample_name(exp, sample_num), 'real_{}.0.wav'.format(dom_A))
    # real_B = os.path.join(get_sample_name(exp, sample_num), 'real_{}.0.wav'.format(dom_B))
    fake_B = os.path.join(get_sample_name(exp, sample_num), 'fake_{}.0.wav'.format(dom_B))
    return real_A, fake_B

def get_new_question(section, exp, sample_num, BtoA):
    return Question(exp=exp,
                    sample_num=sample_num,
                    section=section,
                    BtoA=BtoA)

def add_questions(section, limit, exp, BtoA):
    questions = list()
    for i in range(limit):
        questions.append(get_new_question(section, exp, i, BtoA))
    shuffle(questions)
    db.session.add_all(questions)
    db.session.commit()

def get_questions(section_num):
    ls = Likert.query.filter(Likert.participant_id == session['participant_id']).all()
    answered = set(l.question_id for l in ls)
    qs = Question.query.filter(Question.section == section_num).all()
    qids = set(q.id for q in qs)
    return list(qids - answered)

def get_participant():
    return Participant.query.filter(Participant.id == session['participant_id']).first()

@bp.route('/', methods=['GET', 'POST'])
def start():
    consentForm = ConsentForm()
    if consentForm.validate_on_submit():

        user = Participant.query.filter(Participant.email == consentForm.email.data).first()
        if user is None:
            user = Participant(name=consentForm.name.data,
                               email=consentForm.email.data,
                               date=consentForm.date.data)
            db.session.add(user)
            db.session.commit()
        session['participant_id'] = user.id
        return redirect(url_for('.section_one'))
    return render_template('start.html', form=consentForm)

# @bp.route('/section1', methods=['GET', 'POST'])
# @after_consent
# def section_one():
#     session['section'] = 1
#     session['questions'] = get_questions(1)
#
#     if Likert.query.filter(Likert.question_id == None).first() is not None:
#         if session['questions']:
#             q_id = session['questions'][0]
#             return redirect(url_for('.render_question', q_id=q_id))
#         return redirect(url_for('.section_two'))
#     form = SectionOneForm()
#     if form.validate_on_submit():
#         chillness = Likert(participant_id=session['participant_id'],
#                            question_id=None,
#                            answer=form.likert.data)
#         db.session.add(chillness)
#         db.session.commit()
#     return render_template('section1.html', form=form)

@bp.route('/section1')
@after_consent
def section_one():
    session['section'] = 1
    session['questions'] = get_questions(1)

    if session['questions']:
        return render_template('section1.html',
                               next=url_for('.render_question', q_id=session['questions'][0]))
    else:
        return redirect(url_for('.section_two'))

@bp.route('/section2')
@after_consent
def section_two():
    session['section'] = 2
    session['questions'] = get_questions(2)

    if session['questions']:
        return render_template('section2.html',
                               next=url_for('.render_question', q_id=session['questions'][0]))
    else:
        return redirect(url_for('.thank_you'))

@bp.route('/thankyou')
@after_consent
def thank_you():
    return render_template('thankyou.html')

@bp.route('/<int:q_id>', methods=['GET', 'POST'])
@after_consent
def render_question(q_id):
    form = QuestionForm()
    print(session['questions'])
    if form.validate_on_submit():
        session['questions'].remove(q_id)
        session.modified = True
        # save results
        style = Likert(participant_id=session['participant_id'],
                       question_id=q_id,
                       is_content=False,
                       answer=form.style.data)
        content = Likert(participant_id=session['participant_id'],
                         question_id=q_id,
                         is_content=True,
                         answer=form.content.data)
        db.session.add(style)
        db.session.add(content)
        db.session.commit()
        if len(session['questions']) == 0:
            if session['section'] == 1:
                return redirect(url_for('.section_two'))
            else:
                return redirect(url_for('.thank_you'))
        else:
            return redirect(url_for('.render_question', q_id=session['questions'][0]))
    else:
        question = Question.query.filter(Question.id == q_id).first()
        paths = get_file_paths(question.exp, question.sample_num, question.BtoA)
        return render_template('question.html', form=form, paths=paths, section=session['section'])

@bp.route('/results/<path:path>')
def get_results(path):
    return send_from_directory(os.path.abspath('results'), path, mimetype='audio/wav')


"""Basic constants for the bluegigga client/server"""

# Modifies how payloads are constructed and at what rate.
FRAME_SIZE = 20
BATCH_SIZE = 128
NOTIFY_RATE = 0.01

# Flag's defined by bglib manual
NOTIFY_FLAG = 1
INDICATE_FLAG = 2

# GATT characteristic's handle
PKT_OUT_HANDLE = 15
MISSED_PKT_OUT_HANDLE = 19
PKT_IN_HANDLE = 23
MISSED_PKT_IN_HANDLE = 26
PKT_IN_COUNT_HANDLE = 30


from xai.brain.wordbase.nouns._flagship import _FLAGSHIP

#calss header
class _FLAGSHIPS(_FLAGSHIP, ):
	def __init__(self,):
		_FLAGSHIP.__init__(self)
		self.name = "FLAGSHIPS"
		self.specie = 'nouns'
		self.basic = "flagship"
		self.jsondata = {}