from typing import Optional

from mai_version.representation.TILDE_query import TILDEQuery, TILDEQueryHiddenLiteral
from mai_version.trees.leaf_strategy import LeafStrategy

class TreeNode:
    """A node in a First-Order Logical Decision Tree.

    There are two kinds of nodes: decision nodes and leaf nodes"""
    left_subtree = None  # type: Optional[TreeNode]
    right_subtree = None  # type: Optional[TreeNode]
    nb_of_examples_with_label = None  # type: Optional[int]
    nb_of_examples_in_this_node = None  # type: Optional[int]

    strategy = None  # type: Optional[LeafStrategy]

    query = None  # type: Optional[TILDEQuery]

    def get_left_child_node(self) -> 'TreeNode':
        return self.left_subtree

    def get_right_child_node(self) -> 'TreeNode':
        return self.right_subtree

    def has_both_children(self) -> bool:
        return self.left_subtree is not None and self.right_subtree is not None

    def is_leaf_node(self) -> bool:
        return self.left_subtree is None and self.right_subtree is None

    def to_string_full_query(self, indentation='', current_node_number=0):
        """
        Represents the tree as a string using some layouting
        :param indentation:
        :param current_node_number:
        :return:
        """
        node_indentation = indentation
        child_indentation = indentation

        if current_node_number == 0:
            child_indentation = '\t'
        elif current_node_number == 1:
            node_indentation += '|-'
            child_indentation += '|\t'
        else:
            node_indentation += '\-'
            child_indentation += '\t'

        if self.is_leaf_node():
            result = self.strategy.to_string(node_indentation)
            # result = node_indentation + "Leaf, class label: " + str(self.classification) + ", [" + str(
            #     self.nb_of_examples_with_label) + "/" + str(self.nb_of_examples_in_this_node) + "]" + '\n'
            return result
        else:
            result = node_indentation + 'INode, query: ' + str(self.query) + '\n'

            if self.get_left_child_node() is not None:
                result = result + self.get_left_child_node().to_string_full_query(child_indentation, 1)
            if self.get_right_child_node() is not None:
                result = result + self.get_right_child_node().to_string_full_query(child_indentation, 2)
            return result

    def to_string_compact(self, indentation='', current_node_number=0):
        """
        Represents the tree as a string using some layouting
        :param indentation:
        :param current_node_number:
        :return:
        """
        node_indentation = indentation
        child_indentation = indentation

        if current_node_number == 0:  # root node
            child_indentation = ''
        elif current_node_number == 1:  # this node is the LEFT child node of its parent
            node_indentation += '+--'
            child_indentation += '|       '
        else:  # this node is the RIGHT child node of its parent
            node_indentation += '+--'
            child_indentation += '        '

        if self.is_leaf_node():
            if current_node_number == 0:
                result = self.strategy.to_string_compact()
            elif current_node_number == 1:
                result = node_indentation + 'yes: ' + self.strategy.to_string_compact()
            else:
                result = node_indentation + 'no: ' + self.strategy.to_string_compact()

            # result = self.strategy.to_string(node_indentation)

            # result = node_indentation + "Leaf, class label: " + str(self.classification) + ", [" + str(
            #     self.nb_of_examples_with_label) + "/" + str(self.nb_of_examples_in_this_node) + "]" + '\n'
            return result
        else:
            if current_node_number == 0:
                if self.query.parent is not None and isinstance(self.query.parent, TILDEQueryHiddenLiteral):
                    result = str(self.query.parent.literal) + '\n'
                else:
                    result = ""

                result = result + str(self.query.get_literal()) + ' ?\n'

            elif current_node_number == 1:
                result = node_indentation + 'yes: ' + str(self.query.get_literal()) + ' ?\n'
            else:
                result = node_indentation + 'no: ' + str(self.query.get_literal()) + ' ?\n'

            # result = node_indentation + 'INode, query: ' + str(self.query) + '\n'

            if self.get_left_child_node() is not None:
                result = result + self.get_left_child_node().to_string_compact(child_indentation, 1)
            if self.get_right_child_node() is not None:
                result = result + self.get_right_child_node().to_string_compact(child_indentation, 2)
            return result

    def __str__(self):
        return self.to_string_compact()

    def get_nb_of_nodes(self) -> int:
        return TreeNode._count_nb_of_nodes(self)

    @staticmethod
    def _count_nb_of_nodes(node:Optional['TreeNode'] = None) -> int:

        if node is None:
            return 0
        else:
            count = 1  # count the node itself
            count += TreeNode._count_nb_of_nodes(node.left_subtree)
            count += TreeNode._count_nb_of_nodes(node.right_subtree)

            return count

    def get_nb_of_inner_nodes(self):
        return TreeNode._count_nb_of_inner_nodes(self)

    @staticmethod
    def _count_nb_of_inner_nodes(node:Optional['TreeNode'] = None) -> int:

        if node.is_leaf_node():
            return 0
        else:
            count = 1
            count += TreeNode._count_nb_of_inner_nodes(node.left_subtree)
            count += TreeNode._count_nb_of_inner_nodes(node.right_subtree)
            return count

    def can_classify(self) -> object:
        if self.strategy is None:
            raise AttributeError('TreeNode has no Strategy')
        else:
            return self.strategy.can_classify()


import csv
import json
import logging
import sys
from os.path import join
from typing import TextIO, Tuple, Any
import time
import numpy as np
from collections import OrderedDict
from typing import List
from argparse import Namespace
from collections import namedtuple, deque

from utils import utils

LogCfg = namedtuple(
    "LogCfg", ["key_name", "fixed_window", "processing", "abbreviation", "print_format", "print",
               "plot"]
)
"""
    key_name: Key to uniquely identify data (not print log)
    fixed_window: True, If statistics should be calculated on a fixed number of values defined by
        stats_window_size from cfg.main. If so, than we will keep all previous
        (stats_window_size) values and calculate processing on it.
        If not. list will be cleared after each log.
    processing: How to summarize list (mean, average, etc.)  See LOG_TYPES for all possible types.
    abbreviation: Used for print
    print_format: e.g. for print("string_{:.2f}) -> :.2f
    print: If it should be printed
    plot: If it should be plotted
"""

LOG_TYPES = {
    "a": np.mean,  # average list
    "s": np.std,  # standard deviation list
    "m": np.min,  # min from list
    "M": np.max,  # max from list
    "l": lambda x: x[-1],  # last element in list
}

class DataList:
    def __init__(self, fixed_window, values):
        self.fixed_window = fixed_window
        self.values = values

FILE_LOG = "log.txt"
FILE_CSV = "log.csv"
FILE_VOCAB = "vocab.json"
FILE_STATUS = "status.json"

DEFAULT_TRAIN_LOG = [
    LogCfg("update", False, "l", "U", "", True, True),
    LogCfg("num_frames", False, "l", "F", ":06", True, True),
    LogCfg("num_frames_no_off", False, "l", "F0", ":06", False, True),
    LogCfg("fps", False, "a", "FPS", ":04.0f", True, False),
    LogCfg("duration", False, "l", "D", "", True, False),
]

def extra_log_fields(header: list, log_keys: list) ->list:
    unusable_fields = ['return_per_episode', 'reshaped_return_per_episode',
                       'num_frames_per_episode', 'num_frames']
    extra_fields = []
    for field in log_keys:
        if field not in header and field not in unusable_fields:
            extra_fields.append(field)

    return extra_fields

def get_logger(log_dir: str) -> logging.Logger:
    if log_dir is None:
        logging.basicConfig(
            level=logging.INFO,
            format="%(message)s",
        )
    else:
        path = join(log_dir, FILE_LOG)
        utils.create_folders_if_necessary(path)

        logging.basicConfig(
            level=logging.INFO,
            format="%(message)s",
            handlers=[
                logging.FileHandler(filename=path),
                logging.StreamHandler(sys.stdout)
            ]
        )

    return logging.getLogger()

def get_csv_writer(log_dir: str, csv_file_name: str) -> Tuple[TextIO, Any]:
    csv_path = join(log_dir, csv_file_name)
    utils.create_folders_if_necessary(csv_path)
    csv_file = open(csv_path, "a")
    return csv_file, csv.writer(csv_file)

def load_status(log_dir: str) -> dict:
    path = join(log_dir, FILE_STATUS)
    with open(path) as file:
        return json.load(file)

class Header:
    def __init__(self, header_config: List[LogCfg], window_size: int = 1, max_buffer: int = 9999):
        self.window_size = window_size
        self.header = header = OrderedDict()
        self.data = data = dict()
        self._max_buffer = max_buffer

        for ix, cfg in enumerate(header_config):
            key_name = f"{cfg.key_name}_{cfg.processing}"
            key = cfg.key_name
            header[key_name] = Namespace()
            header[key_name].key = key
            header[key_name].process = LOG_TYPES[cfg.processing]
            header[key_name].abbr = cfg.abbreviation
            header[key_name].print_str = cfg.abbreviation + " {" + cfg.print_format + "}"
            header[key_name].print = cfg.print
            header[key_name].fixed_window = cfg.fixed_window
            header[key_name].plot = cfg.plot

            # data window size is configured according to first cfg
            if key not in self.data:
                if cfg.fixed_window:
                    # TODO List growing too big
                    data[key] = DataList(True, [0])
                else:
                    data[key] = DataList(False, [])

    def __getitem__(self, item):
        return self.header[item]

    def reset(self):
        window_size = self.window_size

        for k, v in self.data.items():
            if v.fixed_window:
                v.values = v.values[-window_size:]
            else:
                v.values = []

    def add_data(self, data: dict):
        local_data = self.data
        for k, v in data.items():
            local_data[k].values += v  # Must add only list to logs!
            if len(local_data[k].values) > self._max_buffer:
                local_data[k].values = local_data[k].values[-self._max_buffer:]

    def get_csv_header(self) -> List[str]:
        return self.header.keys()

    def get_values(self) -> List[Any]:
        vs = []
        data = self.data
        win_size = self.window_size
        for kkk, v in self.header.items():
            k = v.key
            if len(data[k].values) > 0:
                if data[k].fixed_window:
                    vs.append(v.process(data[k].values[-win_size:]))
                else:
                    vs.append(v.process(data[k].values))
            else:
                vs.append(None)
        return vs

    def get_win_value(self, key, default=None) -> Any:
        data = self.data
        if len(data[key].values) > 0:
            return np.mean(data[key].values[-self.window_size:])
        else:
            return default

    def get_log(self) -> Tuple[List[str], str, dict]:
        data = self.data
        win_size = self.window_size

        csv_row = []
        print_row = []
        plot_values = dict()

        for kkk, v in self.header.items():
            k = v.key
            if len(data[k].values) > 0:
                if data[k].fixed_window:
                    rv = v.process(data[k].values[-win_size:])
                else:
                    rv = v.process(data[k].values)
            else:
                rv = None

            csv_row.append(rv)
            if v.print:
                print_row.append(v.print_str.format(rv))

            if v.plot:
                plot_values[kkk] = rv

        return csv_row, " | ".join(print_row), plot_values

class DummyCSV:
    def __init__(self):
        pass

    def flush(self):
        pass

class DummyCSVWrite:
    def __init__(self):
        pass

    def writerow(self):
        pass

class LoggerMaster:
    def __init__(self, log_dir: str, plot: bool = False, plot_x: str = None,
                 stats_window_size: int = 1,
                 logger=None, plotter=None, csv_file_name: str = FILE_CSV,
                 plot_project: str = "project-0", experiment: str = "default",
                 cfg: dict = None):
        self.log_dir = log_dir
        self.plot_x = plot_x
        self.window_size = stats_window_size

        self.logger = get_logger(log_dir) if logger is None else logger
        if log_dir is not None:
            self.csv_file, self.csv_writer = get_csv_writer(log_dir, csv_file_name)
        else:
            self.csv_file, self.csv_writer = DummyCSV(), DummyCSVWrite()

        if plot:
            import wandb
            import os

            if os.path.isfile(".wandb_key"):
                with open(".wandb_key") as f:
                    WANDB_API_KEY = f.readline().strip()

                os.environ['WANDB_API_KEY'] = WANDB_API_KEY

            wandb.init(project=plot_project, name=f"{experiment}")
            if cfg is not None:
                wandb.config.update(cfg)
            self.plotter = wandb
        else:
            self.plotter = plotter

        self.total_start_time = time.time()
        self.update_start_time = time.time()

        self.header = None  # type: Header

    def get_plotter(self):
        return self.plotter

    def set_header(self, update, header_config: List[LogCfg]):
        header_config = DEFAULT_TRAIN_LOG + header_config
        self.header = Header(header_config, window_size=self.window_size)

        if update <= 1:
            self.csv_writer.writerow(self.header.get_csv_header())

        self.csv_file.flush()

    def toggle_print(self, toggle_print: List[str]):
        for x in toggle_print:
            if x in self.header.header.keys():
                self.header[x].print = not self.header[x].print

    def toggle_plot(self, toggle_plot: List[str]):
        for x in toggle_plot:
            if x in self.header.header.keys():
                self.header[x].plot = not self.header[x].plot

    def get_logger(self):
        return self.logger

    def register_log(self, update: int, update_num_frames: int, logs: dict):
        update_start_time = time.time()

        num_frames = logs["num_frames"][0]
        logs["update"] = [update]
        logs["fps"] = [update_num_frames / (update_start_time - self.update_start_time)]
        logs["duration"] = [int(time.time() - self.total_start_time)]

        self.update_start_time = update_start_time

        self.header.add_data(logs)

    def log_logs(self):

        csv_row, print_row, plot_values = self.header.get_log()

        self.csv_writer.writerow(csv_row)
        self.csv_file.flush()
        self.logger.info(print_row)

        self.header.reset()

        if self.plotter:
            self.plotter.log(plot_values)


import sys
from pathlib import Path

from dotenv import load_dotenv

def setup():
    load_dotenv()

    modules = [Path('../utility')]

    for module in modules:
        if module not in sys.path:
            sys.path.insert(0, str(module))
            print(f'Added {module} to sys.path')


import numpy
import pygame
import sys
from pygame.locals import *
from abc import ABC

class Simulation(ABC):
    """ Base Simulation Class
    This class is meant to serve as a parent class for you to create your own
    simulations. It handles the main loop for pygame and all its settings so
    that you don't have to copy/paste them all over the place.
    Simply inherit from this class, override the update() function (main game loop),
    and you're good to go.
    Finally, you call it in a main function like so.

    class ExampleSim(Simulation):

        def update():
            # simulation code that will be executed each frame.

    if __name__ == '__main__':
        ExampleSim().run()
    """

    HEIGHT: int = 1000
    WIDTH: int = 1000
    SCREEN_CENTER: tuple[float, float] = (
        HEIGHT / 2,
        WIDTH / 2
    )
    FPS: int = 60

    def __init__(self):
        self.fps_clock = pygame.time.Clock()
        pygame.init()
        self.display = pygame.display.set_mode((self.WIDTH, self.HEIGHT))
        self.set_caption()
        self.surface = pygame.Surface((self.WIDTH, self.HEIGHT))

    def update(self):
        """ Override this function and add your simulation code here.
        """
        pass

    @staticmethod
    def check_exit():
        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()

    def get_surface_2d(self) -> numpy.ndarray:
        return pygame.surfarray.array2d(self.surface)

    def run(self):
        while True:
            self.display.blit(self.surface, self.SCREEN_CENTER)
            self.update()
            self.check_exit()
            pygame.display.update()
            self.fps_clock.tick(self.FPS)

    def set_caption(self):
        pygame.display.set_caption(str(self.name()))

    def name(self):
        return str(self.__class__).strip("<>'").split(".")[-1]


#!/usr/bin/env python
"""Selection procedures (local and global)"""

import numpy as np
import multiprocessing as mp
import time
import random
import anneal
import itertools
from multiprocessing import pool, Process  # Redundant?

import simulate_data as sim
import summary_stats as sum_stat
import estimators as estim
import assess_sv_estimation as a

theta_initial = []
stats_initial = []
sample_dim = []
theta_out = np.matrix([])
theta_out_sigma = np.matrix([])
theta_in_sigma = np.matrix([])
stats_out = []
local_knn_estimate = []
obs_stats_standard = []
criterion = ""

class BinaryAnnealer(anneal.Annealer):

    def move(self):
        i = 0
        while i == 0 or sum(self.state) == 0:
            n = random.randrange(len(self.state))
            self.state[n] = 1 - self.state[n]
            i += 1

    def energy(self):
        return obj_f(self.state, criterion)

def select_sv(y, s, r, ar, criterion_f):
    """Statistic selection procedure for SV process"""
    # s: In sample size.
    # r: Out sample size.
    # ar:

    # Start timer.
    start = time.time()
    print("Started at:", time.strftime("%H:%M:%S on %d-%m-%Y"))

    t = len(y)

    delta = np.array([1]*22)

    # Calculate statistics from observed sample.
    obs_stats = sum_stat.sv_stats(delta, y)

    # Generate out theta.
    global theta_out
    theta_out = [generate_theta_sv(ar) for i in range(r)]
    theta_out = np.matrix(theta_out).T

    # Generate out sample of time series.
    out_y = [sim.sim_sv(t, theta_out[0, i], theta_out[1, i], theta_out[2, i],
                        theta_out[3, i], 1) for i in range(r)]

    # Generate out sample statistics.
    global stats_out
    stats_out = [sum_stat.sv_stats(delta, out_y[i]) for i
                 in range(len(out_y))]

    # Standardize the out-of-sample parameters.
    theta_out_mean = sum(theta_out.T)/r
    u = sum([np.square(theta_out[:, i]-theta_out_mean.T) for i in range(r)])/r
    theta_out = np.hstack([(theta_out[:, i] - theta_out_mean.T)/np.sqrt(u)
                           for i in range(r)])
    global theta_out_sigma
    theta_out_sigma = np.sqrt(u)

    # Standardize the out-of-sample statistics.
    stats_out_mean = sum(stats_out)/r
    u = sum([np.square(stats_out[i]-stats_out_mean) for i in range(r)])/r
    stats_out = [(stats_out[i] - stats_out_mean)/np.sqrt(u) for i in range(r)]

    # Generate in theta.
    global theta_initial
    theta_initial = [generate_theta_sv(ar) for i in range(s)]
    theta_initial = np.matrix(theta_initial).T

    # Generate in sample of time series.
    in_y = [sim.sim_sv(t, theta_initial[0, i], theta_initial[1, i],
                       theta_initial[2, i], theta_initial[3, i], 1)
            for i in range(s)]

    # Generate in sample statistics.
    global stats_initial
    stats_initial = [sum_stat.sv_stats(delta, in_y[i]) for i
                     in range(s)]

    # Standardize parameters used for estimation.
    theta_initial_mean = sum(theta_initial.T)/s
    u = sum([np.square(theta_initial[:, i]-theta_initial_mean.T)
             for i in range(s)])/s
    theta_initial = np.hstack([(theta_initial[:, i]
                                - theta_initial_mean.T)/np.sqrt(u)
                               for i in range(s)])

    global theta_in_sigma
    global theta_in_mean
    theta_in_sigma = np.sqrt(u)
    theta_in_mean = theta_initial_mean

    # Standardize statistics for estimation.
    stats_initial_mean = sum(stats_initial)/s
    u = sum([np.square(stats_initial[i]-stats_initial_mean)
             for i in range(s)])/s
    stats_initial = [(stats_initial[i] - stats_initial_mean)/np.sqrt(u) for
                     i in range(s)]

    # Standardize statistics calculated from observed sample.
    global obs_stats_standard
    obs_stats_standard = (obs_stats - stats_initial_mean)/np.sqrt(u)

    # Guess delta.
    delta = np.around(np.random.rand(1, 22)).flatten().astype(int)

    global criterion
    criterion = criterion_f

    #  Summary of sample sizes.
    print("Out sample size:", r)
    print("In sample size:", s)
    print("Observed sample size:", len(y))
    print("Number of persistence parameters:", ar)

    # Initiate binary annealing algorithm and assign tuning parameters.
    print("Initial guess for delta is:", delta)
    print("Criterion function is:", criterion)
    opt = BinaryAnnealer(delta)
    opt.Tmax = 0.10  # 0.25
    opt.Tmin = 0.0001  # 0.001
    opt.steps = len(delta)*10
    opt.updates = len(delta)
    print("Simmulated annealing parameters are:")
    print("Tmax:", opt.Tmax)
    print("Tmin:", opt.Tmin)
    print("Beginning simulated annealing at:",
          time.strftime("%H:%M:%S on %d-%m-%Y"))
    results = opt.anneal()

    results_file = open("annealing_results.txt", "a")
    results_file.write("### Results from annealing run finished at: " +
                       time.strftime("%H:%M:%S on %d-%m-%Y") + " ###\n")
    results_file.write(str(results) + "\n")
    results_file.close()

    print("Final selection and minimum are: ", results)
    print("Finished at :", time.strftime("%H:%M:%S on %d-%m-%Y"))
    print("Total time to run (in minutes):", (time.time()-start)/60, "\n")

    return results

def select_lin_reg(obs_y, obs_x, s, r, q, criterion_f):
    """Selection procedure for linear regression"""
    # r: out of sample simulation size.
    # s: within sample simulation size.
    # q: size of initial importance sampler draw.

    # Start timer.
    t = time.time()

    # Sample dimensions.
    global sample_dim
    sample_dim.append(obs_x.shape[0])
    sample_dim.append(obs_x.shape[1])

    # Candidate statistic vector:
    # (6*sample_dim beta) (3*2 intercepts and error term variances)
    # (5 noise random variables)
    n_candidates = 6*sample_dim[1] + 6 + 5
    delta = np.array([0]*n_candidates)
    delta[0:len(delta)] = 1  # Inititalize delta.

    # Print starting message.
    print("### Beginning summary statistic selection from", n_candidates,
          "candidate summary statistics. ###")

    # Calculate full set of summary statistics from observed data.
    obs_stats = sum_stat.lin_reg_stats(delta, obs_y, obs_x)

    # Generate simulated samples from the linear model.
    generate_lin_reg_sample(delta, s, r, sample_dim, obs_stats)

    # Initial guess for delta.
    delta = np.around(np.random.rand(1, n_candidates)).flatten().astype(int)

    #  Summary of sample sizes.
    print("Out sample size:", r)
    print("In sample size:", s)
    print("Observed sample size:", sample_dim[0])
    print("Number of regressors:", sample_dim[1])

    global criterion
    criterion = criterion_f

    # Initiate binary annealing algorithm and assign tuning parameters.
    print("Initial guess for delta is:", delta)
    print("Criterion function is:", criterion)
    opt = BinaryAnnealer(delta)
    opt.Tmax = 0.25  # 0.25
    opt.Tmin = 0.001  # 0.001
    opt.steps = len(delta)*10
    opt.updates = len(delta)
    print("Simmulated annealing parameters are:")
    print("Tmax:", opt.Tmax)
    print("Tmin:", opt.Tmin)
    print("Beginning simulated annealing at:",
          time.strftime("%H:%M:%S on %d-%m-%Y"))
    results = opt.anneal()

    results_file = open("annealing_results.txt", "a")
    results_file.write("### Results from annealing run finished at: " +
                       time.strftime("%H:%M:%S on %d-%m-%Y") + " ###\n")
    results_file.write(str(results) + "\n")
    results_file.close()

    print("Final selection and minimum are: ", results)
    print("Finished at :", time.strftime("%H:%M:%S on %d-%m-%Y"))
    print("Total time to run (in minutes):", (time.time()-t)/60, "\n")
    return

def obj_f(delta, criterion_type):
    """Objective functions for the different types of selection procedures"""
    if criterion_type == 'integrated_kl':
        # Integrated Kullback-Leibler distance criterion.
        criterion = -criterion_f(delta, criterion_type)

    if criterion_type == 'abs_dev_knn':
        # Integrated Bayesian loss of posterior mean estimate
        # (estimated by k-nn).
        a = 0
        criterion = (1+a*sum(delta))*criterion_f(
            delta, criterion_type)

    if criterion_type == 'local_abs_dev':
        # Bayesian loss of posterior mean estimate.
        a = 0  # sum(delta)/len(delta)
        sim_stats = [stats_initial[i][delta > 0]
                     for i in range(len(stats_initial))]
        global local_knn_estimate
        local_knn_estimate = estim.sbil_knn(obs_stats_standard[delta > 0],
                                            sim_stats, theta_initial)
        criterion = (1+a*sum(delta))*criterion_f(
            delta, criterion_type)

    if criterion_type == 'local_kl':
        criterion = -criterion_f(delta, criterion_type)

    return criterion

def criterion_f(delta, criterion_type='abs_dev_knn'):
    """Function calls to the different criteria associated with the different
    selection procedures (with the spawning of multiple processes)"""
    stats_out_filtered = [stats_out[i][delta > 0]
                          for i in range(len(stats_out))]

    if criterion_type == 'local_kl':
        # Find k nearest statistics from R sample (full candidate set).
        norms = [np.linalg.norm(stats_out[i] - obs_stats_standard)
                 for i in range(len(stats_out))]
        k_nearest_index = np.argsort(norms)[0:round(len(  # 0.01
            stats_out_filtered)**.25)]
        k_nearest_out_theta = [theta_out.T[i] for i in k_nearest_index]
        obs_stats_standard_filtered = obs_stats_standard[delta > 0]
        # Estimate posterior density for each of the k nearest thetas.
        p = pool.Pool()

        results = p.map_async(local_kl_star, zip(itertools.repeat(delta),
                                                 k_nearest_out_theta,
                                                 itertools.repeat(
                                                     obs_stats_standard_filtered)))

        p.close()
        p.join()
        results.wait()
        posterior_densities = results.get()
        numerator = sum(np.log(posterior_densities))
        denominator = len(k_nearest_out_theta)

        return numerator/denominator

    if criterion_type == 'local_abs_dev':
        norms = [np.linalg.norm(stats_out_filtered[i] -
                                obs_stats_standard[delta > 0])
                 for i in range(len(stats_out_filtered))]
        k_nearest_stats = np.argsort(norms)[0:round(len(  # 0.01
            stats_out_filtered)**0.25)]
        k_nearest_theta = [theta_out.T[i] for i in k_nearest_stats]
        k_nearest_norms_std = [1 for i in k_nearest_stats]  #[norms[i]/largest_norm for i in k_nearest_stats]
        p = pool.Pool()

        results = p.map_async(local_abs_dev_star, zip(k_nearest_theta,
                                                      itertools.repeat(delta),
                                                      k_nearest_norms_std))
        p.close()
        p.join()
        results.wait()
        num_denom = results.get()
        numerator = [num_denom[i][0] for i in range(len(num_denom))]
        denominator = [num_denom[i][1] for i in range(len(num_denom))]
        criterion = sum(numerator)/sum(denominator)

    if criterion_type == 'integrated_kl':
        p = pool.Pool()
        results = p.map_async(integrated_kl_star, zip(theta_out.T,
                                                      itertools.repeat(delta),
                                                      stats_out_filtered))
        p.close()
        p.join()
        results.wait()
        posterior_densities = results.get()
        criterion = sum(np.log(posterior_densities))/len(posterior_densities)

    if criterion_type == 'abs_dev_knn':
        p = pool.Pool()
        results = p.map_async(abs_dev_knn_star, zip(theta_out.T,  # theta_out.T
                                                    itertools.repeat(delta),
                                                    stats_out_filtered))
        p.close()
        p.join()

        results.wait()
        norms = results.get()
        criterion = sum(norms)/len(norms)

    if criterion_type == '':
        print("Loss function is L2-norm.")
        manager = mp.Manager()
        norms = manager.list()
        procs = []
        chunk_size = int(len(stats_out)/p_n)
        print("Chunk size is: ", chunk_size)
        for i in range(4):
            fargs = (theta_out[:, i*chunk_size:(i+1)*chunk_size], delta,
                     stats_out[i*chunk_size:(i+1)*chunk_size], s,
                     q, i, norms)
            p = Process(target=norm_L2_mp, args=fargs)
            procs.append(p)

        print("Spawned these processes: ", procs)

        for p in procs:
            p.start()

        for p in procs:
            p.join()

    return criterion

def local_kl_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return estim.sbil_pd_accept_reject(*args)

def local_abs_dev_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return local_abs_dev(*args)

def integrated_kl_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return integrated_kl(*args)

def abs_dev_knn_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return abs_dev_knn(*args)

def local_abs_dev(theta_r, delta, norm):
    abs_dev = (1/theta_r.shape[1])*sum(abs(
        theta_r.T-local_knn_estimate))
    #u = np.linalg.norm(stat_r - obs_stats_standard[delta > 0])#/0.01
    #kernel = (1/math.sqrt((2*math.pi)))*math.exp(-u/2)
    #kernel = int(u < 2)
    kernel = 1  # (3/4)*(1-norm**2)
    return abs_dev*kernel, kernel

def integrated_kl(theta_r, delta, stat_r):
    posterior_density = estim.sbil_pd_accept_reject(delta, theta_r.T, stat_r)
    return posterior_density

def abs_dev_knn(theta_out, delta, obs_stat):

    sim_stats = [stats_initial[i][delta > 0]
                 for i in range(len(stats_initial))]
    knn_estimate = estim.sbil_knn(obs_stat, sim_stats, theta_initial)
    abs_dev = (1/theta_out.shape[1])*sum(theta_in_sigma.T*abs(
        theta_out.T-knn_estimate))
    return abs_dev

def norm_L2_mp(theta_out, delta, stats_out, s, q, i, norms):
    L2_norm = []
    for i in range(len(stats_out)):
        L2_norm.append(np.linalg.norm(theta_out[:, i] - np.matrix(estim.sbil_kernel(
        delta, stats_out[i], s, q, i)).T))
    print("Finished one set!")
    print(L2_norm)
    norms.append(sum(L2_norm))
    return

def theta_and_stats(delta, s, sample_size, n):
    """Draw parameter vectors and simulate associated statistics"""
    # Generate draws from the prior.
    theta = sim.prior_draw(n,  s)

    # Generate data for regressors.
    sim_x = np.matrix(np.random.randn(sample_size, s*n))
    sim_x_iter = [sim_x[:, i*n:(i+1)*n] for i in range(s)]

    # Simulate data from the linear model.
    p = pool.Pool()
    theta_iter = [theta[0:n+1, i] for i in range(s)]
    sigma = [theta[n+1, i] for i in range(s)]
    sim_y = p.map(sim_lin_reg_star, zip(theta_iter, sigma, sim_x_iter,
                                        itertools.repeat(1)))
    p.close()
    p.join()

    # Calculate summary statistics from simulated data.
    p = pool.Pool()
    sim_stats = p.map(lin_reg_stats_star, zip(itertools.repeat(delta),
                                              sim_y, sim_x_iter))
    p.close()
    p.join()

    return [theta, sim_stats]

def sim_lin_reg_star(args):
    return sim.sim_lin_reg(*args)

def lin_reg_stats_star(args):
    return sum_stat.lin_reg_stats(*args)

def generate_lin_reg_sample(delta, s, r, sample_dim, obs_stats):
    """Draw from the prior and simulate associated summary statistics for
    cross validation"""
    global theta_out
    global stats_out
    result = theta_and_stats(delta, r, sample_dim[0], sample_dim[1])
    theta_out = result[0]
    stats_out = result[1]

    # Standardize the out-of-sample parameters.
    theta_out_mean = sum(theta_out.T)/r
    u = sum([np.square(theta_out[:, i]-theta_out_mean.T) for i in range(r)])/r
    theta_out = np.hstack([(theta_out[:, i] - theta_out_mean.T)/np.sqrt(u)
                           for i in range(r)])
    global theta_out_sigma
    theta_out_sigma = np.sqrt(u)

    # Standardize the out-of-sample statistics.
    stats_out_mean = sum(stats_out)/r
    u = sum([np.square(stats_out[i]-stats_out_mean) for i in range(r)])/r
    stats_out = [(stats_out[i] - stats_out_mean)/np.sqrt(u) for i in range(r)]

    # Draw from the prior and caclulate summary statistics for estimation.
    result = theta_and_stats(delta, s, sample_dim[0], sample_dim[1])
    global theta_initial
    global stats_initial
    theta_initial = result[0]
    stats_initial = result[1]

    # Standardize parameters used for estimation.
    theta_initial_mean = sum(theta_initial.T)/s
    u = sum([np.square(theta_initial[:, i]-theta_initial_mean.T)
             for i in range(s)])/s
    global theta_in_sigma
    theta_in_sigma = np.sqrt(u)
    theta_initial = np.hstack([(theta_initial[:, i]
                                - theta_initial_mean.T)/np.sqrt(u)
                               for i in range(s)])

    # Standardize statistics for estimation.
    stats_initial_mean = sum(stats_initial)/s
    u = sum([np.square(stats_initial[i]-stats_initial_mean)
             for i in range(s)])/s
    stats_initial = [(stats_initial[i] - stats_initial_mean)/np.sqrt(u) for
                     i in range(s)]

    # Standardize statistics calculated from observed sample.
    global obs_stats_standard
    obs_stats_standard = (obs_stats - stats_initial_mean)/np.sqrt(u)

def generate_theta_sv(ar):
    """Return a parameter vector draw from the prior for the SV process"""
    theta_out = []

    mu_y = 0
    while abs(mu_y) < 0.5:
        mu_y = np.random.uniform(-6, 6)

    theta_out.append(mu_y)  # Expected return.
    theta_out.append(np.random.uniform(0, 3))  # Mu_s.
    theta_out.append(np.random.uniform(0, 0.7))  # Phi_1
    theta_out.append(np.random.uniform(0, 0.7))  # Phi_2

    # Initialize autoregressive coefficients in lag polynomial.
    #p = [-1, -1, 1]

    #while sum(abs(np.roots(p)) < 1.2) != 0:
        #p = [np.random.randn() for i in range(ar)]
        #p = [np.random.uniform(0, 1)*-1 for i in range(ar)]
        #p.append(1)

    #theta_out.append(-1*p[1])  # Phi_1
    #theta_out.append(-1*p[0])  # Phi_2
    #theta_out.append(-1*p[1])
    #theta_out.append(-1*p[0])
    #theta_out.append(np.random.rand()*2)  # Sigma_eta.

    return theta_out


#https://www.coursera.org/learn/python-network-data/lecture/G1up7/worked-example-json-chapter-13
import urllib.request,urllib.parse,urllib.error
import json
data=urllib.request.urlopen(' http://py4e-data.dr-chuck.net/comments_937754.json').read()
info = json.loads(data)
total=0

for tags in info['comments']:
    total=total+tags['count']
print(total)


from tkinter import *
import collections
import communicationInterface
import widgets

class Terminal():
    # Global variables allow access from anywhere in the program
    # not the best practice, but it made it easier than writing functions with unwieldly parameter lists

    widgets = {}

    labels = []
    num_remember = 100

    # A deque (double-ended queue) allows the program to display
    # the most recent (and most relevant) output from the processor
    # When the size of the deque exceeds num_remember, it deletes the oldest entry
    de = collections.deque([], maxlen=num_remember)

    def __init__(self, tabs, tabIndex):
        self.widgets['Terminal Tab'] = tabs[tabIndex]

        # Enable vertical scroll bar
        self.widgets['Terminal Output Generic Frame'] = Frame(self.widgets['Terminal Tab'])
        self.widgets['Terminal Output Generic Frame'].pack(fill=BOTH, expand=YES)

        self.widgets['Canvas'] = Canvas(self.widgets['Terminal Output Generic Frame'], borderwidth=0, highlightthickness=0)
        self.widgets['Canvas'].pack(side='left', fill='both', expand=True)

        vsb = Scrollbar(self.widgets['Terminal Output Generic Frame'], orient='vertical',
                        command=self.widgets['Canvas'].yview)
        vsb.pack(side='right', fill='y')

        self.widgets['Canvas'].configure(yscrollcommand=vsb.set)

        # Create and display a Terminal Output Frame
        self.widgets['Terminal Output'] = Frame(self.widgets['Canvas'])
        self.widgets['Terminal Output'].pack(fill=BOTH, expand=YES)

        self.widgets['Canvas'].create_window((4, 4), window=self.widgets['Terminal Output'], anchor='nw')
        self.widgets['Terminal Output'].bind('<Configure>', self.onFrameConfigure)

        for i in range(self.num_remember):
            lb = Label(self.widgets['Terminal Output'], width=100, anchor='w')
            self.labels.append(lb)

        # Create and display a Terminal Input Text box
        self.widgets['Terminal Input'] = Text(self.widgets['Terminal Tab'], bg='ghost white',
                                              borderwidth=1, relief=SUNKEN)
        self.widgets['Terminal Input'].pack(fill=BOTH, expand=YES)

        # Bind the <Return> key to the Terminal Input Text box
        # On <Return>, call print_to_terminal()
        self.widgets['Terminal Input'].\
            bind('<Return>', lambda e: communicationInterface.custom_write(self.widgets['Terminal Tab'],
                                                                           self.widgets['Terminal Input'],
                                                                           self.widgets['Terminal Input'].get('1.0', END)))

    def print_to_terminal(self, msg, in_out):
        if msg == None:
            return

        serial_dir = ">> "
        if in_out == 'out':
            serial_dir = "<< "
            serial_dir = serial_dir.encode()

        # msg is a binary string with each line terminated by '\r\n'
        # splitlines() is used to prevent these carriage returns and newlines
        # from appearing in the terminal, by making each line in entry
        # an item in a list; each item in the list is added to the deque, de
        for line in msg.splitlines():
            final_line = serial_dir + line
            self.de.append(final_line)

        # Display each item in de according to its position within de

        for i in range(len(self.de)):
            line = self.de[i]
            widgets.edit_message(self.labels[i], None, -1, line)
            self.labels[i].grid(row=i, column=0)

        self.widgets['Canvas'].yview_moveto(1)

    def onFrameConfigure(self, event):
        self.widgets['Canvas'].configure(scrollregion=self.widgets['Canvas'].bbox('all'))


from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from users.views import getEndBalanceOfMonth, getNextDelivery, getNextDeliveryAfterTomorrow
from datetime import datetime, timedelta
from users.models import ExtraLess, Vacation, Subscription, UserType, User, Product
import pytz
import http.client
from django.conf import settings

class Command(BaseCommand):
    help = 'Creates daily transacion for all customers'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def handle(self, *args, **options):
        customers = User.objects.filter(user_type__payment_type='prepaid', user_type__suspend=True)
        for c in customers:
            last_date = timezone.localtime(timezone.now())
            balance = getEndBalanceOfMonth(last_date, c)
            subs = Subscription.objects.filter(user=c)
            vacations = Vacation.objects.filter(user=c)
            extras = ExtraLess.objects.filter(user=c)
            milk = Product.objects.get(id=1)
            next_delivery = getNextDelivery(subs, vacations, extras)
            tomorrow = (last_date + timedelta(days=1)).strftime("%b %d, %Y")
            if next_delivery is not None:
                if tomorrow == next_delivery['date']:
                    price_for_next_delivery = (milk.price + c.user_type.price_variation) * next_delivery['quantity']

                    # check balance for next delivery
                    if balance < price_for_next_delivery:
                        conn = http.client.HTTPSConnection("api.msg91.com")
                        url = "/api/v5/flow?flow_id=" + settings.ENV('MSG91_FLOW_ID') + "&mobile=91" + c.mobile + "&authkey=" + settings.ENV('MSG91_AUTH_KEY')
                        url = url.replace(" ","")
                        conn.request("POST", url)
                        res = conn.getresponse()
                        data = res.read()
                        print(data.decode("utf-8"))
                        continue

                    # check balance for next 2 delivery
                    balance -= price_for_next_delivery
                    next_delivery = getNextDeliveryAfterTomorrow(subs, vacations, extras)
                    price_for_next_delivery = (milk.price + c.user_type.price_variation) * next_delivery['quantity']
                    if balance < price_for_next_delivery:
                        conn = http.client.HTTPSConnection("api.msg91.com")
                        url = "/api/v5/flow?flow_id=" + settings.ENV('MSG91_FLOW_ID') + "&mobile=91" + c.mobile + "&authkey=" + settings.ENV('MSG91_AUTH_KEY')
                        url = url.replace(" ","")
                        conn.request("POST", url)
                        res = conn.getresponse()
                        data = res.read()
                        print(data.decode("utf-8"))


from ipykernel.kernelbase import Kernel
import os
import subprocess

class Flowjupyter(Kernel):
    implementation = 'flowjupyter'
    implementation_version = '0.1'
    language = 'flow command line'
    language_version = '0.1'
    banner = "Flow kernel - flow jupyter! "

    def __init__(self, **kwargs):
        Kernel.__init__(self, **kwargs)
        self.start_flow(**kwargs)

    def start_flow(self, **kwargs):
            cmd = 'flow emulator start'
            print(subprocess.run(["flow", "emulator", "start"], capture_output=True))

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        if not silent:
            process = subprocess.Popen(code, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            output, error = process.communicate()

            if process.returncode != 0:
                error_message = error.decode()
                self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': error_message})
                return {'status': 'error', 'execution_count': self.execution_count, 'ename': '', 'evalue': '', 'traceback': []}

            output_message = output.decode()
            self.send_response(self.iopub_socket, 'stream', {'name': 'stdout', 'text': output_message})

        return {'status': 'ok', 'execution_count': self.execution_count}

if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=Flowjupyter)


import random
from datetime import datetime

from webapp.trip.models import City
from webapp.trip.utils.get_ticket_prices import get_user_ticket
from webapp.trip.utils.get_recommended_hotels import get_best_hotels

def get_random_city(city_outbound, outbound_date, inbound_date, user_money):
    city_list = [city.ru_name for city in City.query.all()]
    random.shuffle(city_list)
    ticket_count = 0
    for city in city_list:
        if city == city_outbound.lower():
            continue

        tickets = get_user_ticket(city_outbound, city, outbound_date, inbound_date, adults_number=2)

        if tickets:
            ticket_count += 1
            cheapest_ticket = min(tickets["recommended"], key=lambda x: x["price"])
            if cheapest_ticket["forward"]["arrival_date"]:
                checkin = cheapest_ticket["forward"]["arrival_date"]
                checkin = datetime.strptime(checkin, "%d-%m-%Y").strftime("%d/%m/%Y")
            else:
                checkin = outbound_date.strftime("%d/%m/%Y")

            checkout = inbound_date.strftime("%d/%m/%Y")

            money_amount = user_money - cheapest_ticket["price"]
            if money_amount <= 0:
                continue
            try:
                hotels = get_best_hotels(city, checkin, checkout, money_amount)
                if hotels:
                    return city
                else:
                    continue
            except Exception as e:
                print(e)
                continue
        else:
            continue
    if ticket_count == 0:
        return "No tickets"
    else:
        return "Not enough money"


import os
import sys
import inspect
import imp
import pkgutil
import re

import logging
logger = logging.getLogger("differ.plugins")

class Plugins(object):
  def __init__(self):
    self.plugins = {}
    self.paths = {}

  def get_plugins(self):
    """Retrieve all the available plugins in the plugins directory"""
    dir_path = os.path.dirname(inspect.getfile(inspect.currentframe()))
    for root, dirs, files in os.walk(dir_path):
      for fname in files:
        path = os.path.join(root, fname)
        if fname == "__init__.py":
          continue
        exten = os.path.splitext(fname)[1]
        if exten == ".py":
          module = imp.load_source('plugin', path)
          for item in dir(module):
            obj = getattr(module, item)
            if inspect.isclass(obj):
              self.add(obj())

  def add(self, plugin):
    """Add the plugin to the dictionary of plugins"""
    name = plugin.__class__.__name__
    self.plugins[name] = plugin

  def add_parsers(self, parsers):
    """Update the parser dictionary

    :param parsers: A passed in dictionary of parsers
    :ptype parsers: dict
    """
    self.paths.update(parsers)

  def get(self, name):
    """Retrieve a plugin by its name

    :return: A plugin object or None if not found
    """
    item = self.plugins.get(name)
    if not item:
      print("No such plugin by name {}".format(name))
      return None
    return item

  def strip_hook(self, path, full_path):
    """Hook to strip the path passed in

    This will perform stripping on the file only if the path
    matches any of the plugins and if the plugin has the strip_hook
    functionality.

    :param path: The path to check
    :param full_path: The full path of the file to strip
    """
    for path_regex, plugin in self.paths.items():
      if re.match(path_regex, path):
        # Run the strip_hook if it exists
        if getattr(plugin, "strip_hook"):
          plugin.strip_hook(full_path)

PLUGINS = Plugins()
PLUGINS.get_plugins()

# Define default paths
PLUGINS.add_parsers({
  '.*iptables.*': PLUGINS.get("IptablesStrip"),
})


from __future__ import division

import sys
import os
from select                                     import select
import argparse
import ast
import copy

import fnmatch

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot                        as plt

import numpy                                    as np
import pandas                                   as pd

from keras.models                               import load_model

from sklearn.model_selection                    import train_test_split
from sklearn.externals                          import joblib

from modules.control                            import config_file_to_dict
from modules.logger                             import logger
from modules.load_model                         import train_model
from modules.data_preparation                   import get_data_dictionary, preprocess, \
                                                       fix_missing_values, get_sub_dictionary
from modules.utils                              import print_dict, split_dictionary, \
                                                       pause_for_input, get_subsample, \
                                                       print_array_in_dictionary_stats, \
                                                       remove_field_name, flatten_dictionary, \
                                                       special_preprocessing, engineer_features, \
                                                       flatten_feature, shape_data
from modules.file_management                    import OutputManager
from modules.evaluation_plots                   import plot_ROCcurve, plot_MVAoutput, \
                                                       plot_cut_efficiencies, plot_all_features, \
                                                       plot_model_loss, plot_autoencoder_output

def main():

    if (sys.version_info < (3, 0)):
        raise OSError('This program needs python3 to work! Currently {} is used.'.format(
            sys.version_info))

    print('run_mode_user: {}'.format(run_mode_user))
    run_params = config_file_to_dict(config_path + 'run_params.conf')
    data_params = config_file_to_dict(config_path + 'data_params.conf')

    data_params = data_params[run_mode_user]
    run_params = run_params[run_mode_user]

    # here is a collection of variables extracted from the config files
    try:
        # ----------- data-parameters --------------
        branches_dic      = data_params['branches']
        evt_id_string     = data_params['evt_id']
        remove_features   = data_params['remove_features']
        print('evt_id_string: {}'.format(evt_id_string))
        # ------------ run-parameters --------------
        do_standard_scale = run_params['do_standard_scale']
        # ------------ model-parametrs -------------
    except KeyError:
        raise KeyError('The variable names in the main either have a typo ' \
                'or do not exist in the config files!')

    try:
        # uncomment the next line if we want to produce the evt-dictionary with the
        # saved pickle files (the event.pkl, etc)
        # raise TypeError('We want to produce the evt_dic again')
        evt_dictionary = get_data_dictionary(inpath)
        print('\n:: Event dictionary loaded from file: {}'.format(inpath))
        # train only on events with no clusters
        indices = np.arange(evt_dictionary['event']['has_no_calo_clusters'].shape[0])[(evt_dictionary['event']['has_no_calo_clusters']==True).ravel()]
        for key in evt_dictionary.keys():
                evt_dictionary[key] = evt_dictionary[key][indices]

        print('::  Saving the id of all events in {}...'.format(model_dir+"all_evts_id.txt"))
        allfile = open(model_dir+"all_evts_id.txt", 'w')
        for item in evt_dictionary['event'][evt_id_string].ravel().tolist():
            allfile.write("%s\n" % item)

    except(OSError, IOError, TypeError, ValueError):
        raise IOError('The event dictionary cannot be loaded from {}!'.format(inpath))

    evt_dictionary = get_sub_dictionary(evt_dictionary, branches_dic)

    if evt_id_string in evt_dictionary['event'].dtype.names:
        print('{} in event category!'.format(evt_id_string))
        print('{}'.format(evt_dictionary['event'].dtype.names))
    else:
        print('{} NOT in event category!'.format(evt_id_string))
        print('{}'.format(evt_dictionary['event'].dtype.names))
    evt_dictionary['event'], evt_id_np = remove_field_name(evt_dictionary['event'], evt_id_string)
    print('\nType evt-id-np: {}'.format(type(evt_id_np)))
    print(evt_id_np)

    if engineer:
        evt_dictionary, list_of_engineered_features = engineer_features(evt_dictionary, replace=False)
    else:
        list_of_engineered_features = []

    # function that extracts the evt-id from each 'event'-array and puts it into a list
    # we do not need the 99 events any more
    not_99_indices = np.arange(evt_dictionary['target'].shape[0])[evt_dictionary['target']!=99]
    for key in evt_dictionary.keys():
        evt_dictionary[key] = evt_dictionary[key][not_99_indices]

    evt_id_list = list(map(int, evt_id_np.ravel().tolist()))
    # remove a feature if it is in the cut_dic and contains no further info
    branches_dic['event'].remove(evt_id_string)
    print(remove_features)
    for key in evt_dictionary.keys():
        if key == 'target':
            continue
        remove_features = list(set(remove_features))
        # have to remove feature from the remove-list that we have engineered!
        remove_features = [x for x in remove_features if x not in list_of_engineered_features]
        for feature_name in remove_features:
            print('key: {}'.format(key))
            evt_dictionary[key] = remove_field_name(evt_dictionary[key], feature_name)[0]
        print('Features left in {}: {}'.format(key, list(evt_dictionary[key].dtype.names)))

    if evt_id_string in evt_dictionary['event'].dtype.names:
        raise KeyError('Attention, the event id key is still in the data! '\
                'By not removing it the machine will treat it as a feature')
    # if plot:
    #     print('\n::  Plotting the features...')
    #     plot_all_features(evt_dictionary, out_path, real_bg=False)

    ######################################################################################
    # STEP 1:
    # ------------------------------- Preprocessing --------------------------------------
    ######################################################################################
    print('\n::  Standarad scaling...')
    # returns a numpy array (due to fit_transform function)
    preprocess(evt_dictionary, model_dir, load_fitted_attributes=True)

    #####################################################################################

    print('\n::  Converting the data from numpy record arrays to standard numpy arrays...')
    shape_data(evt_dictionary)

    if flat:
        flatten_feature(evt_dictionary, 'track')
        evt_dictionary['feature_matrix'] = np.c_[evt_dictionary.pop('track'),
                                                 evt_dictionary.pop('event')]

    # if 'NN' in run_mode_user:
    #     flatten_feature(evt_dictionary, 'track')
    #     evt_dictionary['feature_matrix'] = np.c_[evt_dictionary.pop('track'),
    #                                              evt_dictionary.pop('event')]
    print_array_in_dictionary_stats(evt_dictionary, 'Training data info:')
    # Get the best model
    model = load_model(model_path)
    ######################################################################################
    # STEP 3:
    # ----------------------------- Evaluating the model ---------------------------------
    ######################################################################################
    # save the test dictionary for easy testing later on

    print('\nEvaluating the model on the training sample...')
    y_target = evt_dictionary.pop('target')
    y_score = model.predict(evt_dictionary)
    if isinstance(y_score, list):
        # if one or several aux-outputs exist the main output is on the
        # first position in the list
        y_score = y_score[0]

    full_recon = 0
    feed_down = 0
    sig_list = []
    for idx, val in enumerate(y_score):
        if val >= mva_cut:
            # evt_id_list is created in a function that will be created in the near future
            sig_list.append(evt_id_list[idx])
            if y_target[idx] == 1:
                full_recon += 1
            else:
                feed_down += 1

    num_trueSignal, num_trueBackgr = plot_MVAoutput(y_target, y_score,
                                                    model_dir, label='classify')
    if num_trueSignal is not None:
        MVAcut_opt = plot_cut_efficiencies(num_trueSignal, num_trueBackgr, model_dir, 'classify')
        plot_ROCcurve(y_target, y_score, model_dir, label='classify')
    del num_trueSignal, num_trueBackgr

    print('::  Cut quality:')
    print('   Signal events after cut: {}/{}'.format(full_recon, full_recon+feed_down))
    print('   Signal efficiency: {}'.format(full_recon/(full_recon+feed_down)))

    print('::  Saving the predicted signal evts in {}...'.format(model_dir+outfile))
    thefile = open(model_dir+outfile, 'w')
    for item in sig_list:
        thefile.write("%s\n" % item)

if __name__ == "__main__":

    output_path = 'output/'
    data_path   = '../data/'
    config_path = 'config/'
    om = OutputManager(output_path, keep_sessions=20)
    sys.stdout = logger(om.get_session_folder())

    # fix random seed for reproducibility
    # np.random.seed(7)

    user_argv = None

    if(len(sys.argv) > 1):
        user_argv = sys.argv[1:]

    # commend line parser (right now not a own function as only 2 elements are used)
    parser = argparse.ArgumentParser()
    parser.add_argument('-inpath', '-evtdicpath', '-evtdic',
                        help='path to the event dictionary',
                        action='store',
                        dest='inpath',
                        default='output/evt_dic.pkl',
                        type=str)

    parser.add_argument('-run_mode', '-run_setting',
                        help='keyword to identify which run settings to \
			      choose from the config file (default: "run_params")',
                        action='store',
                        dest='run_mode',
                        default='NN',
                        type=str)

    parser.add_argument('-modelpath',
                        help='str: path where the model can be found',
                        action='store',
                        dest='model_path',
                        default=None,
                        type=str)

    parser.add_argument('-mva_cut',
                        help='float: cut value that seperates sig-bg',
                        action='store',
                        dest='mva_cut',
                        default=None,
                        type=float)

    parser.add_argument('-outfile',
                        help='str: filename where the evt-ids will be saved',
                        action='store',
                        dest='outfile',
                        default='signal_evt_ids.txt',
                        type=str)

    parser.add_argument('-flat',
                        help='bool: has only an effect if run_mode NN is used \
                              if used then the rnn structure of the tracks is flattend',
                        action='store_true',
                        dest='flat',
                        default=False)

    parser.add_argument('-engineer',
                        help='bool: if engineer_features function should be called',
                        action='store_true',
                        dest='engineer',
                        default=False)

    command_line_args = parser.parse_args(user_argv)

    run_mode_user = command_line_args.run_mode
    model_path = command_line_args.model_path
    mva_cut = command_line_args.mva_cut
    outfile = command_line_args.outfile
    inpath = command_line_args.inpath
    flat = command_line_args.flat
    engineer = command_line_args.engineer

    if not os.path.isfile(inpath) or not inpath.endswith('.pkl'):
        raise IOError('No valid "inpath" for the event dictionary provided!\nPlease do so via ' \
                'command line argument: -inpath /path/to/evt_dic_folder/evt_dic.pkl')

    if not outfile.endswith('.txt'):
        outfile += '.txt'

    if not model_path.endswith('/') and not model_path.endswith('.hdf5'):
        model_path += '/'

    if model_path.endswith('/'):
        model_path += 'best_model.h5'

    model_dir = os.path.split(model_path)[0]+'/'

    if model_path is None or not os.path.isfile(model_path):
        raise IOError('No valid "model_path" provided!\nPlease do so via ' \
                'command line argument: -modelpath /path/to/modelfolder/model_file.hdf5')

    if not mva_cut or mva_cut < 0. or mva_cut > 1.:
        raise IOError('MVA cut-value has no valid value: {}'.format(mva_cut))

    main()


from tkinter import*
from PIL import Image,ImageTk
from face_recognition import Face_Recognition
from attendance import Attendance
from Qr_code import Qr_Code

class User:
    def __init__(self,root):
        self.root=root
        self.root.geometry("1000x600+200+50")
        self.root.title("QR Generator")

        #====Background Image
        img3=Image.open(r"college_images\bg.jpg")
        img3=img3.resize((1360,768),Image.ANTIALIAS)
        self.photoimg3=ImageTk.PhotoImage(img3)

        bg_img=Label(self.root,image=self.photoimg3)
        bg_img.place(x=0,y=60,width=1360,height=768)

        title=Label(self.root,text="Student Attendance",font=("times new roman",40),bg='blue',fg='white').place(x=0,y=0,relwidth=1)

        #====STUDENT BUTTOM=========
        img4=Image.open(r"college_images\face_rec.jpg")
        img4=img4.resize((300,300),Image.ANTIALIAS)
        self.photoimg4=ImageTk.PhotoImage(img4)

        b1=Button(bg_img,image=self.photoimg4,command=self.face_data,cursor="hand2")
        b1.place(x=150,y=160,width=300,height=300)

        b1_1=Button(bg_img,text="Face Detector",command=self.face_data,cursor="hand2",font=("times new roman",15,"bold"),bg="white",fg="dark blue")
        b1_1.place(x=150,y=460,width=300,height=40)

        #====DETECT FACE BUTTOM=========
        img5=Image.open(r"college_images\qr1.jpg")
        img5=img5.resize((300,300),Image.ANTIALIAS)
        self.photoimg5=ImageTk.PhotoImage(img5)

        b1=Button(bg_img,image=self.photoimg5,cursor="hand2",command=self.open_qr_code)
        b1.place(x=550,y=160,width=300,height=300)

        b1_1=Button(bg_img,text="QR Code",cursor="hand2",command=self.open_qr_code,font=("times new roman",15,"bold"),bg="white",fg="dark blue")
        b1_1.place(x=550,y=460,width=300,height=40)

    def face_data(self):
        self.new_window=Toplevel(self.root)
        self.app=Face_Recognition(self.new_window)

    def attendance_data(self):
        self.new_window=Toplevel(self.root)
        self.app=Attendance(self.new_window)

    def open_qr_code(self):
        self.new_window=Toplevel(self.root)
        self.app=Qr_Code(self.new_window)

if __name__ == "__main__":
    root=Tk()
    obj=User(root)
    root.mainloop()


import re
from collections import defaultdict,Counter
import time
import pickle

no_of_lines = 0
count_words=0
try:
    file = "address"
except FileNotFoundError:
    print('Correct path of file was not provided')
#count_unique_words = Counter()
#count_unigrams = Counter()
#count_bigrams = Counter()

count_unique_words = defaultdict(int) # dicts used for counting
count_unigrams = defaultdict(int)
count_bigrams = defaultdict(int)

string = "" #this is set zero before it becomes <word> after each line is processed
perf_start = time.perf_counter() #time counters
process_start = time.process_time()

with open(file) as f:
    for line in f:
        list_angular = [] #list of words with angular brackets
        no_of_lines+=1
        modified_line = re.sub(r'^\w+',' ',line) # applying regex for each line in text file and retrieving only words == [a-zA-Z]
        modified_line = re.sub(r'[^a-z]', ' ',modified_line, flags=re.IGNORECASE) # taking out any character which is not a ASCII letter
        modified_lower = modified_line.lower() # converting to lower case
        modified_line_lower = modified_lower.split() # tokenizing, this has better performance than --> tokenize = nltk.word_tokenize(modified_line.lower())
        for w in range(len(modified_line_lower)):
            list_angular.append('<'+modified_line_lower[w]+'>')   #placing angular brackets before the beginning and end of each word
        string = ' '.join(list_angular) # making it a string from list of strings
        for w in modified_line_lower:
            count_words+=1            # counting number of words in the corpus
        for w in modified_line_lower:
            count_unique_words[w]+=1  # unique words in corpus
        for w in re.findall(r'(?=([a-z\<\>]{1}))',string):
            count_unigrams[w]+=1 # unigram count of each word
        for w in re.findall(r'(?=([a-z\<\>]{2}))',string):
            count_bigrams[w]+=1   #bigram count of each word

perf_stop = time.perf_counter() # time counter end
process_stop = time.process_time()
total_time = perf_stop - perf_start
total_cpu_time = process_stop - process_start
print('\n\n')
print('start at elapsed time:{:>5.2f}, cpu time:{:>10}'.format(perf_start,process_start))
print('finish reading at elapsed time:{:>10.2f}, cpu time:{:>10}'.format(perf_stop,process_stop))
print('total elapsed time:{:>5.2f}, cpu time:{:>10}'.format(total_time,total_cpu_time))
print('\n\n')

#class Singleton1:
#     _shared_state = {}
#     def __init__(self):
#        self.__dict__ = self._shared_state
#
#class Singleton2(Singleton1):
#    def __init__(self, arg):
#        Singleton1.__init__(self)
#        self.val = arg
#    def __dict__(self):
#        return self.val
class Spell_correction_singleton:
      def __init__(self,count_unique,count_unigram,count_bigram):
          self.a = count_unique
          self.b = count_unigram
          self.c = count_bigram

object_1 = Spell_correction_singleton(count_unique_words,count_unigrams,count_bigrams)
 #object_1 is the only object of singleton class

output = open('pickled_data.dat', 'wb') #pickling the data structures
pickle.dump(object_1, output,-1)
output.close()

print("Number of words: ",count_words)
print("Number of types (distinct words): ",len(count_unique_words))
print('\n')
print('{:<8}'.format("Unigrams"),'{:>8}'.format("Count"))
for (key,value) in sorted(count_unigrams.items()):
    print("{:<8} {:>10}".format(key,value))
print('\n')

print('{:<8}'.format("Bigrams"),'{:>8}'.format("Count"))
for (key,value) in sorted(count_bigrams.items()):
    print("{:<8} {:>10}".format(key,value))
print('\n')

print("10 most frequent words: ")
print('\n')
print('{:<8}'.format("Word"),'{:>8}'.format("Count"))
for (key,value) in (Counter(count_unique_words).most_common(10)):
    print("{:<8} {:>10}".format(key,value))
print('\n')


from datetime import datetime
import boto3
import cfnresponse
import json

client = boto3.client("cloudwatch")

def log(stack, metric, value):
    # Do it for the stack
    client.put_metric_data(
        Namespace="CloudFormation",
        MetricData=[
            {
                "MetricName": metric,
                "Unit": "Count",
                "Value": value,
                "Timestamp": datetime.now(),
                "Dimensions": [
                    {
                        "Name": "By Stack Name",
                        "Value": stack,
                    },
                ],
            },
        ],
    )

    client.put_metric_data(
        Namespace="CloudFormation",
        MetricData=[
            {
                "MetricName": metric,
                "Unit": "Count",
                "Value": value,
                "Timestamp": datetime.now(),
            },
        ],
    )

def handler(event, context):
    print("Received request:", json.dumps(event, indent=4))

    action = event["RequestType"]

    stack = event["ResourceProperties"]["StackName"]
    resources = int(event["ResourceProperties"]["ResourceCount"])

    try:
        log(stack, action, 1)

        if action == "Create":
            log(stack, "ResourceCount", resources)

        cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "{} metrics".format(stack))
    except Exception as e:
        cfnresponse.send(event, context, cfnresponse.FAILED, {
            "Data": str(e),
        }, "{} metrics".format(stack))


from aeternity.signing import Account
from aeternity import defaults, identifiers, hashing, utils
import pytest
import random
# from aeternity.exceptions import TransactionNotFoundException

blind_auth_contract = """contract BlindAuth =
      record state = { owner : address }

      entrypoint init(owner' : address) = { owner = owner' }

      stateful entrypoint authorize(r: int) : bool =
        // r is a random number only used to make tx hashes unique
        switch(Auth.tx_hash)
          None          => abort("Not in Auth context")
          Some(tx_hash) => true

      entrypoint to_sign(h : hash, n : int) : hash =
        Crypto.blake2b((h, n))
    """

def test_node_spend_native(chain_fixture):
    node_cli = chain_fixture.NODE_CLI
    sender_account = chain_fixture.ALICE
    recipient_id = Account.generate().get_address()
    # with numbers
    tx = node_cli.spend(sender_account, recipient_id, 100)
    print("DATA", tx)
    assert recipient_id == tx.data.tx.data.recipient_id
    assert sender_account.get_address() == tx.data.tx.data.sender_id
    account_balance = node_cli.get_account_by_pubkey(pubkey=recipient_id).balance
    assert account_balance == 100
    # spend some string
    tx = node_cli.spend(sender_account, recipient_id, "0.5ae")

    assert recipient_id == tx.data.tx.data.recipient_id
    assert sender_account.get_address() == tx.data.tx.data.sender_id

    account_balance = node_cli.get_account_by_pubkey(pubkey=recipient_id).balance
    assert account_balance == 500000000000000100

def test_node_spend_burst(chain_fixture):
    sender_account = chain_fixture.ALICE
    # make a new non blocking client
    ae_cli = chain_fixture.NODE_CLI
    ae_cli.config.blocking_mode = False
    # recipient account
    recipient_account = Account.generate().get_address()
    # send 50 consecutive spend
    ths = []
    print(">>"*20, "spend burst start")
    for i in range(50):
        tx = ae_cli.spend(sender_account, recipient_account, "1AE")
        ths.append(tx.hash)
    print("<<"*20, "spend burst end")

    for th in ths:
        ae_cli.wait_for_transaction(th)

    assert(ae_cli.get_balance(recipient_account) == utils.amount_to_aettos("50ae"))

@pytest.mark.parametrize("height,protocol_version", [(0, 1), (1, 1), (2, 2), (3, 2), (4, 3), (5, 3)])
def test_node_get_protocol_version(chain_fixture, height, protocol_version):
    # this test assumes that the configuration of the node bein tested has the follwing configuration:
    #   hard_forks:
    #      "1": 0
    #      "2": 2
    #      "3": 4
    assert(chain_fixture.NODE_CLI.get_consensus_protocol_version(height)) == protocol_version

def test_node_ga_attach(chain_fixture, compiler_fixture):
    ae_cli = chain_fixture.NODE_CLI
    account = chain_fixture.ALICE
    c_cli = compiler_fixture.COMPILER
    # test that the account is not already generalized
    poa_account = ae_cli.get_account_by_pubkey(pubkey=account.get_address())
    assert poa_account.kind == identifiers.ACCOUNT_KIND_BASIC
    # transform the account
    # compile the contract
    ga_contract = c_cli.compile(blind_auth_contract).bytecode
    # now encode the call data
    init_calldata = c_cli.encode_calldata(blind_auth_contract, "init", account.get_address()).calldata
    # this will return an object
    # init_calldata.calldata
    # now we can execute the transaction
    tx = ae_cli.account_basic_to_ga(account, ga_contract, calldata=init_calldata)

    # now check if it is a ga
    ga_account = ae_cli.get_account_by_pubkey(pubkey=account.get_address())
    assert ga_account.kind == identifiers.ACCOUNT_KIND_GENERALIZED

def test_node_ga_meta_spend(chain_fixture, compiler_fixture):

    ae_cli = chain_fixture.NODE_CLI
    account = chain_fixture.ALICE
    c_cli = compiler_fixture.COMPILER
    # make the account poa

    # transform the account
    # compile the contract
    ga_contract = c_cli.compile(blind_auth_contract).bytecode
    # now encode the call data
    init_calldata = c_cli.encode_calldata(blind_auth_contract, "init", account.get_address()).calldata
    # this will return an object
    # init_calldata.calldata
    # now we can execute the transaction
    tx = ae_cli.account_basic_to_ga(account, ga_contract, calldata=init_calldata)

    print("ACCOUNT is now GA", account.get_address())

    # retrieve the account data
    ga_account = ae_cli.get_account(account.get_address())
    # create a dummy account
    recipient_id = Account.generate().get_address()

    # generate the spend transactions
    amount = 54321
    payload = "ga spend tx"
    fee = defaults.FEE
    ttl = defaults.TX_TTL

    spend_tx = ae_cli.tx_builder.tx_spend(account.get_address(), recipient_id, amount, payload, fee, ttl, 0)

    # encode the call data for the transaction
    calldata = c_cli.encode_calldata(blind_auth_contract, ga_account.auth_fun, hashing.randint()).calldata
    # now we can sign the transaction (it will use the auth for do that)
    spend_tx = ae_cli.sign_transaction(ga_account, spend_tx, auth_data=calldata)
    # broadcast
    tx_hash = ae_cli.broadcast_transaction(spend_tx)
    print(f"GA_META_TX {tx_hash}")
    # check that the account received the tokens
    assert ae_cli.get_account_by_pubkey(pubkey=recipient_id).balance == amount


from jobs.search_transcript_job import SearchTranscriptJob
from jobs.clean_video_job import CleanVideoJob
from extensions import aai
from jobs.transcribe_youtube_job import TranscribeYoutubeJob
from jobs.transcribe_upload_job import TranscribeUploadJob
from jobs.yt_download_job import YT_DownloadJob
from werkzeug.utils import secure_filename
import events
from os import getenv
from flask import render_template, request, redirect, url_for, Blueprint, flash
from jobs.filtering_job import FilteringJob
from jobs import rq
import time

http = Blueprint("http", __name__)
ws   = Blueprint("ws", __name__)

default_q = rq.get_queue()

@http.route('/')
def index():
    return render_template("index.html")

@http.route('/webhook', methods=["POST"])
def webhook():
    print('Webhook Received')
    request_json = request.json

    filename = request.args.get("vid_path")

    trans_id = request_json['transcript_id']
    words = request.args.get('words')

    SearchTranscriptJob.queue(trans_id, words, filename)

    return 'Webhook notification received', 200

@http.route('/upload', methods=['POST', 'GET'])
def upload():
    if request.method == 'POST':
        url = request.form['url']
        words = request.form['words']

        fname = str(int(time.time())) + '.mp4'

        if url == "":
            file = request.files['file']
            if file.filename == "":
                flash("No video given")
                return redirect(request.url)
            file.save(secure_filename('/data/' + file.filename))
            job = TranscribeUploadJob.queue('/data/' + file.filename, words)
        else:
            YT_DownloadJob.queue(url, fname)
            job = TranscribeYoutubeJob.queue(url, words, "/data/" + fname)

        # make call to service to get the video
        # job = FilteringJob.queue(url, events.filter_success, meta={"upload_url": url})
        return redirect(url_for("http.show_uploads", job_id=job.id))
    else:
        return render_template("upload/new.html")

@http.route('/uploads/<job_id>')
def show_uploads(job_id):
    job = default_q.fetch_job(job_id)
    if job != None:
        return render_template("upload/show.html", job_id=job_id, job_status=job.get_status(), base_url=getenv("BASE_URL"))
    return "no job", 200


from __future__ import absolute_import #NEEDS TO STAY AS TOP LEVEL MODULE FOR Py2-3 COMPATIBILITY
import vtk
from vmtk import vtkvmtk
import sys

from vmtk import pypes

class vmtkICPRegistration(pypes.pypeScript):

    def __init__(self):

        pypes.pypeScript.__init__(self)

        self.ReferenceSurface = None
        self.Surface = None
        self.DistanceArrayName = ''
        self.SignedDistanceArrayName = ''
        self.FarThreshold = 0.0
        self.Level = 0.0
        self.MaximumMeanDistance = 1E-2
        self.MaximumNumberOfLandmarks = 1000
        self.MaximumNumberOfIterations = 100
        self.Matrix4x4 = None
        self.MatrixCoefficients = None

        self.FlipNormals = 0

        self.SetScriptName('vmtkicpregistration')
        self.SetScriptDoc('register a surface to a reference surface using the ICP algorithm')
        self.SetInputMembers([
            ['Surface','i','vtkPolyData',1,'','the input surface','vmtksurfacereader'],
            ['ReferenceSurface','r','vtkPolyData',1,'','the reference surface','vmtksurfacereader'],
            ['DistanceArrayName','distancearray','str',1,'','name of the array where the distance of the input surface to the reference surface has to be stored'],
            ['SignedDistanceArrayName','signeddistancearray','str',1,'','name of the array where the signed distance of the input surface to the reference surface is stored; distance is positive if distance vector and normal to the reference surface have negative dot product, i.e. if the input surface is outer with respect to the reference surface'],
            ['FarThreshold','farthreshold','float',1,'','threshold distance beyond which points are discarded during optimization'],
            ['FlipNormals','flipnormals','bool',1,'','flip normals to the reference surface after computing them'],
            ['MaximumNumberOfLandmarks','landmarks','int',1,'','maximum number of landmarks sampled from the two surfaces for evaluation of the registration metric'],
            ['MaximumNumberOfIterations','iterations','int',1,'','maximum number of iterations for the optimization problems'],
            ['MaximumMeanDistance','maxmeandistance','float',1,'','convergence threshold based on the maximum mean distance between the two surfaces']
            ])
        self.SetOutputMembers([
            ['Surface','o','vtkPolyData',1,'','the output surface','vmtksurfacewriter'],
            ['MatrixCoefficients','omatrixcoefficients','float',16,'','the output transform matrix coefficients'],
            ['Matrix4x4','omatrix4x4','vtkMatrix4x4',1,'','the output transform matrix']
            ])

    def Execute(self):

        if self.Surface == None:
            self.PrintError('Error: No Surface.')

        if self.ReferenceSurface == None:
            self.PrintError('Error: No ReferenceSurface.')

##         if (self.SignedDistanceArrayName != '') & (self.ReferenceSurface.GetPointData().GetNormals() == None):
        if (self.SignedDistanceArrayName != ''):
            normalsFilter = vtk.vtkPolyDataNormals()
            normalsFilter.SetInputData(self.ReferenceSurface)
            normalsFilter.AutoOrientNormalsOn()
            normalsFilter.ConsistencyOn()
            normalsFilter.SplittingOff()
            normalsFilter.SetFlipNormals(self.FlipNormals)
            normalsFilter.Update()
            self.ReferenceSurface.GetPointData().SetNormals(normalsFilter.GetOutput().GetPointData().GetNormals())

        self.PrintLog('Computing ICP transform.')

        icpTransform = vtkvmtk.vtkvmtkIterativeClosestPointTransform()
        icpTransform.SetSource(self.Surface)
        icpTransform.SetTarget(self.ReferenceSurface)
        icpTransform.GetLandmarkTransform().SetModeToRigidBody()
        icpTransform.StartByMatchingCentroidsOn()
        icpTransform.CheckMeanDistanceOn()
        icpTransform.SetMaximumNumberOfLandmarks(self.MaximumNumberOfLandmarks)
        icpTransform.SetMaximumNumberOfIterations(self.MaximumNumberOfIterations)
        icpTransform.SetMaximumMeanDistance(self.MaximumMeanDistance)
        if self.FarThreshold > 0.0:
            icpTransform.UseFarThresholdOn()
            icpTransform.SetFarThreshold(self.FarThreshold)
        else:
            icpTransform.UseFarThresholdOff()

        transformFilter = vtk.vtkTransformPolyDataFilter()
        transformFilter.SetInputData(self.Surface)
        transformFilter.SetTransform(icpTransform)
        transformFilter.Update()

        self.PrintLog('Mean distance: '+str(icpTransform.GetMeanDistance()))

        self.Surface = transformFilter.GetOutput()
        self.Matrix4x4 = icpTransform.GetMatrix()

        matrix = self.Matrix4x4
        self.MatrixCoefficients = [
            matrix.GetElement(0,0), matrix.GetElement(0,1), matrix.GetElement(0,2), matrix.GetElement(0,3),
            matrix.GetElement(1,0), matrix.GetElement(1,1), matrix.GetElement(1,2), matrix.GetElement(1,3),
            matrix.GetElement(2,0), matrix.GetElement(2,1), matrix.GetElement(2,2), matrix.GetElement(2,3),
            matrix.GetElement(3,0), matrix.GetElement(3,1), matrix.GetElement(3,2), matrix.GetElement(3,3)]
        self.PrintLog('Transform matrix:\n  %f %f %f %f\n  %f %f %f %f\n  %f %f %f %f\n  %f %f %f %f' % tuple(self.MatrixCoefficients))

        if (self.DistanceArrayName != '') | (self.SignedDistanceArrayName != ''):
            self.PrintLog('Computing distance.')
            surfaceDistance = vtkvmtk.vtkvmtkSurfaceDistance()
            surfaceDistance.SetInputData(self.Surface)
            surfaceDistance.SetReferenceSurface(self.ReferenceSurface)
            if (self.DistanceArrayName != ''):
                surfaceDistance.SetDistanceArrayName(self.DistanceArrayName)
            if (self.SignedDistanceArrayName != ''):
                surfaceDistance.SetSignedDistanceArrayName(self.SignedDistanceArrayName)
            surfaceDistance.Update()
            self.Surface = surfaceDistance.GetOutput()

if __name__=='__main__':
    main = pypes.pypeMain()
    main.Arguments = sys.argv
    main.Execute()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

from werkzeug.contrib.atom import AtomFeed
from flask import request, Blueprint, json

from zhihudaily.cache import cache
from zhihudaily.configs import Config
from zhihudaily.models import Zhihudaily
from zhihudaily.utils import Date
from zhihudaily.crawler import Crawler
from zhihudaily._compat import urljoin

# This is the feeds blueprint, but it is deprecated now.
feeds = Blueprint('feeds', __name__, template_folder='templates')
redis_server = Config.redis_server

@feeds.route('/feeds')
@cache.cached(timeout=1200)
def generate_feed():
    """Code snippet from https://flask.pocoo.org/snippets/10/"""
    day = Date()
    feed = AtomFeed('Zhihudaily', feed_url=request.url, url=request.url_root)
    news = Zhihudaily.get(Zhihudaily.date == int(day.today))

    articles = json.loads(news.json_news)
    for article in articles:
        if redis_server.get(article['url']):
            body = redis_server.get(article['url']).decode('utf-8')
        else:
            body = Crawler().send_request(article['url']).json()['body']
            redis_server.setex(article['url'], (60*60*24), body)
        feed.add(article['title'], body,
                 content_type='html',
                 author='zhihudaily',
                 url=urljoin(request.url_root, article['url']),
                 updated=day.now)
    return feed.get_response()


import os
import numpy as np
from PIL import Image
import torch
from net import Net
import json, base64

import isolation_util

BATCH_SIZE = 32

class Isolation(object):

    def __init__(self, checkpoint, gpu_id=0):
        assert os.path.exists(checkpoint)
        self.net = Net(num_classes=5)
        self.net.load_state_dict(torch.load(checkpoint, map_location=lambda storage, loc: storage))

        if torch.cuda.is_available():
            self.net.cuda(device=gpu_id)
            self.dtype_float = torch.cuda.FloatTensor
        else:
            self.dtype_float = torch.FloatTensor
        self.net.eval()
        self.softmax_func = torch.nn.Softmax()

    def eval(self, image):
        image_var = self._image_preprocess(image)
        scores = self.net(image_var)
        scores = self.softmax_func(scores)
        scores = scores.data.numpy()[0].tolist()
        rsp = {
            "Isolated on a white background": scores[0],
            "Isolated on a non-white background": scores[1],
            "Isolated on a contextual background": scores[2],
            "Partial isolation": scores[3],
            "Non-isolation": scores[4]
        }
        return rsp

    def model(self, image_batch):
        self.image_var = torch.autograd.Variable(torch.Tensor(len(image_batch), 3, 224, 224)).type(self.dtype_float).cuda()
        self.image_var.data.copy_(image_batch)
        scores = self.net(self.image_var)
        scores = self.softmax_func(scores)
        scores = scores.cpu().data.numpy()
        return scores

    def preprocess(self, pil_image):
        #pil_image = isolation_util.bytes_to_pil_image(raw_image_bytes)
        image = isolation_util.preprocess_image(pil_image)
        return image

if __name__ == '__main__':
    image_path = 'test.jpg'
    checkpoint = 'isolation_20171204.pth'
    isolation = Isolation(checkpoint=checkpoint)
    with open('test-chunk.json', 'r') as f:
        items = list(f)

    image_batch = []
    for item in items:
        item = json.loads(item)
        raw_image_bytes = base64.b64decode(item['image'])
        image_batch.append(isolation.preprocess(raw_image_bytes))

    images = torch.squeeze(torch.stack(image_batch))
    for i in range(0, images.shape[0], BATCH_SIZE):
        print('processing batch {}'.format(i // 32))
        batch = images[i:i+BATCH_SIZE]
        scores = isolation.model(batch)
        print(scores.shape, scores[0])


from typing import List, Dict
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from backprop.models import HFModel
import torch

class HFNLIModel(HFModel):
    """
    Class for Hugging Face sequence classification models trained on a NLI dataset

    Attributes:
        model_path: path to HF model
        tokenizer_path: path to HF tokenizer
        name: string identifier for the model. Lowercase letters and numbers.
            No spaces/special characters except dashes.
        description: String description of the model.
        tasks: List of supported task strings
        details: Dictionary of additional details about the model
        model_class: Class used to initialise model
        tokenizer_class: Class used to initialise tokenizer
        device: Device for model. Defaults to "cuda" if available.
    """
    def __init__(self, model_path=None, tokenizer_path=None, name: str = None,
                description: str = None, tasks: List[str] = None, details: Dict = None,
                model_class=AutoModelForSequenceClassification,
                tokenizer_class=AutoTokenizer, device=None):
        tasks = tasks or ["text-classification"]

        HFModel.__init__(self, model_path, name=name, description=description,
                    tasks=tasks, details=details, tokenizer_path=tokenizer_path,
                    model_class=model_class, tokenizer_class=tokenizer_class,
                    device=device)

    @torch.no_grad()
    def __call__(self, task_input, task="text-classification"):
        """
        Uses the model for the text-classification task

        Args:
            task_input: input dictionary according to the ``text-classification`` task specification.
                Needs labels (for zero-shot).
            task: text-classification
        """
        if task == "text-classification":
            is_list = False

            text = task_input.get("text")
            labels = task_input.get("labels")
            top_k = task_input.get("top_k", 10000)

            if labels == None:
                raise ValueError("labels must be provided")

            if isinstance(text, list):
                is_list = True
            else:
                text = [text]
                labels = [labels]

            # Must have a consistent amount of examples
            assert(len(text) == len(labels))

            probs = self.classify(text, labels, top_k)

            if not is_list:
                probs = probs[0]

            return probs
        else:
            raise ValueError(f"Unsupported task: {task}")

    @staticmethod
    def list_models():
        from .models_list import models

        return models

    def calculate_probability(self, text, labels):
        batch_features = []

        hypothesis = [f"This example is {l}." for l in labels]
        features = self.tokenizer([text]*len(hypothesis), hypothesis, return_tensors="pt",
                                    truncation=True, padding=True).to(self._model_device)

        logits = self.model(features["input_ids"], features["attention_mask"])[0]
        entail_contradiction_logits = logits[:, [0, 2]]
        probs = entail_contradiction_logits.softmax(dim=1)
        prob_label_is_true = probs[:, 1]
        return prob_label_is_true.tolist()

    def classify(self, text, labels, top_k):
        """
        Classifies text, given a set of labels.
        """
        probabilities = []
        for text, labels in zip(text, labels):
            probs = {}
            probs_list = self.calculate_probability(text, labels)

            for prob, label in zip(probs_list, labels):
                probs[label] = prob

            probs = sorted(probs.items(), key=lambda x: x[1], reverse=True)
            probs = {k: v for k, v in probs[:top_k]}

            probabilities.append(probs)

        return probabilities


from collections import deque

def solution(priorities, location):
    result = 1
    q = deque()
    for i in range(len(priorities)):
        q.append((i, priorities[i]))
    while True:
        inx, prio = q.popleft()
        if prio >= max(q, key=lambda x: x[1])[1]:
            if inx == location:
                break
            result += 1
        else:
            q.append((inx, prio))

    return result


"""This module defines the :class:`QueueFile` class."""

import queue
import io
import itertools

class QueueFile:
    """This class wraps a :class:`queue.Queue` object in the interface
    of a file-like object.

    The :class:`QueueFile` class makes redirecting input and output
    between different parts of a program easier than it would be using
    standard file objects, such as :class:`io.StringIO` or those
    created by :func:`pty.openpty`.
    :class:`QueueFile` is thread-safe, can be read from and written to
    at the same time, and its blocking behaviour can be easily
    controlled.
    (If there is nothing to read, reading from :class:`io.StringIO`
    never blocks, but reading from an object created with
    :func:`pty.openpty` always does.)

    Attributes:
        queue (queue.Queue):
            The wrapped :class:`~queue.Queue` object.
            Strings are saved in the queue one character at a time.

        block (bool):
            A flag indicating whether :meth:`read` and
            :meth:`readline` should block or not.
    """

    def __init__(self, block=False):
        """Intialize a new :class:`QueueFile` with :attr:`block` set
        to the value given as an argument.
        """
        self.queue = queue.Queue()
        self.block = block

    def write(self, string):
        """Write *string* to the queue and return the number of
        characters written.

        This method emulates :meth:`io.TextIOBase.write`.
        """
        # io.TextIOBase.write() doesn't accept keyword arguments,
        # so we can safely change the argument name from *s* to
        # *string* without breaking anything.
        for char in string:
            self.queue.put(char)
        return len(string)

    def read(self, size=-1):
        """Read and return a string of *size* characters from the queue.

        If the queue contains less than *size* characters or *size* is
        ``None`` or less than ``1``, all characters in the queue are
        read.

        This method emulates :meth:`io.TextIOBase.read`.
        """
        return self._read(size)

    def readline(self, size=-1):
        """Like :meth:`read`, but reading characters is stopped if a
        newline character is encountered. If this happens, the string
        returned includes that newline character.

        This method emulates :meth:`io.TextIOBase.readline`.
        """
        return self._read(size, endchar='\n')

    def _read(self, size=-1, endchar=None):
        """This method provides the functionality shared by
        read() and readline().
        """
        buffer = io.StringIO()

        if not size or size < 0:
            # An infinite generator that yields 0,1,2,...
            iterations = itertools.count()
        else:
            iterations = range(size)

        # Read a finite or an infinite number of characters.
        for _ in iterations:
            try:
                char = self.queue.get(block=self.block)
                buffer.write(char)
            except queue.Empty:
                break

            if endchar and char == endchar:
                break

        return buffer.getvalue()

    def flush(self):
        """This method does nothing, but its existence makes it
        possible to :func:`print` to this object with ``flush=True``
        without raising an :class:`AttributeError`."""


#find links (city name + page number)

import requests
from bs4 import BeautifulSoup
from lxml import etree

#find option tags for cities
def cities(type):
    url = f'https://fidilio.com/{type}/'
    reqs = requests.get(url)
    soup = BeautifulSoup(reqs.text, 'html.parser')
    urls = []
    cities=[]
    for link in soup.find_all('option'):
        if(link.get('value')=='0'):
            break
        cities.append(link.get('value'))

    return cities

def cities_page_number(type):
    cities_=cities(type)
    res=dict()
    for city in cities_:
        headers = {'User-Agent': 'Mozilla/5.0'}
        response = requests.get(f"https://fidilio.com/{type}/in/"+city, headers = headers)
        soup = BeautifulSoup(response.text,'html.parser')
        dom = etree.HTML(str(soup))
        pages = dom.xpath('/html/body/div[2]/div/section[1]/section/div[2]/div/div/div[1]/div[2]/div/a')[-1]
        max_number = ((pages.values())[0].split('?')[1].split('=')[1])
        res[city] = max_number
    return res


#i created this
from django.http import HttpResponse
from django.shortcuts import render
#def index(request):
    #return HttpResponse('''
hello Agnisha
Django coderAgnisha''')
def index(request):
    return render(request, 'index.html')
    #return HttpResponse("
home
")
#personal navigator
def ex1(request):
    s='''
Navigation bar

    video

    mail
'''
    return HttpResponse(s)
def analyse(request):
    djtext=request.POST.get('text','default')
    remove_punc=request.POST.get('analyse','default')
    capfirst=request.POST.get('capfirst','default')
    unique=request.POST.get('unique','default')
    print(djtext)
    print(remove_punc)
    print(capfirst)
    #return HttpResponse("
remove punctuation
")
    #analyse=djtext
    if remove_punc=="on":
        punctuation='''"";:?''></.,{}[]|\+=!@#$%^&*()'''
        analyse=""
        for char in djtext:
            if char not in punctuation:
                analyse=analyse + char
        params={'purpose':"removed punctuation",'analysed_text':analyse}
        djtext=analyse
        #return render(request, 'analyse.html', params)
    if capfirst=="on":
        ana=""
        for i in djtext:
            ana=ana+ i.capitalize()
        params={'purpose':"capitalised",'analysed_text':ana}
        #return render(request, 'analyse.html',params)
        djtext=ana
    if unique=="on":
        def check_unique(str):
            for i in range(len(str)):
                for j in range(i + 1,len(str)):
                    if(str[i] == str[j]):
                        return False
                return True
        x=check_unique(djtext)
        params={'purpose':"whether it is unique or not",'analysed_text':x,"originaltext":djtext,"abc":"yes"}
        #return render(request, 'analyse.html', params)
        return render(request, 'analyse.html', params)
    if (remove_punc!="on" and capfirst!="on" and unique!="on"):
     return HttpResponse("error")
'''def cap_first(request):
    return HttpResponse("
capitalise first
")
def newlineremove(request):
    return HttpResponse("
remove newline
back")
def spaceremove(request):
    return HttpResponse("
spaceremove
")'''