# Copyright (c) 2009-2016 Simon van Heeringen <simon.vanheeringen@gmail.com>
#
# This module is free software. You can redistribute it and/or modify it under
# the terms of the MIT License, see the file COPYING included with this
# distribution.

"""
Module to compare DNA sequence motifs (positional frequency matrices)
"""
from __future__ import print_function

# Python imports
import sys
import os
import random
import logging

# External imports
from scipy.stats import norm,entropy,chi2_contingency
from scipy.spatial import distance
import numpy as np

# GimmeMotifs imports
from gimmemotifs.config import MotifConfig
from gimmemotifs.c_metrics import pwmscan,score
from gimmemotifs.motif import parse_motifs
# pool import is at the bottom

try:
    import copy_reg
    import types
    def _pickle_method(m):
        if m.im_self is None:
            return getattr, (m.im_class, m.im_func.func_name)
        else:
            return getattr, (m.im_self, m.im_func.func_name)

    copy_reg.pickle(types.MethodType, _pickle_method)
except:
    pass

# Create random sequence
nucs = []
L = 10 ** 4
for i in range(L):
    nucs.append(random.choice(['A', 'C', 'T', 'G']))
RANDOM_SEQ = "".join(nucs)

# Function that can be parallelized
def _get_all_scores(mc, motifs, dbmotifs, match, metric, combine, pval):

    try:
        scores = {}
        for m1 in motifs:
            scores[m1.id] = {}
            for m2 in dbmotifs:
                scores[m1.id][m2.id] = mc.compare_motifs(m1, m2, match, metric, combine, pval=pval)
        return scores
    except Exception:
        logging.exception("_get_all_scores failed")

def akl(x,y):
    """ Calcylates the average Kullback-Leibler similarity
    See Mahony,  2007
    """
    return 10 - (entropy(x,y) + entropy(y,x)) / 2.0

def chisq(x,y):
    return chi2_contingency([x, y])[1]

def ssd(x,y):
    """ Sum of squared distances
    """
    return np.sum([(a-b)**2 for a,b in zip(x,y)] )

def seqcor(m1, m2, seq=None):
    l1 = len(m1)
    l2 = len(m2)

    l = max(l1, l2)

    if seq is None:
        seq = RANDOM_SEQ

    # Scan random sequence
    result1 = pwmscan(seq, m1.pwm, m1.pwm_min_score(), len(seq), False, True)
    result2 = pwmscan(seq, m2.pwm, m2.pwm_min_score(), len(seq), False, True)
    result1 = np.array(result1)
    result2 = np.array(result2)

    # Return maximum correlation
    c = []
    for i in range(l1):
        c.append(1 - distance.correlation(result1[:L-l-i],result2[i:L-l]))
    for i in range(l2):
        c.append(1 - distance.correlation(result1[i:L-l],result2[:L-l-i]))
    return max(c)

class MotifComparer(object):
    def __init__(self):
        self.config = MotifConfig()
        self.metrics = ["pcc", "ed", "distance", "wic"]
        self.combine = ["mean", "sum"]
        self._load_scores()
        # Create a parallel python job server, to use for fast motif comparison

    def _load_scores(self):
        self.scoredist = {}
        for metric in self.metrics:
            self.scoredist[metric] = {"total": {}, "subtotal": {}}
            for match in ["total", "subtotal"]:
                for combine in ["mean"]:
                    self.scoredist[metric]["%s_%s" % (match, combine)] = {}
                    score_file = os.path.join(self.config.get_score_dir(), "%s_%s_%s_score_dist.txt" % (match, metric, combine))
                    if os.path.exists(score_file):
                        with open(score_file) as f:
                            for line in f:
                                l1, l2, m, sd = line.strip().split("\t")[:4]
                                self.scoredist[metric]["%s_%s" % (match, combine)].setdefault(int(l1), {})[int(l2)] = [float(m), float(sd)]

    def compare_motifs(self, m1, m2, match="total", metric="wic", combine="mean", pval=False):

        if metric == "seqcor":
            return seqcor(m1, m2)
        elif match == "partial":
            if pval:
                return self.pvalue(m1, m2, "total", metric, combine, self.max_partial(m1.pwm, m2.pwm, metric, combine))
            elif metric in ["pcc", "ed", "distance", "wic", "chisq"]:
                return self.max_partial(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_partial(m1.pfm, m2.pfm, metric, combine)

        elif match == "total":
            if pval:
                return self.pvalue(m1, m2, match, metric, combine, self.max_total(m1.pwm, m2.pwm, metric, combine))
            elif metric == "pcc":
                sys.stderr.write("Can't calculate PCC of columns with equal distribution!\n")
                return None
            elif metric in ["ed", "distance", "wic", "chisq"]:
                return self.max_total(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_total(m1.pfm, m2.pfm, metric, combine)

        elif match == "subtotal":
            if metric in ["pcc", "ed", "distance", "wic", "chisq"]:
                return self.max_subtotal(m1.pwm, m2.pwm, metric, combine)
            else:
                return self.max_subtotal(m1.pfm, m2.pfm, metric, combine)

    def _check_length(self, l):
        # Set the length to a length represented in randomly generated JASPAR motifs
        if l < 4:
            return 4
        if l == 13:
            return 14
        if l == 17:
            return 18
        if l == 19:
            return 20
        if l == 21:
            return 22
        if l > 22:
            return 30
        return l

    def pvalue(self, m1, m2, match, metric, combine, score):
        l1, l2 = len(m1.pwm), len(m2.pwm)

        l1 = self._check_length(l1)
        l2 = self._check_length(l2)

        m,s = self.scoredist[metric]["%s_%s" % (match, combine)][l1][l2]

        try:
            [1 - norm.cdf(score[0], m, s), score[1], score[2]]
        except Exception as e:
            print("Error with score: {}\n{}".format(score, e))
            return [1, np.nan, np.nan]
        return [1 - norm.cdf(score[0], m, s), score[1], score[2]]

    def score_matrices(self, matrix1, matrix2, metric, combine):
        if metric in self.metrics and combine in self.combine:
            s = score(matrix1, matrix2, metric, combine)

            if s != s:
                return None
            else:
                return s

        else:
            if metric == "akl":
                func = akl
            elif metric == "chisq":
                func = chisq
            elif metric == "ssd":
                func = ssd
            else:
                try:
                    func = getattr(distance, metric)
                except:
                    raise Exception("Unknown metric '{}'".format(metric))

            scores = []
            for pos1,pos2 in zip(matrix1,matrix2):
                scores.append(func(pos1, pos2))
            if combine == "mean":
                return np.mean(scores)
            elif combine == "sum":
                return np.sum(scores)
            else:
                raise ValueError("Unknown combine")

    def max_subtotal(self, matrix1, matrix2, metric, combine):
        scores = []
        min_overlap = 4

        if len(matrix1) < min_overlap or len(matrix2) < min_overlap:
            return self.max_total(matrix1, matrix2, metric, combine)

        #return c_max_subtotal(matrix1, matrix2, metric, combine)

        for i in range(-(len(matrix2) - min_overlap), len(matrix1) - min_overlap + 1):
            p1,p2 = self.make_equal_length_truncate(matrix1, matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, 1])

        rev_matrix2 = [row[::-1] for row in matrix2[::-1]]
        for i in range(-(len(matrix2) - min_overlap), len(matrix1) - min_overlap + 1):
            p1,p2 = self.make_equal_length_truncate(matrix1, rev_matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, -1])

        if not scores:
            return []
        return sorted(scores, key=lambda x: x[0])[-1]

    def max_partial(self, matrix1, matrix2, metric, combine):

        scores = []

        for i in range(-(len(matrix2) -1), len(matrix1)):
            p1,p2 = self.make_equal_length_truncate_second(matrix1, matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, 1])

        rev_matrix2 = [row[::-1] for row in matrix2[::-1]]
        for i in range(-(len(matrix2) -1), len(matrix1)):
            p1,p2 = self.make_equal_length_truncate_second(matrix1, rev_matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, -1])

        if not scores:
            return []
        return sorted(scores, key=lambda x: x[0])[-1]

    def max_total(self, matrix1, matrix2, metric, combine):
        scores = []

        for i in range(-(len(matrix2) -1), len(matrix1)):
            p1,p2 = self.make_equal_length(matrix1, matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, 1])

        rev_matrix2 = [row[::-1] for row in matrix2[::-1]]
        for i in range(-(len(matrix2) -1), len(matrix1)):
            p1,p2 = self.make_equal_length(matrix1, rev_matrix2, i)
            s = self.score_matrices(p1, p2, metric, combine)
            if s:
                scores.append([s, i, -1])

        if not scores:
            sys.stdout.write("No score {} {}".format(matrix1, matrix2))
            return []
        return sorted(scores, key=lambda x: x[0])[-1]

    def make_equal_length(self, pwm1, pwm2, pos, bg=None):
        if bg is None:
            bg = [0.25,0.25,0.25,0.25]

        p1 = pwm1[:]
        p2 = pwm2[:]

        if pos < 1:
            p1 = [bg for _ in range(-pos)] + p1
        else:
            p2 = [bg for _ in range(pos)] + p2

        diff = len(p1) - len(p2)
        if diff > 0:
            p2 += [bg for _ in range(diff)]
        elif diff < 0:
            p1 += [bg for _ in range(-diff)]

        return p1,p2

    def make_equal_length_truncate(self, pwm1, pwm2, pos):
        p1 = pwm1[:]
        p2 = pwm2[:]

        if pos < 0:
            p2 = p2[-pos:]
        elif pos > 0:
            p1 = p1[pos:]

        if len(p1) > len(p2):
            p1 = p1[:len(p2)]
        else:
            p2 = p2[:len(p1)]
        return p1, p2

    def make_equal_length_truncate_second(self, pwm1, pwm2, pos, bg=None):
        if bg is None:
            bg = [0.25,0.25,0.25,0.25]

        p1 = pwm1[:]
        p2 = pwm2[:]

        if pos < 0:
            p2 = p2[-pos:]
        else:
            p2 = [bg for _ in range(pos)] + p2

        diff = len(p1) - len(p2)
        if diff > 0:
            p2 += [bg for _ in range(diff)]
        elif diff < 0:
            p2 = p2[:len(p1)]
        return p1,p2

    def get_all_scores(self, motifs, dbmotifs, match, metric, combine, pval=False, parallel=True, trim=None, ncpus=None):

        # trim motifs first, if specified
        if trim:
            for m in motifs:
                m.trim(trim)
            for m in dbmotifs:
                m.trim(trim)

        # hash of result scores
        scores ={}

        if parallel:
            # Divide the job into big chunks, to keep parallel overhead to minimum
            # Number of chunks = number of processors available
            if ncpus is None:
                ncpus = int(MotifConfig().get_default_params()["ncpus"])

            pool = Pool(processes=ncpus, maxtasksperchild=1000)

            batch_len = len(dbmotifs) // ncpus
            if batch_len <= 0:
                batch_len = 1
            jobs = []
            for i in range(0, len(dbmotifs), batch_len):
                # submit jobs to the job server

                p = pool.apply_async(_get_all_scores,
                    args=(self, motifs, dbmotifs[i: i + batch_len], match, metric, combine, pval))
                jobs.append(p)

            pool.close()
            for job in jobs:
                # Get the job result
                result = job.get()
                # and update the result score
                for m1,v in result.items():
                    for m2, s in v.items():
                        if m1 not in scores:
                            scores[m1] = {}
                        scores[m1][m2] = s

            pool.join()
        else:
            # Do the whole thing at once if we don't want parallel
            scores = _get_all_scores(self, motifs, dbmotifs, match, metric, combine, pval)

        return scores

    def get_closest_match(self, motifs, dbmotifs=None, match="partial", metric="wic",combine="mean", parallel=True, ncpus=None):
        """Return best match in database for motifs.

        Parameters
        ----------

        motifs : list or str
            Filename of motifs or list of motifs.

        dbmotifs ; list or str, optional
            Database motifs, default will be used if not specified.

        match : str, optional

        metric : str, optional

        combine : str, optional

        ncpus : int, optional
            Number of threads to use.

        Returns
        -------

        closest_match : dict
        """

        if dbmotifs is None:
            pwm = self.config.get_default_params()["motif_db"]
            pwmdir = self.config.get_motif_dir()
            dbmotifs = os.path.join(pwmdir, pwm)

        motifs = parse_motifs(motifs)
        dbmotifs = parse_motifs(dbmotifs)

        dbmotif_lookup = dict([(m.id, m) for m in dbmotifs])

        scores = self.get_all_scores(motifs, dbmotifs, match, metric, combine, parallel=parallel, ncpus=ncpus)
        for motif in scores:
            scores[motif] = sorted(
                    scores[motif].items(),
                    key=lambda x:x[1][0]
                    )[-1]

        for motif in motifs:
            dbmotif, score = scores[motif.id]
            pval, pos, orient = self.compare_motifs(
                    motif, dbmotif_lookup[dbmotif], match, metric, combine, True)

            scores[motif.id] = [dbmotif, (list(score) + [pval])]

        return scores

    def generate_score_dist(self, motifs, match, metric, combine):

        score_file = os.path.join(self.config.get_score_dir(), "%s_%s_%s_score_dist.txt" % (match, metric, combine))
        f = open(score_file, "w")

        all_scores = {}
        for l in [len(motif) for motif in motifs]:
            all_scores[l] = {}

        sorted_motifs = {}
        for l in all_scores.keys():
            sorted_motifs[l] = [motif for motif in motifs if len(motif) == l]

        for l1 in all_scores.keys():
            for l2 in all_scores.keys():
                scores = self.get_all_scores(sorted_motifs[l1], sorted_motifs[l2], match, metric, combine)
                scores = [[y[0] for y in x.values() if y] for x in scores.values()]
                scores = np.array(scores).ravel()
                f.write("%s\t%s\t%s\t%s\n" % (l1, l2, np.mean(scores), np.std(scores)))

        f.close()

# import here is necessary as workaround
# see: http://stackoverflow.com/questions/18947876/using-python-multiprocessing-pool-in-the-terminal-and-in-code-modules-for-django
try:
    from multiprocessing import Pool
except:
    pass


# Copyright 2019-2020 The Kale Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import warnings

from typing import Any, Dict

import nbformat as nb

from kale.config import Field
from kale import Pipeline, Step, PipelineConfig
from kale.static_analysis import dependencies, ast

# fixme: Change the name of this key to `kale_metadata`
KALE_NB_METADATA_KEY = 'kubeflow_notebook'

SKIP_TAG = r'^skip$'
IMPORT_TAG = r'^imports$'
FUNCTIONS_TAG = r'^functions$'
PREV_TAG = r'^prev:[_a-z]([_a-z0-9]*)?$'
# `step` has the same functionality as `block` and is
# supposed to be the new name
STEP_TAG = r'^step:([_a-z]([_a-z0-9]*)?)?$'
# Extension may end up with 'block:' as a tag. We handle
# that as if it was empty.
# TODO: Deprecate `block` tag in future release
BLOCK_TAG = r'^block:([_a-z]([_a-z0-9]*)?)?$'
PIPELINE_PARAMETERS_TAG = r'^pipeline-parameters$'
PIPELINE_METRICS_TAG = r'^pipeline-metrics$'
# Annotations map to actual pod annotations that can be set via KFP SDK
_segment = "[a-zA-Z0-9]+([a-zA-Z0-9-_.]*[a-zA-Z0-9])?"
K8S_ANNOTATION_KEY = "%s([/]%s)?" % (_segment, _segment)
ANNOTATION_TAG = r'^annotation:%s:(.*)$' % K8S_ANNOTATION_KEY
LABEL_TAG = r'^label:%s:(.*)$' % K8S_ANNOTATION_KEY
# Limits map to K8s limits, like CPU, Mem, GPU, ...
# E.g.: limit:nvidia.com/gpu:2
LIMITS_TAG = r'^limit:([_a-z-\.\/]+):([_a-zA-Z0-9\.]+)$'

_TAGS_LANGUAGE = [SKIP_TAG,
                  IMPORT_TAG,
                  FUNCTIONS_TAG,
                  PREV_TAG,
                  BLOCK_TAG,
                  STEP_TAG,
                  PIPELINE_PARAMETERS_TAG,
                  PIPELINE_METRICS_TAG,
                  ANNOTATION_TAG,
                  LABEL_TAG,
                  LIMITS_TAG]
# These tags are applied to every step of the pipeline
_STEPS_DEFAULTS_LANGUAGE = [ANNOTATION_TAG,
                            LABEL_TAG,
                            LIMITS_TAG]

def get_annotation_or_label_from_tag(tag_parts):
    """Get the key and value from an annotation or label tag.

    Args:
        tag_parts: annotation or label notebook tag

    Returns (tuple): key (annotation or label name), values
    """
    # Since value can be anything, merge together everything that's left.
    return tag_parts[0], "".join(tag_parts[1:])

def get_limit_from_tag(tag_parts):
    """Get the key and value from a notebook limit tag.

    Args:
        tag_parts: annotation or label notebook tag

    Returns (tuple): key (limit name), values
    """
    return tag_parts.pop(0), tag_parts.pop(0)

class NotebookConfig(PipelineConfig):
    """Config store for a notebook.

    This config extends the base pipeline config to take into account some
    small differences in the handling of a notebook.
    """
    notebook_path = Field(type=str, required=True)
    # FIXME: Probably this can be removed. The labextension passes both
    #  'experiment_name' and 'experiment', but the latter is not used in the
    #  backend.
    experiment = Field(type=dict)
    # Used in the UI to keep per-notebook state of the volumes snapshot toggle
    snapshot_volumes = Field(type=bool, default=False)
    # override from PipelineConfig: set the default value to False
    autosnapshot = Field(type=bool, default=False)

    @property
    def source_path(self):
        """Get the path to the source notebook."""
        return self.notebook_path

    def _preprocess(self, kwargs):
        kwargs["steps_defaults"] = self._parse_steps_defaults(
            kwargs.get("steps_defaults"))

    def _parse_steps_defaults(self, steps_defaults):
        """Parse common step configuration defined in the metadata."""
        result = dict()

        if not isinstance(steps_defaults, list):
            return steps_defaults

        for c in steps_defaults:
            if any(re.match(_c, c)
                   for _c in _STEPS_DEFAULTS_LANGUAGE) is False:
                raise ValueError("Unrecognized common step configuration:"
                                 " {}".format(c))

            parts = c.split(":")

            conf_type = parts.pop(0)
            if conf_type in ["annotation", "label"]:
                result_key = "{}s".format(conf_type)
                if result_key not in result:
                    result[result_key] = dict()
                key, value = get_annotation_or_label_from_tag(
                    parts)
                result[result_key][key] = value

            if conf_type == "limit":
                if "limits" not in result:
                    result["limits"] = dict()
                key, value = get_limit_from_tag(parts)
                result["limits"][key] = value
        return result

class NotebookProcessor:
    """Convert a Notebook to a Pipeline object."""

    def __init__(self,
                 nb_path: str,
                 nb_metadata_overrides: Dict[str, Any] = None,
                 skip_validation: bool = False):
        """Instantiate a new NotebookProcessor.

        Args:
            nb_path: Path to source notebook
            nb_metadata_overrides: Override notebook config settings
            skip_validation: Set to True in order to skip the notebook's
                metadata validation. This is useful in case the
                NotebookProcessor is used to parse a part of the notebook
                (e.g., retrieve pipeline metrics) and the notebook config (for
                pipeline generation) might still be invalid.
        """
        self.nb_path = os.path.expanduser(nb_path)
        self.notebook = self._read_notebook()

        nb_metadata = self.notebook.metadata.get(KALE_NB_METADATA_KEY, dict())

        # fixme: needed?
        nb_metadata.update({"notebook_path": nb_path})
        if nb_metadata_overrides:
            nb_metadata.update(nb_metadata_overrides)
        # validate and populate defaults
        # FIXME: Maybe improve this by implementing a "skip_validation" flag
        #  in the config class
        self.config = None
        if not skip_validation:
            self.config = NotebookConfig(**nb_metadata)
        self.pipeline = Pipeline(self.config)

    def _read_notebook(self):
        if not os.path.exists(self.nb_path):
            raise ValueError("NotebookProcessor could not find a notebook at"
                             " path %s" % self.nb_path)
        return nb.read(self.nb_path, as_version=nb.NO_CONVERT)

    def to_pipeline(self):
        """Convert an annotated Notebook to a Pipeline object."""
        (pipeline_parameters_source,
         pipeline_metrics_source,
         imports_and_functions) = self.parse_notebook()

        self.parse_pipeline_parameters(pipeline_parameters_source)
        self.pipeline.set_volume_pipeline_parameters()

        # get a list of variables that need to be logged as pipeline metrics
        pipeline_metrics = ast.parse_metrics_print_statements(
            pipeline_metrics_source)

        # run static analysis over the source code
        dependencies.dependencies_detection(
            self.pipeline,
            imports_and_functions=imports_and_functions
        )
        dependencies.assign_metrics(self.pipeline, pipeline_metrics)

        # if there are multiple DAG leaves, add an empty step at the end of the
        # pipeline for final snapshot
        leaf_steps = self.pipeline.get_leaf_steps()
        if self.config.autosnapshot and len(leaf_steps) > 1:
            _name = "final_auto_snapshot"
            self.pipeline.add_step(Step(name=_name, source=[]))
            # add a link from all the last steps of the pipeline to
            # the final auto snapshot one.
            for step in leaf_steps:
                self.pipeline.add_edge(step.name, _name)

        # FIXME: Move this to a base class Processor, to be executed by default
        #  after `to_pipeline`, so that it is agnostic to the type of
        #  processor.
        for step in self.pipeline.steps:
            step.config.update(self.pipeline.config.steps_defaults)

        # TODO: Additional action required:
        #  Run a static analysis over every step to check that pipeline
        #  parameters are not assigned with new values.
        return self.pipeline

    def parse_pipeline_parameters(self, source: str):
        """Get pipeline parameters from source code."""
        pipeline_parameters = ast.parse_assignments_expressions(source)
        for name, (v_type, v_value) in pipeline_parameters.items():
            pipeline_parameters[name] = ast.PipelineParam(v_type, v_value)
        self.pipeline.pipeline_parameters = pipeline_parameters

    def parse_notebook(self):
        """Creates a NetworkX graph based on the input notebook's tags.

        Cell's source code are embedded into the graph as node attributes.
        """
        # will be assigned at the end of each for loop
        prev_step_name = None

        # All the code cells that have to be pre-pended to every pipeline step
        # (i.e., imports and functions) are merged here
        imports_block = list()
        functions_block = list()

        # Variables that will become pipeline parameters
        pipeline_parameters = list()
        # Variables that will become pipeline metrics
        pipeline_metrics = list()

        for c in self.notebook.cells:
            if c.cell_type != "code":
                continue

            tags = self.parse_cell_metadata(c.metadata)

            if len(tags['step_names']) > 1:
                raise NotImplementedError("Kale does not yet support multiple"
                                          " step names in a single notebook"
                                          " cell. One notebook cell was found"
                                          " with %s  step names"
                                          % tags['step_names'])

            step_name = (tags['step_names'][0]
                         if 0 < len(tags['step_names'])
                         else None)

            if step_name == 'skip':
                # when the cell is skipped, don't store `skip` as the previous
                # active cell
                continue
            if step_name == 'pipeline-parameters':
                pipeline_parameters.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'imports':
                imports_block.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'functions':
                functions_block.append(c.source)
                prev_step_name = step_name
                continue
            if step_name == 'pipeline-metrics':
                pipeline_metrics.append(c.source)
                prev_step_name = step_name
                continue

            # if none of the above apply, then we are parsing a code cell with
            # a block names and (possibly) some dependencies

            # if the cell was not tagged with a step name,
            # add the code to the previous cell
            if not step_name:
                if prev_step_name == 'imports':
                    imports_block.append(c.source)
                elif prev_step_name == 'functions':
                    functions_block.append(c.source)
                elif prev_step_name == 'pipeline-parameters':
                    pipeline_parameters.append(c.source)
                elif prev_step_name == 'pipeline-metrics':
                    pipeline_metrics.append(c.source)
                # current_block might be None in case the first cells of the
                # notebooks have not been tagged.
                elif prev_step_name:
                    # this notebook cell will be merged to a previous one that
                    # specified a step name
                    self.pipeline.get_step(prev_step_name).merge_code(c.source)
            else:
                # in this branch we are sure that we are reading a code cell
                # with a step tag, so we must not allow for pipeline-metrics
                if prev_step_name == 'pipeline-metrics':
                    raise ValueError("Tag pipeline-metrics must be placed on a"
                                     " cell at the end of the Notebook."
                                     " Pipeline metrics should be considered"
                                     " as a result of the pipeline execution"
                                     " and not of single steps.")
                # add node to DAG, adding tags and source code of notebook cell
                if step_name not in self.pipeline.nodes:
                    step = Step(name=step_name, source=[c.source],
                                ins=set(), outs=set(),
                                limits=tags.get("limits", {}),
                                labels=tags.get("labels", {}),
                                annotations=tags.get("annotations", {}))
                    self.pipeline.add_step(step)
                    for _prev_step in tags['prev_steps']:
                        if _prev_step not in self.pipeline.nodes:
                            raise ValueError("Step %s does not exist. It was "
                                             "defined as previous step of %s"
                                             % (
                                                 _prev_step,
                                                 tags['step_names']))
                        self.pipeline.add_edge(_prev_step, step_name)
                else:
                    self.pipeline.get_step(step_name).merge_code(c.source)

                prev_step_name = step_name

        # Prepend any `imports` and `functions` cells to every Pipeline step
        for step in self.pipeline.steps:
            step.source = imports_block + functions_block + step.source

        # merge together pipeline parameters
        pipeline_parameters = '\n'.join(pipeline_parameters)
        # merge together pipeline metrics
        pipeline_metrics = '\n'.join(pipeline_metrics)

        imports_and_functions = "\n".join(imports_block + functions_block)
        return pipeline_parameters, pipeline_metrics, imports_and_functions

    def parse_cell_metadata(self, metadata):
        """Parse a notebook's cell's metadata field.

        The Kale UI writes specific tags inside the 'tags' field, as a list
        of string tags. Supported tags are defined by _TAGS_LANGUAGE.

        Args:
            metadata (dict): a dict containing a notebook's cell's metadata

        Returns (dict): parsed tags based on Kale tagging language

        """
        parsed_tags = dict()

        # `step_names` is a list because a notebook cell might be assigned to
        # more than one Pipeline step.
        parsed_tags['step_names'] = list()
        parsed_tags['prev_steps'] = list()
        # define intermediate variables so that dicts are not added to a steps
        # when they are empty
        cell_annotations = dict()
        cell_labels = dict()
        cell_limits = dict()

        # the notebook cell was not tagged
        if 'tags' not in metadata or len(metadata['tags']) == 0:
            return parsed_tags

        for t in metadata['tags']:
            if not isinstance(t, str):
                raise ValueError("Tags must be string. Found tag %s of type %s"
                                 % (t, type(t)))
            # Check that the tag is defined by the Kale tagging language
            if any(re.match(_t, t) for _t in _TAGS_LANGUAGE) is False:
                raise ValueError("Unrecognized tag: {}".format(t))

            # Special tags have a specific effect on the cell they belong to.
            # Specifically:
            #  - skip: ignore the notebook cell
            #  - pipeline-parameters: use the cell to populate Pipeline
            #       parameters. The cell must contain only assignment
            #       expressions
            #  - pipeline-metrics: use the cell to populate Pipeline metrics.
            #       The cell must contain only variable names
            #  - imports: the code of the corresponding cell(s) will be
            #       prepended to every Pipeline step
            #  - functions: same as imports, but the corresponding code is
            #       placed **after** `imports`
            special_tags = ['skip', 'pipeline-parameters', 'pipeline-metrics',
                            'imports', 'functions']
            if t in special_tags:
                parsed_tags['step_names'] = [t]
                return parsed_tags

            # now only `block|step` and `prev` tags remain to be parsed.
            tag_parts = t.split(':')
            tag_name = tag_parts.pop(0)

            if tag_name == "annotation":
                key, value = get_annotation_or_label_from_tag(tag_parts)
                cell_annotations.update({key: value})

            if tag_name == "label":
                key, value = get_annotation_or_label_from_tag(tag_parts)
                cell_labels.update({key: value})

            if tag_name == "limit":
                key, value = get_limit_from_tag(tag_parts)
                cell_limits.update({key: value})

            # name of the future Pipeline step
            # TODO: Deprecate `block` in future release
            if tag_name in ["block", "step"]:
                if tag_name == "block":
                    warnings.warn("`block` tag will be deprecated in a future"
                                  " version, use `step` tag instead",
                                  DeprecationWarning)
                step_name = tag_parts.pop(0)
                parsed_tags['step_names'].append(step_name)
            # name(s) of the father Pipeline step(s)
            if tag_name == "prev":
                prev_step_name = tag_parts.pop(0)
                parsed_tags['prev_steps'].append(prev_step_name)

        if not parsed_tags['step_names'] and parsed_tags['prev_steps']:
            raise ValueError(
                "A cell can not provide `prev` annotations without "
                "providing a `block` or `step` annotation as well")

        if cell_annotations:
            if not parsed_tags['step_names']:
                raise ValueError(
                    "A cell can not provide Pod annotations in a cell"
                    " that does not declare a step name.")
            parsed_tags['annotations'] = cell_annotations

        if cell_limits:
            if not parsed_tags['step_names']:
                raise ValueError(
                    "A cell can not provide Pod resource limits in a"
                    " cell that does not declare a step name.")
            parsed_tags['limits'] = cell_limits
        return parsed_tags

    def get_pipeline_parameters_source(self):
        """Get just pipeline parameters cells from the notebook.

        Returns (str): pipeline parameters source code
        """
        return self._get_reserved_tag_source(PIPELINE_PARAMETERS_TAG)

    def get_pipeline_metrics_source(self):
        """Get just pipeline metrics cells from the notebook.

        Returns (str): pipeline metrics source code
        """
        # check that the pipeline metrics tag is only assigned to cells at
        # the end of the notebook
        detected = False
        tags = _TAGS_LANGUAGE[:]
        tags.remove(PIPELINE_METRICS_TAG)

        for c in self.notebook.cells:
            # parse only source code cells
            if c.cell_type != "code":
                continue

            # if we see a pipeline-metrics tag, set the flag
            if (('tags' in c.metadata
                 and len(c.metadata['tags']) > 0
                 and any(re.match(PIPELINE_METRICS_TAG, t)
                         for t in c.metadata['tags']))):
                detected = True
                continue

            # if we have the flag set and we detect any other tag from the tags
            # language, then raise error
            if (detected
                and 'tags' in c.metadata
                and len(c.metadata['tags']) > 0
                and any([any(re.match(tag, t) for t in c.metadata['tags'])
                         for tag in tags])):
                raise ValueError(
                    "Tag pipeline-metrics tag must be placed on a "
                    "cell at the end of the Notebook."
                    " Pipeline metrics should be considered as a"
                    " result of the pipeline execution and not of"
                    " single steps.")
        return self._get_reserved_tag_source(PIPELINE_METRICS_TAG)

    def _get_reserved_tag_source(self, search_tag):
        """Get just the specific tag's source code.

        When searching for tag x, will return all cells that are tagged with x
        and, if untagged, follow cells with tag x. The result is a multiline
        string containing all the python code associated to x.

        Note: This is designed for 'special' tags, as the STEP_TAG is excluded
              from the match.

        Args:
            search_tag (str): the target tag

        Returns: the unified code of all the cells belonging to `search_tag`
        """
        detected = False
        source = ''

        language = _TAGS_LANGUAGE[:]
        language.remove(search_tag)

        for c in self.notebook.cells:
            # parse only source code cells
            if c.cell_type != "code":
                continue
            # in case the previous cell was a `search_tag` cell and this
            # cell is not any other tag of the tag language:
            if (detected
                and (('tags' not in c.metadata
                      or len(c.metadata['tags']) == 0)
                     or all([not any(re.match(tag, t)
                                     for t in c.metadata['tags'])
                            for tag in language]))):
                source += '\n' + c.source
            elif (('tags' in c.metadata
                   and len(c.metadata['tags']) > 0
                   and any(re.match(search_tag, t)
                           for t in c.metadata['tags']))):
                source += '\n' + c.source
                detected = True
            else:
                detected = False
        return source.strip()


"""
Connect to the FPGA inside of this file
"""

# Import FPGA Lib
# import EggPGA

def get_size(size_in_bytes, suffix="B"):
    """
    Scale bytes to its proper format
    e.g:
        1253656 => '1.20MB'
        1253656678 => '1.17GB'
    """
    factor = 1024
    for unit in ["", "K", "M", "G", "T", "P"]:
        if size_in_bytes < factor:
            return f"{size_in_bytes:.2f}{unit}{suffix}"
        size_in_bytes /= factor

def get_system_stats():
    """
    Collects interesting system stats

    From: https://www.thepythoncode.com/article/get-hardware-system-information-python

    :return: a string containing useful stats
    """
    import psutil
    import platform
    from datetime import datetime

    uname = platform.uname()
    print(f"System: {uname.system}")
    print(f"Node Name: {uname.node}")
    print(f"Release: {uname.release}")
    print(f"Version: {uname.version}")
    print(f"Machine: {uname.machine}")
    print(f"Processor: {uname.processor}")

    ##### Boot Time ########
    print("=" * 40, "Boot Time", "=" * 40)
    boot_time_timestamp = psutil.boot_time()
    bt = datetime.fromtimestamp(boot_time_timestamp)
    print(f"Boot Time: {bt.year}/{bt.month}/{bt.day} {bt.hour}:{bt.minute}:{bt.second}")

    ###### CPU ########
    # let's print CPU information
    print("=" * 40, "CPU Info", "=" * 40)
    # number of cores
    print("Physical cores:", psutil.cpu_count(logical=False))
    print("Total cores:", psutil.cpu_count(logical=True))
    # CPU frequencies
    cpufreq = psutil.cpu_freq()
    print(f"Max Frequency: {cpufreq.max:.2f}Mhz")
    print(f"Min Frequency: {cpufreq.min:.2f}Mhz")
    print(f"Current Frequency: {cpufreq.current:.2f}Mhz")
    # CPU usage
    print("CPU Usage Per Core:")
    for i, percentage in enumerate(psutil.cpu_percent(percpu=True)):
        print(f"Core {i}: {percentage}%")
    print(f"Total CPU Usage: {psutil.cpu_percent()}%")

    ### MEMORY
    # Memory Information
    print("=" * 40, "Memory Information", "=" * 40)
    # get the memory details
    svmem = psutil.virtual_memory()
    print(f"Total: {get_size(svmem.total)}")
    print(f"Available: {get_size(svmem.available)}")
    print(f"Used: {get_size(svmem.used)}")
    print(f"Percentage: {svmem.percent}%")
    print("=" * 20, "SWAP", "=" * 20)
    # get the swap memory details (if exists)
    swap = psutil.swap_memory()
    print(f"Total: {get_size(swap.total)}")
    print(f"Free: {get_size(swap.free)}")
    print(f"Used: {get_size(swap.used)}")
    print(f"Percentage: {swap.percent}%")

    ### Network
    # Network information
    print("=" * 40, "Network Information", "=" * 40)
    # get all network interfaces (virtual and physical)
    if_addrs = psutil.net_if_addrs()
    for interface_name, interface_addresses in if_addrs.items():
        for address in interface_addresses:
            print(f"=== Interface: {interface_name} ===")
            if str(address.family) == 'AddressFamily.AF_INET':
                print(f"  IP Address: {address.address}")
                print(f"  Netmask: {address.netmask}")
                print(f"  Broadcast IP: {address.broadcast}")
            elif str(address.family) == 'AddressFamily.AF_PACKET':
                print(f"  MAC Address: {address.address}")
                print(f"  Netmask: {address.netmask}")
                print(f"  Broadcast MAC: {address.broadcast}")
    # get IO statistics since boot
    net_io = psutil.net_io_counters()
    print(f"Total Bytes Sent: {get_size(net_io.bytes_sent)}")
    print(f"Total Bytes Received: {get_size(net_io.bytes_recv)}")

    # Combine as a string
    os_stats = f"""
    System: {uname.system}
    Node Name: {uname.node}
    Release: {uname.release}
    Version: {uname.version}
    Machine: {uname.machine}
    Processor: {uname.processor}

    {'=' * 40} Boot Time {'=' * 40}
    Boot Time: {bt.year}/{bt.month}/{bt.day} {bt.hour}:{bt.minute}:{bt.second}

    {'=' * 40}    CPU    {'=' * 40}
    Physical cores: {psutil.cpu_count(logical=False)}
    Total cores: {psutil.cpu_count(logical=True)}
    Max Frequency: {cpufreq.max:.2f}Mhz
    Min Frequency: {cpufreq.min:.2f}Mhz
    Current Frequency: {cpufreq.current:.2f}Mhz
    CPU Usage Per Core:
        # ToDo
    Total CPU Usage: {psutil.cpu_percent()}

    {'=' * 40} Memory Information {'=' * 40}
    Total: {get_size(svmem.total)}
    Available: {get_size(svmem.available)}
    Used: {get_size(svmem.used)}
    Percentage: {svmem.percent}

    {'=' * 40} SWAP {'=' * 40}
    Total: {get_size(swap.total)}
    Free: {get_size(swap.free)}
    Used: {get_size(swap.used)}
    Percentage: {swap.percent}
    """

    return os_stats

def get_system_stats_dict():
    import psutil
    import platform
    from datetime import datetime

    uname = platform.uname()
    cpufreq = psutil.cpu_freq()
    if_addrs = psutil.net_if_addrs()
    net_io = psutil.net_io_counters()
    swap = psutil.swap_memory()
    svmem = psutil.virtual_memory()

    stats = {
        "System": uname.system,
        "Node Name": uname.node,
        "Release": uname.release,
        "Version": uname.version,
        "Machine": uname.machine,
        "Processor": uname.processor,
        "Boot": {
            "Boot Time": f"{bt.year}/{bt.month}/{bt.day} {bt.hour}:{bt.minute}:{bt.second}"
        },
        "CPU": {
            "Physical cores": psutil.cpu_count(logical=False),
            "Total cores": psutil.cpu_count(logical=True),
            "Max Frequency": f"{cpufreq.max:.2f}Mhz",
            "Min Frequency": f"{cpufreq.min:.2f}Mhz",
            "Current Frequency": f"{cpufreq.current:.2f}Mhz",
            "Total CPU Usage": psutil.cpu_percent()
        },
        "Memory": {
            "Total": get_size(svmem.total),
            "Available": get_size(svmem.available),
            "Used": get_size(svmem.used),
            "Percentage": svmem.percent,
        },
        "Swap": {
            "Total": get_size(swap.total),
            "Free": get_size(swap.free),
            "Used": get_size(swap.used),
            "Percentage": swap.percent,
        },
        "Network": {
            "Total Bytes Sent": get_size(net_io.bytes_sent),
            "Total Bytes Received": get_size(net_io.bytes_recv)
        }
    }

def get_cpu_load():
    import psutil
    return psutil.cpu_percent(percpu=True)

def get_current_system_name():
    import socket
    return socket.gethostname()

def get_uptime():
    """
    Returns the current system uptime for linux systems
    :return: the uptime as a string and as timedelta object
    """
    from datetime import timedelta

    with open('/proc/uptime', 'r') as f:
        uptime_seconds = float(f.readline().split()[0])
        upt = timedelta(seconds=uptime_seconds)
        uptime_string = str(timedelta(seconds=uptime_seconds))

    return uptime_string, upt


# Basic progress bar for the downloading procedures for urllib.request.urlretrieve

def progress_bar(batch=None, batchsize=None, filesize=None, **kwargs):

    barsize = 50
    prog = batch*batchsize/filesize if not kwargs else kwargs['downloaded']/kwargs['totalsize']
    compl = round(prog*100, 1) if prog < 1 else 100
    prog_size = int(prog * barsize)

    print("\r[{0}{1:3.1f}%{2}]".format(chr(9608)*prog_size, compl, '.'*(barsize-prog_size)),

          "{} / {} Bytes".format(batch*batchsize if prog < 1 else filesize, filesize) if not kwargs

          else "{} / {} Bytes".format(kwargs['downloaded'], kwargs['totalsize']),

          flush=True, end='')


import csv
import numpy as np
import math
import sys

maxInt = sys.maxsize
decrement = True

while decrement:
    # decrease the maxInt value by factor 10
    # as long as the OverflowError occurs.

    decrement = False
    try:
        csv.field_size_limit(maxInt)
    except OverflowError:
        maxInt = int(maxInt/10)
        decrement = True

with open('training_data_abridged.csv') as infile:
    data = csv.DictReader(infile)

    ids = []
    features = []
    classes = []
    for line in data:
        p = line['pixels']
        p = p.split()
        #print (p)

        p_2 = []
        for i in p:
            p_2.append(float(i))

        c = [0,0,0]
        c[int(line['class'])] = 1

        ids.append(line['OBJID'])
        features.append(p_2)
        classes.append(c)

def bounding_box(f):
    threshold = np.median(f) / 2
    #print (f)
    max_dir = 0

    n = len(f)
    sqrt_n = int(math.sqrt(n))
    start = int(n // 2)

    # check right
    s = start
    i = 0
    while(f[s] > threshold):
        i = i+1
        s = s+1
        if (s > start + sqrt_n):
            return sqrt_n

    if (i > max_dir):
        max_dir = i

    # check left
    s = start
    i = 0
    while(f[s] > threshold):
        i = i+1
        s = s-1

        if (s < start - sqrt_n):
            return sqrt_n

    if (i > max_dir):
        max_dir = i

    # check down
    s = start
    i = 0
    while(f[s] > threshold):
        i = i+1
        s = s+sqrt_n
        if (s > n):
            return sqrt_n

    if (i > max_dir):
        max_dir = i

    # check up
    s = start
    i = 0
    while(f[s] > threshold):
        i = i+1
        s = s-sqrt_n
        if (s < 0):
            return sqrt_n

    if (i > max_dir):
        max_dir = i

    return max_dir

for feature in features:
    galaxy = bounding_box(feature)
    print (galaxy)

    n = len(feature)
    sqrt_n = math.sqrt(n)
    start = n // 2

    for i in range(0, len(feature)):
        if (abs((i%sqrt_n)-start) <= galaxy and (i < start + galaxy*sqrt_n + galaxy) and (i > start - galaxy*sqrt_n - galaxy)):
            feature[i] = 0.0

with open('training_data_abridged_blacked.csv', 'w', newline='') as  outputfile:
    writer = csv.DictWriter(outputfile, ['OBJID', 'pixels', 'class'])
    writer.writeheader()

    for i in range(0, len(features)):
        line = features[i]
        line = [str(l) for l in line]
        c = classes[i]
        c = c.index(1)
        pixels_string = ' '.join(line)
        row = {'OBJID': ids[i], 'pixels': pixels_string, 'class': c}
        writer.writerow(row)


# 가장 긴 내림차순
n = int(input())
soldiers = list(map(int, input().split()))
dp = [1]*n

for i in range(1, n):
    for j in range(i):
        if soldiers[i] < soldiers[j]:
            dp[i] = max(dp[j] + 1, dp[i])
print(dp)
print(n - max(dp))


# -*- coding: utf-8 -*-

import hrmcompiler.parser as p
import hrmcompiler.assembler as a
import hrmcompiler.jsonassembler as ja
import hrmcompiler.semantic_check as checker
import hrmcompiler.conversion as conversion
import json

def calculate_optimized_ast(fhandle, args):
    result_ast = p.parse_it(fhandle)

    checker.perform_label_checks(result_ast)
    checker.perform_variable_checks(result_ast)

    result_ast = conversion.convert_ifnz_to_ifez(result_ast)
    result_ast = conversion.convert_iftojump(result_ast)
    if not args.no_jump_compression:
        result_ast = conversion.compress_jumps(result_ast)
    if not args.no_unreachable:
        result_ast = conversion.remove_unreachable_code(result_ast)
    if not args.no_jmp_then_label:
        unchanged = False
        while not unchanged:
            new_ast = conversion.fix_jmp_then_label(result_ast)
            unchanged = new_ast == result_ast
            result_ast = new_ast

    return result_ast

def main(args):
    with open(args.fname) as src:
        result_ast = calculate_optimized_ast(src, args)

        assembler = a.Assembler()
        assembler.convert(result_ast)

        with open(args.fname.replace(".hrm", ".json"), "w") as dst:
            json_assembler = ja.Assembler()
            json_assembler.convert(result_ast)
            json.dump(json_assembler.code, dst)

        print(" ------------------------ ")
        print("\n".join(assembler.code))


# Gets page contents
import urllib.request
import csv

proj_url = "https://devpost.com/software/pill-dispenser-leuqcb"

fp = urllib.request.urlopen(proj_url)
proj_mybytes = fp.read()
proj_mystr = proj_mybytes.decode("utf8")
fp.close()

proj_content = proj_mystr.split("\n")

# Makes sure names found are not in likes section
isOnLikesYet = False
# Prevent double printing
toggle = True
# The makes the Dict for this project
winner_info = {}
winner_info["Awards"] = []
winner_info["Members"] = []

# Goes through each line of projects url
for i in range(len(proj_content)):
    # Breaks if on likes
    if(isOnLikesYet):
        break

    # Display Hackathon
    if("Submitted to" in proj_content[i]):
        winner_info["Hackathon"] = proj_content[i+11].strip().split("")[1].split("")[0]

    # Display Awards
    if("Winner" in proj_content[i]):
        name = proj_content[i+1]
        winner_info["Awards"].append(name.strip())

    # Display Names of people
    if("<a class=\"user-profile-link\" href=" in proj_content[i]):
        name = proj_content[i].split("<a class=\"user-profile-link\" href=\"https://devpost.com/")[1].split("\"")[0]
        if(toggle == True):
            winner_info["Members"].append(name)
            toggle = False
        else:
            toggle = True

    if("" in proj_content[i]):
        isOnLikesYet = True

# Writes to csv
#writer.writerow(winner_info)
print(winner_info)
print()


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from charms.reactive import RelationBase
from charms.reactive import hook
from charms.reactive import scopes

class ClusterDNSRequires(RelationBase):
    scope = scopes.GLOBAL

    @hook('{requires:cluster-dns}-relation-joined')
    def joined(self):
        conv = self.conversation()
        conv.set_state('{relation_name}.joined')

    @hook('{requires:cluster-dns}-relation-changed')
    def changed(self):
        conv = self.conversation()
        if self.get_domain():
            conv.set_state('{relation_name}.domain-ready')
        else:
            conv.remove_state('{relation_name}.domain-ready')

    @hook('{requires:cluster-dns}-relation-departed')
    def departed(self):
        conv = self.conversation()
        conv.remove_state('{relation_name}.ready')
        conv.remove_state('{relation_name}.joined')

    def send_ip(self, host):
        conv = self.conversation()
        conv.set_remote('host', host)

    def get_domain(self):
        """Return the Cluster DNS Domain.
        """
        for conv in self.conversations():
            domain = conv.get_remote('domain')

            return domain


from rake_nltk import Rake
import csv

# Uses stopwords for english from NLTK, and all puntuation characters by
# default
r = Rake()

file = open("Web_Scraping/Keyword_Extraction/test.txt", 'r', encoding="utf-8")

All_The_Words = file.read()

# Extraction given the text.
r.extract_keywords_from_text(All_The_Words)

amount = 0
# To get keyword phrases ranked highest to lowest with scores.
with open('Web_Scraping/Keyword_Extraction/Keyword_Extraction_Method/Keyword_Results/NewYorkTimes_result.csv', mode = 'w', encoding = 'utf-8', newline='') as news_set:
    copier = csv.writer(news_set, delimiter = ',', quotechar = '"', quoting = csv.QUOTE_MINIMAL)
    for row in range(len(titles)):
        string = []
        if first:
            copier.writerow(["Title", "Keyword1", "Keyword2", "Keyword3", "Keyword4", "Keyword5", "Keyword6", "Keyword7", "Keyword8", "Keyword9", "Keyword10"])
            first = False
        else:
            string.append(titles[row])
            amount = 0
            for phrase in r.get_ranked_phrases():
                if amount < 10:
                    copier.append(phrase)
                    amount+=1
                else:
                    break

        copier.writerow(string)


#Modules for Pi
import os
import time
from gpiozero import Button
import RPi.GPIO as GPIO

#Modules for Object Detection
from imutils.video import VideoStream
from imutils.video import FPS
import numpy as np
import argparse
import imutils
import time
import cv2
import pyttsx3 as pt

#Modules for Text Recognition
import logging
import shutil
import subprocess
import sys
import tempfile
from constants import VALID_IMAGE_EXTENSIONS, WINDOWS_CHECK_COMMAND, DEFAULT_CHECK_COMMAND, TESSERACT_DATA_PATH_VAR

GPIO.setmode(GPIO.BCM)

GPIO.setup(5, GPIO.IN, pull_up_down=GPIO.PUD_UP)

GPIO.setup(6, GPIO.IN, pull_up_down=GPIO.PUD_UP)

n = pt.init(driverName='espeak');

def observeMode():
    # construct the argument parse and parse the arguments
    ap = argparse.ArgumentParser()
    ap.add_argument("-c", "--confidence", type=float, default=0.2,
        help="minimum probability to filter weak detections")
    args = vars(ap.parse_args())

    prototext = "MobileNetSSD_deploy.prototxt.txt"
    model = "MobileNetSSD_deploy.caffemodel"

    # initialize the list of class labels MobileNet SSD was trained to
    # detect, then generate a set of bounding box colors for each class
    CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
        "bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
        "dog", "horse", "motorbike", "person", "plant", "sheep",
        "sofa", "train", "monitor"]
    COLORS = np.random.uniform(0, 255, size=(len(CLASSES), 3))

    # load our serialized model from disk
    print("[INFO] loading model...")
    net = cv2.dnn.readNetFromCaffe(prototext, model)
    # initialize the video stream, allow the cammera sensor to warmup,
    # and initialize the FPS counter
    print("[INFO] starting video stream...")
    vs = VideoStream(src=0).start()
    time.sleep(2.0)
    fps = FPS().start()

    # loop over the frames from the video stream
    while True:
        dict = {"L":{"background":0, "aeroplane":0, "bicycle":0, "bird":0, "boat":0,
        "bottle":0, "bus":0, "car":0, "cat":0, "chair":0, "cow":0, "diningtable":0,
        "dog":0, "horse":0, "motorbike":0, "person":0, "plant":0, "sheep":0,
        "sofa":0, "train":0, "monitor":0},"C" : {"background":0, "aeroplane":0, "bicycle":0, "bird":0, "boat":0,
        "bottle":0, "bus":0, "car":0, "cat":0, "chair":0, "cow":0, "diningtable":0,
        "dog":0, "horse":0, "motorbike":0, "person":0, "plant":0, "sheep":0,
        "sofa":0, "train":0, "monitor":0},"R":{"background":0, "aeroplane":0, "bicycle":0, "bird":0, "boat":0,
        "bottle":0, "bus":0, "car":0, "cat":0, "chair":0, "cow":0, "diningtable":0,
        "dog":0, "horse":0, "motorbike":0, "person":0, "plant":0, "sheep":0,
        "sofa":0, "train":0, "monitor":0}}
        # grab the frame from the threaded video stream and resize it
        # to have a maximum width of 400 pixels
        frame = vs.read()
        frame = imutils.resize(frame, width=400)

        #print(frame.shape)
        # grab the frame dimensions and convert it to a blob
        (h, w) = frame.shape[:2]
        blob = cv2.dnn.blobFromImage(frame,
            0.007843, (300, 300), 127.5)
        # print(blob)
        # pass the blob through the network and obtain the detections and
        # predictions
        net.setInput(blob)
        detections = net.forward()
        # print(detections)
        # break
        # loop over the detections
        print("--------------------")
        label_left = []
        label_right = []
        label_center = []
        for i in np.arange(0, detections.shape[2]):
            #count+=1

            # extract the confidence (i.e., probability) associated with
            # the prediction
            confidence = detections[0, 0, i, 2]
            #print(confidence , detections.shape[2])
            # filter out weak detections by ensuring the `confidence` is
            # greater than the minimum confidence
            if confidence > args["confidence"]:
                # extract the index of the class label from the
                # `detections`, then compute the (x, y)-coordinates of
                # the bounding box for the object
                idx = int(detections[0, 0, i, 1])
                box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
                (startX, startY, endX, endY) = box.astype("int")

                # draw the prediction on the frame
                label = "{}: {:.2f}%".format(CLASSES[idx],
                    confidence * 100)
                #print(count)
                label_cut = label.split(':')
                print(label,"----")
                if((startX+endX)/2<=125):
                    print("Left")
                    label_t = label.split(':')
                    label_left.append(label_t[0])
                    if(label_cut[0] in CLASSES):
                        dict['L'][label_cut[0]]+=1
                elif((startX+endX)/2 >=175):
                    label_t = label.split(':')
                    print("Right")
                    label_right.append(label_t[0])
                    if(label_cut[0] in CLASSES):
                        dict['R'][label_cut[0]]+=1
                else:
                    label_t = label.split(':')
                    print("Center")
                    label_center.append(label_t[0])
                    if(label_cut[0] in CLASSES):
                        dict['C'][label_cut[0]]+=1

                cv2.rectangle(frame, (startX, startY), (endX, endY),
                    COLORS[idx], 2)
                y = startY - 15 if startY - 15 > 15 else startY + 15
                cv2.putText(frame, label, (startX, y),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLORS[idx], 2)
                # print("left",label_left)
                # print("right",label_right)
                # print('center',label_center)

                print(dict)

                print("Left label",label_left)
                print("Right label",label_right)
                print("Center label",label_center)
                print("----------------\n",label_left,"\n---------------")
                if(len(label_left)>0):
                    label_for_left = label_left[0].split(':')
                    print(dict['L'][label_for_left[0]])
                if(len(label_center)>0):
                    label_for_center = label_center[0].split(':')
                    print(dict['C'][label_for_center[0]])
                if(len(label_right)>0):
                    label_for_right = label_right[0].split(':')
                    print(dict['R'][label_for_right[0]])
                for j in (label_left):
                    if(dict['L'][j]!=0):
                        cnt = dict['L'][j]
                        if(cnt!=1):
                            if(j == 'person'):
                                textl = "In Left There are "+str(cnt)+" People"
                            elif(j == 'sheep'):
                                textl = "In Left There are "+str(cnt)+"Sheep"
                            elif(j == 'bus'):
                                textl = "In Left There are "+str(cnt)+"buses"
                            else:
                                textl = "In Left There are "+str(cnt)+" "+str(j)+"s"
                        else:
                            textl = "In Left There is "+str(cnt)+" "+str(j)
                    print(str(textl))
                    n.say(str(textl))
                    n.runAndWait()
                for k in (label_center):
                    if(dict['C'][k]!=0):
                        cnt = dict['C'][k]
                        if(cnt!=1):
                            if(k == 'person'):
                                textl = "In Left There are "+str(cnt)+" People"
                            elif(j == 'sheep'):
                                textl = "In Left There are "+str(cnt)+"Sheep"
                            elif(j == 'bus'):
                                textl = "In Left There are "+str(cnt)+"buses"
                            else:
                                textc = "In Front there are "+str(cnt)+" "+str(k)+"s"
                        else:
                            textc = "In Front there is "+str(cnt)+" "+str(k)
                    print(str(textc))
                    n.say(str(textc))
                    n.runAndWait()
                for l in (label_right):
                    if(dict['R'][l]!=0):
                        cnt = dict['R'][l]
                        if(cnt!=1):
                            if(l == 'person'):
                                textl = "In Left There are "+str(cnt)+" People"
                            elif(j == 'sheep'):
                                textl = "In Left There are "+str(cnt)+"Sheep"
                            elif(j == 'bus'):
                                textl = "In Left There are "+str(cnt)+"buses"
                            else:
                                textr = "In Right There are "+str(cnt)+" "+str(l)+"s"
                        else:
                            textr = "In Right There is "+str(cnt)+" "+str(l)
                    print(str(textr))
                    n.say(str(textr))
                    n.runAndWait()
        # show the output frame
        cv2.imshow("Frame", frame)
        key = cv2.waitKey(1) & 0xFF

        # if the `q` key was pressed, break from the loop
        if key == ord("q"):
            break

        # update the FPS counter
        fps.update()

    # stop the timer and display FPS information

    fps.stop()
    print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

    # do a bit of cleanup
    cv2.destroyAllWindows()
    vs.stop()

def create_directory(path):
        """
        Create directory at given path if directory does not exist
        :param path:
        :return:
        """
        if not os.path.exists(path):
            os.makedirs(path)

def check_path(path):
        """
        Check if file path exists or not
        :param path:
        :return: boolean
        """
        return bool(os.path.exists(path))

def get_command():
        """
        Check OS and return command to identify if tesseract is installed or not
        :return:
        """
        if sys.platform.startswith('win'):
            return WINDOWS_CHECK_COMMAND
        return DEFAULT_CHECK_COMMAND

def run_tesseract(filename, output_path, image_file_name):
    # Run tesseract
    filename_without_extension = os.path.splitext(filename)[0]
    # If no output path is provided
    if not output_path:
        temp_dir = tempfile.mkdtemp()
        temp_file = os.path.join(temp_dir, filename_without_extension)
        subprocess.run(['tesseract', image_file_name, temp_file],
                       stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE)
        with open('{}.txt'.format(temp_file), 'r', encoding="utf8") as f:
            text = f.read()
        shutil.rmtree(temp_dir)
        return text
    text_file_path = os.path.join(output_path, filename_without_extension)
    subprocess.run(['tesseract', image_file_name, text_file_path],
                   stdout=subprocess.PIPE,
                   stderr=subprocess.PIPE)
    return

def check_pre_requisites_tesseract():
    """
    Check if the pre-requisites required for running the tesseract application are satisfied or not
    :param : NA
    :return: boolean
    """
    check_command = get_command()
    logging.debug("Running `{}` to check if tesseract is installed or not.".format(check_command))

    result = subprocess.run([check_command, 'tesseract'], stdout=subprocess.PIPE)
    if not result.stdout:
        logging.error("tesseract-ocr missing, install `tesseract` to resolve. Refer to README for more instructions.")
        return False
    logging.debug("Tesseract correctly installed!\n")

    if sys.platform.startswith('win'):
        environment_variables = os.environ
        logging.debug(
            "Checking if the Tesseract Data path is set correctly or not.\n")
        if TESSERACT_DATA_PATH_VAR in environment_variables:
            if environment_variables[TESSERACT_DATA_PATH_VAR]:
                path = environment_variables[TESSERACT_DATA_PATH_VAR]
                logging.debug("Checking if the path configured for Tesseract Data Environment variable `{}` \
                as `{}` is valid or not.".format(TESSERACT_DATA_PATH_VAR, path))
                if os.path.isdir(path) and os.access(path, os.R_OK):
                    logging.debug("All set to go!")
                    return True
                else:
                    logging.error(
                        "Configured path for Tesseract data is not accessible!")
                    return False
            else:
                logging.error("Tesseract Data path Environment variable '{}' configured to an empty string!\
                ".format(TESSERACT_DATA_PATH_VAR))
                return False
        else:
            logging.error("Tesseract Data path Environment variable '{}' needs to be configured to point to\
            the tessdata!".format(TESSERACT_DATA_PATH_VAR))
            return False
    else:
        return True

def main(input_path, output_path):
    # Check if tesseract is installed or not
    if not check_pre_requisites_tesseract():
        return

    # Check if a valid input directory is given or not
    if not check_path(input_path):
        logging.error("Nothing found at `{}`".format(input_path))
        return

    # Create output directory
    if output_path:
        create_directory(output_path)
        logging.debug("Creating Output Path {}".format(output_path))

    # Check if input_path is directory or file
    if os.path.isdir(input_path):
        logging.debug("The Input Path is a directory.")
        # Check if input directory is empty or not
        total_file_count = len(os.listdir(input_path))
        if total_file_count == 0:
            logging.error("No files found at your input location")
            return

        # Iterate over all images in the input directory
        # and get text from each image
        other_files = 0
        successful_files = 0
        logging.info("Found total {} file(s)\n".format(total_file_count))
        for ctr, filename in enumerate(os.listdir(input_path)):
            logging.debug("Parsing {}".format(filename))
            extension = os.path.splitext(filename)[1]

            if extension.lower() not in VALID_IMAGE_EXTENSIONS:
                other_files += 1
                continue

            image_file_name = os.path.join(input_path, filename)
            print(run_tesseract(filename, output_path, image_file_name))
            successful_files += 1

        logging.info("Parsing Completed!\n")
        if successful_files == 0:
            logging.error("No valid image file found.")
            logging.error("Supported formats: [{}]".format(
                ", ".join(VALID_IMAGE_EXTENSIONS)))
        else:
            logging.info(
                "Successfully parsed images: {}".format(successful_files))
            logging.info(
                "Files with unsupported file extensions: {}".format(other_files))

    else:
        filename = os.path.basename(input_path)
        logging.debug("The Input Path is a file {}".format(filename))
        text = run_tesseract(filename, output_path, input_path)
        print(text)
        n.say(text);
        n.runAndWait()

def readMode():
    n = pt.init(driverName='espeak');
    path = "/home/nisarg/image2text-master/realtime.jpg"
    #os.system("cheese")
    os.system("fswebcam -r 640x480 --jpeg 85 -D 0 "+path)
    os.system("eog /home/nisarg/image2text-master/realtime.jpg")

    parser = argparse.ArgumentParser()
    parser._action_groups.pop()
    #required = parser.add_argument_group('required arguments')
    optional = parser.add_argument_group('optional arguments')
    #required.add_argument('-i', '--input', help="Single image file path or images directory path", required=True)
    optional.add_argument('-o', '--output', help="(Optional) Output directory for converted text")
    optional.add_argument('-d', '--debug', action='store_true', help="Enable verbose DEBUG logging")

    args = parser.parse_args()
    #input_path = os.path.abspath(args.input)
    input_path = path
    #print("Args : ",args)
    if args.output:
        output_path = os.path.abspath(args.output)
    else:
        output_path = None

    if args.debug:
        logging.getLogger().setLevel(logging.DEBUG)
    else:
        logging.getLogger().setLevel(logging.INFO)

    logging.debug("Input Path is {}".format(input_path))

    # Check Python version
    if sys.version_info[0] < 3:
        logging.error("You are using Python {0}.{1}. Please use Python>=3".format(
            sys.version_info[0], sys.version_info[1]))
        exit()

    main(input_path, output_path)

while(True):
    if GPIO.input(5)==0:
        observeMode()
        #time.sleep(3)
    elif GPIO.input(6)==0:
        readMode()


import sys
import re
#clipspath = r"D:\Info\I3b\Programare bazata pe reguli\Clips\Proiect\facts.clp"
clipspath = r"..\Clips\facts.clp"
def readFile(filename, mode):
    midiFile = open(filename, mode)
    return midiFile

def readHeader(midiFile):
    MThd = midiFile.read(4).decode()
    headerLength = int.from_bytes(midiFile.read(4), "big")
    format = int.from_bytes(midiFile.read(2), "big")
    tracks = int.from_bytes(midiFile.read(2), "big")
    division = int.from_bytes(midiFile.read(2), "big")
    data = {"format": format, "tracks": tracks, "division": division}
    return {"type": MThd, "length": headerLength, "data": data}

def computeDeltaTime(data, i):
    deltaBytes = []
    while data[i][0] == '1':
        deltaBytes.append(data[i][1:])
        i += 1
    deltaBytes.append(data[i][1:])
    deltaBytes = "".join(deltaBytes)
    delta_time = 0
    for j in range(0, len(deltaBytes)):
        delta_time = delta_time + int(deltaBytes[j]) * pow(2, len(deltaBytes) - 1 - j)
    i = i + 1
    return delta_time, i

def computeData(data):
    events = []
    i = 0
    running_status = None
    while i < len(data):
        delta_time, i = computeDeltaTime(data, i)
        eventID = data[i]
        # meta_event
        if eventID == "11111111":
            i = i + 1
            eventCode = data[i]
            i = i + 1
            eventLengthArray = []
            while data[i][0] == "1":
                eventLengthArray.append(data[i][1:])
                i += 1
            eventLengthArray.append(data[i][1:])
            eventLengthArray = "".join(eventLengthArray)
            eventLength = 0
            for j in range(0, len(eventLengthArray)):
                eventLength = eventLength + int(eventLengthArray[j]) * pow(2, len(eventLengthArray) - 1 - j)
            i += 1
            eventData = []
            for j in range(0, eventLength):
                eventData.append(data[i])
                i += 1
            meta_event = {"eventCode": eventCode, "eventLength": eventLength, "eventData": eventData}
            eventType = {"Delta": delta_time, "eventType": meta_event}
            events.append(eventType)
        # sysex_events
        elif eventID == "11110000" or eventID == "11110111":
            eventCode = data[i]
            i += 1
            sysexLengthArray = []
            while data[i][0] == '1':
                sysexLengthArray.append(data[i][1:])
                i += 1
            sysexLengthArray.append(data[i][1:])
            sysexLengthArray = "".join(sysexLengthArray)
            sysexLength = 0
            for j in range(0, len(sysexLengthArray)):
                sysexLength = sysexLength + int(sysexLengthArray[j]) * pow(2, len(sysexLengthArray) - 1 - j)
            sysexData = []
            for j in range(0, sysexLength):
                sysexData.append(data[i])
                i += 1
            sysex_event = {"eventCode": eventCode, "eventLength": sysexLength, "eventData": sysexData}
            eventType = {"Delta": delta_time, "eventType": sysex_event}
            events.append(eventType)
        # midi_events
        elif "1000" <= eventID[0:4] <= "1110":
            running_status = eventID
            eventCode = data[i][0:4]
            eventChannel = int.from_bytes(int("0b" + data[i][4:], 2).to_bytes(1, "big"), "big") + 1
            eventData = {"Channel": eventChannel}
            i += 1
            if eventCode == "1000" or eventCode == "1001" or eventCode == "1010":
                i, eventData = Note(data, int(i), eventData)
            elif eventCode == "1011":
                i, eventData = ControlChange(data, int(i), eventData)
            elif eventCode == "1100":
                i, eventData = ProgramChange(data, int(i), eventData)
            elif eventCode == "1101":
                i, eventData = ChannelKeyPressure(data, int(i), eventData)
            elif eventCode == "1110":
                i, eventData = PitchBend(data, int(i), eventData)

            midi_event = {"eventCode": eventCode, "eventData": eventData}
            eventType = {"Delta": delta_time, "eventType": midi_event}
            events.append(eventType)

        else:
            eventChannel = int.from_bytes(int("0b" + running_status[4:], 2).to_bytes(1, "big"), "big") + 1
            eventData = {"Channel": eventChannel}
            if running_status[:4] == "1000" or running_status[:4] == "1001" or running_status[:4] == "1010":
                i, eventData = Note(data, i, eventData)
            elif running_status[:4] == "1011":
                i, eventData = ControlChange(data, i, eventData)
            elif running_status[:4] == "1100":
                i, eventData = ProgramChange(data, i, eventData)
            elif running_status[:4] == "1101":
                i, eventData = ChannelKeyPressure(data, i, eventData)
            elif running_status[:4] == "1110":
                i, eventData = PitchBend(data, i, eventData)
            midi_event = {"eventCode": running_status[:4], "eventData": eventData}
            eventType = {"Delta": delta_time, "eventType": midi_event}
            events.append(eventType)

    return events

def Note(data, i, eventData):
    eventNote = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    i += 1
    eventVelocity = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    eventData["Note"] = eventNote
    eventData["Velocity"] = eventVelocity
    i += 1
    return i, eventData

def ControlChange(data, i: int, eventData):
    controllerNumber = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    i += 1
    controllerValue = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    eventData["Control"] = controllerNumber
    eventData["Value"] = controllerValue
    i += 1
    return i, eventData

def ProgramChange(data, i: int, eventData):
    newProgramNumber = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    eventData["Number"] = newProgramNumber
    i += 1
    return i, eventData

def ChannelKeyPressure(data, i: int, eventData):
    channelPressureValue = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    eventData["Pressure"] = channelPressureValue
    i += 1
    return i, eventData

def PitchBend(data, i: int, eventData):
    lsb = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    i += 1
    msb = int.from_bytes(int("0b" + data[i], 2).to_bytes(1, "big"), "big")
    eventData["Lsb"] = lsb
    eventData["Msb"] = msb
    i += 1
    return i, eventData

def computeEvent(event):
    # Start meta-events
    eventName = event["eventType"]["eventCode"]
    if eventName == "00000000":
        event["eventType"]["eventCode"] = "SequenceNumber"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big")
        event["eventType"]["eventData"] = int.from_bytes(
            event["eventType"]["eventData"][0] + event["eventType"]["eventData"][1], "big")
    if eventName == "00000001":
        event["eventType"]["eventCode"] = "TextEvent"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000010":
        event["eventType"]["eventCode"] = "CopyrightNotice"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000011":
        event["eventType"]["eventCode"] = "TrackName"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000100":
        event["eventType"]["eventCode"] = "InstrumentName"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000101":
        event["eventType"]["eventCode"] = "Lyric"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000110":
        event["eventType"]["eventCode"] = "Marker"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00000111":
        event["eventType"]["eventCode"] = "CuePoint"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big").decode()
        event["eventType"]["eventData"] = "".join(event["eventType"]["eventData"])
    if eventName == "00100000":
        event["eventType"]["eventCode"] = "MIDIChannelPrefix"
        event["eventType"]["eventData"] = int("0b" + event["eventType"]["eventData"], 2). \
            to_bytes(1, "big")
        event["eventType"]["eventData"] = int.from_bytes(event["eventType"]["eventData"], "big")
    if eventName == "00101111":
        event["eventType"]["eventCode"] = \
            "EndOfTrack"
    if eventName == "01010001":
        event["eventType"]["eventCode"] = "SetTempo"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big")
        event["eventType"]["eventData"] = int.from_bytes(
            event["eventType"]["eventData"][0] + event["eventType"]["eventData"][1] + event["eventType"]["eventData"][
                2], "big")
    if eventName == "01010100":
        event["eventType"]["eventCode"] = "SMTPEOffset"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big")
            event["eventType"]["eventData"][i] = int.from_bytes(event["eventType"]["eventData"][i], "big")
        event["eventType"]["eventData"] = {"Hours": event["eventType"]["eventData"][0],
                                           "Minutes": event["eventType"]["eventData"][1],
                                           "Seconds": event["eventType"]["eventData"][2],
                                           "Frames": event["eventType"]["eventData"][3],
                                           "Fractional Frame": event["eventType"]["eventData"][4]}
    if eventName == "01011000":
        event["eventType"]["eventCode"] = "TimeSignature"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big")
            event["eventType"]["eventData"][i] = int.from_bytes(event["eventType"]["eventData"][i], "big")
        event["eventType"]["eventData"] = {"Numerator": event["eventType"]["eventData"][0],
                                           "LogDenomitator": event["eventType"]["eventData"][1],
                                           "MIDIClocksPerMetronomeClick": event["eventType"]["eventData"][2],
                                           "ThirtySecondsPer24Clocks": event["eventType"]["eventData"][3]
                                           }
    if eventName == "01011001":
        event["eventType"]["eventCode"] = "KeySignature"
        for i in range(0, event["eventType"]["eventLength"]):
            event["eventType"]["eventData"][i] = int("0b" + event["eventType"]["eventData"][i], 2). \
                to_bytes(1, "big")
            event["eventType"]["eventData"][i] = int.from_bytes(event["eventType"]["eventData"][i], "big")
        event["eventType"]["eventData"] = {"Sharps or flats": event["eventType"]["eventData"][0],
                                           "Major/Minor Key": event["eventType"]["eventData"][1]
                                           }
    if eventName == "01111111":
        event["eventType"]["eventCode"] = \
            "Sequencer-SpecificMeta-event"
    # End meta-events

    # Start sysex-events
    if eventName == "11110000":
        event["eventType"]["eventCode"] = \
            "F0SysexEvent"
    if eventName == "11110111":
        event["eventType"]["eventCode"] = \
            "F7SysexEvent"
    # End sysex-events

    # Start midi-events
    if eventName == "1000":
        event["eventType"]["eventCode"] = "NoteOff"
    if eventName == "1001":
        event["eventType"]["eventCode"] = "NoteOn"
    if eventName == "1010":
        event["eventType"]["eventCode"] = "PolyphonicKeyPressure"
    if eventName == "1011":
        event["eventType"]["eventCode"] = "ControllerChange"
    if eventName == "1100":
        event["eventType"]["eventCode"] = "ProgramChange"
    if eventName == "1101":
        event["eventType"]["eventCode"] = "ChannelKeyPressure"
    if eventName == "1110":
        event["eventType"]["eventCode"] = "PitchBend"

    return event

def readTracks(midiFile, numberOfTracks):
    tracks = {}
    for i in range(0, numberOfTracks):
        type = midiFile.read(4).decode()
        length = int.from_bytes(midiFile.read(4), "big")
        data = []
        for j in range(0, length):
            z = bin(int.from_bytes(midiFile.read(1), "big")).replace('0b', '').rjust(8, "0")
            data.append(z)
        events = computeData(data)
        for event in events:
            computeEvent(event)

        track = {"type": type, "length": length, "data": events}
        tracks[i] = track
    return tracks

def generateClp():
    global clipspath
    clpfile = open(clipspath, "w")
    templateMidiInfo = \
    '''(deftemplate midiInfo
    (slot midiFormat)
    (slot tracks)
    (slot division)
)
    '''
    templateEvent = \
    '''
(deftemplate event
    (slot order (type INTEGER))
    (slot trackID (type INTEGER))
    (slot delta (type INTEGER))
    (slot tip (type STRING))
    (multislot data)
)
    '''
    deffactsInit = \
'''
(deffacts init
'''
    clpfile.write(templateMidiInfo)
    clpfile.write(templateEvent)
    clpfile.write(deffactsInit)
    with open("midi.txt", "r") as inputFile:
        line = inputFile.readline()
        order = 0
        trackNumber = None
        while line:
            line = re.split("[':{,}\n ]", line)
            while "" in line:
                line.remove("")
            while "eventType" in line:
                line.remove("eventType")
            if line[1] == 'MThd':
                format = line[line.index('format') + 1]
                tracks = line[line.index('tracks') + 1]
                division = line[line.index('division') + 1]
                midiInfoFact = "\t(midiInfo (midiFormat {}) (tracks {}) (division {}))\n\n". \
                    format(format, tracks, division)
                clpfile.write(midiInfoFact)
            elif line[0] == 'TrackNumber':
                trackNumber = line[1]
                order = 0
            else:
                delta = line[line.index('Delta') + 1]
                eventCode = line[line.index('eventCode') + 1]
                eventData = switchEventCode(line, eventCode)
                if eventCode in "NoteOn" and eventData[2] == '0':
                    eventCode = "NoteOff"

                eventData = " ".join(eventData)
                if eventCode in ("TextEvent", "CopyrightNotice", "TrackName",
                                 "InstrumentName", "Lyric", "Marker", "CuePoint"):
                    eventData = "\""+eventData+"\""

                eventCode = "\""+eventCode+"\""
                clpEvent = "\t(event (order {}) (trackID {}) (delta {}) (tip {}) (data {}))\n"\
                    .format(order, trackNumber, delta, eventCode, eventData)
                clpfile.write(clpEvent)
            line = inputFile.readline()
            order += 1
    deffactsInit = \
'''
)

'''
    clpfile.write(deffactsInit)

def switchEventCode(line, eventCode):
    eventData = []
    if eventCode in ("TextEvent", "CopyrightNotice", "TrackName",
                     "InstrumentName", "Lyric", "Marker", "CuePoint"):
        index = line.index("eventData") + 1
        data = line[index]
        while index < len(line) - 1:
            index += 1
            data = data + " " + line[index]
        eventData.append(data)
    elif eventCode in ("SequenceNumber", "MIDIChannelPrefix", "SetTempo"):
        eventData.append(line[line.index("eventData") + 1])
    elif eventCode in ("SMTPEOffset", "TimeSignature", "KeySignature", "Sequence-SpecificMeta-Event",
                       "NoteOff", "NoteOn", "PolyphonicKeyPressure", "ControllerChange",
                       "ProgramChange", "ChannelKeyPressure", "PitchBend"):
        index = line.index("eventData") + 2
        eventData.append(line[index])
        while index < len(line) - 1:
            index += 2
            eventData.append(line[index])

    elif eventCode in "EndOfTrack":
        pass

    return eventData

def main():
    outputFile = open("midi.txt", "w")
    midiSource = "MIDI_SAMPLE.mid"
    if len(sys.argv) == 2:
        midiSource = sys.argv[1]
    midiFile = readFile(midiSource, "rb")
    header = readHeader(midiFile)
    print(header, file=outputFile)
    tracks = readTracks(midiFile, header["data"]["tracks"])
    for i in range(0, header["data"]["tracks"]):
        print("TrackNumber:", i, file=outputFile)
        for track in tracks[i]["data"]:
            print(track, file=outputFile)
    generateClp()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Mon May 13 14:00:23 2019

@author: Julia
"""

import os
import pandas as pd
import numpy as np

def read_consensus():
    path = 'C:/Users/Alex/Desktop/Charging/Consensus/'
    os.chdir(path)

    pop_zip = pd.read_csv('Zipcode-ZCTA-Population-Density-And-Area-Unsorted.csv')
    pop_zip = pop_zip.set_index(pop_zip['Zip/ZCTA'])

    # housing ACS_17_5YR_DP04_with_ann
    household = pd.read_csv('ACS_17_5YR_DP04_with_ann.csv',low_memory=False)
    household = household.drop(0)
    household['zip'] = pd.to_numeric(household['GEO.display-label'].str.replace('ZCTA5 ', ''))
    household = household.set_index('zip') # set zip code as label

    get_data = ['03','13','14','15','16','17','18','19','20','21','22','41','42','43','44','45','46','47','48','49',
    '55','56','57','58','59','60','65','66','69','70','84','85','86','87','88','120','121','122','123',
    '124','125','126','127','128','133','159','169','184','185','186','187','188','189','190','191','198']

    sel_household = pd.DataFrame(index=household.index)

    name_c = []
    for i in range(len(get_data)):
        sel_household = sel_household.join(household['HC01_VC'+get_data[i]])
        name_c.append('HC01_VC'+get_data[i])

    with open('name_ACS_17_5YR_DP04.txt')as f:
        names = f.read().split('\n')
        f.close()

    sel_household = sel_household.rename(columns=dict(zip(name_c,names)))

    consensus = pop_zip.join(sel_household)

    #income -  ACS_17_5YR_S1901_with_ann (by household) ACS_17_5YR_B19001_with_ann
    income = pd.read_csv('ACS_17_5YR_B19001_with_ann.csv',low_memory=False)
    income = income.drop(0)
    income['zip'] = pd.to_numeric(income['GEO.display-label'].str.replace('ZCTA5 ', ''))
    income = income.set_index('zip') # set zip code as label

    get_data = ['01','02','03','04','05','06','07','08','09','10','11','13','14','15','16','17']

    sel_income = pd.DataFrame(index=income.index)

    name_c = []
    for i in range(len(get_data)):
        sel_income = sel_income.join(income['HD01_VD'+get_data[i]])
        name_c.append('HD01_VD'+get_data[i])

    with open('name_ACS_17_5YR_B19001.txt')as f:
        names = f.read().split('\n')
        f.close()

    sel_income = sel_income.rename(columns=dict(zip(name_c,names)))

    consensus = consensus.join(sel_income)

    #age and sex  - ACS_17_5YR_S0101_with_ann, DEC_10_SF1_QTP2_with_ann (detail 2010)
    age = pd.read_csv('ACS_17_5YR_S0101_with_ann.csv',low_memory=False)
    age = age.drop(0)
    age['zip'] = pd.to_numeric(age['GEO.display-label'].str.replace('ZCTA5 ', ''))
    age = age.set_index('zip') # set zip code as label

    get_data = ['01','03','04','05','06','07','08','09','10','11','12','13','14','15','16','17','18','19','20']
    ind_data  = ['01','03','04','05','06']

    sel_age = pd.DataFrame(index=age.index)

    name_c = []
    for j in range(len(ind_data)):
        for i in range(len(get_data)):
            sel_age = sel_age.join(age['HC'+ind_data[j]+'_EST_VC'+get_data[i]])
            name_c.append('HC'+ind_data[j]+'_EST_VC'+get_data[i])

    with open('name_ACS_17_5YR_S0101.txt')as f:
        names = f.read().split('\n')
        f.close()

    sel_age = sel_age.rename(columns=dict(zip(name_c,names)))
    sel_age = sel_age.drop(["male_percentage","female_percentage"],axis=1)

    consensus = consensus.join(sel_age)

    #transportation to work - ACS_17_5YR_S0801_with_ann (transport mode and gender) - ACS_17_5YR_S0802_with_ann (by age)
    transport = pd.read_csv('ACS_17_5YR_S0801_with_ann.csv',low_memory=False)
    transport = transport.drop(0)
    transport['zip'] = pd.to_numeric(transport['GEO.display-label'].str.replace('ZCTA5 ', ''))
    transport = transport.set_index('zip') # set zip code as label

    get_data = ['03','04','05','06','07','08','09','10','11','12','13','14','46','47','48','49','50','51','52','53','54','55']

    sel_transport = pd.DataFrame(index=transport.index)

    name_c =[]
    for i in range(len(get_data)):
        transport['HC01_EST_VC'+get_data[i]+'trans'] = transport['HC01_EST_VC'+get_data[i]]
        sel_transport = sel_transport.join(transport['HC01_EST_VC'+get_data[i]+'trans'] )
        name_c.append('HC01_EST_VC'+get_data[i]+'trans')

    with open('name_ACS_17_5YR_S0801.txt')as f:
        names = f.read().split('\n')
        f.close()

    sel_transport = sel_transport.rename(columns=dict(zip(name_c,names)))

    consensus = consensus.join(sel_transport)

    for c in consensus.columns:
        if(type(consensus[c].iloc[1])== str):
            consensus[c] = consensus[c].str.replace('+','').str.replace('-','').str.replace(',','').str.replace('N','0')#.str.replace('(X)','').str.replace('()','')

    return consensus


# import mpi4py
from mpi4py import MPI

# buat COMM
comm = MPI.COMM_WORLD

# dapatkan rank proses
rank = comm.Get_rank()

# dapatkan total proses berjalan
size = comm.Get_size()

# jika saya rank ke 0 maka saya akan mengirimkan pesan ke proses yang mempunyai rank 1 s.d size
if rank == 0:
	msg = 'echo'
	for i in range(1, size):
                comm.send(msg, dest=i)
                print('Proses {} mengirim {} ke proses {}'.format(rank, msg, i))

# jika saya bukan rank 0 maka saya menerima pesan yang berasal dari proses dengan rank 0
else:
    msg = comm.recv(source = 0)
    print('Proses {} menerima {}'.format(rank, msg))