# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from uuid import UUID
from datetime import datetime
from subprocess import Popen, PIPE
from multiprocessing import Process
from os.path import join
from itertools import chain
from collections import defaultdict, Iterable
from json import dumps, loads
from time import sleep

from future.utils import viewitems, viewvalues
import networkx as nx

from qiita_core.qiita_settings import qiita_config
import qiita_db as qdb

def _system_call(cmd):
    """Execute the command `cmd`

    Parameters
    ----------
    cmd : str
        The string containing the command to be run.

    Returns
    -------
    tuple of (str, str, int)
        The standard output, standard error and exist status of the
        executed command

    Notes
    -----
    This function is ported from QIIME (http://www.qiime.org), previously named
    qiime_system_call. QIIME is a GPL project, but we obtained permission from
    the authors of this function to port it to Qiita and keep it under BSD
    license.
    """
    proc = Popen(cmd, universal_newlines=True, shell=True, stdout=PIPE,
                 stderr=PIPE)
    # Communicate pulls all stdout/stderr from the PIPEs
    # This call blocks until the command is done
    stdout, stderr = proc.communicate()
    return_value = proc.returncode
    return stdout, stderr, return_value

def _job_submitter(job_id, cmd):
    """Executes the commands `cmd` and updates the job in case of failure

    Parameters
    ----------
    job_id : str
        The job id that is executed by cmd
    cmd : str
        The command to execute the job
    """
    std_out, std_err, return_value = _system_call(cmd)
    if return_value != 0:
        error = ("Error submitting job:\nStd output:%s\nStd error:%s"
                 % (std_out, std_err))
        # Forcing the creation of a new connection
        qdb.sql_connection.create_new_transaction()
        ProcessingJob(job_id).complete(False, error=error)

class ProcessingJob(qdb.base.QiitaObject):
    r"""Models a job that executes a command in a set of artifacts

    Attributes
    ----------
    user
    command
    parameters
    status
    log
    heartbeat
    step

    Methods
    -------
    exists
    create
    """
    _table = 'processing_job'

    @classmethod
    def exists(cls, job_id):
        """Check if the job `job_id` exists

        Parameters
        ----------
        job_id : str
            The job id

        Returns
        -------
        bool
            True if the job `job_id` exists. False otherwise.
        """
        try:
            UUID(job_id)
        except ValueError:
            return False

        with qdb.sql_connection.TRN:
            sql = """SELECT EXISTS(SELECT *
                                   FROM qiita.processing_job
                                   WHERE processing_job_id = %s)"""
            qdb.sql_connection.TRN.add(sql, [job_id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    @classmethod
    def create(cls, user, parameters, force=False):
        """Creates a new job in the system

        Parameters
        ----------
        user : qiita_db.user.User
            The user executing the job
        parameters : qiita_db.software.Parameters
            The parameters of the job being executed
        force : bool
            Force creation on duplicated parameters

        Returns
        -------
        qiita_db.processing_job.ProcessingJob
            The newly created job

        Notes
        -----
        If force is True the job is gonna be created even if another job
        exists with the same parameters
        """
        TTRN = qdb.sql_connection.TRN
        with TTRN:
            command = parameters.command

            # check if a job with the same parameters already exists
            sql = """SELECT processing_job_id, email, processing_job_status,
                        COUNT(aopj.artifact_id)
                     FROM qiita.processing_job
                     LEFT JOIN qiita.processing_job_status
                        USING (processing_job_status_id)
                     LEFT JOIN qiita.artifact_output_processing_job aopj
                        USING (processing_job_id)
                     WHERE command_id = %s AND processing_job_status IN (
                        'success', 'waiting', 'running', 'in_construction') {0}
                     GROUP BY processing_job_id, email,
                        processing_job_status"""

            # we need to use ILIKE because of booleans as they can be
            # false or False
            params = []
            for k, v in viewitems(parameters.values):
                # this is necessary in case we have an Iterable as a value
                # but that is not unicode or string
                if isinstance(v, Iterable) and not isinstance(v, (str,
                                                                  unicode)):
                    for vv in v:
                        params.extend([k, str(vv)])
                else:
                    params.extend([k, str(v)])

            if params:
                # divided by 2 as we have key-value pairs
                len_params = len(params)/2
                sql = sql.format(' AND ' + ' AND '.join(
                    ["command_parameters->>%s ILIKE %s"] * len_params))
                params = [command.id] + params
                TTRN.add(sql, params)
            else:
                # the sql variable expects the list of parameters but if there
                # is no param we need to replace the {0} with an empty string
                TTRN.add(sql.format(""), [command.id])

            # checking that if the job status is success, it has children
            # [2] status, [3] children count
            existing_jobs = [r for r in TTRN.execute_fetchindex()
                             if r[2] != 'success' or r[3] > 0]
            if existing_jobs and not force:
                raise ValueError(
                    'Cannot create job because the parameters are the same as '
                    'jobs that are queued, running or already have '
                    'succeeded:\n%s' % '\n'.join(
                        ["%s: %s" % (jid, status)
                         for jid, _, status, _ in existing_jobs]))

            sql = """INSERT INTO qiita.processing_job
                        (email, command_id, command_parameters,
                         processing_job_status_id)
                     VALUES (%s, %s, %s, %s)
                     RETURNING processing_job_id"""
            status = qdb.util.convert_to_id(
                "in_construction", "processing_job_status")
            sql_args = [user.id, command.id,
                        parameters.dump(), status]
            TTRN.add(sql, sql_args)
            job_id = TTRN.execute_fetchlast()

            # Link the job with the input artifacts
            sql = """INSERT INTO qiita.artifact_processing_job
                        (artifact_id, processing_job_id)
                     VALUES (%s, %s)"""
            pending = defaultdict(dict)
            for pname, vals in command.parameters.items():
                if vals[0] == 'artifact':
                    artifact_info = parameters.values[pname]
                    # If the artifact_info is a list, then the artifact
                    # still doesn't exists because the current job is part
                    # of a workflow, so we can't link
                    if not isinstance(artifact_info, list):
                        TTRN.add(sql, [artifact_info, job_id])
                    else:
                        pending[artifact_info[0]][pname] = artifact_info[1]
            if pending:
                sql = """UPDATE qiita.processing_job
                         SET pending = %s
                         WHERE processing_job_id = %s"""
                TTRN.add(sql, [dumps(pending), job_id])

            TTRN.execute()

            return cls(job_id)

    @property
    def user(self):
        """The user that launched the job

        Returns
        -------
        qiita_db.user.User
            The user that launched the job
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT email
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            email = qdb.sql_connection.TRN.execute_fetchlast()
            return qdb.user.User(email)

    @property
    def command(self):
        """The command that the job executes

        Returns
        -------
        qiita_db.software.Command
            The command that the job executes
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT command_id
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            cmd_id = qdb.sql_connection.TRN.execute_fetchlast()
            return qdb.software.Command(cmd_id)

    @property
    def parameters(self):
        """The parameters used in the job's command

        Returns
        -------
        qiita_db.software.Parameters
            The parameters used in the job's command
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT command_id, command_parameters
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            res = qdb.sql_connection.TRN.execute_fetchindex()[0]
            return qdb.software.Parameters.load(
                qdb.software.Command(res[0]), values_dict=res[1])

    @property
    def input_artifacts(self):
        """The artifacts used as input in the job

        Returns
        -------
        list of qiita_db.artifact.Artifact
            The artifacs used as input in the job
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT artifact_id
                     FROM qiita.artifact_processing_job
                     WHERE processing_job_id = %s
                     ORDER BY artifact_id"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return [qdb.artifact.Artifact(aid)
                    for aid in qdb.sql_connection.TRN.execute_fetchflatten()]

    @property
    def status(self):
        """The status of the job

        Returns
        -------
        str
            The current status of the job, one of {'queued', 'running',
            'success', 'error', 'in_construction', 'waiting'}

        """
        with qdb.sql_connection.TRN:
            sql = """SELECT processing_job_status
                     FROM qiita.processing_job_status
                        JOIN qiita.processing_job
                            USING (processing_job_status_id)
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    def _set_status(self, value):
        """Sets the status of the job

        Parameters
        ----------
        value : str, {'queued', 'running', 'success', 'error',
                      'in_construction', 'waiting'}
            The new status of the job

        Raises
        ------
        qiita_db.exceptions.QiitaDBStatusError
            - If the current status of the job is 'success'
            - If the current status of the job is 'running' and `value` is
            'queued'
        """
        with qdb.sql_connection.TRN:
            current_status = self.status
            if current_status == 'success':
                raise qdb.exceptions.QiitaDBStatusError(
                    "Cannot change the status of a 'success' job")
            elif current_status == 'running' and value == 'queued':
                raise qdb.exceptions.QiitaDBStatusError(
                    "Cannot revert the status of a 'running' job to 'queued'")

            new_status = qdb.util.convert_to_id(
                value, "processing_job_status")
            sql = """UPDATE qiita.processing_job
                     SET processing_job_status_id = %s
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [new_status, self.id])
            qdb.sql_connection.TRN.execute()

    def _generate_cmd(self):
        """Generates the command to submit the job

        Returns
        -------
        str
            The command to use to submit the job
        """
        job_dir = join(qdb.util.get_work_base_dir(), self.id)
        software = self.command.software
        plugin_start_script = software.start_script
        plugin_env_script = software.environment_script
        # Appending the portal URL so the job requests the information from the
        # portal server that submitted the job
        url = "%s%s" % (qiita_config.base_url, qiita_config.portal_dir)
        cmd = '%s "%s" "%s" "%s" "%s" "%s"' % (
            qiita_config.plugin_launcher, plugin_env_script,
            plugin_start_script, url, self.id, job_dir)
        return cmd

    def submit(self):
        """Submits the job to execution

        Raises
        ------
        QiitaDBOperationNotPermittedError
            If the job is not in 'waiting' or 'in_construction' status
        """
        with qdb.sql_connection.TRN:
            status = self.status
            if status not in {'in_construction', 'waiting'}:
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Can't submit job, not in 'in_construction' or "
                    "'waiting' status. Current status: %s" % status)
            self._set_status('queued')
            # At this point we are going to involve other processes. We need
            # to commit the changes to the DB or the other processes will not
            # see these changes
            qdb.sql_connection.TRN.commit()
        cmd = self._generate_cmd()
        p = Process(target=_job_submitter, args=(self.id, cmd))
        p.start()

    def release(self):
        """Releases the job from the waiting status and creates the artifact

        Returns
        -------
        dict of {int: int}
            The mapping between the job output and the artifact
        """
        with qdb.sql_connection.TRN:
            if self.command.software.type != 'artifact definition':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Only artifact definition jobs can be released")

            # Retrieve the artifact information from the DB
            sql = """SELECT artifact_info
                     FROM qiita.processing_job_validator
                     WHERE validator_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            a_info = qdb.sql_connection.TRN.execute_fetchlast()

            provenance = loads(self.parameters.values['provenance'])
            job = ProcessingJob(provenance['job'])
            if 'data_type' in a_info:
                # This job is resulting from a private job
                parents = None
                params = None
                cmd_out_id = None
                name = None
                data_type = a_info['data_type']
                analysis = qdb.analysis.Analysis(
                    job.parameters.values['analysis'])
                a_info = a_info['artifact_data']
            else:
                # This job is resulting from a plugin job
                parents = job.input_artifacts
                params = job.parameters
                cmd_out_id = provenance['cmd_out_id']
                name = provenance['name']
                analysis = None
                data_type = None

            # Create the artifact
            atype = a_info['artifact_type']
            filepaths = a_info['filepaths']
            a = qdb.artifact.Artifact.create(
                filepaths, atype, parents=parents,
                processing_parameters=params,
                analysis=analysis, data_type=data_type, name=name)

            self._set_status('success')

            mapping = {}
            if cmd_out_id is not None:
                mapping = {cmd_out_id: a.id}

            return mapping

    def release_validators(self):
        """Allows all the validator job spawned by this job to complete"""
        with qdb.sql_connection.TRN:
            if self.command.software.type not in ('artifact transformation',
                                                  'private'):
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Only artifact transformation and private jobs can "
                    "release validators")

            # Check if all the validators are completed. Validator jobs can be
            # in two states when completed: 'waiting' in case of success
            # or 'error' otherwise
            validator_ids = [j.id for j in self.validator_jobs
                             if j.status not in ['waiting', 'error']]

            # Active polling - wait until all validator jobs are completed
            while validator_ids:
                jids = ', '.join([j[0] for j in validator_ids])
                self.step = ("Validating outputs (%d remaining) via "
                             "job(s) %s" % (len(validator_ids), jids))
                sleep(10)
                validator_ids = [j.id for j in self.validator_jobs
                                 if j.status not in ['waiting', 'error']]

            # Check if any of the validators errored
            errored = [j for j in self.validator_jobs
                       if j.status == 'error']
            if errored:
                # At least one of the validators failed, Set the rest of the
                # validators and the current job as failed
                waiting = [j.id for j in self.validator_jobs
                           if j.status == 'waiting']

                common_error = "\n".join(
                    ["Validator %s error message: %s" % (j.id, j.log.msg)
                     for j in errored])

                val_error = "%d sister validator jobs failed: %s" % (
                    len(errored), common_error)
                for j in waiting:
                    ProcessingJob(j)._set_error(val_error)

                self._set_error('%d validator jobs failed: %s'
                                % (len(errored), common_error))
            else:
                mapping = {}
                # Loop through all validator jobs and release them, allowing
                # to create the artifacts. Note that if any artifact creation
                # fails, the rollback operation will make sure that the
                # previously created artifacts are not in there
                for vjob in self.validator_jobs:
                    mapping.update(vjob.release())

                if mapping:
                    sql = """INSERT INTO
                                qiita.artifact_output_processing_job
                                (artifact_id, processing_job_id,
                                command_output_id)
                             VALUES (%s, %s, %s)"""
                    sql_args = [[aid, self.id, outid]
                                for outid, aid in viewitems(mapping)]
                    qdb.sql_connection.TRN.add(sql, sql_args, many=True)

                    self._update_and_launch_children(mapping)
                self._set_status('success')

    def _complete_artifact_definition(self, artifact_data):
        """"Performs the needed steps to complete an artifact definition job

        In order to complete an artifact definition job we need to create
        the artifact, and then start all the jobs that were waiting for this
        artifact to be created. Note that each artifact definition job creates
        one and only one artifact.

        Parameters
        ----------
        artfact_data : {'filepaths': list of (str, str), 'artifact_type': str}
            Dict with the artifact information. `filepaths` contains the list
            of filepaths and filepath types for the artifact and
            `artifact_type` the type of the artifact
        """
        with qdb.sql_connection.TRN:
            atype = artifact_data['artifact_type']
            filepaths = artifact_data['filepaths']
            # We need to differentiate if this artifact is the
            # result of a previous job or uploading
            job_params = self.parameters.values
            if job_params['provenance'] is not None:
                # The artifact is a result from a previous job
                provenance = loads(job_params['provenance'])
                if provenance.get('data_type') is not None:
                    artifact_data = {'data_type': provenance['data_type'],
                                     'artifact_data': artifact_data}

                sql = """UPDATE qiita.processing_job_validator
                         SET artifact_info = %s
                         WHERE validator_id = %s"""
                qdb.sql_connection.TRN.add(
                    sql, [dumps(artifact_data), self.id])
                qdb.sql_connection.TRN.execute()
                # Can't create the artifact until all validators are completed
                self._set_status('waiting')
            else:
                # The artifact is uploaded by the user or is the initial
                # artifact of an analysis
                if ('analysis' in job_params and
                        job_params['analysis'] is not None):
                    pt = None
                    an = qdb.analysis.Analysis(job_params['analysis'])
                    sql = """SELECT data_type
                             FROM qiita.analysis_processing_job
                             WHERE analysis_id = %s
                                AND processing_job_id = %s"""
                    qdb.sql_connection.TRN.add(sql, [an.id, self.id])
                    data_type = qdb.sql_connection.TRN.execute_fetchlast()
                else:
                    pt = qdb.metadata_template.prep_template.PrepTemplate(
                        job_params['template'])
                    an = None
                    data_type = None

                qdb.artifact.Artifact.create(
                    filepaths, atype, prep_template=pt, analysis=an,
                    data_type=data_type, name=job_params['name'])
                self._set_status('success')

    def _complete_artifact_transformation(self, artifacts_data):
        """Performs the needed steps to complete an artifact transformation job

        In order to complete an artifact transformation job, we need to create
        a validate job for each artifact output and submit it.

        Parameters
        ----------
        artifacts_data : dict of dicts
            The generated artifact information keyed by output name.
            The format of each of the internal dictionaries must be
            {'filepaths': list of (str, str), 'artifact_type': str}
            where `filepaths` contains the list of filepaths and filepath types
            for the artifact and `artifact_type` the type of the artifact

        Raises
        ------
        QiitaDBError
            If there is more than one prep information attached to the new
            artifact
        """
        validator_jobs = []
        with qdb.sql_connection.TRN:
            cmd_id = self.command.id
            for out_name, a_data in viewitems(artifacts_data):
                # Correct the format of the filepaths parameter so we can
                # create a validate job
                filepaths = defaultdict(list)
                for fp, fptype in a_data['filepaths']:
                    filepaths[fptype].append(fp)
                atype = a_data['artifact_type']

                # The valdiate job needs a prep information file. In theory,
                # a job can be generated from more that one prep information
                # file, so we check here if we have one or more templates. At
                # this moment, If we allow more than one template, there is a
                # fair amount of changes that need to be done on the plugins,
                # so we are going to restrict the number of templates to one.
                # Note that at this moment there is no way of generating an
                # artifact from 2 or more artifacts, so we can impose this
                # limitation now and relax it later.
                templates = set()
                for artifact in self.input_artifacts:
                    templates.update(pt.id for pt in artifact.prep_templates)
                template = None
                analysis = None
                if len(templates) > 1:
                    raise qdb.exceptions.QiitaDBError(
                        "Currently only single prep template "
                        "is allowed, found %d" % len(templates))
                elif len(templates) == 1:
                    template = templates.pop()
                else:
                    # In this case we have 0 templates. What this means is that
                    # this artifact is being generated in the analysis pipeline
                    # All the artifacts included in the analysis pipeline
                    # belong to the same analysis, so we can just ask the
                    # first artifact for the analysis that it belongs to
                    analysis = self.input_artifacts[0].analysis.id

                # Once the validate job completes, it needs to know if it has
                # been generated from a command (and how) or if it has been
                # uploaded. In order to differentiate these cases, we populate
                # the provenance parameter with some information about the
                # current job and how this artifact has been generated. This
                # does not affect the plugins since they can ignore this
                # parameter
                sql = """SELECT command_output_id
                         FROM qiita.command_output
                         WHERE name = %s AND command_id = %s"""
                qdb.sql_connection.TRN.add(sql, [out_name, cmd_id])
                cmd_out_id = qdb.sql_connection.TRN.execute_fetchlast()
                naming_params = self.command.naming_order
                if naming_params:
                    params = self.parameters.values
                    art_name = "%s %s" % (
                        out_name, ' '.join([str(params[p])
                                            for p in naming_params]))
                else:
                    art_name = out_name

                provenance = {'job': self.id,
                              'cmd_out_id': cmd_out_id,
                              'name': art_name}

                # Get the validator command for the current artifact type and
                # create a new job
                cmd = qdb.software.Command.get_validator(atype)
                values_dict = {
                    'files': dumps(filepaths), 'artifact_type': atype,
                    'template': template, 'provenance': dumps(provenance),
                    'analysis': None}
                if analysis is not None:
                    values_dict['analysis'] = analysis
                validate_params = qdb.software.Parameters.load(
                    cmd, values_dict=values_dict)
                validator_jobs.append(
                    ProcessingJob.create(self.user, validate_params, True))

            # Change the current step of the job

            self.step = "Validating outputs (%d remaining) via job(s) %s" % (
                len(validator_jobs), ', '.join([j.id for j in validator_jobs]))

            # Link all the validator jobs with the current job
            self._set_validator_jobs(validator_jobs)
            # Submit all the validator jobs
            for j in validator_jobs:
                j.submit()

            # Submit the job that will release all the validators
            plugin = qdb.software.Software.from_name_and_version(
                'Qiita', 'alpha')
            cmd = plugin.get_command('release_validators')
            params = qdb.software.Parameters.load(
                cmd, values_dict={'job': self.id})
            job = ProcessingJob.create(self.user, params)
        # Doing the submission outside of the transaction
        job.submit()

    def _set_validator_jobs(self, validator_jobs):
        """Sets the validator jobs for the current job

        Parameters
        ----------
        validator_jobs : list of ProcessingJob
            The validator_jobs for the current job
        """
        with qdb.sql_connection.TRN:
            sql = """INSERT INTO qiita.processing_job_validator
                        (processing_job_id, validator_id)
                     VALUES (%s, %s)"""
            sql_args = [[self.id, j.id] for j in validator_jobs]
            qdb.sql_connection.TRN.add(sql, sql_args, many=True)
            qdb.sql_connection.TRN.execute()

    def complete(self, success, artifacts_data=None, error=None):
        """Completes the job, either with a success or error status

        Parameters
        ----------
        success : bool
            Whether the job has completed successfully or not
        artifacts_data : dict of dicts, optional
            The generated artifact information keyed by output name.
            The format of each of the internal dictionaries must be
            {'filepaths': list of (str, str), 'artifact_type': str}
            where `filepaths` contains the list of filepaths and filepath types
            for the artifact and `artifact_type` the type of the artifact
        error : str, optional
            If the job was not successful, the error message

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the job is not in running state
        """
        with qdb.sql_connection.TRN:
            if success:
                if self.status != 'running':
                    # If the job is not running, we only allow to complete it
                    # if it did not succeed
                    raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                        "Can't complete job: not in a running state")
                if artifacts_data:
                    if self.command.software.type == 'artifact definition':
                        # There is only one artifact created
                        _, a_data = artifacts_data.popitem()
                        self._complete_artifact_definition(a_data)
                    else:
                        self._complete_artifact_transformation(artifacts_data)
                else:
                    self._set_status('success')
            else:
                self._set_error(error)

    @property
    def log(self):
        """The log entry attached to the job if it failed

        Returns
        -------
        qiita_db.logger.LogEntry or None
            If the status of the job is `error`, returns the LogEntry attached
            to the job
        """
        with qdb.sql_connection.TRN:
            res = None
            if self.status == 'error':
                sql = """SELECT logging_id
                         FROM qiita.processing_job
                         WHERE processing_job_id = %s"""
                qdb.sql_connection.TRN.add(sql, [self.id])
                log_id = qdb.sql_connection.TRN.execute_fetchlast()
                res = qdb.logger.LogEntry(log_id)
        return res

    def _set_error(self, error):
        """Attaches a log entry to the job

        Parameters
        ----------
        error : str
            The error message

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the status of the job is 'success'
        """
        with qdb.sql_connection.TRN:
            if self.status == 'success':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Can only set up the log for jobs whose status is 'error'")

            self._set_status('error')
            log = qdb.logger.LogEntry.create('Runtime', error)

            sql = """UPDATE qiita.processing_job
                     SET logging_id = %s
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [log.id, self.id])
            qdb.sql_connection.TRN.execute()

            # All the children should be marked as failure
            for c in self.children:
                c.complete(False, error="Parent job '%s' failed." % self.id)

    @property
    def heartbeat(self):
        """The timestamp of the last heartbeat received from the job

        Returns
        -------
        datetime
            The last heartbeat timestamp
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT heartbeat
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    def update_heartbeat_state(self):
        """Updates the heartbeat of the job

        In case that the job is in `queued` status, it changes the status to
        `running`.

        Raises
        ------
        QiitaDBOperationNotPermittedError
            If the job is already completed
        """
        with qdb.sql_connection.TRN:
            status = self.status
            if status == 'queued':
                self._set_status('running')
            elif status != 'running':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Can't execute heartbeat on job: already completed")
            sql = """UPDATE qiita.processing_job
                     SET heartbeat = %s
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [datetime.now(), self.id])
            qdb.sql_connection.TRN.execute()

    @property
    def step(self):
        """Returns the current step of the job

        Returns
        -------
        str
            The current step of the job
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT step
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    @step.setter
    def step(self, value):
        """Sets the current step of the job

        Parameters
        ----------
        value : str
            The new current step of the job

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the status of the job is not 'running'
        """
        with qdb.sql_connection.TRN:
            if self.status != 'running':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Cannot change the step of a job whose status is not "
                    "'running'")
            sql = """UPDATE qiita.processing_job
                     SET step = %s
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [value, self.id])
            qdb.sql_connection.TRN.execute()

    @property
    def children(self):
        """The children jobs

        Returns
        -------
        generator of qiita_db.processing_job.ProcessingJob
            The children jobs
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT child_id
                     FROM qiita.parent_processing_job
                     WHERE parent_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            for jid in qdb.sql_connection.TRN.execute_fetchflatten():
                yield ProcessingJob(jid)

    @property
    def validator_jobs(self):
        """The validators of this job

        Returns
        -------
        generator of qiita_db.processing_job.ProcessingJob
            The validators of this job
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT validator_id
                     FROM qiita.processing_job_validator pjv
                     JOIN qiita.processing_job pj
                         ON pjv.validator_id = pj.processing_job_id
                     JOIN qiita.processing_job_status USING (
                        processing_job_status_id)
                     WHERE pjv.processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            for jid in qdb.sql_connection.TRN.execute_fetchflatten():
                yield ProcessingJob(jid)

    def _update_children(self, mapping):
        """Updates the children of the current job to populate the input params

        Parameters
        ----------
        mapping : dict of {int: int}
            The mapping between output parameter and artifact

        Returns
        -------
        list of qiita_db.processing_job.ProcessingJob
            The list of childrens that are ready to be submitted
        """
        ready = []
        with qdb.sql_connection.TRN:
            sql = """SELECT command_output_id, name
                     FROM qiita.command_output
                     WHERE command_output_id IN %s"""
            sql_args = [tuple(mapping.keys())]
            qdb.sql_connection.TRN.add(sql, sql_args)
            res = qdb.sql_connection.TRN.execute_fetchindex()
            new_map = {name: mapping[oid] for oid, name in res}

            sql = """SELECT command_parameters, pending
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            sql_update = """UPDATE qiita.processing_job
                            SET command_parameters = %s,
                                pending = %s
                            WHERE processing_job_id = %s"""
            sql_link = """INSERT INTO qiita.artifact_processing_job
                            (artifact_id, processing_job_id)
                          VALUES (%s, %s)"""
            for c in self.children:
                qdb.sql_connection.TRN.add(sql, [c.id])
                params, pending = qdb.sql_connection.TRN.execute_fetchflatten()
                for pname, out_name in viewitems(pending[self.id]):
                    a_id = new_map[out_name]
                    params[pname] = str(a_id)
                    del pending[self.id]
                    # Link the input artifact with the child job
                    qdb.sql_connection.TRN.add(sql_link, [a_id, c.id])

                # Force to insert a NULL in the DB if pending is empty
                pending = pending if pending else None
                qdb.sql_connection.TRN.add(sql_update,
                                           [dumps(params), pending, c.id])
                qdb.sql_connection.TRN.execute()

                if pending is None:
                    # The child already has all the parameters
                    # Add it to the ready list
                    ready.append(c)
        return ready

    def _update_and_launch_children(self, mapping):
        """Updates the children of the current job to populate the input params

        Parameters
        ----------
        mapping : dict of {int: int}
            The mapping between output parameter and artifact
        """
        ready = self._update_children(mapping)
        # Submit all the children that already have all the input parameters
        for c in ready:
            c.submit()

    @property
    def outputs(self):
        """The outputs of the job

        Returns
        -------
        dict of {str: qiita_db.artifact.Artifact}
            The outputs of the job keyed by output name
        """
        with qdb.sql_connection.TRN:
            if self.status != 'success':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Can't return the outputs of a non-success job")

            sql = """SELECT artifact_id, name
                     FROM qiita.artifact_output_processing_job
                        JOIN qiita.command_output USING (command_output_id)
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return {
                name: qdb.artifact.Artifact(aid)
                for aid, name in qdb.sql_connection.TRN.execute_fetchindex()}

    @property
    def processing_job_workflow(self):
        """The processing job worflow

        Returns
        -------
        ProcessingWorkflow
            The processing job workflow the job
        """
        with qdb.sql_connection.TRN:
            # Retrieve the workflow root jobs
            sql = """SELECT get_processing_workflow_roots
                     FROM qiita.get_processing_workflow_roots(%s)"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            res = qdb.sql_connection.TRN.execute_fetchindex()
            if res:
                sql = """SELECT processing_job_workflow_id
                         FROM qiita.processing_job_workflow_root
                         WHERE processing_job_id = %s"""
                qdb.sql_connection.TRN.add(sql, [res[0][0]])
                r = qdb.sql_connection.TRN.execute_fetchindex()
                return (qdb.processing_job.ProcessingWorkflow(r[0][0]) if r
                        else None)
            else:
                return None

    @property
    def pending(self):
        """A dictionary with the information about the predecessor jobs

        Returns
        -------
        dict
            A dict with {job_id: {parameter_name: output_name}}"""
        with qdb.sql_connection.TRN:
            sql = """SELECT pending
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            res = qdb.sql_connection.TRN.execute_fetchlast()
            return res if res is not None else {}

    @property
    def hidden(self):
        """Whether the job is hidden or not

        Returns
        -------
        bool
            Whether the jobs is hidden or not
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT hidden
                     FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    def hide(self):
        """Hides the job from the user

        Raises
        ------
        QiitaDBOperationNotPermittedError
            If the job is not in the error status
        """
        with qdb.sql_connection.TRN:
            status = self.status
            if status != 'error':
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    'Only jobs in error status can be hidden. Current status: '
                    '%s' % status)
            sql = """UPDATE qiita.processing_job
                     SET hidden = %s
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [True, self.id])
            qdb.sql_connection.TRN.execute()

class ProcessingWorkflow(qdb.base.QiitaObject):
    """Models a workflow defined by the user

    Parameters
    ----------
    user : qiita_db.user.User
        The user that modeled the workflow
    root : list of qiita_db.processing_job.ProcessingJob
        The first job in the workflow
    """
    _table = "processing_job_workflow"

    @classmethod
    def _common_creation_steps(cls, user, root_jobs, name=None):
        """Executes the common creation steps

        Parameters
        ----------
        user : qiita_db.user.User
            The user creating the workflow
        root_jobs : list of qiita_db.processing_job.ProcessingJob
            The root jobs of the workflow
        name : str, optional
            The name of the workflow. Default: generated from user's name
        """
        with qdb.sql_connection.TRN:
            # Insert the workflow in the processing_job_workflow table
            name = name if name else "%s's workflow" % user.info['name']
            sql = """INSERT INTO qiita.processing_job_workflow (email, name)
                     VALUES (%s, %s)
                     RETURNING processing_job_workflow_id"""
            qdb.sql_connection.TRN.add(sql, [user.email, name])
            w_id = qdb.sql_connection.TRN.execute_fetchlast()
            # Connect the workflow with it's initial set of jobs
            sql = """INSERT INTO qiita.processing_job_workflow_root
                        (processing_job_workflow_id, processing_job_id)
                     VALUES (%s, %s)"""
            sql_args = [[w_id, j.id] for j in root_jobs]
            qdb.sql_connection.TRN.add(sql, sql_args, many=True)
            qdb.sql_connection.TRN.execute()

        return cls(w_id)

    @classmethod
    def from_default_workflow(cls, user, dflt_wf, req_params, name=None,
                              force=False):
        """Creates a new processing workflow from a default workflow

        Parameters
        ----------
        user : qiita_db.user.User
            The user creating the workflow
        dflt_wf : qiita_db.software.DefaultWorkflow
            The default workflow
        req_params : dict of {qdb.software.Command: dict of {str: object}}
            The required parameters values for the source commands in the
            workflow, keyed by command. The inner dicts are keyed by
            parameter name.
        name : str, optional
            Name of the workflow. Default: generated from user's name
        force : bool
            Force creation on duplicated parameters

        Returns
        -------
        qiita_db.processing_job.ProcessingWorkflow
            The newly created workflow
        """
        with qdb.sql_connection.TRN:
            dflt_g = dflt_wf.graph

            # Find the roots of the workflow. That is, the nodes that do not
            # have a parent in the graph (in_degree = 0)
            in_degrees = dflt_g.in_degree()

            # We can potentially access this information from the nodes
            # multiple times, so caching in here
            all_nodes = {n: (n.command, n.parameters)
                         for n in in_degrees}
            roots = {n: (n.command, n.parameters)
                     for n, d in viewitems(in_degrees) if d == 0}

            # Check that we have all the required parameters
            root_cmds = set(c for c, _ in viewvalues(roots))
            if root_cmds != set(req_params):
                error_msg = ['Provided required parameters do not match the '
                             'initial set of commands for the workflow.']
                missing = [c.name for c in root_cmds - set(req_params)]
                if missing:
                    error_msg.append(
                        ' Command(s) "%s" are missing the required parameter '
                        'set.' % ', '.join(missing))
                extra = [c.name for c in set(req_params) - root_cmds]
                if extra:
                    error_msg.append(
                        ' Paramters for command(s) "%s" have been provided, '
                        'but they are not the initial commands for the '
                        'workflow.' % ', '.join(extra))
                raise qdb.exceptions.QiitaDBError(''.join(error_msg))

            # Start creating the root jobs
            node_to_job = {
                n: ProcessingJob.create(
                    user,
                    qdb.software.Parameters.from_default_params(
                        p, req_params[c]), force)
                for n, (c, p) in viewitems(roots)}
            root_jobs = node_to_job.values()

            # SQL used to create the edges between jobs
            sql = """INSERT INTO qiita.parent_processing_job
                        (parent_id, child_id)
                     VALUES (%s, %s)"""

            # Create the rest of the jobs. These are different form the root
            # jobs because they depend on other jobs to complete in order to be
            # submitted
            for n in nx.topological_sort(dflt_g):
                if n in node_to_job:
                    # We have already visited this node
                    # (because it is a root node)
                    continue

                cmd, dflt_params = all_nodes[n]
                job_req_params = {}
                parent_ids = []

                # Each incoming edge represents an artifact that is generated
                # by the source job of the edge
                for source, dest, data in dflt_g.in_edges(n, data=True):
                    # Retrieve the id of the parent job - it already exists
                    # because we are visiting the nodes in topological order
                    source_id = node_to_job[source].id
                    parent_ids.append(source_id)
                    # Get the connections between the job and the source
                    connections = data['connections'].connections
                    for out, in_param in connections:
                        # We take advantage of the fact the parameters are
                        # stored in JSON to encode the name of the output
                        # artifact from the previous job
                        job_req_params[in_param] = [source_id, out]

                # At this point we should have all the requried parameters for
                # the current job, so create it
                new_job = ProcessingJob.create(
                    user, qdb.software.Parameters.from_default_params(
                        dflt_params, job_req_params), force)
                node_to_job[n] = new_job

                # Create the parent-child links in the DB
                sql_args = [[pid, new_job.id] for pid in parent_ids]
                qdb.sql_connection.TRN.add(sql, sql_args, many=True)

            return cls._common_creation_steps(user, root_jobs, name)

    @classmethod
    def from_scratch(cls, user, parameters, name=None, force=False):
        """Creates a new processing workflow from scratch

        Parameters
        ----------
        user : qiita_db.user.User
            The user creating the workflow
        parameters : qiita_db.software.Parameters
            The parameters of the first job in the workflow
        name : str, optional
            Name of the workflow. Default: generated from user's name
        force : bool
            Force creation on duplicated parameters

        Returns
        -------
        qiita_db.processing_job.ProcessingWorkflow
            The newly created workflow
        """
        job = ProcessingJob.create(user, parameters, force)
        return cls._common_creation_steps(user, [job], name)

    @property
    def name(self):
        """"The name of the workflow

        Returns
        -------
        str
            The name of the workflow
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT name
                     FROM qiita.processing_job_workflow
                     WHERE processing_job_workflow_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            return qdb.sql_connection.TRN.execute_fetchlast()

    @property
    def user(self):
        """The user that created the workflow

        Returns
        -------
        qdb.user.User
            The user that created the workflow
        """
        with qdb.sql_connection.TRN:
            sql = """SELECT email
                     FROM qiita.processing_job_workflow
                     WHERE processing_job_workflow_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            email = qdb.sql_connection.TRN.execute_fetchlast()
            return qdb.user.User(email)

    @property
    def graph(self):
        """Returns the graph of jobs that represent the workflow

        Returns
        -------
        networkx.DiGraph
            The graph representing the workflow
        """
        g = nx.DiGraph()
        with qdb.sql_connection.TRN:
            # Retrieve all graph workflow nodes
            sql = """SELECT parent_id, child_id
                     FROM qiita.get_processing_workflow_edges(%s)"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            edges = qdb.sql_connection.TRN.execute_fetchindex()
            nodes = {}
            if edges:
                nodes = {jid: ProcessingJob(jid)
                         for jid in set(chain.from_iterable(edges))}
                edges = [(nodes[s], nodes[d]) for s, d in edges]
                g.add_edges_from(edges)
            # It is possible that there are root jobs that doesn't have any
            # child, so they do not appear on edge list
            sql = """SELECT processing_job_id
                     FROM qiita.processing_job_workflow_root
                     WHERE processing_job_workflow_id = %s"""
            sql_args = [self.id]
            if nodes:
                sql += " AND processing_job_id NOT IN %s"
                sql_args.append(tuple(nodes))
            qdb.sql_connection.TRN.add(sql, sql_args)
            nodes = [
                ProcessingJob(jid)
                for jid in qdb.sql_connection.TRN.execute_fetchflatten()]
            g.add_nodes_from(nodes)

        return g

    def _raise_if_not_in_construction(self):
        """Raises an error if the workflow is not in construction

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the workflow is not in construction
        """
        with qdb.sql_connection.TRN:
            # To know if the workflow is in construction or not it suffices
            # to look at the status of the root jobs
            sql = """SELECT DISTINCT processing_job_status
                     FROM qiita.processing_job_workflow_root
                        JOIN qiita.processing_job USING (processing_job_id)
                        JOIN qiita.processing_job_status
                            USING (processing_job_status_id)
                     WHERE processing_job_workflow_id = %s"""
            qdb.sql_connection.TRN.add(sql, [self.id])
            res = qdb.sql_connection.TRN.execute_fetchflatten()
            # If the above SQL query returns a single element and the value
            # is different from in construction, it means that all the jobs
            # in the workflow are in the same status and it is not
            # 'in_construction', hence raise the error. If the above SQL query
            # returns more than value (len(res) > 1) it means that the workflow
            # is no longer in construction cause some jobs have been submited
            # for processing. Note that if the above query doesn't retrun any
            # value, it means that no jobs are in the workflow and that means
            # that the workflow is in construction.
            if (len(res) == 1 and res[0] != 'in_construction') or len(res) > 1:
                # The workflow is no longer in construction, raise an error
                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                    "Workflow not in construction")

    def add(self, dflt_params, connections=None, req_params=None,
            opt_params=None, force=False):
        """Adds a new job to the workflow

        Parameters
        ----------
        dflt_params : qiita_db.software.DefaultParameters
            The DefaultParameters object used
        connections : dict of {qiita_db.processing_job.ProcessingJob:
                               {str: str}}, optional
            Dictionary keyed by the jobs in which the new job depends on,
            and values is a dict mapping between source outputs and new job
            inputs
        req_params : dict of {str: object}, optional
            Any extra required parameter values, keyed by parameter name.
            Default: None, all the requried parameters are provided through
            the `connections` dictionary
        opt_params : dict of {str: object}, optional
            The optional parameters to change from the default set, keyed by
            parameter name. Default: None, use the values in `dflt_params`
        force : bool
            Force creation on duplicated parameters

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the workflow is not in construction
        """
        with qdb.sql_connection.TRN:
            self._raise_if_not_in_construction()

            if connections:
                # The new Job depends on previous jobs in the workflow
                req_params = req_params if req_params else {}
                # Loop through all the connections to add the relevant
                # parameters
                for source, mapping in viewitems(connections):
                    source_id = source.id
                    for out, in_param in viewitems(mapping):
                        req_params[in_param] = [source_id, out]

                new_job = ProcessingJob.create(
                    self.user, qdb.software.Parameters.from_default_params(
                        dflt_params, req_params, opt_params=opt_params), force)

                # SQL used to create the edges between jobs
                sql = """INSERT INTO qiita.parent_processing_job
                            (parent_id, child_id)
                         VALUES (%s, %s)"""
                sql_args = [[s.id, new_job.id] for s in connections]
                qdb.sql_connection.TRN.add(sql, sql_args, many=True)
                qdb.sql_connection.TRN.execute()
            else:
                # The new job doesn't depend on any previous job in the
                # workflow, so it is a new root job
                new_job = ProcessingJob.create(
                    self.user, qdb.software.Parameters.from_default_params(
                        dflt_params, req_params, opt_params=opt_params), force)
                sql = """INSERT INTO qiita.processing_job_workflow_root
                            (processing_job_workflow_id, processing_job_id)
                         VALUES (%s, %s)"""
                sql_args = [self.id, new_job.id]
                qdb.sql_connection.TRN.add(sql, sql_args)
                qdb.sql_connection.TRN.execute()

            return new_job

    def remove(self, job, cascade=False):
        """Removes a given job from the workflow

        Parameters
        ----------
        job : qiita_db.processing_job.ProcessingJob
            The job to be removed
        cascade : bool, optional
            If true, remove the also the input job's children. Default: False.

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the workflow is not in construction
            If the job to be removed has children and `cascade` is `False`
        """
        with qdb.sql_connection.TRN:
            self._raise_if_not_in_construction()

            # Check if the given job has children
            children = list(job.children)
            if children:
                if not cascade:
                    raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                        "Can't remove job '%s': it has children" % job.id)
                else:
                    # We need to remove all job's children, remove them first
                    # and then remove the current job
                    for c in children:
                        self.remove(c, cascade=True)

            # Remove any edges (it can only appear as a child)
            sql = """DELETE FROM qiita.parent_processing_job
                     WHERE child_id = %s"""
            qdb.sql_connection.TRN.add(sql, [job.id])

            # Remove as root job
            sql = """DELETE FROM qiita.processing_job_workflow_root
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [job.id])

            # Remove the input reference
            sql = """DELETE FROM qiita.artifact_processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [job.id])

            # Remove the job
            sql = """DELETE FROM qiita.processing_job
                     WHERE processing_job_id = %s"""
            qdb.sql_connection.TRN.add(sql, [job.id])

            qdb.sql_connection.TRN.execute()

    def submit(self):
        """Submits the workflow to execution

        Raises
        ------
        qiita_db.exceptions.QiitaDBOperationNotPermittedError
            If the workflow is not in construction
        """
        with qdb.sql_connection.TRN:
            self._raise_if_not_in_construction()

            g = self.graph
            # In order to avoid potential race conditions, we are going to set
            # all the children in 'waiting' status before submitting
            # the root nodes
            in_degrees = g.in_degree()
            roots = []
            for job, degree in viewitems(in_degrees):
                if degree == 0:
                    roots.append(job)
                else:
                    job._set_status('waiting')

            for job in roots:
                job.submit()


ALLOW_OPTION_VRF = 'vrf'
ALLOW_OPTION_KNOWN = 'known-client'
ALLOW_OPTION_LEGACY = 'legacy'

ALLOW_OPTIONS = {
    ALLOW_OPTION_VRF: 'Allow members of VRF',
    ALLOW_OPTION_KNOWN: 'Allow known-clients',
    ALLOW_OPTION_LEGACY: 'Allow Ctnr: Legacy Option',
}

# Some ranges have no allow statements so this option should be able to be
# null. There are a collection of such subnets documented in the migration.
DENY_OPTION_UNKNOWN = 'deny-unknown'
DENY_OPTIONS = {
    DENY_OPTION_UNKNOWN: 'Deny dynamic unknown-clients',
}

STATIC = "st"
DYNAMIC = "dy"
RANGE_TYPE = {
    STATIC: 'Static',
    DYNAMIC: 'Dynamic',
}

DHCP_KEY_VALUES = ("static_interface_kv", "dynamic_interface_kv", "range_kv",
                   "network_kv", "workgroup_kv", "vlan_kv", "vrf_kv",
                   "site_kv")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov  4 17:12:29 2019

@author: ppxct2
"""
import numpy as np
import matplotlib.pyplot as plt
# better-looking plots
plt.rcParams['font.family'] = 'serif'
plt.rcParams['figure.figsize'] = (12.0, 8.0)
plt.rcParams['font.size'] = 14

x = np.load('/home/ppxct2/Documents/CANDELS/predict.npy')
y = np.load('/home/ppxct2/Documents/CANDELS/true.npy')
plt.plot(x, y, linestyle = 'none', marker = 'x', color = 'k', alpha = 1)
'''
#### Asymmetry #####
plt.plot((-0.1,0.6), (-0.1,0.6))
plt.xlim(-0.1, 0.6)
plt.ylim(-0.1, 0.6)
plt.xlabel('Predicted Asymmetry Values')
plt.ylabel('True Asymmetry values')
plt.savefig('/home/ppxct2/Documents/CANDELS/pred_vs_true_A.png')
'''
### Concentration ####
plt.plot((0.0, 7), (0.0,7))
plt.xlabel('Predicted Concentration Values')
plt.ylabel('True Concentration values')
plt.xlim(0, max(x)+1)
plt.ylim(0, max(y)+1)
#plt.show()
plt.savefig('/home/ppxct2/Documents/CANDELS/pred_vs_true_C.png')
plt.close()

#Plotting histogram of predicted values against true values
#Creating bin widths so that predicitons and true values can be compared
xbins = np.arange(0,4.5,0.25)
#printing any predictions outside range of true values
for i in x:
    if i < 0.0:
        print('Prediction of concentration outside range plotted:', i)

plt.hist(x, color = 'g', alpha = 0.75, label = 'Predicted Values', bins = (xbins))
plt.xlabel('Concentration Values')
plt.ylabel('Frequency')
plt.hist(y, color= 'k', alpha = 0.7, label = 'True Values', bins=(xbins), histtype = 'step', linewidth = 3)
plt.legend()
plt.savefig('/home/ppxct2/Documents/CANDELS/hist_predictions.png')


import torch.nn as nn

from ptutils.base import Base

class Net(nn.Module, Base):

    def __init__(self, *args, **kwargs):
        super(Net, self).__init__()
        Base.__init__(self, *args, **kwargs)
        for arg in args:
            if isinstance(arg, nn.Module):
                # self.__setattr__(type(arg), arg)
                self = arg

    # def to_params(self):
    #     params = {}
    #     for name, value in self.__dict__.items():
    #         if not name.startswith('_'):
    #             params[name] = value
    #     # for name, mod in self._modules.items():
    #     try:
    #         for name, mod in self.modules():
    #             net = Net(mod)
    #             params[name] = net.to_params()
    #     except Exception as e:
    #         print(e)
    #     try:
    #         for name, mod in self.children():
    #             net = Net(mod)
    #             params[name] = net.to_params()
    #     except Exception as e:
    #         print(e)
    #     return params

    def __setattr__(self, name, value):
        if isinstance(value, nn.Module):
            self._bases[name] = value
        super(Net, self).__setattr__(name, value)

class MNIST(Net):
    __name__ = 'MNIST'

    def __init__(self,):
        super(MNIST, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.fc = nn.Linear(7 * 7 * 32, 10)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.fc(out)
        return out

# m = MNIST()
# conv = m.layer1[0]
# cn = Net(conv)
# print(cn.to_params())


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 29 13:07:11 2019

@author: Kyle Odom
"""

import pandas

data=pandas.read_csv("example.txt")
throughput=list(data['throughput'])
throughput.sort()
key=throughput[0]
interface = data.loc[data['throughput'] == key, 'interfaceID']
print(interface[1])


import click
from click_shell import shell
import time

from DHTNodeDataValue import DHTNodeDataValue
from DHTServer import DHTServer
from DHTServerManager import dhtServerManager
from DHTMessage import DHTMessage, DHTMessageType
from DHTSocket import DHTSocketAddress

KeySpaceRange = None

# ------------------------------------------------------------------------------- private functions

def getDHTNodeDataValueListFromRange(range):
    datalist = []
    for i in range:
        datalist.append(DHTNodeDataValue(i))
    return datalist

def initialize(file):
    ## Key space
    key_space = "#key-space"
    key_space_range = None

    ## Nodes
    nodes = "#nodes"
    node_array_str = None

    ## Shortcuts
    shortcuts = "#shortcuts"
    shortcuts_array = []

    file_name = file
    file_object = open(file_name , "r")
    file_list = file_object.read().splitlines()

    my_list = list(filter(lambda x: x is not None and x != " " and x != "", file_list))
    for index, line in enumerate(my_list):
        if(line.lower() == key_space):
            key = my_list[index + 1]
            key_array = key.split(',')
            if len(key_array) > 0:
                key_space_range = range(int(key_array[0]), int(key_array[1]) + 1)

        if(line.lower() == nodes):
            key = my_list[index + 1]
            node_array_str = key.split(',')

        if(line.lower() == shortcuts):
            key = my_list[index + 1]
            if key.startswith("#") == False:
                shortcuts_array = key.split(',')

    node_array = []
    for i in node_array_str:
        if i != " " and i != "":
            value = int(i)
            if value <= key_space_range[-1]:
                node_array.append(value)

    node_array.sort()
    file_dictionary = dict()
    for i, value in enumerate(node_array):
        dht_server = DHTServer()
        dhtServerManager.AssignHostAndPort(dht_server)
        socket_address = dht_server.GetDHTSocketAddress()
        file_dictionary[value] = (dht_server, socket_address)
        servers = [dht_server]

    lastIndex = len(node_array) - 1

    for i, value in enumerate(node_array):

        prevIndex = (i - 1) if i != 0 else lastIndex
        nextIndex = 0 if i == lastIndex else i + 1

        if i == lastIndex - 1:
            nextNextIndex = 0
        elif i == lastIndex:
            nextNextIndex = 1
        else:
            nextNextIndex = i + 2

        if lastIndex == 0:
            nextNextIndex = 0

        prevServerAddress = file_dictionary[node_array[prevIndex]][1]
        successorServerAddress = file_dictionary[node_array[nextIndex]][1]
        nextSuccessorServerAddress = file_dictionary[node_array[nextNextIndex]][1]

        shortcutsArray = []
        for shortcut in shortcuts_array:
            shortcutInfo = shortcut.split(':')
            if int(shortcutInfo[0]) == node_array[i]:
                if int(shortcutInfo[1]) in file_dictionary:
                    shortcutsArray.append(file_dictionary[int(shortcutInfo[1])][1])

        dataList = None
        if i == 0:
            dataList = getDHTNodeDataValueListFromRange(range(node_array[lastIndex] + 1, key_space_range[-1] + 1))
            dataList.extend(getDHTNodeDataValueListFromRange(range(key_space_range[0], node_array[i] + 1)))
        else:
            firstData = node_array[i - 1] + 1
            lastData = node_array[i]
            dataList = getDHTNodeDataValueListFromRange(range(firstData, lastData + 1))

        server = file_dictionary[node_array[i]][0]
        server.Upload(DHTNodeDataValue(node_array[i]), dataList, prevServerAddress, successorServerAddress, nextSuccessorServerAddress, shortcutsArray)

    servers = []
    for key in file_dictionary:
        servers.append(file_dictionary[key][0])

    return (servers, key_space_range)

def runDHTServers(dhtServers):
    servers = []
    for dhtServer in dhtServers:
        dhtServer = dhtServerManager.StartServer(dhtServer)
        servers.append(dhtServer)

    return servers

def AddShortcutToNode(nodeValue, shortcutNodeValue):

        if (not nodeValue) or nodeValue == '':
            return "the node to add the shortcut to, is not valid"

        if (not shortcutNodeValue) or shortcutNodeValue == '':
            return "the shortcut node to add the shortcut to, is not valid"

        try:
            nodeValue = int(nodeValue)
        except:
            return "the node data value for the node to add the shortcut to, is not valid"

        try:
            shortcutNodeValue = int(shortcutNodeValue)
        except:
            return "the shortcut node data value, is not valid"

        server = dhtServerManager.GetServerByDataValue(nodeValue)

        if server==None:
            return "the node to add the shortcut to, was not found"

        shortcutserver = dhtServerManager.GetServerByDataValue(shortcutNodeValue)
        if shortcutserver == None:
            return "the node is supposed to be added as a shortcut, was not found"

        msg=DHTMessage(DHTMessageType.Shortcut)
        msg.Argument=shortcutserver.Host
        msg.Tag=str(shortcutserver.Port)
        return dhtServerManager.SendRequestTo(server,msg)

def joinTheData(param):
    if param =='':
        return "the node is not valid"

    try:
        param=int(param)
    except:
        return "the join node data value, is not valid"

    server = dhtServerManager.GetServerByDataValue(param)

    if server ==None:

        dhtServer4= DHTServer()
        dhtServer4.DHTNodeDataValue=DHTNodeDataValue(param)
        dhtServerManager.AssignHostAndPort(dhtServer4)
        socketAddress4 = dhtServer4.GetDHTSocketAddress()

        serverLovestValue= dhtServerManager.GetTheServerWithTheLowestDataValue()

        msg=DHTMessage(DHTMessageType.Join)
        msg.Argument=dhtServer4.Host
        msg.Tag=str(dhtServer4.Port)
        msg.Nodevalue=str(param)

        return dhtServerManager.SendRequestTo(serverLovestValue,msg)

    else:
        return 'chord table already has this node'
#def initialize(file):

#    dhtServer5 = DHTServer()
#    dhtServerManager.AssignHostAndPort(dhtServer5)
#    socketAddress5 = dhtServer5.GetDHTSocketAddress()

#    dhtServer17 = DHTServer()
#    dhtServerManager.AssignHostAndPort(dhtServer17)
#    socketAddress17 = dhtServer17.GetDHTSocketAddress()

#    datalist5 = getDHTNodeDataValueListFromRange(range(93, 101))
#    datalist5.extend(getDHTNodeDataValueListFromRange(range(1, 6)))
#    dhtServer5.Upload(DHTNodeDataValue(5), datalist5, socketAddress17, socketAddress17, socketAddress5, [])

#    dhtServer17.Upload(DHTNodeDataValue(17), getDHTNodeDataValueListFromRange(range(6, 18)), socketAddress5, socketAddress5, socketAddress17, [])

#    servers = [dhtServer5, dhtServer17]
#    keySpace = range(1, 101)

#    return (servers, keySpace)

def validateLeaveParam(param):
    result = False

    # TODO result = ????

    return result

# ------------------------------------------------------------------------------- private functions

@shell(prompt='dht > ')
def main():
    pass

@main.command()
@click.argument('file')
def init(file):

    (servers, keySpace) = initialize(file)
    KeySpaceRange = keySpace
    dhtServers = runDHTServers(servers)

    click.echo("Initialized Done... Server(s) created and you can ping them. Server(s) is/are as follow:")
    for dhtServer in dhtServers:
         click.echo("Host: " + dhtServer.Host + " , Port: " + str(dhtServer.Port))

@main.command()
@click.argument('host')
@click.argument('port')
def ping(host, port):
    result = dhtServerManager.PingTheHostAndPort(host, int(port), DHTMessage(DHTMessageType.Ping))
    click.echo(result)

@main.command()
@click.argument('param')

def join(param):
    result = ""
    param=param

    result= joinTheData(param)

    click.echo(result)

@main.command()
@click.argument('param')
def lookup(param):
    #msg = DHTMessage(DHTMessageType.Ping)
    #msg.DataValue = 7
    result = ""
    click.echo(result)

@main.command()
@click.argument('shortcut')

def shortcut(shortcut):
    result = ""
    nodes = shortcut.split(':')
    if len(nodes) != 2:
        result="the input format is not satisfied, e.g. <Node_Value>:<ShortcutNode_Value>"
    else:
        result= AddShortcutToNode(nodes[0], nodes[1])

    click.echo(result)
    List()

@main.command()
def List():
    server = dhtServerManager.GetTheServerWithTheLowestDataValue()
    if server == None:
        click.echo("the Ring is not initialized!!")
        return

    result = dhtServerManager.SendRequestTo(server, DHTMessage(DHTMessageType.List))
    click.echo(result)

@main.command()
@click.argument('param')
def leave(param):

    #if validateLeaveParam(param):
    #    dataValue = int(param)
    #    dhtServer = dhtServerManager.ShutdownDHTServerByKey(dataValue)

    #result = ""
    #click.echo(result)
    pass

if __name__ == '__main__':
    main()


from datetime import datetime

from flask import Flask

from .blueprints import data_management, enc_cells, heatmaps, tracks, depth_map
from .containers import Container
from simplejson import JSONEncoder

# https://stackoverflow.com/a/56562567
class DateTimeEncoder(JSONEncoder):
    def default(self, z):
        if isinstance(z, datetime):
            return str(z)
        else:
            return super().default(z)

container = Container()
container.wire(modules=[data_management, enc_cells, heatmaps, tracks, depth_map])

app = Flask(__name__)
app.container = container
app.register_blueprint(data_management.blueprint, url_prefix="/data_management")
app.register_blueprint(enc_cells.blueprint, url_prefix="/enc")
app.register_blueprint(heatmaps.blueprint, url_prefix="/heatmaps")
app.register_blueprint(tracks.blueprint, url_prefix="/tracks")
app.register_blueprint(depth_map.blueprint, url_prefix="/depth_map")
app.json_encoder = DateTimeEncoder

@app.route("/")
def index():
    return "Ok"

@app.after_request
def after_request(response):
    header = response.headers
    header["Access-Control-Allow-Origin"] = "*"
    return response


#! /usr/bin/python
import rospy
import cv2
import sys
import numpy as np
from sensor_msgs.msg import Image as KinectImage
from cv_bridge import CvBridge
from math import sqrt, atan2, sin, cos, exp, degrees
import termios, fcntl, sys, os, rospy, roslib
from std_msgs.msg import String
from geometry_msgs.msg import Twist, Point
from matplotlib import pyplot as plt
import message_filters
from corobot_common.msg import Pose, Goal
from sensor_msgs.msg import Image, CameraInfo
import sensor_msgs.point_cloud2 as pc2
from sensor_msgs.msg import LaserScan
from laser_geometry import LaserProjection
import rospy
from nav_msgs.msg import Odometry
from tf.transformations import euler_from_quaternion, quaternion_from_euler
import math

# Range for which Kinect v1 worked
CAMERA_RANGE = 48

class Polar:  # commands
    __slots__ = ['d', 'a']

    def __init__(self, distance, angle):
        self.d = distance
        self.a = angle

    def __str__(self):
        return '(d={},a={})'.format(self.d, self.a)

    def valid(self):
        return self.d > 0

class Robot:

    def __init__(self, tracker):
        self.bridge = CvBridge()  # OpenCV API
        self.image = None  # Start with no image
        self.depth_image = None  # Start with no image
        self.tracker = tracker  # tacker OBJECT
        self.tracker_success = False  # tacker read flag set to False
        self.tracker_bbox = None  # No initial bounding box
        self.tracker_init = False
        self.count_num_linear = -1
        self.count_num_angular = -1

        self.prev_center_x = None
        self.prev_center_y = None
        self.prev_area = None
        self.controller = Controller()
        self.rgb_image_queue = []
        self.d_image_queue = []
        self.object_centers_queue = []
        self.bouding_box_queue = []
        self.pose = None
        self.scan = None
        self.laser_projector = LaserProjection()
        self.detecting = True
        self.disable_continuous_motion = False
        self.wp_publisher = rospy.Publisher('waypoints', Point)
        self.screens_count = 0
        # Polar points
        self.cache = []
        self.last_success_depth = None
        self.roll = self.pitch = self.yaw = 0.0
        self.target = 90
        self.kp = 0.5

    def histogram_equalize(self, img):
        b, g, r = cv2.split(img)
        red = cv2.equalizeHist(r)
        green = cv2.equalizeHist(g)
        blue = cv2.equalizeHist(b)
        return cv2.merge((blue, green, red))

    def save_images(self, rgb_image, depth_image):
        self.image = self.bridge.imgmsg_to_cv2(rgb_image, "passthrough")
        self.image = self.histogram_equalize(self.image)
        self.depth_image = self.bridge.imgmsg_to_cv2(depth_image, "32FC1")

    def convertRobotCoorToGlobalCoor(self, polarPoint):
        sp = Pose()
        sp.x = self.pose.x + polarPoint.d * cos(self.pose.theta + math.radians(polarPoint.a))
        sp.y = self.pose.y + polarPoint.d * sin(self.pose.theta + math.radians(polarPoint.a))
        sp.theta = 0
        return sp

    def main_orchestrator(self, rgb_image, depth_image):
        # Solve all of perception here...
        # get both rgb and depth image and save
        self.save_images(rgb_image, depth_image)
        self.tracker_success, self.tracker_bbox = self.tracker.update(self.image)
        if (self.tracker_init):
            if self.tracker_success:
                curr_center_x = int(self.tracker_bbox[0] + (self.tracker_bbox[2] // 2))
                curr_center_y = int(self.tracker_bbox[1] + (self.tracker_bbox[3] // 2))
                temp_d = np.array(self.depth_image, dtype=np.float32)
                temp_i = np.array(self.image, dtype=np.float32)

                if( self.tracker_bbox[0] + (self.tracker_bbox[2] // 2) >= temp_i.shape[1] ):
                    cv2.putText(self.image, "Tracking Failure", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    return
                if( self.tracker_bbox[1] + (self.tracker_bbox[3] // 2) >= temp_i.shape[0] ):
                    cv2.putText(self.image, "Tracking Failure", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    return

                depth = temp_d[curr_center_y][curr_center_x]

                calculated_depth = depth / 1000.0

                if( temp_i.shape[0] > temp_i.shape[1] ):
                    max_dimension = temp_i.shape[0] * 1.0
                else:
                    max_dimension = temp_i.shape[1] * 1.0

                actual_center = max_dimension // 2

                difference = actual_center - curr_center_x

                angle_per_pixel = CAMERA_RANGE / max_dimension
                total_angle = angle_per_pixel * difference
                destination = self.convertRobotCoorToGlobalCoor(Polar(calculated_depth, total_angle))

                if abs(destination.y - self.pose.y) <= 0.099 and abs(destination.x - self.pose.x) <= 0.099:
                    pass
                else:
                    if self.detecting or not self.disable_continuous_motion:
                        self.wp_publisher.publish(Point(destination.x, destination.y, 0))
                self.detecting = False

                p1 = (int(self.tracker_bbox[0]), int(self.tracker_bbox[1]))
                p2 = (int(self.tracker_bbox[0] + self.tracker_bbox[2]),
                      int(self.tracker_bbox[1] + self.tracker_bbox[3]))
                cv2.rectangle(self.image, p1, p2, (255, 0, 0), 2, 1)

                if self.image is not None:
                    print("in image Tracking")
                    cv2.putText(self.image, "Tracking ...", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                else:
                    cv2.putText(self.image, "Failure ...", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                    cv2.imshow("Tracking ...", self.image)
                    cv2.waitKey(1)
                    rospy.loginfo("Nothing to visualize!")

            else:
                print("Tracking Failure")
                pass

    def waypointsReached(self, point):
        print("reached" + str(point))
        self.detecting = True

    # to select Object
    def selectROI(self, raw_image):
        # convert raw image to cv2 image format
        self.image = self.bridge.imgmsg_to_cv2(raw_image, "passthrough")
        if not self.tracker_init:
            # initialize parameters for robot to start moving
            self.controller.start_moving()
            # get tracker/bounding box
            self.tracker_bbox = cv2.selectROI(self.image, False)
            # boolean variable indicating if object was found
            self.tracker_success = self.tracker.init(self.image, self.tracker_bbox)
            # tracker initialized
            self.tracker_init = True

    def reduce_covariance(self, cov):
        """Convert a flat 6x6 covariance matrix into a flat 3x3."""
        """ 6 variances are for linear x, y, z and rotate x, y, z """
        return (cov[0], cov[1], cov[5],
                cov[6], cov[7], cov[11],
                cov[30], cov[31], cov[35])

    def odom_to_pose(self, odom):
        """Utility function to convert an Odometry message into a Pose message."""
        pose = Pose()
        pose.header = odom.header
        pose.x = odom.pose.pose.position.x
        pose.y = odom.pose.pose.position.y
        qz = odom.pose.pose.orientation.z
        qw = odom.pose.pose.orientation.w
        pose.theta = atan2(2 * qw * qz, 1 - 2 * qz * qz)
        pose.cov = self.reduce_covariance(odom.pose.covariance)
        return pose

    # Pose -> None
    def poseCallback(self, odom):
        self.pose = self.odom_to_pose(odom)
        print(self.pose.theta)

def main():
    if len(sys.argv) != 2:
        raise AssertionError("Usage: rosrun project_name tracker.py tracker_type")

    # Create the tracker
    tracker = getattr(cv2, "Tracker{}_create".format(sys.argv[1]))()

    # Initialize node
    rospy.init_node("sy4923_tracker", anonymous=False)
    robot = Robot(tracker)  # create the robot object

    # to get robots current pose
    rospy.Subscriber('odom', Odometry, robot.poseCallback)

    # Synchronous callbacks of rgb and depth images
    image_rgb_sub = message_filters.Subscriber('/camera/rgb/image_color', KinectImage)
    image_depth_sub = message_filters.Subscriber('/camera/depth_registered/image_raw', KinectImage)
    ts = message_filters.ApproximateTimeSynchronizer([image_rgb_sub, image_depth_sub],
                                                     queue_size=2, slop=0.1)
    ts.registerCallback(robot.main_orchestrator)

    # Callback to select ROI
    rospy.Subscriber("/camera/rgb/image_color", KinectImage, robot.selectROI, queue_size=1)

    rospy.Subscriber('waypoints_reached', Point, robot.waypointsReached)
    rospy.Subscriber('waypoints_failed', Point, robot.waypointsReached)

    print("Done")
    rospy.spin()

class Controller:
    __slots__ = ['commandVelocityPublisher', 'wp_publisher']

    def __init__(self):
        self.commandVelocityPublisher = rospy.Publisher('mobile_base/commands/velocity', Twist)
        rospy.Subscriber('handle_twist', Twist, self.handle_twist)

    def handle_twist(self, twist_callback):
        self.commandVelocityPublisher.publish(twist_callback)

    def start_moving(self):
        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)
        try:
            self.twist = Twist()
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
            fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)

        pass

    def move_Left_By(self, amount):
        print('move left {} m/s'.format(amount))
        self.twist.angular.z = amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Right_By(self, amount):
        print('move right {} m/s'.format(amount))
        self.twist.angular.z = -amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Forward_By(self, amount):
        print('move forward {} m/s'.format(amount))
        self.twist.linear.x = amount
        # self.commandVelocityPublisher.publish(self.twist)

    def move_Backward_By(self, amount):
        print('move backward {} m/s'.format(amount))
        self.twist.linear.x = -amount
        # self.commandVelocityPublisher.publish(self.twist)

    def test(self):
        p = Point(0.0, 0.5, 0.0)
        p.x = 1900

        while (True):
            self.wp_publisher.publish(p)

    def run(self):

        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)

        speed = .5

        try:
            while True:
                twist = Twist()
                try:
                    c = sys.stdin.read(1)
                    if c == '\x1b':
                        c = sys.stdin.read(2)
                    if c == '[A' or c == 'w':
                        print('move forward {} m/s'.format(speed))
                        twist.linear.x = speed
                    elif c == 's' or c == '[B':
                        print('move backward {} m/s'.format(-speed))
                        twist.linear.x = -speed
                    elif c == '[C' or c == 'd':
                        print('rotate right {} m/s'.format(speed))
                        twist.angular.z = -speed
                    elif c == '[D' or c == 'a':
                        print('rotate left {} m/s'.format(speed))
                        twist.angular.z = speed
                    elif c == '-':
                        speed = max(.1, speed - .1)
                        print('decreasing linear speed to ', speed)
                    elif c == '+':
                        speed = min(.5, speed + .1)
                        print('increasing linear speed to ', speed)
                    elif c == 'q':
                        return
                    else:
                        print('Controls:')
                        print('          w or up    : move forward')
                        print('          a or left  : rotate counterclockwise')
                        print('          d or right : rotate clockwise')
                        print('          -          : decrease linear speed by .1 m/s (.1 m/s minimum)')
                        print('          +          : increase linear speed by .1 m/s (.5 m/s maximum)')
                        print('          q          : quit')
                    # more seamless with multiple requests
                    self.commandVelocityPublisher.publish(twist)
                    self.commandVelocityPublisher.publish(twist)
                    self.commandVelocityPublisher.publish(twist)
                except IOError:
                    pass
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
            fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)

if __name__ == "__main__":
    try:
        main()
    except rospy.ROSInterruptException:
        rospy.loginfo("Thank you for using the tracker! Exiting ...")


#! /usr/bin/env python3

import os
import os.path
import sys
import json
import git
import common
import logging
from optparse import OptionParser

logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger("main")

def find_plan(plan_name):
    candidates = [plan_name,
                  os.path.join(os.getcwd(), "plans", plan_name),
                  os.path.join(os.getcwd(), "plans", plan_name + ".json")]
    for c in candidates:
        _logger.debug("Probe path {0} for plan file".format(c))
        if os.path.exists(c):
            _logger.debug("Open plan file at {0}".format(c))
            with open(c, 'rt') as f:
                planjson = json.load(f)
            return planjson

    return None

def main():
    parser = OptionParser(usage='usage: %prog [options] [plan]')

    optlist, args = parser.parse_args()

    if len(args) < 1:
        print('Incorrect parameters. Use option "-h" to get some help.')
        sys.exit(1)

    # Load plan file
    _logger.info("Loading plan " + args[0])
    planjson = find_plan(args[0])
    if planjson == None:
        _logger.error('The plan {0} does not exist.'.format(args[0]))
        sys.exit(1)

    settings = {}
    settings['working_dir'], settings['app_dir'] = common.set_up_workspace()

    # Load apps settings
    _logger.info('Load apps ' + json.dumps(planjson['app']))
    appSetting = planjson['app']
    settings['appsrc_type'] = appSetting['type']
    settings['app_proj'] = appSetting['project']
    if appSetting['type'] == 'github':
        _logger.info("Load test app from github")
        import git
        giturl = appSetting['uri']
        _logger.info("Github: " + giturl)
        settings['app_name'] = git.extract_repo_name(giturl)

        def app_setup():
            git.setup(giturl, settings['app_dir'])
    else:
        _logger.error('Unknown app source type ' + appSetting['type'])

    # Load cases
    _logger.info('Load test cases ' + json.dumps(planjson['cases']))
    settings['cases'] = planjson['cases']

    # Load runner
    _logger.info('Load test runner ' + json.dumps(planjson['runner']))
    settings['runner'] = planjson['runner']['module']
    def runner_setup():
        _logger.info('Setup runner')
        import_statement = 'from ' + settings['runner'] + ' import get_runner_name, run_case'
        _logger.info('Import statement: ' + import_statement)
        exec(import_statement, globals())

    # Execution!
    app_setup()
    runner_setup()

    resultFiles = [run_case(case, settings) for case in settings['cases']]
    print('Done. Results are saved in:\n' + '\n'.join(resultFiles))

if (__name__ == '__main__'):
    main()