import numpy as np
import pytest
from pytest import approx

from gym_datums.envs.portfolio_env import DatumsError, PortfolioResetNeeded
from tests.aux import assert_that, follows_contract, assert_obs_eq, unpack_obs, unpack_done, unpack_reward, until_done, \
    unpack_info, idle_step, make_box

def test_adherence_to_gym_contract(make_env, gym_interface, gym_properties):
    assert_that(make_env(), follows_contract(gym_interface, gym_properties))

@pytest.mark.parametrize('series, minmax', [
    ([[1]], (1, 1)),
    ([[2], [1], [3]], (1, 3)),
])
def test_single_value_single_datum_observation_space(make_env, datums, series, minmax):
    datums.add().rows(*series)
    assert make_env().observation_space == make_box(minmax[0], minmax[1], shape=(1,))

def test_multiple_values_observation_space(make_env, datums):
    datums.add().rows([2, 3, 1], [3, 4, 2])
    assert make_env().observation_space == make_box(1, 4, shape=(3,))

def test_multiple_datums_observation_space(make_env, datums):
    datums.add().rows([1], [2], [3])
    datums.add().rows([-1], [0], [1])
    assert make_env().observation_space == make_box(-1, 3, shape=(1, 2))

def test_windowed_observation_space(make_env, datums):
    datums.add().rows([1], [2], [3])
    assert make_env(window_size=2).observation_space == make_box(1, 3, shape=(1, 2))

@pytest.mark.parametrize("num_values, assets, window_size, shape", [
    (3, 2, 1, (3, 2)),
    (3, 1, 3, (3, 3)),
    (2, 5, 3, (2, 5, 3)),
])
def test_combined_setup_observation_space(make_env, datums, num_values, assets, window_size, shape):
    fill_datums(datums, assets, num_values)
    assert make_env(window_size).observation_space == make_box(1, 1, shape)

def fill_datums(datums, assets, num_values, value=1):
    for _ in range(assets):
        series = [[value] * num_values] * 10
        datums.add().rows(*series)

def test_reset_returns_first_value_from_datums(make_env, datums):
    datums.add().rows([1], [2])
    assert_obs_eq(make_env().reset(), [1])

@pytest.mark.parametrize('num_values, assets, window_size', [
    (3, 1, 1),
    (3, 2, 1),
    (1, 3, 1),
    (3, 1, 2),
    (1, 1, 5),
])
def test_observations_follow_observation_space_shape(make_env, datums, num_values, assets, window_size):
    fill_datums(datums, assets, num_values)
    env = make_env(window_size)
    assert env.reset().shape == env.observation_space.shape

def test_reset_returns_properly_windowed_observation(make_env, datums):
    datums.add().rows([1], [2], [3])
    assert_obs_eq(make_env(window_size=2).reset(), [[1, 2]])

def test_reset_combined_setup_observation(make_env, datums):
    datums.add().rows(
        [2, 1, 3],
        [3, 2, 4],
        [1, 1, 1],
    )
    datums.add().rows(
        [1e-6, 1e-9, 1],
        [1, 1e-6, 2],
        [1, 1, 1],
    )
    assert_obs_eq(make_env(window_size=2).reset(), [[[2, 3], [1e-6, 1]],
                                                    [[1, 2], [1e-9, 1e-6]],
                                                    [[3, 4], [1, 2]]])

def test_stepping_the_environment_returns_next_observation(make_ready_env, datums):
    datums.add().rows([1], [2], [3])
    env = make_ready_env()
    assert_obs_eq(unpack_obs(idle_step(env)), [2])
    assert_obs_eq(unpack_obs(idle_step(env)), [3])

def test_resetting_the_environment_resets_observations(make_ready_env, datums):
    datums.add().rows([1], [2], [3])
    env = make_ready_env()
    assert_obs_eq(unpack_obs(idle_step(env)), [2])
    env.reset()
    assert_obs_eq(unpack_obs(idle_step(env)), [2])

def test_environment_is_done_at_the_last_observation(make_ready_env, datums):
    datums.add().rows([1], [2], [3])
    env = make_ready_env()
    assert not unpack_done(idle_step(env))
    assert unpack_done(idle_step(env))

def test_raise_an_error_when_not_enough_data_for_a_single_step(make_env, datums):
    datums.add().rows([1], [2])
    env = make_env(window_size=2)
    with pytest.raises(DatumsError):
        env.reset()

def test_raise_an_error_when_stepping_past_done(make_ready_env, datums):
    datums.add().rows([1], [2])
    env = make_ready_env()
    list(until_done(env, [1, 0]))
    with pytest.raises(PortfolioResetNeeded):
        idle_step(env)

@pytest.mark.parametrize('invalid', [0, np.nan, np.inf, -1e-5])
def test_raise_error_when_upcoming_invalid_datum_is_encountered(make_ready_env, datums, invalid):
    datums.add().rows([2], [1], [invalid])
    env = make_ready_env()
    with pytest.raises(DatumsError):
        idle_step(env)

@pytest.mark.parametrize('num_assets', [1, 2])
def test_action_shape_is_dependent_on_number_of_assets(make_ready_env, datums, num_assets):
    fill_datums(datums, num_assets, 1)
    env = make_ready_env()
    assert env.action_space == make_box(low=-1, high=1, shape=(num_assets + 1,))

def test_idle_reward(make_ready_env, datums):
    datums.add().rows([1], [1], [2])
    env = make_ready_env(cash=10)
    assert unpack_reward(idle_step(env)) == 1.0

def test_positive_reward(make_ready_env, datums):
    datums.add().rows([1], [1], [2], [1], [2])
    env = make_ready_env(cash=10)
    assert unpack_reward(env.step([0, 1])) == 1.0
    assert unpack_reward(env.step([1, 0])) == 2.0
    assert unpack_reward(env.step([0, 1])) == 2.0
    assert unpack_reward(env.step([1, 0])) == 4.0

def test_negative_reward(make_ready_env, datums):
    datums.add().rows([1], [1], [0.5], [1], [0.5])
    env = make_ready_env(cash=10)
    assert unpack_reward(env.step([0, 1])) == 1.0
    assert unpack_reward(env.step([1, 0])) == 0.5
    assert unpack_reward(env.step([0, 1])) == 0.5
    assert unpack_reward(env.step([1, 0])) == 0.25

def test_reset_returns_portfolio_to_original_state(make_ready_env, datums):
    datums.add().rows([1], [2])
    env = make_ready_env(cash=10)
    env.step([0, 1])
    assert_portfolio(env.portfolio, np.array([0, 5]))
    env.reset()
    assert_portfolio(env.portfolio, np.array([10, 0]))

def assert_portfolio(actual, expected):
    np.testing.assert_almost_equal(actual.assets, expected)

def test_by_default_use_buy_and_hold_baseline(make_ready_env, datums):
    datums.add().rows([1], [2], [0.5], [4])
    env = make_ready_env(cash=10)
    assert unpack_info(idle_step(env))['baseline'] == 2
    assert unpack_info(idle_step(env))['baseline'] == 0.5
    assert unpack_info(idle_step(env))['baseline'] == 4

def test_buy_and_hold_baseline_with_multiple_assets(make_ready_env, datums):
    datums.add().rows([1], [2], [0.5], [4])
    datums.add().rows([1], [1], [3], [2])
    env = make_ready_env(cash=10)
    assert unpack_info(idle_step(env))['baseline'] == 1.5
    assert unpack_info(idle_step(env))['baseline'] == 3.5 / 2
    assert unpack_info(idle_step(env))['baseline'] == 3

def test_configure_datum_as_baseline(make_ready_env, datums, baseline_datums):
    datums.add().rows([1], [2], [0.5])
    baseline_datums.rows(10, 5, 20)
    env = make_ready_env(cash=10, baseline=baseline_datums)
    assert unpack_info(idle_step(env))['baseline'] == 0.5
    assert unpack_info(idle_step(env))['baseline'] == 2

def test_reset_baseline_datums(make_ready_env, datums, baseline_datums):
    datums.add().rows([1], [2], [0.5])
    baseline_datums.rows(10, 5, 20)
    env = make_ready_env(cash=10, baseline=baseline_datums)
    assert unpack_info(idle_step(env))['baseline'] == 0.5
    assert unpack_info(idle_step(env))['baseline'] == 2
    env.reset()
    assert unpack_info(idle_step(env))['baseline'] == 0.5
    assert unpack_info(idle_step(env))['baseline'] == 2

def test_commission_fees_are_deducted(make_ready_env, datums):
    datums.add().rows([1], [1], [2], [1], [2])
    env = make_ready_env(cash=10, commission=0.1)
    assert unpack_reward(env.step([0, 1])) == 0.9
    assert unpack_reward(env.step([1, 0])) == approx((2 * 0.9) * 0.9)
    assert unpack_reward(env.step([0, 1])) == approx((2 * 0.9) * 0.9 ** 2)
    assert unpack_reward(env.step([1, 0])) == approx((2 * (2 * 0.9) * 0.9 ** 2) * 0.9)

def test_commission_fees_are_not_deducted_when_assets_are_not_moved(make_ready_env, datums):
    datums.add().rows([1], [1], [2])
    env = make_ready_env(cash=10, commission=0.1)
    assert unpack_reward(idle_step(env)) == 1.0
    assert unpack_reward(idle_step(env)) == 1.0

def test_commission_fees_are_deducted_on_all_shifted_assets(make_ready_env, datums):
    datums.add().rows([1], [1], [3])
    datums.add().rows([1], [1], [0.5])
    env = make_ready_env(cash=10, commission=0.1)
    assert unpack_reward(env.step([0, 0.6, 0.4])) == 0.9
    assert unpack_reward(env.step([1, 0, 0])) == approx((2 * 0.9) * 0.9)

def test_commission_fees_on_real_example(make_ready_env, datums):
    datums.add().rows([6425.8], [6425.8])
    env = make_ready_env(cash=100, commission=0.0016)
    env.step([0.0997454, 0.9002546])
    assert_portfolio(env.portfolio, np.array([9.97454, 0.01398758]))


import logging
import os
import subprocess
import stat
import shutil

# ours
import validationfunctions
from validationscript import Script

class Cluster:
    """
    A class that provides the controls for running jobs on a (remote)
    Sun Grid Engine cluster. It provides two methods:
    - is_job_finished(job): Returns True or False, depending on whether the job
        has finished execution
    - execute(job): Takes a job and executes it by sending it to the cluster
    """

    @staticmethod
    def is_supported():
        """
        Check if qsub is available
        """
        return shutil.which("qsub") is not None

    @staticmethod
    def name():
        """
        Returns name of this job contol
        """
        return "cluster-sge"

    @staticmethod
    def description():
        """
        Returns description of this job control
        """
        return "Batch submission via command line to Grid Engine"

    def __init__(self):
        """!
        The default constructor.
        - Holds the current working directory, which is also the location of
          the shellscripts that are being sent to the cluster.
        - Initializes a logger which writes to validate_basf2.py's log.
        - Finds the revision of basf2 that will be set up on the cluster.
        """

        #: The command to submit a job. 'LOGFILE' will be replaced by the
        # actual log file name
        self.submit_command = (
            "qsub -cwd -l h_vmem={requirement_vmem}G,"
            "h_fsize={requirement_storage}G "
            "-oo {logfile} -q {queuename} -V"
        )

        #: required vmem by the job in GB, required on DESY NAF, otherwise
        #: jobs get killed due to memory consumption
        self.requirement_vmem = 4

        #: the storage IO in GB which can be performed by each job. By
        #: default, this is 3GB at DESY which is to small for some validation
        #:  scripts
        self.requirement_storage = 50

        #: Queue best suitable for execution at DESY NAF
        self.queuename = "short.q"

        #: The path, where the help files are being created
        #: Maybe there should be a special subfolder for them?
        self.path = os.getcwd()

        #: Contains a reference to the logger-object from validate_basf2
        #: Set up the logging functionality for the 'cluster execution'-Class,
        #: so we can log to validate_basf2.py's log what is going on in
        #: .execute and .is_finished
        self.logger = logging.getLogger("validate_basf2")

        #: We need to set up the same environment on the cluster like on the
        #: local machine. The information can be extracted from $BELLE2_TOOLS,
        #: $BELLE2_RELEASE_DIR and $BELLE2_LOCAL_DIR

        #: Path to the basf2 tools and central/local release
        self.tools = self.adjust_path(os.environ["BELLE2_TOOLS"])
        belle2_release_dir = os.environ.get("BELLE2_RELEASE_DIR", None)
        belle2_local_dir = os.environ.get("BELLE2_LOCAL_DIR", None)

        #: The command for b2setup (and setoption)
        self.b2setup = "b2setup"
        if belle2_release_dir is not None:
            self.b2setup += " " + belle2_release_dir.split("/")[-1]
        if belle2_local_dir is not None:
            self.b2setup = (
                "MY_BELLE2_DIR="
                + self.adjust_path(belle2_local_dir)
                + " "
                + self.b2setup
            )
        if os.environ.get("BELLE2_OPTION") != "debug":
            self.b2setup += "; b2code-option " + os.environ.get("BELLE2_OPTION")

        # Write to log which revision we are using
        self.logger.debug(f"Setting up the following release: {self.b2setup}")

        # Define the folder in which the log of the cluster messages will be
        # stored (same folder like the log for validate_basf2.py)
        clusterlog_dir = "./html/logs/__general__/"
        if not os.path.exists(clusterlog_dir):
            os.makedirs(clusterlog_dir)

        #: The file object to which all cluster messages will be written
        self.clusterlog = open(clusterlog_dir + "clusterlog.log", "w+")

    # noinspection PyMethodMayBeStatic
    def adjust_path(self, path: str):
        """!
        This method can be used if path names are different on submission
        and execution hosts.
        @param path: The past that needs to be adjusted
        @return: The adjusted path
        """

        return path

    # noinspection PyMethodMayBeStatic
    def available(self):
        """!
        The cluster should always be available to accept new jobs.
        @return: Will always return True if the function can be called
        """

        return True

    def execute(self, job: Script, options="", dry=False, tag="current"):
        """!
        Takes a Script object and a string with options and runs it on the
        cluster, either with ROOT or with basf2, depending on the file type.

        @param job: The steering file object that should be executed
        @param options: Options that will be given to the basf2 command
        @param dry: Whether to perform a dry run or not
        @param tag: The folder within the results directory
        @return: None
        """

        # Define the folder in which the results (= the ROOT files) should be
        # created. This is where the files containing plots will end up. By
        # convention, data files will be stored in the parent dir.
        # Then make sure the folder exists (create if it does not exist) and
        # change to cwd to this folder.
        output_dir = os.path.abspath(f"./results/{tag}/{job.package}")
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        # Path where log file is supposed to be created
        log_file = output_dir + "/" + os.path.basename(job.path) + ".log"

        # Remove any left over done files
        donefile_path = f"{self.path}/script_{job.name}.done"
        if os.path.isfile(donefile_path):
            os.remove(donefile_path)

        # Now we need to distinguish between .py and .C files:
        extension = os.path.splitext(job.path)[1]
        if extension == ".C":
            # .c files are executed with root
            command = "root -b -q " + job.path
        else:
            # .py files are executed with basf2
            # 'options' contains an option-string for basf2, e.g. '-n 100'
            params = validationfunctions.basf2_command_builder(
                job.path, options.split()
            )
            command = subprocess.list2cmdline(params)

        # Create a helpfile-shellscript, which contains all the commands that
        # need to be executed by the cluster.
        # First, set up the basf2 tools and perform b2setup with the correct
        # revision. The execute the command (i.e. run basf2 or ROOT on a
        # steering file). Write the return code of that into a *.done file.
        # Delete the helpfile-shellscript.
        tmp_name = self.path + "/" + "script_" + job.name + ".sh"
        with open(tmp_name, "w+") as tmp_file:
            tmp_file.write(
                "#!/bin/bash \n\n"
                + "BELLE2_NO_TOOLS_CHECK=1 \n"
                + f"source {self.tools}/b2setup \n"
                + "cd {} \n".format(self.adjust_path(output_dir))
                + f"{command} \n"
                + "echo $? > {}/script_{}.done \n".format(self.path, job.name)
                + f"rm {tmp_name} \n"
            )

        # Make the helpfile-shellscript executable
        st = os.stat(tmp_name)
        os.chmod(tmp_name, st.st_mode | stat.S_IEXEC)

        # Prepare the command line command for submission to the cluster
        params = self.submit_command.format(
            queuename=self.queuename,
            requirement_storage=self.requirement_storage,
            requirement_vmem=self.requirement_vmem,
            logfile=log_file,
        ).split() + [tmp_name]

        # Log the command we are about the execute
        self.logger.debug(subprocess.list2cmdline(params))

        # Submit it to the cluster. The steering
        # file output will be written to 'log_file' (see above).
        # If we are performing a dry run, don't send anything to the cluster
        # and just create the *.done file right away and delete the *.sh file.
        if not dry:
            process = subprocess.Popen(
                params, stdout=self.clusterlog, stderr=subprocess.STDOUT
            )

            # Check whether the submission succeeded
            if process.wait() != 0:
                job.status = "failed"
        else:
            os.system(f"echo 0 > {self.path}/script_{job.name}.done")
            os.unlink(tmp_name)

    def is_job_finished(self, job: Script):
        """!
        Checks whether the '.done'-file has been created for a job. If so, it
        returns True, else it returns False.
        Also deletes the .done-File once it has returned True.

        @param job: The job of which we want to know if it finished
        @return: True if the job has finished, otherwise False
        """

        # If there is a file indicating the job is done, that is its name:
        donefile_path = f"{self.path}/script_{job.name}.done"

        # Check if such a file exists. If so, this means that the job has
        # finished.
        if os.path.isfile(donefile_path):

            # Read the returncode/exit_status for the job from the *.done-file
            with open(donefile_path) as f:
                try:
                    returncode = int(f.read().strip())
                except ValueError:
                    returncode = -654

            # Delete the *.done file
            os.remove(donefile_path)

            # Return that the job is finished + the return code for it
            return [True, returncode]

        # If no such file exists, the job has not yet finished
        else:
            return [False, 0]

    # noinspection PyMethodMayBeStatic
    def terminate(self, job: Script):
        """!
        Terminate a running job, not support with this backend so ignore the
        call.
        """
        self.logger.error("Script termination not supported.")


#!/usr/bin/env python3

import osmfilter.osm_colors           as CC
import osmfilter.pre_filter           as pre_filter
import osmfilter.osm_info             as osm_info
import osmfilter.osm_pickle           as osm_pickle
import os
from osmfilter.element_filter     import GetOSM_IDs

import json
import sys
import pprint
import logging

logging.basicConfig()
logger=logging.getLogger(__name__)
#logger.setLevel(logging.INFO)
logger.setLevel(logging.WARNING)

def write_overview(elements,elementname,outputdir):
    """
    **Purpose**
        writes overview of elements to outputdir

    **Input**
        elements:
            list of filtered element items
        elementname:
            name of the elements (e.g. 'pipelines')
        outputdir:
            relative name of outputdir
    """

    original = sys.stdout
    if outputdir  != "":
        filename   = elementname+"_short.json"
        logdirbase = (os.path.dirname(outputdir)+"/Analysis/Elements")
        logfile    = os.path.join(logdirbase,filename)
#        print("Outputfile: ", logfile)
        os.makedirs(os.path.dirname(logfile), exist_ok=True)
        f = open(logfile, 'w')
        sys.stdout = f#Tee(f)
    sys.stdout.write("Overview OSM "+elementname +"\n")
    sys.stdout.write("===========\n")
    sys.stdout.write("Nodes:       " + "{:>26}".format(str(len(elements["Node"])))+"\n")
    sys.stdout.write("Ways:        " + "{:>26}".format(str(len(elements["Way"])))+"\n")
    sys.stdout.write("Relations:   " + "{:>26}".format(str(len(elements["Relation"])))+"\n")
    sys.stdout.write("===========\n\n\n")
    counter=0

    for entry in elements["Node"]:
        sys.stdout.write('\n------Node--------\n')
        counter+=1
        sys.stdout.write('\n'+'Nr. '+str(counter)+" of "+ elementname+"\n")
        sys.stdout.write('Id: ' +str(elements["Node"][entry]['id'])+"\n")
        sys.stdout.write('Tags:\n')
        dictstring = pprint.pformat(elements["Node"][entry]['tags'],width=35)
        sys.stdout.write(dictstring +"\n")
    counter=0
    for entry in elements["Way"]:
        sys.stdout.write('\n------Way--------\n')
        counter+=1
        sys.stdout.write('\n'+'Nr. '+str(counter)+" of "+ elementname+"\n")
        sys.stdout.write('Id: ' +str(elements["Way"][entry]['id'])+"\n")
        sys.stdout.write('Tags:\n')
        dictstring = pprint.pformat(elements["Way"][entry]['tags'],width=35)
        sys.stdout.write(dictstring +"\n")
    counter=0
    for entry in elements["Relation"]:
        sys.stdout.write('\n------Relation--------\n')
        counter+=1
        sys.stdout.write('\n'+'Nr. '+str(counter)+" of "+ elementname+"\n")
        sys.stdout.write('Id: ' +str(elements["Relation"][entry]['id'])+"\n")
        sys.stdout.write('Tags:\n')
        dictstring = pprint.pformat(elements["Relation"][entry]['tags'],width=35)
        sys.stdout.write(dictstring +"\n")
    sys.stdout = original
    pass

def write_elements(elements,elementname,outputdir):
    """
    **Purpose**
        writes elements to JSON file in outputdir

    **Input**
        elements:
            list of filtered element items
        elementname:
            name of the elements (e.g. 'pipelines')

    **Return**
        writes elements to jsonfile at outputdir
    """

    write_overview(elements,elementname,outputdir)
    filename   = elementname+".json"
    logdirbase = (os.path.dirname(outputdir)+"/Analysis/Elements/")
    logfile    = os.path.join(logdirbase,filename)

    os.makedirs(os.path.dirname(logfile), exist_ok=True)
    with open(logfile, 'w') as target:
            json.dump(elements, target, indent=4, separators=(',', ':'))
    pass

def find_elements(data, ID_type_list):
    """
    **Purpose**
        use ID_type_list to find items in the prefiltered data and return
        them in a dict

    **Input**
        data:
            dictionary of prefiltered Data
        ID_type_list:
            [({OSM_id},{OSM_item_type}),]

            e.g. [('457320703','Node'),('457320703','Node')]

    **Output**
        elements:
            dictionary of Nodes, Ways and Relation
    """

    elements={'Node':{},'Way':{},'Relation':{}}

    for i in range(len(ID_type_list)):
        if ID_type_list[i][1] == "Way":
                elements['Way'][ID_type_list[i][0]]=data["Way"][ID_type_list[i][0]]
        elif ID_type_list[i][1] == "Node":
                elements['Node'][ID_type_list[i][0]]=data["Node"][ID_type_list[i][0]]
        elif ID_type_list[i][1] == "Relation":
                elements['Relation'][ID_type_list[i][0]]=data["Relation"][ID_type_list[i][0]]
    return elements

def create_single_element(data,JSON_outputfile,whitefilter,blackfilter):
    '''
    **Purpose**
        the prefiltered OSM data are filtered for a specific element.

    **Input**
        data:
            dictionary of prefiltered data
        whitefilter:
            (see doc: filter)
        blackfilter:
            (see doc: filter)

    **Return**
        elements:
           dictionaries with filtered elements
    '''

    elementIDs = GetOSM_IDs(data,whitefilter,blackfilter)
    elements = find_elements(data,elementIDs)
    write_elements(elements,"pipelines",JSON_outputfile)

    logging.info('Save elements as jsonfiles at\n'+CC.Cyan+ os.path.dirname(JSON_outputfile)+"/Analysis/Elements\n"+CC.End)
    return elements

def run_filter(elementname,countrycode,PBF_inputfile,JSON_outputfile,
                    prefilter,whitefilter,blackfilter, NewPreFilterData=True,
                    CreateElements=True, LoadElements=True,verbose=False):
    '''
    **Purpose**
        Reads OSM pbf-files, filters data and storages the output as elements in JSON files`

    **Input**
        elementname:
            name of the element (e.g. 'pipelines')
        countrycode:
            two digits, (e.g. 'EU')
        PBF_inputfile:
            filepathname of inputfile
        JSON_outputfile:
            filedir of output
        prefilter:
            prefilter data (see doc: filter)
        whitefilter:
            whitefilter data (see doc:   :ref:`filter <filter>' )
        blackfilter:
            blackfilter data (see doc: filter)
        NewPreFilterData:
            prefilter and store Data from pbf
        CreateElements:
            filter and store Elements from Data
        LoadElements:
            load Elements

    **Return**
       Data:
           prefiltered Data of pbf
       Elements:
           filtered Elements from Data

    **Example**

    Access output:
        Data['Node']['6037838916']['lonlat']
    '''
    if verbose == False: logging.disable(logging.INFO)

    element         = ()
    Data           = {}

    if NewPreFilterData==True:
        ###Create and Save Gas Data
        logger.info(CC.Script+'Filter Gas Data for country ' + countrycode+ CC.End)
        logger.info('Load OSM data from '+CC.Cyan+ PBF_inputfile+'\n'+CC.End)
        pre_filter.filter_pbf(PBF_inputfile, JSON_outputfile,prefilter)
        logger.info('\nRead gas data from \n'+CC.Cyan+ JSON_outputfile+CC.End)
        Data = osm_info.ReadJason(JSON_outputfile,verbose='no')
        ###Save Gas Data (Pickle)
        DataDict = {"Data":Data}
        logger.info('\nPickle gas data to \n'+os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile))+CC.End)
        osm_pickle.picklesave(DataDict,os.path.realpath(os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile))))

    # loading of OSM Gas daten filtered data
    DataDict      = {"Data":Data}
    DataDict      = osm_pickle.pickleload(DataDict,os.path.realpath(os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile))))
    Data      = DataDict["Data"]

    if CreateElements==True:
        ###Create Elements
        logger.info(CC.Script+'\nCreate the OSM-elements\n'+CC.End)
        element = create_single_element(Data,JSON_outputfile,whitefilter,blackfilter)
        ###Save Elements(Pickle)
    Elements = {
                  elementname:element
                }

    if CreateElements==True:
        logger.info('\nPickle OSM-elements to \n'+CC.Cyan+ os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile), 'Elements/')+'\n'+CC.End)
        osm_pickle.picklesave(Elements,os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile), 'Elements/'))

    if LoadElements==True:
        logger.info('\nUnpickle OSM-elements from \n'+CC.Cyan +os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile), 'Elements/')+CC.End)

        Elements = osm_pickle.pickleload(Elements,os.path.join(os.getcwd(),os.path.dirname(JSON_outputfile), 'Elements/'))

    return Data, Elements


from flask import Flask, render_template, request, redirect, url_for

app = Flask(__name__)

@app.route("/")
def home():
  return render_template("index.html")

@app.route("/result", methods=['POST'])
def result():
    num1 = request.form['num1']
    num2 =request.form['num2']

    num1 = float(num1)
    num2 = float(num2)
    result = 0
    operation = request.form['operation']
    if operation == 'add':
      result = num1 + num2
      content = 'sum'
    elif operation == 'subtract':
      result = num1 - num2
      content = 'difference'
    elif operation == 'multiply':
      result = num1 * num2
      content = 'product'
    elif operation == 'divide':
      content = 'quotient'
      if num2 != 0:
        result = num1 / num2
      else:
        result = "Error: Division by zero"
    return render_template('result.html', result=result, content=content)

if __name__ == "__main__":
  app.run(debug=True)


"""
 Pygame base template for opening a window

 Sample Python/Pygame Programs
 Simpson College Computer Science
 http://programarcadegames.com/
 http://simpson.edu/computer-science/

 Explanation video: http://youtu.be/vRB_983kUMc
"""

import pygame
import random

# Define some colors
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
GREEN = (0, 255, 0)
RED = (255, 0, 0)
BLUE = (0, 0, 255)
colors = [BLACK, GREEN, RED, BLUE]
def random_color():
	return random.coice(colors)
pygame.init()

# Set the width and height of the screen [width, height]
screen_width = 700
screen_height = 500
screen = pygame.display.set_mode((screen_width, screen_height))

x_speed = 3
y_speed = 3

pygame.display.set_caption("Documentation Hunt")

class Circle():
	def __init__(self, mouse_position, Circle_size):
		self.mouse_position = mouse_position
		x_pos = mouse_position[0]
		y_pos = mouse_position[1]
		self.x_pos = x_pos
		self.y_pos = y_pos
		self.Circle_size = Circle_size
	def draw(self):
		global screen
		global RED
		pygame.draw.circle(screen,RED, self.mouse_position,self.Circle_size)
	def move(self, x_pos, y_pos, y_speed, x_speed):
		self.x_speed = x_speed
		self.y_speed = y_speed
		self.x_pos = x_pos								#make a list of positions?
		self.y_pos = y_pos

		if self.x_pos >= screen_width - self.Circle_size or self.x_pos < self.Circle_size:
			self.x_speed = x_speed * -1

		if self.y_pos >= screen_height - self.Circle_size or self.y_pos < self.Circle_size:
			self.y_speed = self.y_speed * -1

		self.x_pos += self.x_speed
		self.y_pos += self.y_speed

# Loop until the user clicks the close button.
done = False

# Used to manage how fast the screen updates
clock = pygame.time.Clock()

circle_list = []

screen.fill(WHITE)

# -------- Main Program Loop -----------
while not done:
	# --- Main event loop
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			done = True

	# --- Game logic should go here
	pressed = pygame.mouse.get_pressed()
	if pressed[0] == 1:			  #checks if mouse click is happening
		print("Here!")
		screen.fill(WHITE)
		mouse_position = pygame.mouse.get_pos()			#takes position where mouse is pressed
		circle = Circle(mouse_position,20)					#makes circle object
		circle_list.append(circle)
		for circle in circle_list:
			circle.draw()								#draws circles
			circle.move(1,1,1,1)

	# --- Screen-clearing code goes here

	# Here, we clear the screen to white. Don't put other drawing commands
	# above this, or they will be erased with this command.

	# If you want a background image, replace this clear with blit'ing the
	# background image

	# --- Drawing code should go here

	# --- Go ahead and update the screen with what we've drawn.
	pygame.display.flip()

	# --- Limit to 60 frames per second
	clock.tick(60)

# Close the window and quit.
pygame.quit()
exit() # Needed when using IDLE


from kivy.properties import (
    NumericProperty,
    ObjectProperty,
    BooleanProperty,
    AliasProperty,
)
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.image import Image
from kivy.uix.behaviors import ButtonBehavior

class PlayerSelector(BoxLayout):
    gameObject = ObjectProperty()
    numPlayers = NumericProperty(2)
    numAI = NumericProperty(1)
    minPlayers = NumericProperty(1)
    maxPlayers = NumericProperty(2)
    maxAI = NumericProperty(1)
    hasAI = BooleanProperty()

    numPlayersText = AliasProperty(
        lambda self: f"Number of players: {self.numPlayers:.0f}", bind=["numPlayers"]
    )
    numAIText = AliasProperty(
        lambda self: f"Number of AIs: {self.numAI:.0f}", bind=["numAI"]
    )

    def __init__(self, **kwargs):
        BoxLayout.__init__(self, **kwargs)
        self.register_event_type("on_choose")

    def on_gameObject(self, instance, gameObject):
        self.minPlayers = gameObject.minPlayers
        self.maxPlayers = gameObject.maxPlayers
        self.numPlayers = gameObject.maxPlayers
        self.hasAI = gameObject.hasAI
        if self.hasAI:
            self.maxAI = self.maxPlayers - 1
            self.numAI = self.maxPlayers - 1

    def submit(self):
        players = []
        numAI = self.numAI if self.hasAI else 0
        players += [False] * int(self.numPlayers - numAI)
        players += [True] * int(numAI)
        self.dispatch("on_choose", tuple(players))

    def on_choose(self, players):
        pass


import argparse
import boto3
import os
import subprocess
import sys
import tempfile

def zip(src_folder, zip_name):
    """Zip all the files and folders in src_folder.

    Note that the zip command is run _within_ src_folder so the folder,
    itself, will not appear in the zip.

    Args:
        src_folder (string): Folder to zip.
        zip_name (string): Name of output zip file.

    Returns:
        (bool): True on success.
    """

    #args = ('/usr/bin/zip', '--symlinks', '-r', '-q', zip_name, '.')
    args = ('/usr/bin/zip', '-r', '-q', zip_name, '.')
    popen = subprocess.Popen(args, cwd=src_folder, stdout=subprocess.PIPE)
    exit_code = popen.wait()
    output = popen.stdout.read()
    if not exit_code == 0:
        print(str(output))
        return False
    return True

def upload_to_s3(session, zip_file, bucket):
    """Upload the zip file to the given S3 bucket.

    Args:
        session (Session): Boto3 Session.
        zip_file (string): Name of zip file.  The name (after any path is stripped) is used as the key.
        bucket (string): Name of bucket to use.
    """
    key = os.path.basename(zip_file)
    s3 = session.client('s3')
    try:
        s3.create_bucket(Bucket=bucket)
    except s3.exceptions.BucketAlreadyOwnedByYou:
        pass # Only us-east-1 will not throw an exception if the bucket already exists
    s3.put_object(Bucket=bucket, Key=key, Body=open(zip_file, 'rb'))

def setup_parser():
    parser = argparse.ArgumentParser(
        description='Script for deploying lambda functions to S3.')
    parser.add_argument(
        '--aws-credentials', '-a',
        metavar = '<file>',
        default = os.environ.get('AWS_CREDENTIALS'),
        type = argparse.FileType('r'),
        help = 'File with credentials for connecting to AWS (default: AWS_CREDENTIALS)')
    parser.add_argument(
        '--upload-only', '-u',
        action = 'store_const',
        const = True,
        default = False,
        help = 'Don\'t re-create the zip.  Just upload to S3.')
    parser.add_argument(
        'source_folder',
        help = 'Folder containing lambda handlers and the virtualenv.')
    parser.add_argument(
        'zip_file',
        help = 'source_folder will be zipped and stored in this filename.')
    parser.add_argument(
        'bucket',
        help = 'Name of S3 bucket to upload lambdas to.')

    return parser

if __name__ == '__main__':
    parser = setup_parser()
    args = parser.parse_args()

    cwd = os.getcwd()

    # Build an absolute path to the zip file if one not provided.
    zip_file_abs_path = os.path.join(cwd, args.zip_file)

    if not args.upload_only:
        if not zip(args.source_folder, zip_file_abs_path):
            sys.exit(1)

    if args.aws_credentials is None:
        # This allows aws roles to be used to create sessions.
        session = boto3.session.Session()
    else:
        # moved so the script can be independent if not passing credentials
        from lambdautils import create_session
        session = create_session(args.aws_credentials)
    upload_to_s3(session, args.zip_file, args.bucket)

    print('Done.\n')


import torch
import time
import glob
import os
import csv
import sys
import argparse
import numpy as np

tmp_path = os.path.dirname(os.path.abspath(__file__))
root_path = '/'.join(tmp_path.split('/')[:-2])
sys.path.append(root_path)
from utils.config import cfg
from model.main_msnet import MsNet
from workflow.train.dataset_loader import fill_feature_cuda
from utils.polar_mask import get_point_instance, get_final_masks
from utils.ms_compatibility import get_mz_fwhm

class MsNetEvaluator:
    def __init__(self, exp, epoch):
        self.net = MsNet(cfg)
        self.net.backbone.load_state_dict(
            torch.load(root_path + '/%s/%s/%s_%.3d.pth' % ('experiment', exp, 'backbone', epoch)))
        self.net.sem_net.load_state_dict(
            torch.load(root_path + '/%s/%s/%s_%.3d.pth' % ('experiment', exp, 'sem_net', epoch)))
        self.net.center_net.load_state_dict(
            torch.load(root_path + '/%s/%s/%s_%.3d.pth' % ('experiment', exp, 'box_center_net', epoch)))
        self.net.polar_mask_net.load_state_dict(
            torch.load(root_path + '/%s/%s/%s_%.3d.pth' % ('experiment', exp, 'polar_mask_net', epoch)))
        self.net.backbone.eval()
        self.net.sem_net.eval()
        self.net.center_net.eval()
        self.net.polar_mask_net.eval()

    def eval(self, eval_dir, mass_analyzer, mz_resolution, resolution_mz, rt_fwhm, center_threshold=0.5, block_rt_width=None, block_mz_width=None, target_id=None):
        start_time = time.time()
        print('Evaluating on ', eval_dir)
        eval_file_paths = glob.glob(os.path.join(eval_dir, '*.csv'))
        print('Point cloud count: {}'.format(len(eval_file_paths)))
        result_list = []

        for file_dir in eval_file_paths:
            file_name = file_dir.split('/')[-1].split('.csv')[0]
            pc_id = int(file_name.split('_')[0])
            block_mz_center = float(file_name.split('_')[1])
            block_rt_center = float(file_name.split('_')[2])

            if target_id is not None and target_id != -1 and pc_id != target_id:
                continue

            reader = csv.reader(open(file_dir, 'r'))
            data = np.array(list(reader), dtype=np.float32)
            raw_points = data[:, :3]

            points = self.normalize_points(raw_points, mass_analyzer, mz_resolution, resolution_mz, rt_fwhm)
            # points = fill_feature_cuda(points)

            pc = torch.tensor(points.reshape(1, len(points), -1), dtype=torch.float32).cuda()

            """ feature extraction """
            point_features = self.net.backbone(pc[:, :, :3])
            """ semantic segmentation """
            pre_sem = self.net.sem_net(point_features)
            """ center prediction """
            pre_center = self.net.center_net(point_features).squeeze(-1)
            """ mask prediction """
            pre_masks = self.net.polar_mask_net(point_features)

            pre_sem = pre_sem[0].cpu().detach().numpy()
            pre_center = pre_center[0].cpu().detach().numpy()
            pre_masks = pre_masks[0].cpu().detach().numpy()

            center_idx = ((pre_center * pre_sem) > center_threshold).nonzero()[0]

            candidate_masks = pre_masks[center_idx]
            final_center_idx, final_masks = get_final_masks(points, pre_masks, center_idx, candidate_masks)

            if len(final_center_idx) == 0:
                continue
            arg_idx = np.argsort(-points[final_center_idx][:, 2])
            final_center_idx = final_center_idx[arg_idx]
            final_masks = np.array(final_masks)[arg_idx]

            # manage result
            point_instance = get_point_instance(points, final_center_idx, final_masks)
            for i in range(len(final_center_idx)):
                instance_idxes = (point_instance == i).nonzero()[0]
                if len(instance_idxes) < 10:
                    continue
                instance_points = raw_points[instance_idxes]

                # Intensity calculation
                total_intensity = self.get_instance_volume(instance_points)
                # total_intensity = np.sum(instance_points[:, 2])

                # Center point selection
                # apex_idx = instance_idxes[np.argmax(instance_points[:, 2])]
                # apex_raw_point = raw_points[apex_idx]
                apex_raw_point = raw_points[final_center_idx[i]]
                if block_rt_width is not None and block_mz_width is not None:
                    if apex_raw_point[0] >= block_rt_center + block_rt_width / 2 \
                            or apex_raw_point[0] < block_rt_center - block_rt_width / 2 \
                            or apex_raw_point[1] >= block_mz_center + block_mz_width / 2 \
                            or apex_raw_point[1] < block_mz_center - block_mz_width / 2:
                        continue

                result_list += [[pc_id, apex_raw_point[1], np.min(instance_points[:, 1]), np.max(instance_points[:, 1]),
                                apex_raw_point[0], np.min(instance_points[:, 0]), np.max(instance_points[:, 0]),
                                apex_raw_point[2], total_intensity, len(instance_points)]]

                # id, mz, mz_start, mz_end, rt, rt_start, rt_end, apex_intensity, volume, point_count, mask_len_0, ..., mask_len_35
                # print(total_intensity, apex_raw_point[0], apex_raw_point[1])

            print(file_dir, len(final_center_idx))

            # debug mode
            if target_id is not None:
                if target_id == -1 or pc_id == target_id:
                    from utils.visualize import Plot
                    print(len(center_idx))
                    Plot.draw_pc_heatmap(pc_xyz=points, idx=pc_id, heatmap=pre_center)
                    Plot.draw_pc_heatmap(pc_xyz=points, idx=pc_id, heatmap=pre_sem)
                    center_map = np.zeros(pre_center.shape)
                    center_map[center_idx] = 1
                    # Plot.draw_pc_heatmap(pc_xyz=points, idx=pc_id, heatmap=center_map)
                    Plot.draw_pc_polar(pc_xyzrgb=points, idx=pc_id, center_idxes=center_idx, polar_masks=candidate_masks)
                    Plot.draw_pc_polar(pc_xyzrgb=points, idx=pc_id, center_idxes=final_center_idx, polar_masks=final_masks)
                    print(pc_id)
        time_cost = time.time() - start_time
        print('Time Cost:', time_cost)

        if target_id is None:
            # output result
            output_file_name = eval_dir.split('/')[-1] + '-result-{}-{}.csv'.format(
                time.strftime("%Y%m%d_%H%M%S", time.localtime()), int(time_cost))
            output_file_dir = os.path.join(os.path.dirname(eval_dir), 'result')
            output_file_path = os.path.join(output_file_dir, output_file_name)
            if not os.path.exists(output_file_dir):
                os.mkdir(output_file_dir)

            output_file = open(output_file_path, 'w')
            writer = csv.writer(output_file)
            writer.writerows(result_list)
            output_file.close()
            print('Finish on ', output_file_path)

    def normalize_points(self, raw_points, mass_analyzer, mz_resolution, resolution_mz, rt_fwhm):
        points = raw_points.copy()
        min_rt = np.min(points[:, 0])
        max_rt = np.max(points[:, 0])
        min_mz = np.min(points[:, 1])
        max_mz = np.max(points[:, 1])
        mid_mz = (min_mz + max_mz) / 2
        mid_rt = (min_rt + max_rt) / 2

        mz_fwhm = get_mz_fwhm(mid_mz, mass_analyzer, mz_resolution, resolution_mz)
        rt_factor = 1.0 / rt_fwhm
        mz_factor = 1.0 / mz_fwhm
        points[:, 0] = (points[:, 0] - mid_rt) * rt_factor
        points[:, 1] = (points[:, 1] - mid_mz) * mz_factor
        points[:, 2] = np.log2(points[:, 2])
        min_intensity = np.min(points[:, 2]) - 1
        points[:, 2] -= min_intensity
        return points

    def get_instance_volume(self, instance_points):
        rt_sorted_points = instance_points[np.argsort(instance_points[:, 0])]
        rt_map = {}
        for point in rt_sorted_points:
            if not rt_map.__contains__(point[0]):
                rt_map[point[0]] = []
            rt_map[point[0]] += [point]
        rt_list = list(rt_map.keys())
        if len(rt_list) < 2:
            return 0
        areas = []
        for i in range(len(rt_list) - 1, -1, -1):
            rt = rt_list[i]
            tmp_points = np.array(rt_map[rt])
            if len(tmp_points) < 2:
                rt_list.remove(rt)
                continue
            mz_sorted_points = tmp_points[np.argsort(tmp_points[:, 1])]
            mz_interval = mz_sorted_points[1:, 1] - mz_sorted_points[:-1, 1]
            mid_intensity = (mz_sorted_points[1:, 2] + mz_sorted_points[:-1, 2]) / 2
            area = np.sum(mz_interval * mid_intensity)
            areas += [area]
        rt_list = np.array(rt_list)
        areas = np.array(areas)
        rt_interval = rt_list[1:] - rt_list[:-1]
        mid_area = (areas[1:] + areas[:-1]) / 2
        volume = np.sum(rt_interval * mid_area)
        return volume

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Untargeted feature extraction')

    parser.add_argument('--data_dir', type=str, help='dataset dir', required=True)
    parser.add_argument('--mass_analyzer', type=str, help='orbitrap or tof', required=True)
    parser.add_argument('--mz_resolution', type=float, help='the resolution of mass analyzer', required=True)
    parser.add_argument('--resolution_mz', type=float, help='the m/z value at the resolution', required=True)
    parser.add_argument('--rt_fwhm', type=float, help='median of feature RT FWHM', required=True)
    parser.add_argument('--experiment', type=str, help='choose a pretrained model', default='msnet_20220427_141044')
    parser.add_argument('--epoch', type=int, help='choose the epoch of saved model', default=1000)
    parser.add_argument('--center_threshold', type=float, help='feature center selection threshold', default=0.5)
    parser.add_argument('--block_rt_width', type=float, help='point cloud rt window width', default=6)
    parser.add_argument('--block_mz_width', type=float, help='point cloud m/z window width', default=0.8)
    parser.add_argument('--target_id', type=int, help='None, not visualize. -1, visualize each point cloud. Integer, visualize a specific point cloud', default=None)
    args = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = '0'

    data_dir = glob.glob(os.path.join(args.data_dir, '*arget-*'))
    print(data_dir)
    evaluator = MsNetEvaluator(exp=args.experiment, epoch=args.epoch)
    for eval_dir in data_dir:
        evaluator.eval(eval_dir=eval_dir, mass_analyzer=args.mass_analyzer, mz_resolution=args.mz_resolution,
                       resolution_mz=args.resolution_mz, rt_fwhm=args.rt_fwhm, center_threshold=args.center_threshold,
                       block_rt_width=args.block_rt_width, block_mz_width=args.block_mz_width, target_id=args.target_id)


import numpy as np
import pandas as pd
from tqdm import tqdm
import itertools, argparse, os, sys, random, logging, config, torch, torchvision, utils
import torchvision.models as models
import torch.nn as nn

def run_inference(args, model, input_data, device):

	model.eval()
	softmax = nn.Softmax(dim=1)

	starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)

	inference_time_list = []

	with torch.no_grad():
		for i in tqdm(range(args.n_rounds)):

			starter.record()

			_ = model(input_data)

			ender.record()
			torch.cuda.synchronize()
			curr_time = starter.elapsed_time(ender)

			print(curr_time)

			inference_time_list.append(curr_time)

			torch.cuda.empty_cache()

	results = {"inference_time": inference_time_list}

	return results

def save_result(result, save_path):
	df = pd.DataFrame(np.array(list(result.values())).T, columns=list(result.keys()))
	df.to_csv(save_path, mode='a', header=not os.path.exists(save_path) )

def extracting_inference_time_backbone(args, model, input_data, input_dim, dim, save_path, device):

	result = run_inference(args, model, input_data, device)

	save_result(result, save_path)

def main(args):

	model_save_path = os.path.join(config.DIR_NAME, "models", config.dataset_name, config.model_name,
		"%s_backbone_model_%s_%s.pth"%(args.distortion_type, config.model_name, args.model_id))

	inference_time_path = os.path.join(config.DIR_NAME, "inference_data", args.dataset_name, args.model_name,
		"inference_time_backbone_id_%s_final_final.csv"%(args.model_id))

	device = torch.device('cuda' if (torch.cuda.is_available() and args.cuda) else 'cpu')

	n_classes = 257

	#Load the trained backbone DNN model.
	model = models.mobilenet_v2(pretrained=True)
	model.classifier[1] = nn.Linear(1280, n_classes)

	model.load_state_dict(torch.load(model_save_path, map_location=device)["model_state_dict"])
	model = model.to(device)

	dummy_input = torch.randn(1, 3, 224,224, dtype=torch.float).to(device)

	for _ in range(10):
		_ = model(dummy_input)

	extracting_inference_time_backbone(args, model, dummy_input, args.input_dim, args.dim, inference_time_path, device)

if (__name__ == "__main__"):

	parser = argparse.ArgumentParser(description='UCB using MobileNet')
	parser.add_argument('--model_id', type=int, default=config.model_id, help='Model Id.')
	parser.add_argument('--dataset_name', type=str, default=config.dataset_name, help='Dataset Name.')
	parser.add_argument('--model_name', type=str, default=config.model_name, help='Model name.')
	parser.add_argument('--cuda', type=bool, default=config.cuda, help='Cuda ?')
	parser.add_argument('--seed', type=int, default=config.seed, help='Seed.')
	parser.add_argument('--batch_size_train', type=int, default=config.batch_size_train, help='Size of train batch.')
	parser.add_argument('--split_ratio', type=float, default=config.split_ratio, help='Split Ratio')
	parser.add_argument('--distortion_prob', type=float, default=1)
	parser.add_argument('--distortion_type', type=str, default=config.distortion_type, help='Distortion Type.')
	parser.add_argument('--input_dim', type=int, default=config.input_dim, help='Input Dim. Default: %s'%config.input_dim)
	parser.add_argument('--dim', type=int, default=config.dim, help='Dim. Default: %s'%(config.dim))
	parser.add_argument('--n_rounds', type=float, default=10000)

	args = parser.parse_args()
	main(args)


import hmac
from psycopg2.extensions import quote_ident
from psycopg2.errorcodes import UNDEFINED_TABLE
from django.db import connection, ProgrammingError, InternalError
from django.db.transaction import atomic
from django.utils import timezone

def number_generator(
    tenant_id: str,
    sequence_name: str,
    number_format: str,
    checksum_salt: str,
    checksum_algorithm: str,
    checksum_length: int,
    checksum_format: str,
) -> str:
    if not tenant_id:
        raise ValueError("Tenant-ID not given")

    sequence_name = f'{sequence_name}_{tenant_id}'
    val = None
    i = 0

    while not val and i < 2:
        try:
            with connection.cursor() as cursor:
                with atomic():
                    cursor.execute("SELECT nextval(%s);", (quote_ident(sequence_name, cursor.cursor),))
                    val = cursor.fetchone()
                    break

        except (ProgrammingError, InternalError) as error:
            if error.__cause__.pgcode != UNDEFINED_TABLE:
                raise

            i += 1
            with connection.cursor() as cursor:
                with atomic():
                    cursor.execute("CREATE SEQUENCE IF NOT EXISTS %s START 1;" % quote_ident(sequence_name, cursor.cursor))

    if not val:
        raise ValueError('cannot_obtain_next_number')

    nextnumber = int(val[0])
    now = timezone.now()
    number = number_format % {
        'year': now.year,
        'month': now.month,
        'number': nextnumber,
    }

    if checksum_length > 0:
        checksum_value = hmac.new(checksum_salt, bytes(number, 'utf-8'), digestmod=checksum_algorithm)
        checksum_int = int(checksum_value.hexdigest(), 16)
        checksum_number = checksum_int % 10 ** checksum_length
        checksum = '%0*d' % (checksum_length, checksum_number)
        number = checksum_format % {
            'number': number,
            'checksum': checksum,
        }

    return number


from django import forms

from .models import Feria

class FeriaCreateForm(forms.ModelForm):
	"""docstring for FeriaCreateForm"""
	feria_id = forms.CharField(label = "ID", required=True,
					widget = forms.TextInput(
						attrs={
							"placeholder":"Ingrese la id de la feria",
							"minlength":3
						}
					)
				)
	feria_calle = forms.CharField(label="Nombre calle", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese el nombre de la calle de la feria",
                            "minlength":5
                        }
                )
            )
	feria_comuna = forms.CharField(label="Comuna feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese el comuna de la feria",
                            "minlength":5
                        }
                	)
                )
	feria_direccion_ini = forms.CharField(label="Direccion inicio feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese la direccion de inicio de la feria",
                            "minlength":5
                        }
                	)
                )
	feria_direccion_fin = forms.CharField(label="Direccion fin feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese la direccion del termino de la feria",
                            "minlength":5
                        }
                	)
            	)
	feria_horario_ini = forms.CharField(label="Horario inicio feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese el horario de inicio de la feria",
                            "minlength":5
                        }
                	)
            	)
	feria_horario_fin = forms.CharField(label="Horario fin feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese el horario de termino de la feria",
                            "minlength":5
                        }
                	)
            	)
	feria_dias = forms.CharField(label="Dias feria", required=True,
                widget=forms.TextInput(
                        attrs={
                            "placeholder":"Ingrese dias de funcionamiento de la feria",
                            "minlength":5
                        }
                	)
            	)
	feria_imagen = forms.ImageField(label="Imagen principal")

	class Meta:
		"""docstring for Meta"""
		model = Feria
		fields = [
			'feria_id',
			'feria_calle',
			'feria_comuna',
			'feria_direccion_ini',
			'feria_direccion_fin',
			'feria_horario_ini',
			'feria_horario_fin',
			'feria_dias',
			'feria_imagen'
		]


from django.core.exceptions import ObjectDoesNotExist
from rest_framework import serializers

from hotel_app.models import (
    Booking,
    Hotel,
    City,
)
from hotel_app.utils.common import find_room

class HotelSerializer(serializers.ModelSerializer):
    rooms_beds = serializers.ReadOnlyField(source="get_rooms_beds")

    class Meta:
        model = Hotel
        fields = ("title", "city", "rooms_beds")

class BookingSerializer(serializers.ModelSerializer):
    city = serializers.CharField(read_only=True)
    hotel = serializers.CharField(read_only=True)

    class Meta:
        model = Booking
        fields = ("guest_name", "booking_start", "booking_end", "city", "hotel", "persons")

    @staticmethod
    def get_hotel(city, hotel):
        try:
            city = City.objects.get(title=city)
            hotel = city.hotels.get(title=hotel)
        except ObjectDoesNotExist as e:
            raise serializers.ValidationError(
                {"error": f"{city}:{hotel} - {e}"}
            )
        print(f"Hotel Detected: {hotel}")
        return hotel

    def to_internal_value(self, data):
        print(f"Raw Booking data: {data}")
        self.hotel = self.get_hotel(data.get("city"), data.get("hotel"))
        internal_data = super().to_internal_value(data)
        print(f"Internal Booking data: {internal_data}")
        return internal_data

    def create(self, validated_data):
        print(f"Validated data: {validated_data}")
        booking_start = validated_data.pop("booking_start")
        booking_end = validated_data.pop("booking_end")
        persons = validated_data.pop("persons")
        room = find_room(persons, booking_start, booking_end, self.hotel)
        booking = Booking.objects.create(
            guest_name=validated_data.pop("guest_name"),
            persons=persons,
            booking_start=booking_start,
            booking_end=booking_end,
            room=room,
        )
        print(f"Booking Instance: {booking}")
        return booking


from os.path import dirname, join as os_path_join, exists, isfile
from os import listdir, makedirs, remove

import csv
from datetime import date

from xlsxwriter.workbook import Workbook

class Logger:
    '''Logger which create an ".xlsx" file which could be set with:

    Parameters:
        logging_name in order to create a file with the following format:
            TodayDate_logging_name_xxxx.xlsx (xxxx is interger)

        logging_folder relative path from logger.py folder path
        where create the log files
        (default: create "logging" folder in project root)

        columns_width in order to manage the width of each column
        (default: not column width modification)
            [40, 10]  ->  set the first colum to 40, the second to 10...

    Functions:
        add_lines_to_logging_file(list):
            each list's items are respectively write in columns of a new row

        stop_logging(sheet_name):
            when all needed data are stored, create the xlsx file
            with a sheet named 'sheet_name'
            (default: default excel file sheet name)

        set_logging_name(logging_name):
            in order to change the logging file name
            after Logger instantiation

        set_column_width(columns_width):
            in order to change excel sheet columns width
            after Logger instantiation
    '''

    def __init__(self,
                 logging_name='multimeter_Measurements',
                 logging_folder="../../logging",
                 columns_width=[]):
        self._logging_folder = logging_folder
        self._logging_logging_name = '_' + logging_name + '_'
        self._set_logging_folder()
        self._generate_logging_file_path()
        self.set_column_width(columns_width)

    def add_lines_to_logging_file(self, data: list):
        with open(
                self._logging_file_path,
                'a',
                newline='',
                encoding='utf8') as file:
            writer = csv.writer(file)
            writer.writerow(data)

    def stop_logging(self, sheet_name=None):
        '''After logging, convert csv to xlsx'''

        csvfile = self._logging_file_path
        # Create Workbook with worksheet named 'sheet_name'
        self._workbook = Workbook(csvfile[:-4] + '.xlsx')
        worksheet = self._workbook.add_worksheet(name=sheet_name)
        # Set cells format
        cell_format = self._set_cell_format()
        cell_format_ok = self._set_cell_format('green')
        cell_format_nok = self._set_cell_format('red')
        cell_format_board_number = self._set_cell_format('cyan')
        # Update column width
        for colum_index, width in enumerate(self._columns_width):
            worksheet.set_column(colum_index, colum_index, width)
        # Check if csv file has been created
        if not exists(csvfile):
            return
        # Write csv in xlsx
        with open(csvfile, 'rt', encoding='utf8') as file:
            reader = csv.reader(file)
            for r, row in enumerate(reader):
                for c, col in enumerate(row):
                    if col == 'Test OK':
                        choice_format = cell_format_ok
                    elif col == 'Test NOK' or col == 'Exception occurs':
                        choice_format = cell_format_nok
                    elif col == 'Board number':
                        choice_format = cell_format_board_number
                    else:
                        choice_format = cell_format
                    worksheet.write(r, c, col, choice_format)
        self._workbook.close()
        # Delete temporary csv file
        remove(csvfile)
        # Update logging file's name
        self._generate_logging_file_path()

    def set_logging_name(self, logging_name: str):
        self._logging_logging_name = '_' + logging_name + '_'
        self._generate_logging_file_path()

    def set_column_width(self, columns_width: list = []):
        self._columns_width = columns_width

    def _set_cell_format(self, bg_color: str = ''):
        cell_format = self._workbook.add_format()
        cell_format.set_align('right')
        cell_format.set_align('vcenter')
        cell_format.set_indent(1)
        if not bg_color == '':
            cell_format.set_bg_color(bg_color)
        return cell_format

    def _files_in_folder(self, folder_path: str) -> list:
        '''Return the list of all files in folder_path and its subfolder'''
        files_in_folder_path = []
        # Loop on all files or directories inside folder_path
        for file_or_dir in listdir(folder_path):
            file_or_dir_path = os_path_join(folder_path, file_or_dir)
            # If file_or_dir is a file
            if isfile(file_or_dir_path):
                # Add it to the files list
                files_in_folder_path.append(file_or_dir)
            else:
                # Loop inside the subfolder to find all files
                files_in_subfolder = self._files_in_folder(file_or_dir_path)
                # Add relative path from folder_path
                for index, file in enumerate(files_in_subfolder.copy()):
                    files_in_subfolder[index] = file_or_dir + "/" + file
                # Add this files to the files list
                files_in_folder_path += files_in_subfolder
        return files_in_folder_path

    def _set_logging_folder(self):
        module_dir = dirname(__file__)
        logging_folder_path = os_path_join(
            module_dir,
            self._logging_folder)
        self._check_folder_exist_or_create(logging_folder_path)
        self._logging_folder_path = logging_folder_path

    def _get_logging_files_list(self):
        logging_files_list = self._files_in_folder(
            self._logging_folder_path)
        return logging_files_list

    def _generate_logging_file_path(self) -> None:
        '''Generate a new default logging file path'''
        logging_files_list = self._get_logging_files_list()
        today = date.today().strftime('%Y-%m-%d')
        for i in range(10000):
            file_name = (
                today
                + self._logging_logging_name
                + self._file_number_padding(i)
                )
            file_name_csv = file_name + ".csv"
            file_name_xlsx = file_name + ".xlsx"
            if (
                    file_name_csv not in logging_files_list
                    and
                    file_name_xlsx not in logging_files_list):
                self._logging_file_path = os_path_join(
                    self._logging_folder_path,
                    file_name_csv)
                break

    def _file_number_padding(self, index: int) -> str:
        if index < 10:
            return "000" + str(index)
        elif index < 100:
            return "00" + str(index)
        elif index < 1000:
            return "0" + str(index)
        else:
            return str(index)

    def _check_folder_exist_or_create(self, dir: str):
        '''
        From an absolute path, create the folder if it doesn't already exist
        '''
        # Check if this folder exist
        if not exists(dir):
            # Created the folders
            makedirs(dir)


from typing import List

class Codec:
    def encode(self, strs: List[str]) -> str:
        """Encodes a list of strings to a single string.
        """
        encoded = ""
        for item in strs:
            # eg. "5#Hello5#World"
            encoded += str(len(item)) + "#" + item
        return encoded

    def decode(self, s: str) -> List[str]:
        """Decodes a single string to a list of strings.
        """
        index_s = 0
        decoded = []
        while index_s < len(s):
            index_num = index_s
            # Find out how long is the string by getting a number
            # comes before #, and a number can be more than one digit
            while s[index_num] != "#":
                index_num += 1
            # eg. "5#Hello5#World" index num would be 1, 8
            # [0:1] [7:8]
            length_item = int(str(s[index_s:index_num]))
            # +1 to indexes to skip # added
            decoded.append(s[index_num+1:index_num+1+length_item])
            index_s = index_num + 1 + length_item
        return decoded

strs = ["Hello","World"]
codec = Codec()
print("Result of encode is", codec.encode(strs))
print("Result of encode -> decode is", codec.decode(codec.encode(strs)))


import os
import shutil
from time import *
from datetime import *
from ch.systemsx.cisd.openbis.generic.shared.api.v1.dto import SearchCriteria

MarkerHiSeqComplete = 'RTAComplete.txt'

def createOpenbisTimeStamp(file):
  '''
  Creates a openBIS compatible time stamp of a file time stamp
  '''
  mtime = os.path.getmtime(file)
  lt = localtime(mtime)
  tz = localtime().tm_hour - gmtime().tm_hour
  return (strftime("%Y-%m-%d %H:%M:%S GMT" + "%+.2d" % tz + ":00", lt))

# -----------------------------------------------------------------------------

def extractFlowCellName (runFolderName):
  return runFolderName.split('_')[-1][1:]

# -----------------------------------------------------------------------------

def process(transaction):

  incomingPath = transaction.getIncoming().getAbsolutePath()
  # Get the incoming name
  runFolderName = transaction.getIncoming().getName()
  flowCellName = extractFlowCellName(runFolderName)

  Markerfile = incomingPath + "/" + MarkerHiSeqComplete

  # Search for the sample and check if there is already sample with this name
  search_service = transaction.getSearchService()
  sc = SearchCriteria()
  sc.addMatchClause(SearchCriteria.MatchClause.createAttributeMatch(SearchCriteria.MatchClauseAttribute.CODE, flowCellName));
  foundSamples = search_service.searchForSamples(sc)

  if foundSamples.size() > 0:
      sa = transaction.getSampleForUpdate(foundSamples[0].getSampleIdentifier())
      sa.setPropertyValue("SEQUENCER_FINISHED", createOpenbisTimeStamp(Markerfile))


import torch
from torch import nn
from random import shuffle
import numpy as np
import matplotlib.pyplot as plt

class ff_network(nn.Module):

    def __init__(self):
        """
        Fully Connected layers
        """
        super(ff_network, self).__init__()

        self.semnet = nn.Sequential(
            nn.Linear(17, 50), # 17 properties
            nn.ReLU(),
            nn.Linear(50, 10), # very small network for tests
            nn.ReLU(),
            nn.Linear(10, 1)
        )

    def forward(self, x):
        """
        Pass throught network
        """
        res = self.semnet(x)

        return res

def train_model(data_train0, data_test0, data_train1, data_test1, lr_enc, batch_size):

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    model_semnet = ff_network().to(device)
    optimizer_predictor = torch.optim.Adam(model_semnet.parameters(), lr=lr_enc)

    data_train0=torch.tensor(data_train0, dtype=torch.float).to(device)
    data_test0=torch.tensor(data_test0, dtype=torch.float).to(device)

    data_train1=torch.tensor(data_train1, dtype=torch.float).to(device)
    data_test1=torch.tensor(data_test1, dtype=torch.float).to(device)

    test_loss_total=[]
    for epoch in range(500): # should be much larger, with good early stopping criteria
        small_train_len=min(len(data_train0),len(data_train1))
        for batch_iteration in range(int(small_train_len/batch_size)):
            idx = torch.randint(0, len(data_train0), (batch_size,))
            data_train0_samples = data_train0[idx]

            idx = torch.randint(0, len(data_train1), (batch_size,))
            data_train1_samples = data_train1[idx]

            calc_properties0 = model_semnet(data_train0_samples)
            criterion = torch.nn.MSELoss()
            curr_pred=torch.tensor([0] * batch_size, dtype=torch.float).to(device)
            real_loss0 = criterion(calc_properties0, curr_pred)
            loss0 = torch.clamp(real_loss0, min = 0., max = 50000.).double()

            calc_properties1 = model_semnet(data_train1_samples)

            criterion = torch.nn.MSELoss()
            curr_pred=torch.tensor([1] * batch_size, dtype=torch.float).to(device)
            real_loss1 = criterion(calc_properties1, curr_pred)
            loss1 = torch.clamp(real_loss1, min = 0., max = 50000.).double()

            loss=loss0+loss1
            optimizer_predictor.zero_grad()
            loss.backward()
            optimizer_predictor.step()

        # calculate train set
        model_semnet.eval()

        calc_properties0 = model_semnet(data_train0[1:3000])
        criterion = torch.nn.MSELoss()

        curr_pred=torch.tensor([0] * len(data_train0), dtype=torch.float).to(device)

        real_loss0 = criterion(calc_properties0, curr_pred)

        real_loss_train_num0=real_loss0.detach().cpu().numpy()

        calc_properties1 = model_semnet(data_train1)
        criterion = torch.nn.MSELoss()
        curr_pred=torch.tensor([1] * len(data_train1), dtype=torch.float).to(device)
        real_loss1 = criterion(calc_properties1, curr_pred)
        real_loss_train_num1=real_loss1.detach().cpu().numpy()

        # calculate test set
        calc_properties0 = model_semnet(data_test0[1:3000])
        criterion = torch.nn.MSELoss()
        curr_pred=torch.tensor([0] * len(data_test0), dtype=torch.float).to(device)
        real_loss0 = criterion(calc_properties0, curr_pred)
        real_loss_test_num0=real_loss0.detach().cpu().numpy()

        calc_properties1 = model_semnet(data_test1)
        criterion = torch.nn.MSELoss()
        curr_pred=torch.tensor([1] * len(data_test1), dtype=torch.float).to(device)
        real_loss1 = criterion(calc_properties1, curr_pred)
        real_loss_test_num1=real_loss1.detach().cpu().numpy()

        test_loss_total.append(np.mean(real_loss_test_num0)+np.mean(real_loss_test_num1))

        if epoch%10==0:
            info_str='epoch: '+str(epoch)+' - totalloss: ',np.mean(real_loss_train_num0)+np.mean(real_loss_train_num1) ,'; l1/l2: '+str(np.mean(real_loss_train_num0))+'/'+str(np.mean(real_loss_train_num1))+'; total: ',np.mean(real_loss_test_num0)+np.mean(real_loss_test_num1) ,' ts1/ts2: '+str(np.mean(real_loss_test_num0))+'/'+str(np.mean(real_loss_test_num1))
            print('train_model: ',info_str)
        model_semnet.train()

        if len(test_loss_total)>30: # early stopping
            if (test_loss_total[-10]-test_loss_total[-1])<=0 and (test_loss_total[-11]-test_loss_total[-1])<=0 and (test_loss_total[-12]-test_loss_total[-1])<=0:
                break

    plt.plot(test_loss_total)
    plt.pause(0.05)
    plt.show

    return True

def train_nn_one_instance(data_0,data_1,model_semnet):
    batch_size=100
    lr_enc=0.01

    model_semnet.train()

    train_valid_test_size=[0.5, 0.5, 0.0]
    x = [i for i in range(len(data_0))]  # random shuffle input
    shuffle(x)
    data_0 = data_0[x]
    idx_traintest=int(len(data_0)*train_valid_test_size[0])
    data_train0=data_0[0:idx_traintest]
    data_test0=data_0[idx_traintest:]

    x = [i for i in range(len(data_1))]  # random shuffle input
    shuffle(x)
    data_1 = data_1[x]
    idx_traintest=int(len(data_1)*train_valid_test_size[0])
    data_train1=data_1[0:idx_traintest]
    data_test1=data_1[idx_traintest:]

    print('train_nn_one_instance - start training')
    bestval_no=train_model(data_train0, data_test0, data_train1, data_test1, lr_enc, batch_size)

    print('train_nn_one_instance - Calculate ROC')

    return bestval_no

def calculate_ROC(data_0,data_1,model_semnet):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    model_semnet.eval()
    data_0=torch.tensor(data_0, dtype=torch.float).to(device)
    data_1=torch.tensor(data_1, dtype=torch.float).to(device)
    future_data_0=model_semnet(data_0).detach().cpu().numpy()
    future_data_1=model_semnet(data_1).detach().cpu().numpy()

    print(future_data_0.shape)
    print(future_data_1.shape)

    all_pred=np.concatenate([future_data_0,future_data_1])
    corr_answers=np.array([0] * len(future_data_0) + [1] * len(future_data_1))
    idx1=np.flip(np.argsort(all_pred,axis=0)) # indices from strongest predicted connection to shortest one

    print(corr_answers.shape)
    corr_answers_sorted=corr_answers[idx1]

    xpos=[0]
    ypos=[0]
    ROC_vals=[]
    for ii in range(len(corr_answers_sorted)):
        if corr_answers_sorted[ii]==1:
            xpos.append(xpos[-1])
            ypos.append(ypos[-1]+1)
        if corr_answers_sorted[ii]==0:
            xpos.append(xpos[-1]+1)
            ypos.append(ypos[-1])
            ROC_vals.append(ypos[-1])

    ROC_vals=np.array(ROC_vals)/max(ypos)
    ypos=np.array(ypos)/max(ypos)
    xpos=np.array(xpos)/max(xpos)

    plt.plot(xpos, ypos)
    plt.show()

    AUC=sum(ROC_vals)/len(ROC_vals)
    print('Area Under Curve: ', AUC,'\n\n\n')

    return future_data_0, future_data_1

def train_nn(all_data_0, all_data_1, prediction_distance, start_year):
    for y in range(len(all_data_0[1:3])):
        print('train_nn - year ',start_year+y+2)

        model_semnet = ff_network()
        train_nn_one_instance(all_data_0[y],all_data_1[y],model_semnet)
        print('train_nn - finished - year ',start_year+y+2)
        print('Calculate ROC & AUC')

        if y+prediction_distance<len(all_data_0):
            future_data_0, future_data_1 = calculate_ROC(all_data_0[y+prediction_distance],all_data_1[y+prediction_distance],model_semnet)

    return True


from bottle import route,run,response,request,static_file
from LINE_Messaging import LINE,HookExecuter,Event,Message,Command

class Events(object):
    @Event("message")
    def got_message(self,msg):
        self.cl.setReplyToken(msg["replyToken"])
        self.trace(msg,type="Message")
    @Event("follow")
    def got_follow(self,msg):
        print("FOLLOW")
        self.cl.setReplyToken(msg["replyToken"])
        self.cl.addMessage("Thanks for add me!")
        self.cl.replyMessage()
    @Event("unfollow")
    def got_unfollow(self,msg):
        print("UNFOLLOW")
        #self.cl.setReplyToken(msg["replyToken"])
        #self.cl.addMessage("Why you unfollow me!!!!!!!")
        #self.cl.replyMessage()
    @Event("join")
    def got_join(self,msg):
        print("JOIN")
        self.cl.setReplyToken(msg["replyToken"])
        self.cl.addMessage("Thanks for invite me to this group!")
        self.cl.replyMessage()
    @Event("leave")
    def got_leave(self,msg):
        print("LEAVE")
        print(msg)
    @Event("postback")
    def got_postback(self,msg):
        print("POSTBACK")
        self.cl.setReplyToken(msg["replyToken"])
        print(msg)

class Messages(object):
    @Message("text")
    def got_text(self,msg):
        print(msg)
        self.trace(msg,type="Command")
    @Message("image")
    def got_image(self,msg):
        print(msg)
        self.cl.addMessage("Kawaii!")
        self.cl.replyMessage()
    @Message("video")
    def got_video(self,msg):
        print(msg)
        self.cl.addMessage("Nice Video!")
        self.cl.replyMessage()
    @Message("audio")
    def got_audio(self,msg):
        print(msg)
        self.cl.addMessage("Nice audio!")
        self.cl.replyMessage()
    @Message("file")
    def got_file(self,msg):
        print(msg)
        self.cl.addMessage("I can't see the file!")
        self.cl.replyMessage()
    @Message("location")
    def got_location(self,msg):
        print(msg)
        self.cl.addMessage("Oh, you are at there?")
        self.cl.replyMessage()
    @Message("sticker")
    def got_sticker(self,msg):
        print(msg)
        cl.addSticker(self,1,2)
        cl.replyMessage()

class Commands(object):
    @Command(alt=["ハロー","hello"],users=["ALL"])
    def hi(self,msg):
        '''Check the bot Alive'''
        print(msg)
        self.cl.addMessage("Hi too!")
        self.cl.replyMessage()
    @Command(users=["ALL"])
    def help(self,msg):
        '''Display this help message'''
        self.cl.addMessage(self.help())
        self.cl.replyMessage()

cl = LINE(channelAccessToken="TOKEN")
tracer = HookExecuter(cl,prefix="!")
tracer.addClass(Events(),type="Events")
tracer.addClass(Messages(),type="Messages")
tracer.addClass(Commands(),type="Commands")

@route('/callback',method='POST')
def bot():
    data = request.json
    tracer.trace(data)
    return "OK"

if __name__ == "__main__":
    run(host="localhost", port=8080,reload=False)


'''

Author: @vaibhav.garg
Date: 16th July 2020

'''

import numpy as np
import sys
import argparse
import cv2
import matplotlib.pyplot as plt
import tqdm
np.set_printoptions(threshold=sys.maxsize)

def findSeam(I):

	# Getting the gradient matrix
	grad = np.abs(cv2.Laplacian(I, cv2.CV_64F))

	# Calculating cumulative grad matrix
	for i in range(1, grad.shape[0]):
		option1 = grad[i-1:i, :]
		option2 = np.append(grad[i-1, 1:], [np.inf]).reshape(1, -1)
		option3 = np.append([np.inf], grad[i-1, :-1]).reshape(1, -1)
		options = np.concatenate((option1, option2, option3), axis=0)

		grad[i, :] = grad[i, :] + np.min(options, axis=0)

	# Finding smoothest path
	seam = []
	seam.append(np.argmin(grad[-1, :]))
	for i in range(grad.shape[0]-2, -1, -1):
		if seam[-1] == 0:
			seam.append( np.argmin(grad[i, :2]) )
		elif seam[-1] == grad.shape[1]-1:
			seam.append( np.argmin(grad[i, -2:])-1+seam[-1] )
		else:
			seam.append( np.argmin(grad[i, seam[-1]-1:seam[-1]+2])-1+seam[-1] )

	seam = np.array(seam[::-1]).reshape(-1, 1)
	return seam

def removeSeam(I, seam):
	mask = np.zeros((I.shape[0], I.shape[1]), dtype=np.bool)
	mask[(np.arange(I.shape[0]), seam.flatten())] = True
	newI = I[np.where(mask==False)].reshape(I.shape[0], I.shape[1]-1)
	return newI

def removeMultipleSeams(I, seams):
	mask = np.zeros((I.shape[0], I.shape[1]), dtype=np.bool)

	# Removing from all channels at once
	ind1 = np.repeat(np.arange(I.shape[0]), seams.shape[1])
	ind2 = seams.flatten()
	mask[(ind1, ind2)] = True

	newI = I[np.where(mask==False)].reshape(I.shape[0], I.shape[1]-seams.shape[1], 3)
	I[mask] = [0, 0, 255]
	return I, newI

def addMultipleSeams(I, seams):
	newI = np.zeros((I.shape[0], I.shape[1]+seams.shape[1], 3), dtype=np.uint8)
	for i in range(I.shape[0]):
		newI[i] = np.insert(I[i], seams[i], I[i, seams[i]], axis=0)
		I[i, seams[i]] = [0, 255, 255]

	return I, newI

def runSeamCarving(I, n):

	# Convert to single channel
	try:
		gray = cv2.cvtColor(I, cv2.COLOR_BGR2GRAY)
	except:
		print("Image given is not RGB")

	# Finding and removing seams
	print("Finding optimal seams")
	seamOffset = np.zeros(gray.shape, dtype=np.int)
	for _ in tqdm.tqdm(range(n)):
		seam = findSeam(gray)
		gray = removeSeam(gray, seam)

		try:
			seams = np.concatenate((seams, seam + seamOffset[(np.arange(I.shape[0]), seam.flatten())].reshape(-1, 1) ), axis=1)
		except:
			seams = seam

		for i in range(I.shape[0]):
			seamOffset[i, seam[i, 0]+1:] = seamOffset[i, seam[i, 0]+1:] + 1
		mask = np.zeros(seamOffset.shape, dtype=np.bool)
		mask[(np.arange(I.shape[0]), seam.flatten())] = True
		seamOffset = seamOffset[np.where(mask==False)].reshape(I.shape[0], seamOffset.shape[1]-1)

	return seams

# Parsing arguments
parser = argparse.ArgumentParser()
parser.add_argument("image", help="image path")
valid_width = list(range(1, 201))
valid_width.remove(100)
parser.add_argument("-W", "--width", help="width change in percentage", type=int, default=100, choices = valid_width)
args = parser.parse_args()
print("Arguments given", args)

# Loading image
try:
	I = cv2.imread(args.image)
	print("Loaded image of shape", I.shape)
except:
	print("Error: Unable to load image")

# Core processing
pixel_change = int(abs(100-args.width)*I.shape[1]/100)
print("Change in width of", pixel_change, "pixels")

seams = runSeamCarving(I, pixel_change)

# Expansion or reduction
if args.width > 100:
	orig, carved_img = addMultipleSeams(I, seams)
else:
	orig, carved_img = removeMultipleSeams(I, seams)

# Plotting
plt.subplot(1, 2, 1)
plt.imshow(cv2.cvtColor(orig, cv2.COLOR_BGR2RGB))
plt.title("Original")
plt.subplot(1, 2, 2)
plt.imshow(cv2.cvtColor(carved_img, cv2.COLOR_BGR2RGB))
plt.title("Carved")
plt.show()

# Saving
try:
	cv2.imwrite('out.jpg', carved_img)
	print("Image Saved as out.jpg")
except:
	print("Error while saving the image")


#This class is responsible for structuring the flight data into Google sheets.
import requests
import os

class FlightData:
    def __init__(self, object_id, code, lowest_price ):
        edit_sheety_endpoint = f"https://api.sheety.co/{os.environ['SHEET_ID']}/flightDeals/prices/{object_id}"
        edit_row = {
            "price": {
                "iataCode": code,
                "lowestPrice": lowest_price
            }
        }
        header = {
            "Authorization": os.envrion["SHEETY_AUTH"]
        }
        edit_sheety_response = requests.put(url=edit_sheety_endpoint, headers=header, json= edit_row)
        edit_sheety_response.text

Day 39 & 40 - Flight Deal Finder/flight_data.py


#pinheirocfc@gmail.com
#Orientacoes em: https://youtu.be/vjXsa0I_dtc

#Essenciais
#pip install SpeechRecognition

#Extras
#pip install pipwin
#pipwin install pyaudio
#pip install pydub

#sudo apt update
#sudo apt install ffmpeg

import speech_recognition as sr
from pydub import AudioSegment
import os

directory = os.path.abspath('.') + "/audios/16327880741157603-audio.mp3"# current directory
# convert mp3 file to wav
sound = AudioSegment.from_mp3(directory)
sound.export(directory.replace(".mp3",".wav"), format="wav")

def reconhecer_fala():
    microfone = sr.Recognizer() #Habilita o microfone
    sample_audio = sr.AudioFile(directory.replace(".mp3",".wav"))

    with sample_audio as source:
        audio_text = microfone.record(source)
        try:
            print(microfone.recognize_google(audio_text,language='pt-BR'))
        except:
            print("Não entendi o que você disse!")
reconhecer_fala()


""" Standard calculator input number module.
"""

import math
import enum
from collections import deque

class NumberModel:
    """ Standard calculator input number object.
    """

    class Symbols(enum.Enum):
        ZERO = "0"
        ONE = "1"
        TWO = "2"
        THREE = "3"
        FOUR = "4"
        FIVE = "5"
        SIX = "6"
        SEVEN = "7"
        EIGHT = "8"
        NINE = "9"
        NEGATIVE = "-"
        DECIMAL_PT = "."

    def __init__(self):
        self._appended = False
        self._digits = deque([self.Symbols.ZERO.value])

    def new(self):
        """ Create a new number.
        """
        self._digits.clear()
        self._appended = False
        self._digits.append(self.Symbols.ZERO.value)

    def append(self, symbol):
        """ Append NumberModel.Symbol to number.
        """
        assert isinstance(symbol, self.Symbols)

        # replace (default) initial zero (0) value.
        # this does not apply when appending a decimal point
        if (self._is_zero()) and (symbol != self.Symbols.DECIMAL_PT):
            self._digits.pop()

        # number should only contain one decimal point
        if (self._digits.count(self.Symbols.DECIMAL_PT.value) > 0) and\
                (symbol is self.Symbols.DECIMAL_PT):
            return

        self._appended = True
        self._digits.append(symbol.value)

    def backspace(self):
        """ Delete last added digit.
        """
        if self._is_zero():
            return

        is_one_digit_pos = len(self._digits) == 1 and\
            (self._digits[0] != self.Symbols.ZERO.value)

        is_one_digit_neg = len(self._digits) == 2 and\
            (self._digits[0] == self.Symbols.NEGATIVE.value)

        if is_one_digit_pos or is_one_digit_neg:
            self.new()
            return

        self._digits.pop()

    def change_sign(self):
        """ Change sign from positive to negative, and vice versa.
        """
        if self._is_zero():
            return

        if self._digits[0] != self.Symbols.NEGATIVE.value:
            self._digits.appendleft(self.Symbols.NEGATIVE.value)
        else:
            self._digits.popleft()

    def join(self):
        """ Join digits and return as string.
        """
        return "".join(self._digits)

    def is_appended(self):
        """ Return true if a value, zero or non-zero, has been appended.
        """
        return self._appended

    def _is_zero(self):
        """ Return True if number is zero (initial value).
        """
        return (len(self._digits) == 1) and\
            (self._digits[0] == self.Symbols.ZERO.value)


#!/usr/bin/env python
import os
import os.path as osp
import shutil
import sys
import tempfile
import warnings
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
from dataclasses import dataclass
from typing import Tuple

from esgpull import Esgpull, Query

from cmip6_utils.file import download_file
from cmip6_utils.misc import BC, verify_checksum

@dataclass
class ESGFFile:
    filename: str
    url: str
    checksum: str
    local_path: str
    data_node: str

def cmip_path_to_query(path: str) -> Query:
    if not path.startswith("CMIP6"):
        raise RuntimeError("path should be described with root directory CMIP6")

    path = path.strip().split("/")

    query = Query()
    query.selection.project = path[0]
    query.selection.activity_id = path[1]
    query.selection.institution_id = path[2]
    query.selection.source_id = path[3]
    query.selection.experiment_id = path[4]
    query.selection.variant_label = path[5]
    query.selection.table_id = path[6]
    query.selection.variable_id = path[7]
    query.selection.grid_label = path[8]

    return query

def cli() -> Namespace:
    parser = ArgumentParser(
        formatter_class=ArgumentDefaultsHelpFormatter,
        description=(
            "This script can be used to download all available files in a dataset. "
            "For example for /the/full/local/path/to/dataset, the script will query all "
            "all available files and download the missing files to that location."
        ),
    )
    parser.add_argument("dataset", type=str, help="Full local path to the dataset to download")
    args = parser.parse_args(args=None if sys.argv[1:] else ["--help"])
    args.dataset = args.dataset.rstrip("/")

    if args.dataset.endswith(".nc"):
        raise ValueError("The supplied argument should be the path to the datset, not a specific file")

    return args

def split_dataset_path(path: str) -> Tuple[str, str]:
    i = path.find("CMIP6")
    return path[:i].rstrip("/"), path[i:].strip("/")

def dataset_version(path: str) -> str:
    version = path.strip().split("/")[-1]
    assert version.startswith("v")
    return version

def print_query(query: Query):
    print(BC.bold("Query selection:"))
    for facet in query.selection._facets:
        print(f"{facet.name:15s}: {facet.value}")
    print("\n")

def main():
    args = cli()
    find_download_missing_files(args.dataset)

def find_download_missing_files(dataset: str):
    # args = cli()

    path_to_cmip6_data, cmip6_dataset_structure = split_dataset_path(dataset)
    version = dataset_version(dataset)

    # print(f"{BC.bold('Dataset path:')} {args.dataset}")
    print(f"{BC.bold('Local path to CMIP6 data:')} {path_to_cmip6_data}")
    print(f"{BC.bold('CMIP6 dataset structure :')} {cmip6_dataset_structure}")
    print(f"{BC.bold('Dataset version         :')} {version}")
    print("\n")

    query = cmip_path_to_query(cmip6_dataset_structure)

    print_query(query)

    query.options.distrib = True  # default=False
    query.options.replica = True

    esg = Esgpull(path="/home/dchandan/esgpull_profiles/scratch")
    search_results = esg.context.files(query, max_hits=None)

    dataset_files = {}

    print(f"{BC.warn('# of files found from search: ')}{len(search_results)}")

    for res in search_results:
        if version == res.version:
            filename = res.filename
            data_ = (filename, res.url, res.checksum, res.local_path, res.data_node)
            if filename in dataset_files:
                dataset_files[filename].append(ESGFFile(*data_))
            else:
                dataset_files[filename] = [ESGFFile(*data_)]

    print(f"{BC.warn('# of unique files: ')}{len(dataset_files)}")

    changes_made = False
    for filename, entries in dataset_files.items():
        if not osp.exists(osp.join(dataset, filename)):
            success = False
            print(f"Found missing file: {filename}")

            tf_ = tempfile.NamedTemporaryFile()
            for item in entries:
                print(f" ---> Attempting download from: {item.data_node}")
                err = download_file(item.url, tf_.name)

                if err == 200:
                    print(f" ---> Download {BC.okgreen('successful')}")
                    vrfy = verify_checksum(tf_.name, item.checksum)
                    if vrfy:
                        print(f" ---> Checksum {BC.okgreen('PASS')}")
                        success = True
                        localpath = osp.join(path_to_cmip6_data, item.local_path)
                        print(f" ---> Copying downloaded file to {localpath}")
                        os.makedirs(localpath, exist_ok=True)
                        local_filename = osp.join(localpath, filename)
                        shutil.copy2(tf_.name, local_filename)
                        vrfy2 = verify_checksum(local_filename, item.checksum)
                        if not vrfy2:
                            print(f" ---> {BC.fail('Checksum of copied file failed')}")
                            os.remove(local_filename)
                            continue

                        changes_made = True
                        break
                    else:
                        print(f" ---> Checksum {BC.fail('FAIL')}")
                else:
                    print(f" ---> Download {BC.fail('unsuccessful')}")

            if not success:
                print(f" ---> {BC.fail('Unable to download this file from any source')}")

    if not changes_made:
        print(f"{BC.warn('No missing files were found')}")

if __name__ == "__main__":
    main()


# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-

# This file is part of Guadalinex
#
# This software is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this package; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

__author__ = "David Amian <damian@emergya.com>"
__copyright__ = "Copyright (C) 2011, Junta de Andalucía <devmaster@guadalinex.org>"
__license__ = "GPL-2"

import os
import shlex
import subprocess
from gi.repository import Gtk
from firstboot_lib import PageWindow
from firstboot import serverconf

import gettext
from gettext import gettext as _
gettext.textdomain('firstboot')

import firstboot.pages

__REQUIRED__ = False

__TITLE__ = _('Auto Configuration')

def get_page(main_window):

    page = AutoConfigPage(main_window)
    return page

class AutoConfigPage(PageWindow.PageWindow):
    __gtype_name__ = "AutoConfigPage"

    def finish_initializing(self):
        self.set_status(None)

    def load_page(self, params=None):
        self.emit('status-changed', 'autoConfig', not __REQUIRED__)

        self.ui.chkAutoconf.set_visible(False)
        url_config = self.fbe.get_url()
        url = self.cmd_options.url

        if url == None or len(url) == 0:
            url = url_config

        if url == None or len(url) == 0:
            url = ''

        self.ui.txtAutoconf.set_text(url)
        if serverconf.json_is_cached():
            self.ui.boxCheckAutoconf.set_visible(True)
            self.ui.chkAutoconf.set_visible(True)
            self.ui.txtAutoconf.set_sensitive(False)
            self.serverconf = serverconf.get_server_conf(None)

    def translate(self):
        desc = _('Parameters can be filled automatically if an autoconfiguration file is available in your GECOS server.')

        self.ui.lblDescription.set_text(desc)
        self.ui.chkAutoconf.set_label(_('Download a new file with default configuration parameters.'))
        self.ui.lblAutoconf.set_label(_('Autoconfig file URL'))

    def on_chkAutoconf_toggled(self, widget):
        self.ui.txtAutoconf.set_sensitive(self.ui.chkAutoconf.get_active())

    def set_status(self, code, description=''):

        self.ui.imgStatus.set_visible(code != None)
        self.ui.lblStatus.set_visible(code != None)

        if code == None:
            return

        if code == 0:
            icon = Gtk.STOCK_YES

        else:
            icon = Gtk.STOCK_DIALOG_ERROR

        self.ui.imgStatus.set_from_stock(icon, Gtk.IconSize.MENU)
        self.ui.lblStatus.set_label(description)

    def previous_page(self, load_page_callback):

        load_page_callback(firstboot.pages.network)

    def next_page(self, load_page_callback):
        if not serverconf.json_is_cached() or (serverconf.json_is_cached() and self.ui.chkAutoconf.get_active()):
           url = self.ui.txtAutoconf.get_text()
           if url != '' and url != None:
               try:
                   if serverconf.json_is_cached():
                       serverconf.clean_json_cached()
                   self.serverconf = serverconf.get_server_conf(url)

               except Exception as e:
                    self.set_status(1, str(e))
                    return

        load_page_callback(firstboot.pages.dateSync)