from __future__ import absolute_import, print_function, division

import sys
import subprocess
import itertools
import ConfigParser
import fnmatch
from PyQt5.QtCore import QObject, pyqtSignal
from idc import *

from .index import Index

class Project(QObject):
    """Represents a continuum project and it's properties."""
    META_DIR_NAME = '.continuum'
    CFG_FILE_NAME = 'project.conf'

    def __init__(self):
        super(Project, self).__init__()
        self.conf = None
        self.index = None
        self.proj_dir = None
        self.meta_dir = None
        self.ignore_changes = False
        self.files = []

    def open(self, root, skip_analysis=False):
        """Opens an existing project by its root path."""
        if self.proj_dir:
            raise Exception("A project is already opened")

        # Find meta directory and read config.
        meta_dir = os.path.join(root, self.META_DIR_NAME)
        conf = ConfigParser.SafeConfigParser()
        if not conf.read(os.path.join(meta_dir, self.CFG_FILE_NAME)):
            raise Exception("Project is lacking its config file")

        # Determine project files.
        file_patterns = conf.get('project', 'file_patterns')
        if file_patterns is None:
            raise Exception("Project configuration lacks `file_patterns` directive")
        files = list(self.find_project_files(root, file_patterns))

        # Everything fine, put info into `self`.
        self.conf = conf
        self.proj_dir = root
        self.meta_dir = meta_dir
        self.files = files
        self.index = Index(self)

        if not skip_analysis:
            # Is index for *this* IDB built? If not, do so.
            if not self.index.is_idb_indexed(GetIdbPath()):
                self.index.index_types_for_this_idb()
                self.index.index_symbols_for_this_idb()

            # Analyze other files, if required.
            self._analyze_project_files()

    def _analyze_project_files(self):
        """Launches background analysis instances for binaries that aren't indexed yet."""
        plugin_root = os.path.dirname(os.path.realpath(__file__))
        procs = []

        for cur_file in self.files:
            # IDB already indexed? Skip.
            if self.index.is_idb_indexed(self.file_to_idb(cur_file)):
                continue

            procs.append(subprocess.Popen([
                sys.executable,
                '-A',
                '-S"{}"'.format(os.path.join(plugin_root, 'analyze.py')),
                '-L{}.log'.format(cur_file),
                cur_file,
            ]))

        return procs

    @staticmethod
    def file_to_idb(file):
        """Obtains the IDB path for a binary."""
        return os.path.splitext(file)[0] + '.idb'

    @classmethod
    def find_project_dir(cls, start_path):
        """
        Traverses up the directory tree, searching for a project root.
        If one is found, returns the path, else `None`.
        """
        tail = object()
        head = start_path
        while tail:
            head, tail = os.path.split(head)
            cur_meta_path = os.path.join(head, tail, cls.META_DIR_NAME)
            if os.path.exists(cur_meta_path):
                return os.path.join(head, tail)

    @staticmethod
    def find_project_files(root, file_patterns):
        """Locates all binaries that are part of a project."""
        file_patterns = [x.strip() for x in file_patterns.split(';')]
        for dirpath, _, filenames in os.walk(root):
            relevant_files = itertools.chain.from_iterable(
                fnmatch.filter(filenames, x) for x in file_patterns
            )

            # Py2 Y U NO SUPPORT "yield from"? :(
            for cur_file in relevant_files:
                yield os.path.join(dirpath, cur_file)

    @classmethod
    def create(cls, root, file_patterns):
        """Creates a new project."""
        # Create meta directory.
        cont_dir = os.path.join(root, cls.META_DIR_NAME)
        if os.path.exists(cont_dir):
            raise Exception("Directory is already a continuum project")
        os.mkdir(cont_dir)

        # Create config file.
        config = ConfigParser.SafeConfigParser()
        config.add_section('project')
        config.set('project', 'file_patterns', file_patterns)
        with open(os.path.join(cont_dir, cls.CFG_FILE_NAME), 'w') as f:
            config.write(f)

        # Create initial index.
        project = Project()
        project.open(root)

        return project


import re
import unittest

from trac.tests.functional import FunctionalTestCaseSetup, \
                                  internal_error, tc

#TODO: split this into multiple smaller testcases
class TestPreferences(FunctionalTestCaseSetup):
    def runTest(self):
        """Set preferences for admin user"""
        self._tester.go_to_preferences()
        try:
            tc.notfind('Your preferences have been saved.')
            tc.formvalue('userprefs', 'name', ' System Administrator ')
            tc.formvalue('userprefs', 'email', ' admin@example.com ')
            tc.submit()
            tc.find('Your preferences have been saved.')
            self._tester.go_to_preferences("Localization")
            tc.formvalue('userprefs', 'tz', 'GMT -10:00')
            tc.submit()
            tc.find('Your preferences have been saved.')
            self._tester.go_to_preferences()
            tc.notfind('Your preferences have been saved.')
            tc.find('value="System Administrator"')
            tc.find(r'value="admin@example\.com"')
            self._tester.go_to_preferences("Localization")
            tc.find('GMT -10:00')
        finally:
            self._tester.go_to_preferences()
            tc.formvalue('userprefs', 'name', '')
            tc.formvalue('userprefs', 'email', '')
            tc.submit()
            tc.find('Your preferences have been saved.')

class TestDefaultHandler(FunctionalTestCaseSetup):
    def runTest(self):
        """Set default handler."""

        # Project default_handler is selected.
        self._tester.go_to_preferences("User Interface")
        tc.notfind(r'<option[^>]+selected="selected"')
        tc.find("Default: WikiModule")

        try:
            # Project default handler still selected after value is changed.
            hint = r"SearchModule is not a valid IRequestHandler or is not" \
                   r" enabled\."
            self._testenv.set_config('trac', 'default_handler',
                                     'SearchModule')
            self._tester.go_to_preferences("User Interface")
            tc.notfind('<option[^>]+selected="selected"')
            tc.find("Default: SearchModule")
            tc.notfind(hint)

            # Project default handler still selected after module is disabled.
            component = 'trac.search.web_ui.*'
            self._testenv.set_config('components', component, 'disabled')
            self._tester.go_to_preferences("User Interface")
            try:
                tc.notfind('<option[^>]+selected="selected"')
                tc.find(r"Default: SearchModule")
                tc.find(hint)
            finally:
                self._testenv.remove_config('components', component)
        finally:
            self._testenv.set_config('trac', 'default_handler', 'WikiModule')

        # Set session default handler and navigate to base URL.
        tc.formvalue('userprefs', 'default_handler', 'TimelineModule')
        tc.submit()
        tc.find(r"Your preferences have been saved\.")
        tc.find('<option[^>]+selected="selected"[^>]+TimelineModule')
        self._tester.go_to_front()
        tc.find("
Timeline
")

        # Clear session default handler.
        self._tester.go_to_preferences("User Interface")
        tc.formvalue('userprefs', 'default_handler', '')
        tc.submit()
        tc.find(r"Your preferences have been saved\.")
        tc.notfind(r'<option[^>]+selected="selected"')
        tc.find("Default: WikiModule")

class RegressionTestRev5785(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of the fix in r5785"""
        self._tester.go_to_preferences()
        tc.submit('logout', 'logout')
        tc.notfind(internal_error)  # See [5785]
        tc.follow('Login')

class RegressionTestTicket5765(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/5765
        Unable to turn off 'Enable access keys' in Preferences
        """
        self._tester.go_to_preferences("User Interface")
        tc.formvalue('userprefs', 'accesskeys', True)
        tc.submit(formname='userprefs')
        tc.find('name="accesskeys".*checked="checked"')
        tc.formvalue('userprefs', 'accesskeys', False)
        tc.submit(formname='userprefs')
        tc.notfind('name="accesskeys".*checked="checked"')

class RegressionTestTicket11319(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11319
        Only alphanumeric characters can be used for session key in advanced
        panel.
        """
        try:
            self._tester.logout()
            self._tester.go_to_preferences('Advanced')
            tc.formvalue('userprefs', 'newsid', 'śeśśion_id')
            tc.submit('change')
            tc.notfind(internal_error)
            tc.find('Session ID must be alphanumeric')
            self._tester.go_to_preferences('Advanced')
            tc.formvalue('userprefs', 'loadsid', 'śeśśion_id')
            tc.submit('restore')
            tc.notfind(internal_error)
            tc.find('Session ID must be alphanumeric')
        finally:
            self._tester.login('admin')

class RegressionTestTicket11337(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11337
        The language select will be disabled if Babel is not installed and a
        hint will be shown. The text of the hint is dependent on whether the
        user has TRAC_ADMIN and the message catalogs have been compiled.
        """
        from trac.util.translation import has_babel, get_available_locales

        babel_hint = "Install Babel for extended language support."
        catalog_hint = "Message catalogs have not been compiled."
        nonadmin_hint = r'\s+'.join("Please contact your Trac administrator "
                                    "to enable existing translations".split())
        language_select = '<select id="language" name="language">'
        disabled_language_select = \
            '<select id="language" name="language" disabled="disabled" ' \
            'title="Translations are currently unavailable">'

        self._tester.go_to_preferences("Localization")
        if has_babel:
            tc.notfind(babel_hint)
            if get_available_locales():
                tc.find(language_select)
                tc.notfind(babel_hint)
                tc.notfind(catalog_hint)
            else:
                tc.find(disabled_language_select)
                tc.find(catalog_hint)
                tc.notfind(babel_hint)
        else:
            tc.find(disabled_language_select)
            tc.find(babel_hint)
            tc.notfind(catalog_hint)
        tc.notfind(nonadmin_hint)

        # For users without TRAC_ADMIN, the Language tab should only be
        # present when Babel is installed
        self._tester.logout()
        self._tester.go_to_preferences("Localization")
        try:
            if has_babel and get_available_locales():
                tc.find(language_select)
                tc.notfind(nonadmin_hint)
            elif has_babel:
                tc.find(disabled_language_select)
                tc.find(nonadmin_hint)
            else:
                tc.find(disabled_language_select)
                tc.find(nonadmin_hint)
            tc.notfind(catalog_hint)
            tc.notfind(babel_hint)
        finally:
            self._tester.login('admin')

class RegressionTestTicket11515(FunctionalTestCaseSetup):
    def runTest(self):
        """Test for regression of https://trac.edgewall.org/ticket/11515
        Show a notice message with new language setting after it is changed.
        """
        from trac.util.translation import has_babel, get_available_locales
        from pkg_resources import resource_exists, resource_filename

        if not has_babel:
            print("SKIP: RegressionTestTicket11515 (Babel not installed)")
            return
        if not resource_exists('trac', 'locale'):
            return
        locale_dir = resource_filename('trac', 'locale')
        from babel.support import Translations
        string = 'Your preferences have been saved.'
        translated = None
        for second_locale_id in get_available_locales():
            tx = Translations.load(locale_dir, second_locale_id)
            translated = tx.dgettext('messages', string)
            if string != translated:
                break  # the locale has a translation
        else:
            print("SKIP: RegressionTestTicket11515 "
                  "(Message catalogs not compiled)")
            return

        try:
            self._tester.go_to_preferences('Localization')
            tc.formvalue('userprefs', 'language', second_locale_id)
            tc.submit()
            tc.find(re.escape(translated))
            tc.find('<option selected="selected" value="%s">'
                    % second_locale_id)
        finally:
            tc.formvalue('userprefs', 'language', '')  # revert to default
            tc.submit()
            tc.find('Your preferences have been saved')

class RegressionTestTicket11531(FunctionalTestCaseSetup):
    """Test for regression of https://trac.edgewall.org/ticket/11531
    PreferencesModule can be set as the default_handler."""
    def runTest(self):
        default_handler = self._testenv.get_config('trac', 'default_handler')
        self._testenv.set_config('trac', 'default_handler',
                                 'PreferencesModule')
        try:
            tc.go(self._tester.url)
            tc.notfind(internal_error)
            tc.find(r"\bPreferences\b")
        finally:
            self._testenv.set_config('trac', 'default_handler',
                                     default_handler)

def functionalSuite(suite=None):
    if not suite:
        import trac.tests.functional
        suite = trac.tests.functional.functionalSuite()
    suite.addTest(TestPreferences())
    suite.addTest(TestDefaultHandler())
    suite.addTest(RegressionTestRev5785())
    suite.addTest(RegressionTestTicket5765())
    suite.addTest(RegressionTestTicket11319())
    suite.addTest(RegressionTestTicket11337())
    suite.addTest(RegressionTestTicket11515())
    suite.addTest(RegressionTestTicket11531())
    return suite

test_suite = functionalSuite

if __name__ == '__main__':
    unittest.main(defaultTest='test_suite')

{'has_babel': 'trac.util.translation.has_babel', 'get_available_locales': 'trac.util.translation.get_available_locales'}

{'has_babel': 'trac.util.translation.has_babel', 'get_available_locales': 'trac.util.translation.get_available_locales', 'resource_exists': 'pkg_resources.resource_exists', 'resource_filename': 'pkg_resources.resource_filename', 'Translations': 'babel.support.Translations'}


# Pytorch Tabular
# Author: Manu Joseph <manujoseph@gmail.com>
# For license information, see LICENSE.TXT
"""Tabular Data Module."""
import re
from pathlib import Path
from typing import Iterable, List, Optional, Tuple, Union

import category_encoders as ce
import joblib
import numpy as np
import pandas as pd
import pytorch_lightning as pl
import torch
from omegaconf import DictConfig
from pandas.tseries import offsets
from pandas.tseries.frequencies import to_offset
from sklearn.base import copy, TransformerMixin
from sklearn.preprocessing import (
    FunctionTransformer,
    LabelEncoder,
    PowerTransformer,
    QuantileTransformer,
    StandardScaler,
)
from torch.utils.data import DataLoader, Dataset

from pytorch_tabular.config import InferredConfig
from pytorch_tabular.utils import get_logger

from .categorical_encoders import OrdinalEncoder

logger = get_logger(__name__)

class TabularDatamodule(pl.LightningDataModule):
    CONTINUOUS_TRANSFORMS = {
        "quantile_uniform": {
            "callable": QuantileTransformer,
            "params": {"output_distribution": "uniform", "random_state": None},
        },
        "quantile_normal": {
            "callable": QuantileTransformer,
            "params": {"output_distribution": "normal", "random_state": None},
        },
        "box-cox": {
            "callable": PowerTransformer,
            "params": {"method": "box-cox", "standardize": False},
        },
        "yeo-johnson": {
            "callable": PowerTransformer,
            "params": {"method": "yeo-johnson", "standardize": False},
        },
    }

    def __init__(
        self,
        train: pd.DataFrame,
        config: DictConfig,
        validation: pd.DataFrame = None,
        test: pd.DataFrame = None,
        target_transform: Optional[Union[TransformerMixin, Tuple]] = None,
        train_sampler: Optional[torch.utils.data.Sampler] = None,
        seed: Optional[int] = 42,
    ):
        """The Pytorch Lightning Datamodule for Tabular Data.

        Args:
            train (pd.DataFrame): The Training Dataframe

            config (DictConfig): Merged configuration object from ModelConfig, DataConfig,
                TrainerConfig, OptimizerConfig & ExperimentConfig

            validation (pd.DataFrame, optional): Validation Dataframe.
                If left empty, we use the validation split from DataConfig to split a random sample as validation.
                Defaults to None.

            test (pd.DataFrame, optional): Holdout DataFrame to check final performance on.
                Defaults to None.

            target_transform (Optional[Union[TransformerMixin, Tuple(Callable)]], optional):
                If provided, applies the transform to the target before modelling and inverse the transform during
                prediction. The parameter can either be a sklearn Transformer which has an inverse_transform method, or
                a tuple of callables (transform_func, inverse_transform_func)
        """
        super().__init__()
        self.train = train.copy()
        self.validation = validation
        self._set_target_transform(target_transform)
        self.test = test if test is None else test.copy()
        self.target = config.target
        self.batch_size = config.batch_size
        self.train_sampler = train_sampler
        self.config = config
        self.seed = seed
        self._fitted = False

    def _set_target_transform(self, target_transform: Union[TransformerMixin, Tuple]) -> None:
        if target_transform is not None:
            if isinstance(target_transform, Iterable):
                target_transform = FunctionTransformer(func=target_transform[0], inverse_func=target_transform[1])
            self.do_target_transform = True
        else:
            self.do_target_transform = False
        self.target_transform_template = target_transform

    def update_config(self, config) -> InferredConfig:
        """Calculates and updates a few key information to the config object.

        Args:
            config (DictConfig): The config object

        Returns:
            InferredConfig: The updated config object
        """
        categorical_dim = len(config.categorical_cols)
        continuous_dim = len(config.continuous_cols)
        if config.task == "regression":
            output_dim = len(config.target)
        elif config.task == "classification":
            output_dim = len(self.train[config.target[0]].unique())
        else:
            output_dim = None
        categorical_cardinality = None
        embedding_dims = None
        if not self.do_leave_one_out_encoder():
            categorical_cardinality = [
                int(self.train[col].fillna("NA").nunique()) + 1 for col in config.categorical_cols
            ]
            embedding_dims = [(x, min(50, (x + 1) // 2)) for x in categorical_cardinality]
            if hasattr(config, "embedding_dims"):
                if config.embedding_dims is not None:
                    embedding_dims = config.embedding_dims
        return InferredConfig(
            categorical_dim=categorical_dim,
            continuous_dim=continuous_dim,
            output_dim=output_dim,
            categorical_cardinality=categorical_cardinality,
            embedding_dims=embedding_dims,
        )

    def do_leave_one_out_encoder(self) -> bool:
        """Checks the special condition for NODE where we use a LeaveOneOutEncoder to encode categorical columns
        DEPRECATED: Automatically encoding categorical columns using LeaveOneOutEncoder is deprecated.

        Returns:
            bool: True if LeaveOneOutEncoder is used
        """
        if hasattr(self.config, "_model_name"):
            return (self.config._model_name == "NODEModel") and (not self.config.embed_categorical)
        else:
            return False

    def _encode_date_columns(self, data: pd.DataFrame) -> pd.DataFrame:
        added_features = []
        for field_name, freq in self.config.date_columns:
            data = self.make_date(data, field_name)
            data, _new_feats = self.add_datepart(data, field_name, frequency=freq, prefix=None, drop=True)
            added_features += _new_feats
        return data, added_features

    def _encode_categorical_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            if self.do_leave_one_out_encoder():
                logger.debug("Encoding Categorical Columns using LeavOneOutEncoder")
                self.categorical_encoder = ce.LeaveOneOutEncoder(
                    cols=self.config.categorical_cols, random_state=self.seed
                )
                # Multi-Target Regression uses the first target to encode the categorical columns
                if len(self.config.target) > 1:
                    logger.warning(
                        f"Multi-Target Regression: using the first target({self.config.target[0]})"
                        f" to encode the categorical columns"
                    )
                data = self.categorical_encoder.fit_transform(data, data[self.config.target[0]])
            else:
                logger.debug("Encoding Categorical Columns using OrdinalEncoder")
                self.categorical_encoder = OrdinalEncoder(
                    cols=self.config.categorical_cols,
                    handle_unseen="impute" if self.config.handle_unknown_categories else "error",
                    handle_missing="impute" if self.config.handle_missing_values else "error",
                )
                data = self.categorical_encoder.fit_transform(data)
        else:
            data = self.categorical_encoder.transform(data)
        return data

    def _transform_continuous_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            transform = self.CONTINUOUS_TRANSFORMS[self.config.continuous_feature_transform]
            if "random_state" in transform["params"] and self.seed is not None:
                transform["params"]["random_state"] = self.seed
            self.continuous_transform = transform["callable"](**transform["params"])
            # TODO implement quantile noise
            data.loc[:, self.config.continuous_cols] = self.continuous_transform.fit_transform(
                data.loc[:, self.config.continuous_cols]
            )
        else:
            data.loc[:, self.config.continuous_cols] = self.continuous_transform.transform(
                data.loc[:, self.config.continuous_cols]
            )
        return data

    def _normalize_continuous_columns(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if stage == "fit":
            self.scaler = StandardScaler()
            data.loc[:, self.config.continuous_cols] = self.scaler.fit_transform(
                data.loc[:, self.config.continuous_cols]
            )
        else:
            data.loc[:, self.config.continuous_cols] = self.scaler.transform(data.loc[:, self.config.continuous_cols])
        return data

    def _label_encode_target(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if self.config.task == "classification":
            if stage == "fit":
                self.label_encoder = LabelEncoder()
                data[self.config.target[0]] = self.label_encoder.fit_transform(data[self.config.target[0]])
            else:
                if self.config.target[0] in data.columns:
                    data[self.config.target[0]] = self.label_encoder.transform(data[self.config.target[0]])
        return data

    def _target_transform(self, data: pd.DataFrame, stage: str) -> pd.DataFrame:
        if self.config.task == "regression":
            # target transform only for regression
            if all(col in data.columns for col in self.config.target):
                if self.do_target_transform:
                    if stage == "fit":
                        target_transforms = []
                        for col in self.config.target:
                            _target_transform = copy.deepcopy(self.target_transform_template)
                            data[col] = _target_transform.fit_transform(data[col].values.reshape(-1, 1))
                            target_transforms.append(_target_transform)
                        self.target_transforms = target_transforms
                    else:
                        for col, _target_transform in zip(self.config.target, self.target_transforms):
                            data[col] = _target_transform.transform(data[col].values.reshape(-1, 1))
        return data

    def preprocess_data(self, data: pd.DataFrame, stage: str = "inference") -> Tuple[pd.DataFrame, list]:
        """The preprocessing, like Categorical Encoding, Normalization, etc. which any dataframe should undergo before
        feeding into the dataloder.

        Args:
            data (pd.DataFrame): A dataframe with the features and target
            stage (str, optional): Internal parameter. Used to distinguisj between fit and inference.
                Defaults to "inference".

        Returns:
            Returns the processed dataframe and the added features(list) as a tuple
        """
        added_features = None
        if self.config.encode_date_columns:
            data, added_features = self._encode_date_columns(data)
        # The only features that are added are the date features extracted
        # from the date which are categorical in nature
        if (added_features is not None) and (stage == "fit"):
            logger.debug(f"Added {added_features} features after encoding the date_columns")
            self.config.categorical_cols += added_features
            # Update the categorical dimension in config
            self.config.categorical_dim = (
                len(self.config.categorical_cols) if self.config.categorical_cols is not None else 0
            )
        # Encoding Categorical Columns
        if len(self.config.categorical_cols) > 0:
            data = self._encode_categorical_columns(data, stage)

        # Transforming Continuous Columns
        if (self.config.continuous_feature_transform is not None) and (len(self.config.continuous_cols) > 0):
            data = self._transform_continuous_columns(data, stage)
        # Normalizing Continuous Columns
        if (self.config.normalize_continuous_features) and (len(self.config.continuous_cols) > 0):
            data = self._normalize_continuous_columns(data, stage)
        # Converting target labels to a 0 indexed label
        data = self._label_encode_target(data, stage)
        # Target Transforms
        data = self._target_transform(data, stage)
        return data, added_features

    def setup(self, stage: Optional[str] = None) -> None:
        """Data Operations you want to perform on all GPUs, like train-test split, transformations, etc. This is called
        before accessing the dataloaders.

        Args:
            stage (Optional[str], optional):
                Internal parameter to distinguish between fit and inference. Defaults to None.
        """
        if stage == "fit" or stage is None:
            logger.info(f"Setting up the datamodule for {self.config.task} task")
            if self.validation is None:
                logger.debug(
                    f"No validation data provided."
                    f" Using {self.config.validation_split*100}% of train data as validation"
                )
                val_idx = self.train.sample(
                    int(self.config.validation_split * len(self.train)),
                    random_state=self.seed,
                ).index
                self.validation = self.train[self.train.index.isin(val_idx)]
                self.train = self.train[~self.train.index.isin(val_idx)]
            else:
                self.validation = self.validation.copy()
            # Preprocessing Train, Validation
            self.train, _ = self.preprocess_data(self.train, stage="fit")
            self.validation, _ = self.preprocess_data(self.validation, stage="inference")
            if self.test is not None:
                self.test, _ = self.preprocess_data(self.test, stage="inference")
            self._fitted = True

    # adapted from gluonts
    @classmethod
    def time_features_from_frequency_str(cls, freq_str: str) -> List[str]:
        """Returns a list of time features that will be appropriate for the given frequency string.

        Args:
            freq_str (str): Frequency string of the form `[multiple][granularity]` such as "12H", "5min", "1D" etc.

        Returns:
            List of added features
        """

        features_by_offsets = {
            offsets.YearBegin: [],
            offsets.YearEnd: [],
            offsets.MonthBegin: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
            ],
            offsets.MonthEnd: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
            ],
            offsets.Week: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week",
            ],
            offsets.Day: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
            ],
            offsets.BusinessDay: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
            ],
            offsets.Hour: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
                "Hour",
            ],
            offsets.Minute: [
                "Month",
                "Quarter",
                "Is_quarter_end",
                "Is_quarter_start",
                "Is_year_end",
                "Is_year_start",
                "Is_month_start",
                "Week" "Day",
                "Dayofweek",
                "Dayofyear",
                "Hour",
                "Minute",
            ],
        }

        offset = to_offset(freq_str)

        for offset_type, feature in features_by_offsets.items():
            if isinstance(offset, offset_type):
                return feature

        supported_freq_msg = f"""
        Unsupported frequency {freq_str}

        The following frequencies are supported:

            Y, YS   - yearly
                alias: A
            M, MS   - monthly
            W   - weekly
            D   - daily
            B   - business days
            H   - hourly
            T   - minutely
                alias: min
        """
        raise RuntimeError(supported_freq_msg)

    # adapted from fastai
    @classmethod
    def make_date(cls, df: pd.DataFrame, date_field: str) -> pd.DataFrame:
        """Make sure `df[date_field]` is of the right date type.

        Args:
            df (pd.DataFrame): Dataframe

            date_field (str): Date field name

        Returns:
            Dataframe with date field converted to datetime
        """
        field_dtype = df[date_field].dtype
        if isinstance(field_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
            field_dtype = np.datetime64
        if not np.issubdtype(field_dtype, np.datetime64):
            df[date_field] = pd.to_datetime(df[date_field], infer_datetime_format=True)
        return df

    # adapted from fastai
    @classmethod
    def add_datepart(
        cls,
        df: pd.DataFrame,
        field_name: str,
        frequency: str,
        prefix: str = None,
        drop: bool = True,
    ) -> Tuple[pd.DataFrame, List[str]]:
        """Helper function that adds columns relevant to a date in the column `field_name` of `df`.

        Args:
            df (pd.DataFrame): Dataframe

            field_name (str): Date field name

            frequency (str): Frequency string of the form `[multiple][granularity]` such as "12H", "5min", "1D" etc.

            prefix (str, optional): Prefix to add to the new columns. Defaults to None.

            drop (bool, optional): Drop the original column. Defaults to True.

        Returns:
            Dataframe with added columns and list of added columns
        """
        field = df[field_name]
        prefix = (re.sub("[Dd]ate$", "", field_name) if prefix is None else prefix) + "_"
        attr = cls.time_features_from_frequency_str(frequency)
        added_features = []
        for n in attr:
            if n == "Week":
                continue
            df[prefix + n] = getattr(field.dt, n.lower())
            added_features.append(prefix + n)
        # Pandas removed `dt.week` in v1.1.10
        if "Week" in attr:
            week = field.dt.isocalendar().week if hasattr(field.dt, "isocalendar") else field.dt.week
            df.insert(3, prefix + "Week", week)
            added_features.append(prefix + "Week")
        # TODO Not adding Elapsed by default. Need to route it through config
        # mask = ~field.isna()
        # df[prefix + "Elapsed"] = np.where(
        #     mask, field.values.astype(np.int64) // 10 ** 9, None
        # )
        # added_features.append(prefix + "Elapsed")
        if drop:
            df.drop(field_name, axis=1, inplace=True)

        # Removing features woth zero variations
        # for col in added_features:
        #     if len(df[col].unique()) == 1:
        #         df.drop(columns=col, inplace=True)
        #         added_features.remove(col)
        return df, added_features

    def train_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the train set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Train dataloader
        """
        dataset = TabularDataset(
            task=self.config.task,
            data=self.train,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=True if self.train_sampler is None else False,
            num_workers=self.config.num_workers,
            sampler=self.train_sampler,
            pin_memory=self.config.pin_memory,
        )

    def val_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the validation set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Validation dataloader
        """
        dataset = TabularDataset(
            task=self.config.task,
            data=self.validation,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=False,
            num_workers=self.config.num_workers,
            pin_memory=self.config.pin_memory,
        )

    def test_dataloader(self, batch_size: Optional[int] = None) -> DataLoader:
        """Function that loads the validation set.

        Args:
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: Test dataloader
        """
        if self.test is not None:
            dataset = TabularDataset(
                task=self.config.task,
                data=self.test,
                categorical_cols=self.config.categorical_cols,
                continuous_cols=self.config.continuous_cols,
                embed_categorical=(not self.do_leave_one_out_encoder()),
                target=self.target,
            )
            return DataLoader(
                dataset,
                batch_size if batch_size is not None else self.batch_size,
                shuffle=False,
                num_workers=self.config.num_workers,
                pin_memory=self.config.pin_memory,
            )

    def _prepare_inference_data(self, df: pd.DataFrame) -> pd.DataFrame:
        """Prepare data for inference."""
        # TODO Is the target encoding necessary?
        if len(set(self.target) - set(df.columns)) > 0:
            if self.config.task == "classification":
                df.loc[:, self.target] = np.array([self.label_encoder.classes_[0]] * len(df)).reshape(-1, 1)
            else:
                df.loc[:, self.target] = np.zeros((len(df), len(self.target)))
        df, _ = self.preprocess_data(df, stage="inference")
        return df

    def prepare_inference_dataloader(self, df: pd.DataFrame, batch_size: Optional[int] = None) -> DataLoader:
        """Function that prepares and loads the new data.

        Args:
            df (pd.DataFrame): Dataframe with the features and target
            batch_size (Optional[int], optional): Batch size. Defaults to `self.batch_size`.

        Returns:
            DataLoader: The dataloader for the passed in dataframe
        """
        df = df.copy()
        df = self._prepare_inference_data(df)
        dataset = TabularDataset(
            task=self.config.task,
            data=df,
            categorical_cols=self.config.categorical_cols,
            continuous_cols=self.config.continuous_cols,
            embed_categorical=(not self.do_leave_one_out_encoder()),
            target=self.target if all(col in df.columns for col in self.target) else None,
        )
        return DataLoader(
            dataset,
            batch_size if batch_size is not None else self.batch_size,
            shuffle=False,
            num_workers=self.config.num_workers,
        )

    def save_dataloader(self, path: Union[str, Path]) -> None:
        """Saves the dataloader to a path.

        Args:
            path (Union[str, Path]): Path to save the dataloader
        """
        if isinstance(path, str):
            path = Path(path)
        joblib.dump(self, path)

    @classmethod
    def load_datamodule(cls, path: Union[str, Path]):
        """Loads a datamodule from a path.

        Args:
            path (Union[str, Path]): Path to the datamodule

        Returns:
            TabularDatamodule (TabularDatamodule): The datamodule loaded from the path
        """
        if isinstance(path, str):
            path = Path(path)
        if not path.exists():
            raise FileNotFoundError(f"{path} does not exist.")
        datamodule = joblib.load(path)
        return datamodule

class TabularDataset(Dataset):
    def __init__(
        self,
        data: pd.DataFrame,
        task: str,
        continuous_cols: List[str] = None,
        categorical_cols: List[str] = None,
        embed_categorical: bool = True,
        target: List[str] = None,
    ):
        """Dataset to Load Tabular Data.

        Args:
            data (pd.DataFrame): Pandas DataFrame to load during training
            task (str):
                Whether it is a classification or regression task. If classification, it returns a LongTensor as target
            continuous_cols (List[str], optional): A list of names of continuous columns. Defaults to None.
            categorical_cols (List[str], optional): A list of names of categorical columns.
                These columns must be ordinal encoded beforehand. Defaults to None.
            embed_categorical (bool):
                Flag to tell the dataset whether to convert categorical columns to LongTensor or retain as float.
                If we are going to embed categorical cols with an embedding layer,
                we need to convert the columns to LongTensor
            target (List[str], optional): A list of strings with target column name(s). Defaults to None.
        """

        self.task = task
        self.n = data.shape[0]

        if target:
            self.y = data[target].astype(np.float32).values
            if isinstance(target, str):
                self.y = self.y.reshape(-1, 1)  # .astype(np.int64)
        else:
            self.y = np.zeros((self.n, 1))  # .astype(np.int64)

        if task == "classification":
            self.y = self.y.astype(np.int64)
        self.categorical_cols = categorical_cols if categorical_cols else []
        self.continuous_cols = continuous_cols if continuous_cols else []

        if self.continuous_cols:
            self.continuous_X = data[self.continuous_cols].astype(np.float32).values

        if self.categorical_cols:
            self.categorical_X = data[categorical_cols]
            if embed_categorical:
                self.categorical_X = self.categorical_X.astype(np.int64).values
            else:
                self.categorical_X = self.categorical_X.astype(np.float32).values

    def __len__(self):
        """Denotes the total number of samples."""
        return self.n

    def __getitem__(self, idx):
        """Generates one sample of data."""
        return {
            "target": self.y[idx],
            "continuous": self.continuous_X[idx] if self.continuous_cols else torch.Tensor(),
            "categorical": self.categorical_X[idx] if self.categorical_cols else torch.Tensor(),
        }


from django.views.generic import ListView, CreateView, UpdateView, DeleteView
from django.urls import reverse_lazy
from django.contrib.auth.mixins import LoginRequiredMixin
from helpers.mixins import PermissionMixin
from setting.models import OrganizationalStructure, ITEM_CHOICES
from django.shortcuts import redirect
from helpers.functions import get_organizational_structure

class OrganizationalStructureList(LoginRequiredMixin, PermissionMixin, ListView):
    permission_required = [
        'view_organizationalstructure',
        'add_organizationalstructure',
        'update_organizationalstructure',
        'delete_organizationalstructure'
    ]
    template_name = 'setting/organizational-structure/list.html'
    model = OrganizationalStructure

    def get_unused_options(self, existing_qs):
        items = ITEM_CHOICES
        return_items = list(ITEM_CHOICES)
        for item in items:
            counter = 0
            for existing_item in existing_qs:
                if item[0] == existing_item.item:
                    break
                counter += 1
            if counter != len(existing_qs):
                return_items.remove(item)
        if ('company', 'Company') in return_items:
            return_items.remove(('company', 'Company'))
        return return_items

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        org_structure_qs = OrganizationalStructure.objects.all().order_by('order')
        context['orgs'] = org_structure_qs
        context['unused_items'] = self.get_unused_options(existing_qs=org_structure_qs)
        return context

    def post(self, request, *args, **kwargs):
        data = request.POST['org-structure']
        if data == '':
            data = 'company'
        data_list = data.split(",")

        if '' in data_list:
            data_list.remove('')

        existing_structure_qs = OrganizationalStructure.objects.all()
        to_delete_items_list = []
        for obj_item in existing_structure_qs:
            if obj_item.item not in data_list:
                to_delete_items_list.append(obj_item.item)
        print(to_delete_items_list)
        for item in to_delete_items_list:
            try:
                OrganizationalStructure.objects.get(item=item).delete()
            except:
                pass

        i = 0
        parent = None
        while i < len(data_list):
            if i == 0:
                try:
                    parent, created = OrganizationalStructure.objects.update_or_create(
                        item=data_list[i], defaults={'parent_item': None, 'order': i+1}
                    )
                except:
                    break
            else:
                try:
                    parent, created = OrganizationalStructure.objects.update_or_create(
                        item=data_list[i], defaults={'parent_item': parent, 'order': i+1}
                    )
                except:
                    break
            i += 1

        return redirect(request.META['HTTP_REFERER'])

class OrganizationalStructureCreate(LoginRequiredMixin, PermissionMixin, CreateView):
    permission_required = 'add_organizationalstructure'
    template_name = 'setting/organizational-structure/create.html'
    model = OrganizationalStructure
    fields = '__all__'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context

class OrganizationalStructureUpdate(LoginRequiredMixin, PermissionMixin, UpdateView):
    permission_required = 'update_organizationalstructure'
    template_name = 'setting/organizational-structure/update.html'
    model = OrganizationalStructure
    fields = '__all__'
    context_object_name = 'orgs'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context

class OrganizationalStructureDelete(LoginRequiredMixin, PermissionMixin, DeleteView):
    permission_required = 'update_organizationalstructure'
    template_name = 'setting/organizational-structure/delete.html'
    model = OrganizationalStructure
    fields = '__all__'
    success_url = reverse_lazy('beehive_admin:setting:organizational_structure_list')

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['permissions'] = self.get_current_user_permission_list()
        context['org_items_list'] = get_organizational_structure()
        return context


from django import forms
from django.utils.translation import ugettext_lazy as _
from .models import TipoActividad

class TipoActividadForm(forms.ModelForm):
    class Meta:
        model = TipoActividad
        exclude = ['']
        labels = {
            'horasRequeridas': _('Required hours for 1 credit'),
        }


from sklearn.feature_extraction.text import TfidfVectorizer

X = ["the cat in the hat",
     "the cat wearing a hat",
     "the cat's moving"
     "the cat is moving"
     ]
X_val = ["the cat is my hat"]
y = [1, 0]
y_val = [1]
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(X)
X_val = vectorizer.transform(X_val)

print(vectorizer.get_feature_names_out())
print(X_val)


"""Pass JSON 'data' to HMAC service"""

from binascii import hexlify
import socket
from flask import Flask, request
import os

app = Flask(__name__)

@app.route("/")
def home_page():
    """Useless home page"""
    return "
Crypto
"

@app.route("/hmac", methods=['POST'])
def build_mac():
    """Handle HMAC requests"""
    data = request.get_json()

    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
        # Connect to server and send data
        sock.connect(os.environ['SOCKET_PATH'])
        sock.sendall(bytes(data['data'] + '\n', 'utf-8'))

        # Receive data from the server and shut down
        return {
            "hmac": hexlify(sock.recv(1024))
        }


"""This module supports loading JSON documents when memory is tight.

Attributes:
  load_data (function): Loads JSON data from a file.
"""

import json
from typing import Any, Dict, List, Union

JsonType = Union[
    Dict[str, Any],
    List[Any],
]

def load_data() -> JsonType:
    """Loads JSON data from a file.

    Data is taken from small.json.

    Returns:
      The deserialized data.
    """

    try:
        with open("small.json", encoding="utf8") as json_file:
            return json.load(json_file)
    except MemoryError:
        return []


import os
import sys
import json
import math
from collections import defaultdict
import itertools
import re
import argparse
import fnmatch

import pandas as pd
import numpy as np
from statsmodels.tsa.stattools import grangercausalitytests, adfuller
from scipy.stats import linregress

import metadata
from kshape import zscore, _sbd

from preprocess import interpolate_missing

def preferred_cluster(clusters):
    preferred = 0
    preferred_value = -1
    for k, v in clusters.items():
        if v["silhouette_score"] > preferred_value:
            preferred = int(k)
            preferred_value = v["silhouette_score"]
    return preferred

def natural_sort(l):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

def best_column_of_cluster(filenames, path):
    best_column_of_clusters = []
    index = None
    for filename in natural_sort(filenames):
        best_distance = np.inf
        best_column = None
        cluster_path = os.path.join(path, filename)
        df = pd.read_csv(cluster_path, sep="\t", index_col='time', parse_dates=True)
        for c in df.columns:
            if c == "centroid":
                continue
            distance = _sbd(df.centroid, df[c])[0]
            if distance < best_distance:
                best_distance = distance
                best_column = c
        best_column_of_clusters.append(best_column)
    return best_column_of_clusters

def read_service(srv, path):
    srv_path = os.path.join(path, srv["preprocessed_filename"])
    df = pd.read_csv(srv_path, sep="\t", index_col='time', parse_dates=True)
    for c in df.columns:
        df[c] = zscore(df[c])

    preferred = preferred_cluster(srv["clusters"])
    cluster = srv["clusters"][str(preferred)]
    selected_metrics = best_column_of_cluster(cluster["filenames"], path)
    with metadata.update(path) as data:
        for s in data["services"]:
            if s["name"] == srv["name"]:
                s["clusters"][str(preferred)]["grangercausality-metrics"] = selected_metrics

    return Service(srv["name"], selected_metrics, df)

def grangercausality(service_a, service_b, df, stats, lags):
    c1 = df.columns[0]
    c2 = df.columns[1]
    try:
        res = grangercausalitytests(df, lags, verbose=False)
    except Exception as e:
        if df[c1].var() < 1e-30:
            print("low variance for %s, got: %s" % (c1, e))
        if df[c2].var() < 1e-30:
            print("low variance for %s, got: %s" % (c2, e))
        else:
            print("error while processing %s -> %s, got: %s" % (c1, c2, e))
        return
    reg = linregress(df[c1], df[c2])
    reg_a = linregress(df[c1].index.astype("int64"), df[c1])
    reg_b = linregress(df[c2].index.astype("int64"), df[c2])

    stats["perpetrator_service"].append(service_a.name)
    stats["perpetrator_metric"].append(c1)
    stats["consequence_service"].append(service_b.name)
    stats["consequence_metric"].append(c2)
    stats["selected_by_clustering"].append(c1 in service_a.selected_metrics and c2 in service_b.selected_metrics)
    stats["quotient"].append(reg_a.slope / reg_b.slope)
    stats["slope_a"].append(reg_a.slope)
    stats["slope_b"].append(reg_b.slope)
    stats["p_value_a"].append(reg_a[3])
    stats["p_value_b"].append(reg_b[3])

    for i, type in enumerate(["slope", "intercept", "r_value", "p_value", "std_err"]):
        stats[type].append(reg[i])

    for i in range(lags):
        lag = i + 1
        stats["p_for_lag_%d" % lag].append(res[lag][0]['ssr_ftest'][1])

def combine(a, b):
    for x in a:
        for y in b:
            yield(x,y)

class Service():
    def __init__(self, name, selected_metrics, df):
        self.name = name
        self.selected_metrics = set(selected_metrics)
        self.df = df

def _compare_services(service_a, service_b, path, sampling_rate):
    stats = defaultdict(list)
    df = interpolate_missing(pd.concat([service_a.df, service_b.df]), sampling_rate)

    for c1, c2 in combine(service_a.df.columns, service_b.df.columns):
        if c1 == c2:
            continue
        grangercausality(service_a, service_b, df[[c1, c2]], stats, 5)
        grangercausality(service_b, service_a, df[[c2, c1]], stats, 5)
    return pd.DataFrame(stats)

#def read_slas_metrics(service_sla, path):
#    for file, metrics in service_sla.items():
#        metric_path = os.path.join(path, file)
#        import pdb; pdb.set_trace()

def compare_services(srv_a, srv_b, path, sampling_rate, slas=None):
    print("%s -> %s" % (srv_a["name"], srv_b["name"]))
    causality_file = os.path.join(path, "%s-%s-causality-callgraph-best.tsv.gz" % (srv_a["name"], srv_b["name"]))
    if os.path.exists(causality_file):
        print("skip %s" % causality_file)
        return

    service_a = read_service(srv_a, path)
    service_b = read_service(srv_b, path)

    #if slas is not None:
    #    slas_a = read_slas_metrics(slas[srv_a["name"]], path)
    #    slas_b = read_slas_metrics(slas[srv_b["name"]], path)
    #    df1 = _compare_services(srv_a["name"], df_a, srv_b["name"], slas_b, path, sampling_rate)
    #    df2 = _compare_services(srv_a["name"], slas_a, srv_b["name"], df_b, path, sampling_rate)
    #else:

    df = _compare_services(service_a, service_b, path, sampling_rate)
    df.to_csv(causality_file, sep="\t", compression="gzip")

def undirected_callgraph(callgraph):
    uniq = {}
    for edge in callgraph:
        a = edge["from"]
        b = edge["to"]
        if a > b:
            a, b = b, a
        uniq[a + b] = (a, b)
    return uniq.values()

def match_field(service_name, field, sla_metrics):
    for assigned_srv, patterns in sla_metrics["services"].items():
        for wildcard in patterns["wildcards"]:
            if fnmatch.fnmatch(field, wildcard):
                return assigned_srv
    for wildcard in sla_metrics["generic"]["wildcards"]:
        if fnmatch.fnmatch(field, wildcard):
            return service_name

def assign_slas(sla_metrics, measurement_data):
    services = defaultdict(lambda: defaultdict(list))
    for srv in measurement_data["services"]:
        preprocessed_filename = srv["preprocessed_filename"]
        for field in srv["preprocessed_fields"]:
            service_name = match_field(srv["name"], field, sla_metrics)
            if service_name is not None:
                services[service_name][preprocessed_filename].append(field)
    import pdb; pdb.set_trace()
    return services

def find_causality(path, services_data, args):
    measurement_data = metadata.load(path)
    call_pairs = undirected_callgraph(services_data["edges"])

    services = {}
    for srv in measurement_data["services"]:
        services[srv["name"]] = srv
    if args.compare_slas:
        slas = assign_slas(services_data["sla_metrics"], measurement_data)
    else:
        slas = None

    if args.parallel:
        from submit_jobs import lview
        def _compare_services(args):
            import grangercausality
            from imp import reload
            reload(grangercausality)
            grangercausality.compare_services(*args)

        ids = []
        for srv_a, srv_b in call_pairs:
            lview.block = True
            res = lview.apply_async(_compare_services, (services[srv_a], services[srv_b], path, args.sampling_rate, slas))
            ids.extend(res.msg_ids)
        return ids
    else:
        for srv_a, srv_b in call_pairs:
            compare_services(services[srv_a], services[srv_b], path, args.sampling_rate, slas)

def parse_args():
    parser = argparse.ArgumentParser(prog='grangercausality', usage='%(prog)s [options]')
    parser.add_argument('services_metadata', help='json file with callgraph and sla_metrics')
    parser.add_argument('measurement_directories', nargs="+", help="measurement directories")
    parser.add_argument('--compare-slas', action='store_true', help="compare metrics with slas")
    parser.add_argument('--sampling-rate', default=(1/2), help="how often data is updated per second")
    parser.add_argument('--parallel', action='store_true', help="submit job to ipython parallel cluster")
    return parser.parse_args()

def main():
    args = parse_args()
    ids = []
    for path in args.measurement_directories:
        if not os.path.exists(os.path.join(path, "metadata.json")):
            sys.stderr.write("skip '%s': cannot find metadata.json\n" % path)
            continue
        services_data = json.load(open(args.services_metadata))
        if args.parallel:
            ids.extend(find_causality(path, services_data, args))
        else:
            find_causality(path, services_data, args)

    if args.parallel:
        from submit_jobs import lview
        lview.get_result(ids, owner=False).wait_interactive()

if __name__ == '__main__':
    main()


from django.conf.urls import patterns, url

from tickerwatch import views

urlpatterns = patterns('',
    url(r'^$', views.index, name='index'),
    url(r'^register/$', views.register, name='register'),
    url(r'^login/$', views.user_login, name='login'),
    url(r'^logout/$', views.user_logout, name='logout'),
    url(r'^add_stock/$', views.add_stock, name='add_stock'),
    url(r'^profile/$', views.profile, name='profile'),
    url(r'^text/$', views.text, name='text'),
    url(r'^text_demo/$', views.text_demo, name='text_demo'),
)


from functools import reduce

def num(lista):
    resultado = list(filter((lambda x: x % 2), lista))
    print("Lista de numeros: \n",resultado)

lista = list(range(50))
num(lista)

suma = reduce( (lambda x, y: x + y), lista)
print("Sumatoria de los elementos: ",suma)


import colorgram
from turtle import Turtle,Screen
import random
colors=colorgram.extract('img.jpg',25)
col=colors[0]
s=Screen()
t=Turtle()
s.colormode(255)
cols=[]
for color in colors:
    r = color.rgb.r
    g = color.rgb.g
    b = color.rgb.b
    nc=(r,g,b)
    cols.append(nc)

t.pensize(20)
t.penup()
t.setx(-350)
t.sety(300)
for i in range(16):
    for j in range(18):
        t.pencolor(cols[random.randint(0,24)])
        t.pendown()
        t.forward(1)
        t.penup()
        t.forward(40)
    t.back(738)
    t.right(90)
    t.forward(40)
    t.left(90)

t.forward(1)
s.exitonclick()


import dulwich
from dulwich import porcelain
import os
git_repo = porcelain.Repo("/home/lukaszwelnic/Decentralization/Photos_cloud")
from datetime import datetime

# commits = []
# commit = git_repo.get_object(git_repo.head())
# commit.message
# commit.author
# commit.tree

# tree = git_repo.get_object(commit.tree)

def show_changes(commit_id: bytes):
    commit = git_repo.get_object(commit_id)
    if not commit.parents:
        return
    new_files = git_repo.get_object(commit.tree).items()

    old_files = git_repo.get_object(git_repo.get_object(commit.parents[0]).tree).items()

    for i in range(0, len(new_files)):
        if i >= len(old_files):
            break
        if not [file for file in old_files if file.path == new_files[i].path]:
            print(f"file {new_files[i].path} is new")
        elif new_files[i].sha != old_files[i].sha:
            print(f"file {new_files[i].path} was changed")

def get_commit_history():
    commits = []
    # that commit object gives me the list of files
    # this is how i can know what files have been changed
    commit = git_repo.get_object(git_repo.head())
    commits = [commit.id]
    while True:
        if not commit.parents:
            break
        commits.append(commit.parents[0])
        commit = git_repo.get_object(commit.parents[0])

    commits.reverse()
    return commits

# commits_list = ['a20ca157a4c69881be82592e3a098a331e6ce4c7', 'eff93cc7dad0952b7ee07eaed444aab004aea65e',
#                 'f05a4d33a32533171726df6ced60b99379444db0']
#
# commit_mess = ['Specified 1', 'Specified 2', 'Specified 3']
#
#
commits = get_commit_history()

# version_control(git_repo, commits_list, commit_mess)

# status = porcelain.status(git_repo)

# print(commits)
#
# for com in commits:
#     print("\nCommit", com, ":")
#     show_changes(com)

# print(os.path.getmtime('/home/lukaszwelnic/SharedFolder/HostShare/spotify-party-planner-service/Dockerfile'))
print((datetime.utcnow() - datetime.fromtimestamp(os.path.getmtime('/home/lukaszwelnic/SharedFolder/HostShare/spotify-party-planner-service/Dockerfile'))).total_seconds())


import pandas as pd
import datetime
import numpy as np
import os

prices = pd.DataFrame.from_csv("data/prices.csv", index_col=False)

tmp = pd.to_datetime(prices.marketdate)
prices["date"] = [d.date() for d in tmp]

def date_filter(df, date_col, start, end):
    print(type(end))
    print(type(df[date_col][1]))
    res = df[np.logical_and(df[date_col] >= start,
                            df[date_col] < end)]
    return (res)

def sum_max(df, group_cols, value_col):
    grp = group_cols + [value_col]
    res = df[grp].groupby(group_cols).sum().max()
    return (res)

def qtile_mean(df, group_cols, value_col, qtile):
    grp = group_cols + [value_col]
    res = df[grp].groupby(group_cols).quantile(qtile).mean()
    return (res)

def gross_period_returns(df, date_col, value_col, id_col, start, end):
    cols = [date_col, value_col, id_col]

    tmp = df[cols][df[date_col] <= start]
    tmp.sort_values(by=date_col, inplace=True)
    tmp = tmp.drop_duplicates(subset=id_col, keep="last")
    tmp.set_index(id_col, inplace=True)
    start_vals = tmp[value_col]

    tmp = df[cols][df[date_col] >= end]
    tmp.sort_values(by=date_col, inplace=True)
    tmp = tmp.drop_duplicates(subset=id_col, keep="first")
    tmp.set_index(id_col, inplace=True)
    end_vals = tmp[value_col]

    res = (end_vals - start_vals) / start_vals
    return (res)

def gross_nday_returns(df, date_col, id_col, value_col, start, ndays=30):
    end = start + datetime.timedelta(days=ndays)
    res = gross_period_returns(df=df, date_col=date_col, value_col=value_col, id_col=id_col,
                               start=start, end=end)
    return (res)

def date_iterator(first, last, incr_days=20 ):
    res = first
    while res <= last:
        yield res
        res = res + datetime.timedelta(days=incr_days)

min_date = prices.date.min()
max_date = prices.date.max()
start = min_date

fulldf = pd.DataFrame()
diter = date_iterator(first=min_date, last=max_date, incr_days=30)
for date in diter:
    print (date)
    datedf = pd.DataFrame()
    for ndays in [30, 90, 180, 360]:
        ser = gross_nday_returns(df=prices,
                               id_col="gempermid",
                               date_col="date",
                               value_col="close_",
                               start=date,
                               ndays=ndays)
        df = pd.DataFrame(ser)
        df.columns = ["ret%d" % (ndays)]
        datedf = (df if (datedf.shape[0] == 0)
                else pd.merge(datedf, df, how="outer", left_index=True, right_index=True))
        print(datedf.shape)
    datedf["date"] = date
    fulldf = (datedf if fulldf.shape[0] == 0 else
               pd.concat([fulldf, datedf]))
    print (fulldf.tail())

    print('')
if (not os.path.isdir("output")):
    os.makedirs("output")
fulldf.to_csv("output/returns.csv")
print ("done")


#
# @lc app=leetcode id=508 lang=python3
#
# [508] Most Frequent Subtree Sum
#
# https://leetcode.com/problems/most-frequent-subtree-sum/description/
#
# algorithms
# Medium (59.65%)
# Total Accepted:    86.2K
# Total Submissions: 144.5K
# Testcase Example:  '[5,2,-3]'
#
# Given the root of a binary tree, return the most frequent subtree sum. If
# there is a tie, return all the values with the highest frequency in any
# order.
#
# The subtree sum of a node is defined as the sum of all the node values formed
# by the subtree rooted at that node (including the node itself).
#
#
# Example 1:
#
#
# Input: root = [5,2,-3]
# Output: [2,-3,4]
#
#
# Example 2:
#
#
# Input: root = [5,2,-5]
# Output: [2]
#
#
#
# Constraints:
#
#
# The number of nodes in the tree is in the range [1, 10^4].
# -10^5 <= Node.val <= 10^5
#
#
#
# Definition for a binary tree node.

# find compute the subtree sum. Then compute the mode using 501.

from typing import Dict, List, no_type_check

class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def findFrequentTreeSum(self, root: TreeNode) -> List[int]:
        freq = {}
        self.findFrequentTreeSumHelper(root, freq)
        modes = []
        max_freq = -1
        for val, curr_freq in freq.items():
            if curr_freq > max_freq:
                modes = [val]
                max_freq = curr_freq
            elif curr_freq == max_freq:
                modes.append(val)

        return modes

    def findFrequentTreeSumHelper(self, root: TreeNode, freq: Dict[int, int]) -> int:
        if root is None:
            return 0

        # compute sum recursively
        left_sum = self.findFrequentTreeSumHelper(root.left, freq)
        right_sum = self.findFrequentTreeSumHelper(root.right, freq)

        node_sum = left_sum + right_sum + root.val

        if node_sum not in freq:
            freq[node_sum] = 0

        freq[node_sum] += 1

        return node_sum


import pygame
from pygame import Color
from models import Spaceship, Asteroid, Bullet
from utils import load_sprite , get_random_position, print_text

class SpaceRocks:
    MIN_ASTEROID_DISTANCE = 250
    SPAWN_TIME = 120
    SHIELD_PULSE = 60

    def __init__(self):
        self._init_pygame()
        self.screen = pygame.display.set_mode((800, 600))
        self.background = load_sprite("space", False)
        self.clock = pygame.time.Clock()
        self.font = pygame.font.Font(None, 64)
        self.play_again = True
        self.setup_new_game()

    def setup_new_game(self):
        self.shield_pulse_time = 0
        self.time_to_spawn = 0
        self.destroyed_asteroids = 0
        self.message = ""
        self.asteroids = []
        self.bullets = []
        self.spaceship = Spaceship((400, 300), self.bullets.append)

        for _ in range(1):
            self.spawn_asteroid()

    def spawn_asteroid(self):
        while True:
            position = get_random_position(self.screen)
            if (
                position.distance_to(self.spaceship.position)
                > self.MIN_ASTEROID_DISTANCE
            ):
                break
        self.asteroids.append(Asteroid(position, self.asteroids.append))

    def main_loop(self):
        while self.play_again == True:
            self._handle_input()
            self._process_game_logic()
            self._draw()

    def _init_pygame(self):
        pygame.init()
        pygame.display.set_caption("Space Rocks")

    def _handle_input(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT or (
                event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE
            ):
                quit()
            elif (
                self.spaceship
                and event.type == pygame.KEYDOWN
                and event.key == pygame.K_SPACE
            ):
                self.spaceship.shoot()
            elif event.type == pygame.KEYDOWN and event.key == pygame.K_a and self.spaceship:
                self.spawn_asteroid()

            elif event.type == pygame.KEYDOWN and event.key == pygame.K_e and not self.spaceship :
                self.spaceship = Spaceship((400, 300), self.bullets.append)
                self.message = ""

            elif event.type == pygame.KEYDOWN and event.key == pygame.K_r:
                self.setup_new_game()

        is_key_pressed = pygame.key.get_pressed()
        if self.spaceship:
            if is_key_pressed[pygame.K_RIGHT]:
                self.spaceship.rotate(clockwise=True)
            elif is_key_pressed[pygame.K_LEFT]:
                self.spaceship.rotate(clockwise=False)
            if is_key_pressed[pygame.K_UP]:
                self.spaceship.accelerate()
            elif is_key_pressed[pygame.K_DOWN]:
                self.spaceship.decelerate()
            if is_key_pressed[pygame.K_b]:
                self.spaceship.emergency_brake()

    def _process_game_logic(self):
        for game_object in self._get_game_objects():
            game_object.move(self.screen)

        if self.spaceship:
            for asteroid in self.asteroids:
                if asteroid.collides_with(self.spaceship):
                    if self.spaceship.shield == 0:
                        self.spaceship = None
                        self.message = "Przegrałeś!"
                        break
                    else:
                        self.asteroids.remove(asteroid)
                        self.spaceship.shield = 0
                        asteroid.split()
                        self.destroyed_asteroids += 1

        #self.asteroid.move()
        for bullet in self.bullets[:]:
            for asteroid in self.asteroids[:]:
                if asteroid.collides_with(bullet):
                    self.asteroids.remove(asteroid)
                    self.bullets.remove(bullet)
                    asteroid.split()
                    self.destroyed_asteroids += 1
                    break

            if not self.screen.get_rect().collidepoint(bullet.position):
                self.bullets.remove(bullet)

        if not self.asteroids and self.spaceship:
            self.message = "Wygrałeś!"

        self.time_to_spawn += 1
        if self.time_to_spawn == self.SPAWN_TIME and self.spaceship:
            self.spawn_asteroid()
            self.time_to_spawn = 0

        self.shield_pulse_time += 1
        if self.shield_pulse_time == 60:
            self.shield_pulse_time = 0

    def _draw(self):
        self.screen.blit(self.background, (0, 0))
        for game_object in self._get_game_objects():
            game_object.draw(self.screen)

        if self.spaceship and self.spaceship.shield == 1 :
            color = (255, 100 + 155 * (self.shield_pulse_time/self.SHIELD_PULSE), 0+ 255 * (self.shield_pulse_time/self.SHIELD_PULSE))
            pygame.draw.circle(self.screen, color, self.spaceship.position, self.spaceship.radius+3, 2)

        if self.message:
            print_text(self.screen, self.message, self.font)

        print_text(self.screen, str(self.destroyed_asteroids), self.font, (770,30))
        # if self.spaceship:
        #     print_text(self.screen, str(self.spaceship.shield), self.font, (570,30))

        #print_text(self.screen, str(self.SPAWN_TIME - self.time_to_spawn), self.font, (570,30))

        print_text(self.screen, str(self.shield_pulse_time), self.font, (570,30))

        pygame.display.flip()
        self.clock.tick(60)

    def _get_game_objects(self):
        game_objects = [*self.asteroids, *self.bullets]

        if self.spaceship:
            game_objects.append(self.spaceship)

        return game_objects


from agent import Agent
import argparse
from collections import deque
from env import Environment
import numpy as np
import torch

from model import QNetwork, Small, Large, Dropout
import matplotlib
matplotlib.use("TkAgg")
from matplotlib import pyplot as plt

def main(args):
    if args.examine:
        examine()
    if args.random:
        random()
    if args.train:
        train_multiple(summary_interval=50)
    if args.test:
        test()

def examine():
    with Environment() as env:
        # reset the environment
        env_info = env.reset()

        # number of agents in the environment
        print('Number of agents:', len(env_info.agents))

        # number of actions
        action_size = env.action_space_size
        print('Number of actions:', action_size)

        # examine the state space
        state = env_info.vector_observations[0]
        print('States look like:', state)
        state_size = len(state)
        print('States have length:', state_size)

def random():
    with Environment(no_graphics=False) as env:
        env_info = env.reset()
        action_size = env.action_space_size
        score = 0
        while True:
            action = np.random.randint(action_size)
            env_info = env.step(action)
            reward = env_info.rewards[0]
            done = env_info.local_done[0]
            score += reward
            if done:
                break
            print('\rScore: {:.2f}'.format(score), end="")
        print('\rScore: {:.2f}'.format(score))

def test():
    with Environment(no_graphics=False) as env:
        env_info = env.reset(train_mode=False)
        action_size = env.action_space_size
        state_size = len(env_info.vector_observations[0])
        state = env_info.vector_observations[0]
        agent = Agent(model=QNetwork, state_size=state_size, action_size=action_size, seed=0)
        agent.qnetwork_local.load_state_dict(torch.load('results/checkpoint.pth'))
        score = 0
        while True:
            action = agent.act(state)
            env_info = env.step(action)
            state = env_info.vector_observations[0]
            reward = env_info.rewards[0]
            done = env_info.local_done[0]
            score += reward
            if done:
                break
            print('\rScore: {:.2f}'.format(score), end="")
        print('\rScore: {:.2f}'.format(score))

def train_multiple(summary_interval):
    models = [QNetwork, Small, Large, Dropout]
    with Environment() as env:
        summary_scores = {}
        for model in models:
            summary = train(model, env, summary_interval)
            summary_scores[model.__name__] = summary
        plot_summary(summary_scores, summary_interval)

def train(model, env, summary_interval=100):
    env_info = env.reset(train_mode=True)
    action_size = env.action_space_size
    state_size = len(env_info.vector_observations[0])
    agent = Agent(model=model, state_size=state_size, action_size=action_size, seed=0)
    model_name = model.__name__
    summary_scores = _train(env, agent, model_name=model_name, summary_interval=summary_interval)
    return summary_scores

def _train(env, agent, model_name='model', summary_interval=100,
           n_episodes=2000, max_t=1000, eps_start=1.0, eps_end=0.01):
    """Deep Q-Learning.

    Params
    ======
        n_episodes (int): maximum number of training episodes
        max_t (int): maximum number of timesteps per episode
        eps_start (float): starting value of epsilon, for epsilon-greedy action selection
        eps_end (float): minimum value of epsilon
    """
    scores = []                        # list containing scores from each episode
    summary_scores = []
    scores_window = deque(maxlen=summary_interval)  # last 100 scores
    eps = eps_start                    # initialize epsilon
    eps_decay = (eps_end ** (1 / n_episodes)) / eps_start
    for i_episode in range(1, n_episodes + 1):
        env_info = env.reset(train_mode=True)
        state = env_info.vector_observations[0]
        score = 0
        for t in range(max_t):
            action = agent.act(state, eps)
            env_info = env.step(action)
            next_state = env_info.vector_observations[0]
            reward = env_info.rewards[0]
            done = env_info.local_done[0]
            agent.step(state, action, reward, next_state, done)
            state = next_state
            score += reward
            if done:
                break
        scores_window.append(score)       # save most recent score
        scores.append(score)              # save most recent score
        eps = max(eps_end, eps_decay * eps)  # decrease epsilon
        print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end="")
        if i_episode % summary_interval == 0:
            summary_scores.append(np.mean(scores_window))
            print('\rEpisode {}\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))
            torch.save(agent.qnetwork_local.state_dict(), 'results/' + model_name + '.pth')
            plot_scores(scores, summary_scores, summary_interval, model_name)
        # if np.mean(scores_window) >= 20.0:
        #     print('\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'.format(
        #         i_episode - summary_interval, np.mean(scores_window)))
        #     break
    torch.save(agent.qnetwork_local.state_dict(), 'results/' + model_name + '.pth')
    plot_scores(scores, summary_scores, summary_interval, model_name)
    return summary_scores

def plot_scores(scores, summary_scores, summary_interval, file_name=''):
    plt.ylabel('Score')
    plt.xlabel('Episode #')
    plt.plot(np.arange(len(scores)), scores)
    plt.plot(np.arange(len(summary_scores) * summary_interval), summary_scores)
    if file_name:
        plt.savefig('results' + file_name + '.png', bbox_inches='tight')
    else:
        plt.show()

def plot_summary(summary_scores, summary_interval):
    plt.ylabel('Score')
    plt.xlabel('Episode #')
    for key, scores in summary_scores.items():
        plt.plot(np.arange(len(scores)) * summary_interval, scores, label=key)
    plt.legend()
    plt.savefig('results/summary.png', bbox_inches='tight')
    plt.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='DeepRL - Q-learning project')
    parser.add_argument('--examine',
                        action="store_true",
                        dest="examine",
                        help='Print environment information')
    parser.add_argument('--random',
                        action="store_true",
                        dest="random",
                        help='Start a random agent')
    parser.add_argument('--train',
                        action="store_true",
                        dest="train",
                        help='Train a new network')
    parser.add_argument('--test',
                        action="store_true",
                        dest="test",
                        help='Load an existing network and test it')
    args = parser.parse_args()
    if not any(vars(args).values()):
        parser.error('No arguments provided.')
    main(args)


from tkinter import *
from tkinter import ttk
from pytube import YouTube

import unicodedata
import re

import os

def slugify(value, allow_unicode=False):
    """
    Taken from https://github.com/django/django/blob/master/django/utils/text.py
    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
    dashes to single dashes. Remove characters that aren't alphanumerics,
    underscores, or hyphens. Convert to lowercase. Also strip leading and
    trailing whitespace, dashes, and underscores.
    """
    value = str(value)
    if allow_unicode:
        value = unicodedata.normalize('NFKC', value)
    else:
        value = unicodedata.normalize('NFKD',
                                      value).encode('ascii',
                                                    'ignore').decode('ascii')
    value = re.sub(r'[^\w\s-]', '', value.lower())
    return re.sub(r'[-\s]+', '-', value).strip('-_')

def download():
    try:

        link = link_var.get()
        format = format_var.get()
        print(link)
        print(format)

        yt = YouTube(link)
        # print(yt.title)
        yt.title = slugify(yt.title)
        # yt.title = str(uuid.uuid4())

        if format == "1" or format == "mp3":
            yt.streams.first().download()
            os.rename(f'{yt.title}.3gpp', f'{yt.title}.mp3')
            print("MP3 downloaded")
        elif format == "2" or format == "mp4":
            yt.streams.filter(progressive=True, file_extension='mp4').order_by(
                'resolution').desc().first().download()
            print("MP4 downloaded")
        elif format == "3" or format == "both":
            yt.streams.first().download()
            yt.streams.filter(progressive=True, file_extension='mp4').order_by(
                'resolution').desc().first().download()
            os.rename(f'{yt.title}.3gpp', f'{yt.title}.mp3')
            print("MP3 and MP4 downloaded")
        else:
            print("Invalid format")

    except Exception as e:
        print(e)
    link_var.set("")

root = Tk()
root.title("YouTube Video Converter")

frm = ttk.Frame(root, padding=10)
link_var = StringVar(frm)
format_var = StringVar(frm, "1")
frm.grid()
ttk.Entry(frm, textvariable=link_var).grid(column=0, row=0)
ttk.Button(frm, text="Download", command=download).grid(column=2, row=0)

ttk.Radiobutton(frm, text="MP3", value="1", variable=format_var).grid(column=0, row=1)
ttk.Radiobutton(frm, text="MP4", value="2", variable=format_var).grid(column=1, row=1)
ttk.Radiobutton(frm, text="BOTH", value="3", variable=format_var).grid(column=2, row=1)

# ttk.Button(frm, text="Quit", command=root.destroy).grid(column=1, row=2)

root.mainloop()


import sqlite3
import string
import os
import click
from flask import Flask, request, render_template, redirect, jsonify
from flask import send_from_directory
from flask.ext.cors import CORS, cross_origin
from sqlite3 import OperationalError
from urllib.parse import urlparse

__version__ = '1.1.4'
db_path = 'var/urls.db'
host = os.environ.get("SHORTENER_DOMAIN", 'localhost:5000')
protocol = os.environ.get("PROTOCOL", 'http://')

BASE = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
BASE.extend(list(string.ascii_lowercase))
BASE.extend(list(string.ascii_uppercase))
BASE_LEN = len(BASE)

# Assuming urls.db is in your app root folder
app = Flask(__name__)

cors = CORS(app, resources={r"/": {"origins": "*"}})

def get_base_next(char):
    if char == '':
        return False, '0'
    char_index = BASE.index(char)
    char_index += 1
    return (False, BASE[char_index]) if \
        char_index < BASE_LEN else (True, '0')

def next_id(id_=None):
    new_id = id_
    if id_ is None:
        new_id = '0'
    else:
        index = -1
        to_inc = new_id[index]
        final, next = get_base_next(to_inc)
        new_id = new_id[:index] + next
        index -= 1
        len_id = len(new_id)
        while index + 6 >= 0 and final:
            if index + len_id >= 0:
                to_inc = new_id[index]
                final, next = get_base_next(to_inc)
                new_id = new_id[:index] + next + new_id[index + 1:]
            else:
                to_inc = ''
                final, next = get_base_next(to_inc)
                new_id = next + new_id[index + 1:]

            index -= 1

    return new_id

def table_check():
    create_table = """
        CREATE TABLE WEB_URL(
        ID INTEGER PRIMARY KEY     AUTOINCREMENT,
        NUM TEXT NOT NULL UNIQUE,
        URL  TEXT  NOT NULL UNIQUE
        );
        """
    with sqlite3.connect(db_path) as conn:
        cursor = conn.cursor()
        try:
            cursor.execute(create_table)
        except OperationalError:
            pass

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'favicon.ico',
                               mimetype='image/vnd.microsoft.icon')

@app.route('/', methods=['GET', 'POST'])
@cross_origin(origin='localhost', headers=['Content-Type', 'Authorization'])
def home():
    method = request.method
    with sqlite3.connect(db_path) as conn:
        try:
            cursor = conn.cursor()
            rows_query = """
                SELECT NUM, max(ID) FROM WEB_URL"""
            result_cursor = cursor.execute(rows_query)
            result_fetch = result_cursor.fetchone()
            last_num = result_fetch[0]
            number_of_rows = result_fetch[1]
            number_of_rows = 0 if number_of_rows is None else number_of_rows
            if (method == 'GET' and request.args.get('url', None)) or \
               method == 'POST':
                original_url = request.args.get('url') if \
                    method == 'GET' else request.form.get('url')
                if original_url:
                    if urlparse(original_url).scheme == '':
                        original_url = 'http://' + original_url
                    exist_row = """
                        SELECT NUM FROM WEB_URL
                            WHERE URL='{url}'
                        """.format(url=original_url)
                    result_cursor = cursor.execute(exist_row)
                    result_fetch = result_cursor.fetchone()
                    if result_fetch:
                        new_num = result_fetch[0]
                    else:
                        new_num = next_id(last_num)
                        insert_row = """
                            INSERT INTO WEB_URL (URL, NUM)
                                VALUES ('{url}', '{num}')
                            """.format(url=original_url, num=new_num)
                        cursor.execute(insert_row)
                        number_of_rows += 1
                    encoded_string = new_num
                    short_url = '/'.join([host, encoded_string])
                    if method == 'GET':
                        return jsonify({'short_url': short_url,
                                        'code': 'SUCCESS',
                                        'original_url': original_url,
                                        'version': __version__})
                    else:
                        return render_template(
                            'home.html', short_url=short_url,
                            protocol=protocol,
                            number_of_rows=number_of_rows,
                            version=__version__)

            return render_template('home.html',
                                   protocol=protocol,
                                   number_of_rows=number_of_rows,
                                   version=__version__)
        except Exception as error:
            if method == 'GET':
                return jsonify(**{'code': 'ERROR',
                                  'error': str(error),
                                  'original_url': original_url
                                  })
            else:
                return render_template(
                    'home.html',
                    number_of_rows=number_of_rows,
                    error=True)

@app.route('/<short_url>')
def redirect_short_url(short_url):
    decoded_string = short_url
    with sqlite3.connect(db_path) as conn:
        cursor = conn.cursor()
        select_row = """
                SELECT URL FROM WEB_URL
                    WHERE NUM='{num}'
                """.format(num=decoded_string)
        result_cursor = cursor.execute(select_row)
        try:
            return redirect(result_cursor.fetchone()[0])
        except Exception:
            pass

    return render_template(
        'home.html',
        error=True)

@click.option('--version', help='Print Version and exit',
              is_flag=True)
@click.option('--debug', help='Enable debug option',
              is_flag=True)
@click.command()
def main(debug, version):
    if version:
        import sys
        sys.exit('Your Version is: %s' % __version__)
    table_check()
    app.run(host='0.0.0.0', debug=debug)

if __name__ == '__main__':
    main()


import sqlite3

TRANSLATION_TABLE = "translation_table"
LEMMA_ADDS_TABLE = "lemma_adds_table"
WORD_KEY = "word"
LEMMA_KEY = "lemma"
ADDS_KEY = "adds"
TRANSLATION_KEY ="translation"

class DatabaseManager:
    def __init__(self, db_name="vocabs.db"):
        self.conn = sqlite3.connect(db_name)

    def create_lemma_adds_table(self):
        cursor = self.conn.cursor()

        create_table = f"CREATE TABLE IF NOT EXISTS {LEMMA_ADDS_TABLE} " \
                       f"({WORD_KEY} text, {LEMMA_KEY} text, {ADDS_KEY} text, PRIMARY KEY ({WORD_KEY}))"
        cursor.execute(create_table)

        self.conn.commit()

    def create_translation_table(self):
        cursor = self.conn.cursor()

        create_table = f"CREATE TABLE IF NOT EXISTS {TRANSLATION_TABLE} " \
                       f"({LEMMA_KEY} text, {ADDS_KEY} text, {TRANSLATION_KEY} text, " \
                       f"CONSTRAINT lemma_adds PRIMARY KEY ({LEMMA_KEY}, {ADDS_KEY}))"
        cursor.execute(create_table)

        self.conn.commit()

    def insert_lemma_adds(self, word, lemma, adds):
        self.insert_lemma_adds_multiple([(word, lemma, adds)])

    def insert_lemma_adds_multiple(self, word_lemma_adds_list):
        """
        Insert into lemma_adds table
        Args:
            word_lemma_adds_list(list((str, str, str))): list of tuples of word, lemma and adds
        """
        self.create_lemma_adds_table()
        cursor = self.conn.cursor()

        for params in word_lemma_adds_list:
            try:
                insert = f"INSERT INTO {LEMMA_ADDS_TABLE} VALUES (?, ?, ?)"
                cursor.execute(insert, params)
            except sqlite3.IntegrityError:
                update = f"UPDATE {LEMMA_ADDS_TABLE} SET {LEMMA_KEY}=?, {ADDS_KEY}=? WHERE {WORD_KEY}=?"
                cursor.execute(update, (params[1], params[2], params[0]))

        self.conn.commit()

    def insert_translation(self, lemma, adds, translation):
        self.insert_translation_multiple([(lemma, adds, translation)])

    def insert_translation_multiple(self, lemma_adds_translation_list):
        """
        Insert into translation table
        Args:
            lemma_adds_translation_list(list((str, str, str))): list of tuples of lemma, adds and translation
        """
        self.create_translation_table()
        cursor = self.conn.cursor()

        for params in lemma_adds_translation_list:
            try:
                insert = f"INSERT INTO {TRANSLATION_TABLE} VALUES (?, ?, ?)"
                cursor.execute(insert, params)
            except sqlite3.IntegrityError:
                update = f"UPDATE {TRANSLATION_TABLE} SET {TRANSLATION_KEY}=? WHERE {LEMMA_KEY}=? and {ADDS_KEY}=?"
                cursor.execute(update, (params[2], params[0], params[1]))

        self.conn.commit()

    def retrieve_lemma_adds(self, word):
        return self.retrieve_lemma_adds_multiple([word])[0]

    def retrieve_lemma_adds_multiple(self, words):
        self.create_lemma_adds_table()
        cursor = self.conn.cursor()

        select = f"SELECT * FROM {LEMMA_ADDS_TABLE} WHERE {WORD_KEY}=?"

        result = []
        for word in words:
            cursor.execute(select, (word, ))
            row = cursor.fetchone()
            if row is None:
                row = (word, None, None)
            result.append(row)
        return result

    def retrieve_translation(self, lemma, adds):
        return self.retrieve_translation_multiple([(lemma, adds)])[0]

    def retrieve_translation_multiple(self, lemma_adds_list):
        self.create_translation_table()
        cursor = self.conn.cursor()

        select = f"SELECT * FROM {TRANSLATION_TABLE} WHERE {LEMMA_KEY}=? AND {ADDS_KEY}=?"

        result = []
        for params in lemma_adds_list:
            cursor.execute(select, params)
            row = cursor.fetchone()
            if row is None:
                row = (params[0], params[1], None)
            result.append(row)
        return result

    def close(self):
        self.conn.close()

if __name__ == '__main__':
    manager = DatabaseManager("test_vocabs")
    manager.create_lemma_adds_table()
    manager.create_translation_table()
    manager.insert_lemma_adds_multiple([("wort1", "lemma1", "adds1"), ("wort2", "lemma2", "adds2"), ("wort3", "lemma3", "adds3")])
    print(manager.retrieve_lemma_adds_multiple(["wort1", "wort3", "wort5"]))
    manager.insert_lemma_adds_multiple([("wort1", "lemma13", "adds13"), ("wort4", "lemma4", "adds4"), ("wort2", "lemma2", "adds2"), ("wort1", "lemma14", "adds14")])
    print(manager.retrieve_lemma_adds_multiple(["wort1", "wort3", "wort2", "wort4"]))
    manager.insert_translation_multiple([("lemma1", "adds1", "trans1"), ("lemma2", "adds2", "trans2")])
    print(manager.retrieve_translation_multiple([("lemma2", "adds2"), ("lemma1", "adds3"), ("lemma1", "adds1"), ("lemma3", "adds3")]))
    manager.insert_translation_multiple([("lemma1", "adds13", "trans1"), ("lemma3", "adds3", "trans3")])
    print(manager.retrieve_translation_multiple([("lemma3", "adds3"), ("lemma1", "adds1"), ("lemma1", "adds13")]))