import numpy
import torch
import pandas
from typing import Dict, List, Sequence, Any
from neptune.new.types import File
from pytorch_lightning import LightningModule

from sklearn.base import is_regressor, is_classifier
from sklearn.model_selection import RepeatedStratifiedKFold, RepeatedKFold
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.linear_model import RidgeClassifierCV, RidgeCV

from tissue_purifier.data.dataset import MetadataCropperDataset
from tissue_purifier.plots.plot_images import show_raw_all_channels
from tissue_purifier.plots.plot_embeddings import plot_embeddings
from tissue_purifier.utils.nms_util import NonMaxSuppression
from tissue_purifier.utils.dict_util import (
    concatenate_list_of_dict,
    subset_dict)

from tissue_purifier.utils.validation_util import (
    SmartPca,
    SmartUmap)

def classify_and_regress(
        input_dict: dict,
        feature_keys: List[str],
        regress_keys: List[str] = None,
        classify_keys: List[str] = None,
        regressor: "sklearn_like_regressor" = None,
        classifier: "sklearn_like_classifier" = None,
        n_splits: int = 5,
        n_repeats: int = 1,
        verbose: bool = False) -> [pandas.DataFrame, pandas.DataFrame]:
    """
    Train a Classifier and a Regressor to use some features to classify/predict other annotations.

    Args:
        input_dict: dict with both the features and the annotations
        regressor: the regressor to train
        classifier: the classifier to train
        feature_keys: keys corresponding to the independent variables in the :attr:`input_dict`.
        regress_keys: keys corresponding to the variables to regress in the :attr:`input_dict`.
        classify_keys: keys corresponding to the variables to classify in the :attr:`input_dict`.
        n_splits: int, number of splits for RepeatedKFold (regressor) or RepeatedStratifiedKFold (classifier).
            If n_splits is 5 (defaults) then train_test_split is 80% - 20%.
        n_repeats: int, number of repeats for RepeatedKFold (regressor) or RepeatedStratifiedKFold (classifier).
            The total number of trained model is n_plists * n_repeats.
        verbose: if True (default is False) prints some intermediate statements

    Returns:
        A dataframe. Each row is a different X,y combination with the metrics describing the quality of the
        regression/classification.
    """
    if regress_keys is not None:
        assert is_regressor(regressor) or regressor.is_regressor, "Please pass in a regressor"

    if classify_keys is not None:
        assert is_classifier(classifier) or classifier.is_classifier, "Please pass in a classifier"

    assert isinstance(n_splits, int) and isinstance(n_repeats, int) and n_splits >= 1 and n_repeats >= 1, \
        "Error. n_splits = {0} and n_repeats = {1} must be integers >= 1.".format(n_splits, n_repeats)
    assert n_splits > 1 or (n_splits == 1 and n_repeats == 1), \
        "Misconfiguration error. It does not make sense to have n_splits == 1 and n_repeats != 1"

    assert isinstance(feature_keys, list), \
        "Feature_keys need to be a list. Received {0}".format(type(feature_keys))
    assert regress_keys is None or isinstance(regress_keys, list), \
        "Regress_keys need to be a list. Received {0}".format(type(regress_keys))
    assert classify_keys is None or isinstance(classify_keys, list), \
        "Classify_keys need to be a list. Received {0}".format(type(classify_keys))

    assert set(feature_keys).issubset(input_dict.keys()), \
        "Feature keys are not present in input dictionary."
    assert regress_keys is None or set(regress_keys).issubset(set(input_dict.keys())), \
        "Regress keys are not present in input dictionary."
    assert classify_keys is None or set(classify_keys).issubset(set(input_dict.keys())), \
        "Classify keys are not present in input dictionary."

    def _manual_shuffle(_X, _y):
        assert _X.shape[0] == _y.shape[0]
        random_index = numpy.random.permutation(_y.shape[0])
        return _X[random_index], _y[random_index]

    def _preprocess_to_numpy(x, len_shape: int):
        """ convert the features into a 2D numpy tensor (n, p) and the targets into a 1D numpy tensor (n) """
        assert isinstance(len_shape, int) and (len_shape == 1 or len_shape == 2)
        if isinstance(x, torch.Tensor):
            x = x.flatten(end_dim=-len_shape)
            assert len(x.shape) == len_shape
            return x.cpu().numpy()
        elif isinstance(x, numpy.ndarray):
            assert len(x.shape) == len_shape
            return x
        elif isinstance(x, list):
            assert len_shape == 1
            return numpy.array(x)

    def _do_regression(_X, _y, x_key, y_key):
        _X, _y = _manual_shuffle(_X, _y)
        _tmp_dict = {}

        if n_splits > 1:
            rkf = RepeatedKFold(n_splits=n_splits, n_repeats=n_repeats)
            for train_index, test_index in rkf.split(_X, _y):
                _X_train, _X_test, _y_train, _y_test = _X[train_index], _X[test_index], _y[train_index], _y[test_index]
                regressor.fit(_X_train, _y_train)

                _tmp_dict["x_key"] = _tmp_dict.get("x_key", []) + [x_key]
                _tmp_dict["y_key"] = _tmp_dict.get("y_key", []) + [y_key]
                _tmp_dict["r2_train"] = _tmp_dict.get("r2_train", []) + [regressor.score(_X_train, _y_train)]
                _tmp_dict["r2_test"] = _tmp_dict.get("r2_test", []) + [regressor.score(_X_test, _y_test)]
            _df_tmp = pandas.DataFrame(_tmp_dict, index=numpy.arange(rkf.get_n_splits()))
        elif n_splits == 1 and n_repeats == 1:
            regressor.fit(_X, _y)
            _tmp_dict = {
                "x_key": x_key,
                "y_key": y_key,
                "r2": regressor.score(_X, _y)
            }
            _df_tmp = pandas.DataFrame(_tmp_dict, index=[0])
        else:
            raise Exception("Does not make sense to have n_splits = {0} and n_repeats = {1}".format(n_splits,
                                                                                n_repeats))
        return _df_tmp

    def _do_classification(_X, _y, x_key, y_key):
        _X, _y = _manual_shuffle(_X, _y)
        _tmp_dict = {}

        if n_splits > 1:

            rkf = RepeatedStratifiedKFold(n_splits=n_splits, n_repeats=n_repeats)
            for train_index, test_index in rkf.split(_X, _y):
                _X_train, _X_test, _y_train, _y_test = _X[train_index], _X[test_index], _y[train_index], _y[test_index]
                classifier.fit(_X_train, _y_train)

                _tmp_dict["x_key"] = _tmp_dict.get("x_key", []) + [x_key]
                _tmp_dict["y_key"] = _tmp_dict.get("y_key", []) + [y_key]
                _tmp_dict["accuracy_train"] = _tmp_dict.get("accuracy_test", []) + [classifier.score(_X_train, _y_train)]
                _tmp_dict["accuracy_test"] = _tmp_dict.get("accuracy_test", []) + [classifier.score(_X_test, _y_test)]
            _df_tmp = pandas.DataFrame(_tmp_dict, index=numpy.arange(rkf.get_n_splits()))

        elif n_splits == 1 and n_repeats == 1:
            classifier.fit(_X, _y)
            _tmp_dict = {
                "x_key": x_key,
                "y_key": y_key,
                "accuracy": classifier.score(_X, _y)
            }
            _df_tmp = pandas.DataFrame(_tmp_dict, index=[0])
        else:
            raise Exception("Does not make sense to have n_splits = {0} and n_repeats = {1}".format(n_splits,
                                                                                n_repeats))
        return _df_tmp

    # loop over everything to make the predictions
    df = None
    for feature_key in feature_keys:
        X_all = _preprocess_to_numpy(input_dict[feature_key], len_shape=2)

        if classify_keys is not None:
            for kc in classify_keys:
                if verbose:
                    print("{0} classify {1}".format(feature_key, kc))
                y_all = _preprocess_to_numpy(input_dict[kc], len_shape=1)
                tmp_df = _do_classification(X_all, y_all, x_key=feature_key, y_key=kc)
                df = tmp_df if df is None else df.merge(tmp_df, how='outer')

        if regress_keys is not None:
            for kr in regress_keys:
                if verbose:
                    print("{0} regress {1}".format(feature_key, kr))
                y_all = _preprocess_to_numpy(input_dict[kr], len_shape=1)
                tmp_df = _do_regression(X_all, y_all, x_key=feature_key, y_key=kr)
                df = tmp_df if df is None else df.merge(tmp_df, how='outer')

    return df

def knn_classification_regression(world_dict: dict, val_iomin_threshold: float):
    """
    Utility function to perform knn-based classification and regression.
    It takes a dictionaries with path-level features and annotations.
    A set of (weakly) overlapping patches (with Intersection over Minimum smaller than the assigned threshold)
    is selected. A knn classifier/regressor is trained to
    predict the various annotations starting from the features.
    This process is repeated multiple times for all feature-annotation pairs to produce a confidence interval.

    Args:
        world_dict: a dictionaries with path-level features and annotations.
            The dict must contains the keys "patches_xywh" and "classify_tissue_label".
            It may contain additional keys starting in "feature_", "classify_" or "regress_".
        val_iomin_threshold: threshold for the Intersection over Minimum. It must be in [0.0, 1.0).
    """

    # compute the patch_to_patch overlap just one at the beginning
    assert {"patches_xywh", "classify_tissue_label"}.issubset(world_dict.keys())
    patches = world_dict["patches_xywh"]
    initial_score = torch.rand_like(patches[:, 0].float())
    tissue_ids = world_dict["classify_tissue_label"]
    nms_mask_n, overlap_nn = NonMaxSuppression.compute_nm_mask(
        score=initial_score,
        ids=tissue_ids,
        patches_xywh=patches,
        iom_threshold=val_iomin_threshold)
    binarized_overlap_nn = (overlap_nn > val_iomin_threshold).float()
    # print("# non-overlapping patches->", nms_mask_n.sum())

    # figure out the keys for the features, regression and classification
    feature_keys, regress_keys, classify_keys = [], [], []
    for key in world_dict.keys():
        if key.startswith("regress"):
            regress_keys.append(key)
        elif key.startswith("classify"):
            classify_keys.append(key)
        elif key.startswith("pca_") or key.startswith("umap_") or key.startswith("feature"):
            feature_keys.append(key)

    # define regressor and classifier
    def exclude_self(d):
        # This has shape: d.shape = (n_points, n_neighbours)
        # This function relies on the fact that the distances are sorted in increasing order
        w = numpy.ones_like(d)
        w[:, 0] = 0.0
        return w

    kn_kargs = {
        "n_neighbors": 5,
        "weights": exclude_self,
    }

    regressor = KNeighborsRegressor(**kn_kargs)
    classifier = KNeighborsClassifier(**kn_kargs)

    # loop over subset made of non-overlapping patches
    df_tot = None
    for n in range(20):
        # create a dictionary with only non-overlapping patches to test kn-regressor/classifier
        nms_mask_n = NonMaxSuppression.perform_nms_selection(mask_overlap_nn=binarized_overlap_nn,
                                                             score_n=torch.rand_like(initial_score),
                                                             possible_n=torch.ones_like(initial_score).bool())
        world_dict_subset = subset_dict(input_dict=world_dict, mask=nms_mask_n)

        df_tmp = classify_and_regress(
            input_dict=world_dict_subset,
            feature_keys=feature_keys,
            regress_keys=regress_keys,
            classify_keys=classify_keys,
            regressor=regressor,
            classifier=classifier,
            n_repeats=1,
            n_splits=1,
            verbose=False,
        )
        df_tot = df_tmp if df_tot is None else df_tot.merge(df_tmp, how='outer')

    df_tot["combined_key"] = df_tot["x_key"] + "_" + df_tot["y_key"]
    df_mean = df_tot.groupby("combined_key").mean()
    df_std = df_tot.groupby("combined_key").std()
    return df_mean, df_std

def linear_classification_regression(world_dict: dict, val_iomin_threshold: float):
    """
    Utility function to perform linear classification and regression.
    It takes a dictionaries with path-level features and annotations.
    A set of (weakly) overlapping patches (with Intersection over Minimum smaller than the assigned threshold)
    is selected. A linear classifier/regressor is trained to
    predict the various annotations starting from the features.
    This process is repeated multiple times for all feature-annotation pairs to produce a confidence interval.

    Args:
        world_dict: a dictionaries with path-level features and annotations.
            The dict must contains the keys "patches_xywh" and "classify_tissue_label".
            It may contain additional keys starting in "feature_", "classify_" or "regress_".
        val_iomin_threshold: threshold for the Intersection over Minimum. It must be in [0.0, 1.0).
    """

    # compute the patch_to_patch overlap just one at the beginning
    assert {"patches_xywh", "classify_tissue_label"}.issubset(world_dict.keys())
    patches = world_dict["patches_xywh"]
    initial_score = torch.rand_like(patches[:, 0].float())
    tissue_ids = world_dict["classify_tissue_label"]
    nms_mask_n, overlap_nn = NonMaxSuppression.compute_nm_mask(
        score=initial_score,
        ids=tissue_ids,
        patches_xywh=patches,
        iom_threshold=val_iomin_threshold)
    binarized_overlap_nn = (overlap_nn > val_iomin_threshold).float()
    # print("# non-overlapping patches->", nms_mask_n.sum())

    # figure out the keys for the features, regression and classification
    feature_keys, regress_keys, classify_keys = [], [], []
    for key in world_dict.keys():
        if key.startswith("regress"):
            regress_keys.append(key)
        elif key.startswith("classify"):
            classify_keys.append(key)
        elif key.startswith("pca_") or key.startswith("umap_") or key.startswith("feature"):
            feature_keys.append(key)

    # define regressor and classifier
    ridge_kargs = {
        "alphas": (1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1.0),
    }

    regressor = RidgeCV(**ridge_kargs)
    classifier = RidgeClassifierCV(**ridge_kargs)

    # loop over subset made of non-overlapping patches
    df_tot = None
    for n in range(20):
        # create a dictionary with only non-overlapping patches to test kn-regressor/classifier
        nms_mask_n = NonMaxSuppression.perform_nms_selection(mask_overlap_nn=binarized_overlap_nn,
                                                             score_n=torch.rand_like(initial_score),
                                                             possible_n=torch.ones_like(initial_score).bool())
        world_dict_subset = subset_dict(input_dict=world_dict, mask=nms_mask_n)

        df_tmp = classify_and_regress(
            input_dict=world_dict_subset,
            feature_keys=feature_keys,
            regress_keys=regress_keys,
            classify_keys=classify_keys,
            regressor=regressor,
            classifier=classifier,
            n_repeats=1,
            n_splits=5,
            verbose=False,
        )
        df_tot = df_tmp if df_tot is None else df_tot.merge(df_tmp, how='outer')

    df_tot["combined_key"] = df_tot["x_key"] + "_" + df_tot["y_key"]
    df_mean = df_tot.groupby("combined_key").mean()
    df_std = df_tot.groupby("combined_key").std()
    return df_mean, df_std

class SslModelBase(LightningModule):
    """
    Base class for the self-supervised learning (ssl) models (Vae, Dino, Barlow, Simclr).
    This base class is responsible for the validation (which is common to all ssl models) and some logging.
    The child classes need to implement :meth:`head_and_backbone_embeddings_step`, :meth:`forward` and
    :meth:`training_step`.
    """
    def __init__(self, val_iomin_threshold: float):
        super(SslModelBase, self).__init__()
        self.val_iomin_threshold = val_iomin_threshold
        self.neptune_run_id = None

    def head_and_backbone_embeddings_step(self, x) -> (torch.Tensor, torch.Tensor):
        # must be overwritten by child class
        # this generates both head and backbone embeddings
        raise NotImplementedError

    def forward(self, x) -> torch.Tensor:
        # must be overwritten by child class
        # this is the stuff that will generate the backbone embeddings
        raise NotImplementedError

    def training_step(self, batch, batch_idx) -> dict:
        # must be overwritten by child class
        raise NotImplementedError

    def get_metadata_to_regress(self, metadata) -> dict:
        try:
            return self.trainer.datamodule.get_metadata_to_regress(metadata)
        except AttributeError:
            return dict()

    def get_metadata_to_classify(self, metadata) -> dict:
        try:
            return self.trainer.datamodule.get_metadata_to_classify(metadata)
        except AttributeError:
            return dict()

    @property
    def n_global_crops(self):
        return self.trainer.datamodule.n_global_crops

    @property
    def n_local_crops(self):
        return self.trainer.datamodule.n_local_crops

    @property
    def trsfm_train_local(self):
        return self.trainer.datamodule.trsfm_train_local

    @property
    def trsfm_train_global(self):
        return self.trainer.datamodule.trsfm_train_global

    @property
    def trsfm_test(self):
        return self.trainer.datamodule.trsfm_test

    def __log_example_images__(self, which_loaders: str, n_examples: int = 10, n_cols: int = 5):
        if which_loaders == "val":
            loaders = self.trainer.datamodule.val_dataloader()
            log_name = "val_imgs"
        elif which_loaders == "train":
            loaders = self.trainer.datamodule.train_dataloader()
            log_name = "train_imgs"
        elif which_loaders == "predict":
            loaders = self.trainer.datamodule.predict_dataloader()
            log_name = "predict_imgs"
        else:
            raise Exception("Invalid value for which_loaders. Expected 'val' or 'train' or 'predict'. \
            Received={0}".format(which_loaders))

        if not isinstance(loaders, Sequence):
            loaders = [loaders]

        for idx_dataloader, loader in enumerate(loaders):
            indeces = torch.randperm(n=loader.dataset.__len__())[:n_examples]
            list_imgs, _, _ = loader.load(index=indeces)
            list_imgs = list_imgs[:n_examples]

            tmp_ref = self.trsfm_test(list_imgs)
            tmp_ref_plot = show_raw_all_channels(tmp_ref, n_col=n_cols, show_colorbar=True)
            self.logger.run[log_name + "/ref_" + str(idx_dataloader)].log(File.as_image(tmp_ref_plot))

            if which_loaders == 'train':
                tmp_global = self.trsfm_train_global(list_imgs)
                tmp_global_plot = show_raw_all_channels(tmp_global, n_col=n_cols, show_colorbar=True)
                self.logger.run[log_name+"/global"].log(File.as_image(tmp_global_plot))

                tmp_local = self.trsfm_train_local(list_imgs)
                tmp_local_plot = show_raw_all_channels(tmp_local, n_col=n_cols, show_colorbar=True)
                self.logger.run[log_name + "/local"].log(File.as_image(tmp_local_plot))

    def on_train_start(self) -> None:
        if self.global_rank == 0:
            self.__log_example_images__(n_examples=10, n_cols=5, which_loaders="train")

    def on_predict_start(self) -> None:
        if self.global_rank == 0:
            self.__log_example_images__(n_examples=10, n_cols=5, which_loaders="predict")

    def validation_step(self, batch, batch_idx, dataloader_idx: int = -1) -> Dict[str, torch.Tensor]:
        list_imgs: List[torch.sparse.Tensor]
        list_labels: List[int]
        list_metadata: List[MetadataCropperDataset]
        list_imgs, list_labels, list_metadata = batch

        # Compute the embeddings
        img = self.trsfm_test(list_imgs)
        z, y = self.head_and_backbone_embeddings_step(img)

        # Collect the xywh for the patches in the validation
        w, h = img.shape[-2:]
        patch_x = torch.tensor([metadata.loc_x for metadata in list_metadata], dtype=z.dtype, device=z.device)
        patch_y = torch.tensor([metadata.loc_y for metadata in list_metadata], dtype=z.dtype, device=z.device)
        patch_w = w * torch.ones_like(patch_x)
        patch_h = h * torch.ones_like(patch_x)
        patches_xywh = torch.stack([patch_x, patch_y, patch_w, patch_h], dim=-1)

        # Create the validation dictionary. Note that all the entries are torch.Tensors
        val_dict = {
            "features_bbone": y,
            "features_head": z,
            "patches_xywh": patches_xywh
        }

        # Add to this dictionary the things I want to classify and regress
        dict_classify = concatenate_list_of_dict([self.get_metadata_to_classify(metadata)
                                                  for metadata in list_metadata])
        for k, v in dict_classify.items():
            val_dict["classify_"+k] = torch.tensor(v, device=self.device)

        dict_regress = concatenate_list_of_dict([self.get_metadata_to_regress(metadata)
                                                 for metadata in list_metadata])
        for k, v in dict_regress.items():
            val_dict["regress_" + k] = torch.tensor(v, device=self.device)

        return val_dict

    def validation_epoch_end(self, list_of_val_dict) -> None:
        """ You can receive a list_of_valdict or, if you have multiple val_datasets a list_of_list_of_valdict """
        print("inside validation epoch end")

        if isinstance(list_of_val_dict[0], dict):
            list_dict = [concatenate_list_of_dict(list_of_val_dict)]
        elif isinstance(list_of_val_dict[0], list):
            list_dict = [concatenate_list_of_dict(tmp_list) for tmp_list in list_of_val_dict]
        else:
            raise Exception("In validation epoch end. I received an unexpected input")

        for loader_idx, total_dict in enumerate(list_dict):
            print("rank {0} dataloader_idx {1}".format(self.global_rank, loader_idx))

            # gather dictionaries from the other processes and flatten the extra dimension.
            world_dict = self.all_gather(total_dict, sync_grads=False)
            all_keys = list(world_dict.keys())
            for key in all_keys:
                if len(world_dict[key].shape) == 1 + len(total_dict[key].shape):
                    world_dict[key] = world_dict[key].flatten(end_dim=1)
            print("done dictionary. rank {0}".format(self.global_rank))

            # DO operations ONLY on rank 0.
            # ADD "rank_zero_only=True" to avoid deadlocks on synchronization.
            if self.global_rank == 0:

                # plot the UMAP colored by all available annotations
                smart_pca = SmartPca(preprocess_strategy='z_score')
                smart_umap = SmartUmap(n_neighbors=25, preprocess_strategy='raw',
                                       n_components=2, min_dist=0.5, metric='euclidean')

                print("starting with embeddings")
                embedding_keys = []
                annotation_keys = []
                umap_keys = []
                all_keys = list(world_dict.keys())
                for k in all_keys:
                    if k.startswith("feature"):
                        embedding_keys.append(k)
                        # print("working on feature", k)
                        input_features = world_dict[k]
                        embeddings_pca = smart_pca.fit_transform(input_features, n_components=0.95)
                        embeddings_umap = smart_umap.fit_transform(embeddings_pca)
                        world_dict['pca_' + k] = embeddings_pca
                        world_dict['umap_' + k] = embeddings_umap
                        umap_keys.append('umap_' + k)
                    elif k.startswith("regress") or k.startswith("classify"):
                        annotation_keys.append(k)
                print("done with embeddings")

                print("starting to make umaps")
                all_files = []
                for umap_key in umap_keys:
                    fig_tmp = plot_embeddings(
                        input_dictionary=world_dict,
                        embedding_key=umap_key,
                        annotation_keys=annotation_keys,
                        n_col=2,
                        sup_title="{0} epoch= {1}".format(umap_key, self.current_epoch)
                    )
                    all_files.append(File.as_image(fig_tmp))
                print("done making umaps")

                print("starting to log the umaps")
                for file_tmp, key_tmp in zip(all_files, embedding_keys):
                    self.logger.run["maps/" + key_tmp].log(file_tmp)
                print("printed the embeddings")

                # knn classification/regression
                print("starting knn classification/regression")
                df_mean_knn, df_std_knn = knn_classification_regression(world_dict, self.val_iomin_threshold)
                # print("df_mean_knn ->", df_mean_knn)

                for row in df_mean_knn.itertuples():
                    for k, v in row._asdict().items():
                        if isinstance(v, float) and numpy.isfinite(v):
                            name = "kn/" + row.Index + "/" + k + "/mean"
                            self.log(name=name, value=v, batch_size=1, rank_zero_only=True)

                for row in df_std_knn.itertuples():
                    for k, v in row._asdict().items():
                        if isinstance(v, float) and numpy.isfinite(v):
                            name = "kn/" + row.Index + "/" + k + "/std"
                            self.log(name=name, value=v, batch_size=1, rank_zero_only=True)

                # linear classification/regression
                print("starting linear classification/regression")
                df_mean_linear, df_std_linear = linear_classification_regression(world_dict, self.val_iomin_threshold)
                # print("df_mean_linear ->", df_mean_linear)

                for row in df_mean_linear.itertuples():
                    for k, v in row._asdict().items():
                        if isinstance(v, float) and numpy.isfinite(v):
                            name = "linear/" + row.Index + "/" + k + "/mean"
                            self.log(name=name, value=v, batch_size=1, rank_zero_only=True)

                for row in df_std_linear.itertuples():
                    for k, v in row._asdict().items():
                        if isinstance(v, float) and numpy.isfinite(v):
                            name = "linear/" + row.Index + "/" + k + "/std"
                            self.log(name=name, value=v, batch_size=1, rank_zero_only=True)

    def optimizer_zero_grad(self, epoch, batch_idx, optimizer, optimizer_idx):
        optimizer.zero_grad(set_to_none=True)

    def on_load_checkpoint(self, checkpoint: Dict[str, Any]) -> None:
        """ Loading and resuming is handled automatically. Here I am dealing only with the special variables """
        self.neptune_run_id = checkpoint.get("neptune_run_id", None)

    def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None:
        """ Loading and resuming is handled automatically. Here I am dealing only with the special variables """
        checkpoint["neptune_run_id"] = getattr(self.logger, "_run_short_id", None)


# psutil wird zum Auslesen der CPU-Last unter Windows verwendet (Muss jedoch installiert werden).
# Unter pycharm (File > Settings > Project: pythonProject >Python interpreter > +psutil).
# Unter Python direkt oder anderen IDEs (CMD> "py -m pip install psutil").
import psutil
import os

# Die Zeit gibt, in Sekunden, an wie lange das Interval zur Messung der CPU-Last ist (Minimum=0.1)
zeit = 1

# Gibt die Anzahl der Durchläufe des Programms an (Minimum=1 für einen Durchlauf)
loop = 60

# Variablen zur bestimmung der CPU Belastungsgrenzen (Minimum und Maximum der moderaten/gelben Grenze) maxgelb>mingelb!
mingelb = 50
maxgelb = 85

def cpulast():
    # Kommando aus der PSutil Bibliothek um die Auslastung der CPU in einem Intervall anzugeben.
    cpulast = psutil.cpu_percent(interval=zeit)
    # Kommando aus os um die Konsole zu leeren (funktioniert nicht in der IDE-Konsole)
    os.system('cls')
    print(cpulast, "% CPU-Auslastung")
    return cpulast

def belastung(cpulast):
    # Wenn die CPU-Last unter der gelben Mindestgrenze somit unter gelb/moderat
    if cpulast < mingelb:
        print("Die CPU-Auslastung liegt unter", mingelb, "%, alles im grünen Bereich!")
    # Wenn die CPU-Last unter der gelben Maximum-Grenze somit innerhalb von gelb/moderat
    elif cpulast < maxgelb:
        print("Die CPU-Auslastung liegt zwischen ", mingelb, "%  und ", maxgelb, "%. moderate Auslastung!")
    else:
        print("Über ", maxgelb, "% CPU-Auslastung, starke Auslastung!")

# Visualisierung der CPU-Last mit 100 Strichen welche Parallel zur oberen Auswertung ablaufen
def visualize(cpulast):
    # Neue Variable um diese von float auf int zu setzen und +0,5 um mathematisch korrekt zu runden
    cpulastint = cpulast + 0.5
    cpulastint = int(cpulastint)
    # visuello ist zur Visualisierung der nicht genutzte CPU-Ressourcen
    # visuell1,2,3 für die Stärken der Auslastungen um diese farblich zu markieren 1=grün 2=gelb 3=rot
    # '\033[32m' stehen für Farbcodes um die Visualisierung farblich darzustellen '\x1b[0m' zum reset der farben.
    visuello = 100 - cpulastint
    visuello = visuello * "-"
    if cpulastint < mingelb:
        visuell1 = cpulastint * "|"
        print("[" + '\033[92m' + visuell1 + '\x1b[0m' + visuello + "]")
    elif cpulastint < maxgelb:
        visuell1 = mingelb * "|"
        calc = cpulastint - mingelb
        visuell2 = calc * "|"
        print("[" + '\033[92m' + visuell1 + '\033[33m' + visuell2 + '\x1b[0m' + visuello + "]")
    else:
        visuell1 = mingelb * "|"
        gelb = maxgelb-mingelb
        visuell2 = gelb * "|"
        calc = cpulastint - maxgelb
        visuell3 = calc * "|"
        print("[" + '\033[92m' + visuell1 + '\033[33m' + visuell2 + '\033[91m' + visuell3 + '\x1b[0m' + visuello + "]")

# Loop welche alle funktionen chronologisch durchgeht und diese anspricht
while loop > 0:
    loop -= 1
    # Zieht, die Variable last aus der Funktion cpulast heraus
    last = cpulast()
    # Beide funktionen nehmen die Variable Last hinein
    belastung(last)
    visualize(last)


import time

date = time.strptime("2019-06-24 06:32:48.0", "%Y-%m-%d %H:%M:%S.%f")
print(int(time.mktime(date))*1000)

class Temp:
    number = None

    @classmethod
    def get_number(cls):
        if cls.number is None:
            print('Number is None')
            cls.number = 5
        else:
            print('Number not none')
        return cls.number

temp1 = Temp()
temp1.get_number()

temp2 = Temp()
temp2.get_number()

temp3 = Temp()
temp3.get_number()

class Rule:
    def __init__(self, a, b, c):
        self.a = a
        self.b = b
        self.c = c

rules = []

for i in range(1,5):
    rules.append(Rule(i, i+2, i+3))

rules = [{"a": dat.a, "b": dat.b} for dat in rules]
print("rules: {}".format(rules))


# This script takes the following GET params:
#   * pid: A pcap ID
#   * host: The hostname of the device to get the pcap from
#   * stime: The search_time window, formatted as YYYY/MM/DD HH:MM:SS
#   * serial: Optional.  Serial number of the firewall if host is panorama

import os
import sys

libpath = os.path.dirname(os.path.abspath(__file__))
sys.path[:0] = [os.path.join(libpath, 'lib')]
sys.path[:0] = [os.path.join(libpath, 'lib', 'pan-python', 'lib')]
sys.path[:0] = [os.path.join(libpath, 'lib', 'pandevice')]

import time
import common
import json
import splunk
import splunk.mining.dcutils as dcu

import pandevice.base
import pandevice.panorama
import pandevice.firewall

logger = dcu.getLogger()

class Cfg(object):
    """Config is a separate object for testing purposes."""
    def __init__(self):
        self.skey = None
        self.hostname = None
        self.search_time = ''
        self.serial = None
        self.pcap_id = 0
        self.api_key = None

    def parse(self, skey, req):
        # Initialize all config.
        self.skey = skey

        #sc = common.SplunkConnector(self.skey, logger)
        self.hostname = req['query']['host']
        self.search_time = req['query']['stime']
        self.serial = req['query'].get('serial', None)
        self.pcap_id = req['query']['pid']
        if self.pcap_id <= 0:
            raise ValueError('{0} is not a valid pcap ID'.format(self.pcap_id))

        self.api_key = common.apikey(self.skey, self.hostname)

class MasterP(splunk.rest.BaseRestHandler):
    def handle_GET(self):
        # Get and process query params.
        logger.info('Initiated pcap retrieval, parsing request config')
        cfg = Cfg()
        cfg.parse(self.sessionKey, self.request)

        # Connect to the firewall / panorama.
        logger.info('Creating connection object for given hostname')
        o = pandevice.base.PanDevice.create_from_device(
            cfg.hostname, api_key=cfg.api_key)

        # Set the serial target (if present and this is a panorama).
        if isinstance(o, pandevice.panorama.Panorama):
            if not cfg.serial:
                raise ValueError('Host is panorama, serial number is required')
            logger.info('This is a panorama device, creating sub-object')
            dev = pandevice.firewall.Firewall(serial=cfg.serial)
            o.add(dev)
        else:
            dev = o

        # Retrieve the pcap file.
        logger.info('Retrieving pcap file')
        dev.xapi.export(
            'threat-pcap', search_time=cfg.search_time, pcapid=cfg.pcap_id)
        if not dev.xapi.export_result or not dev.xapi.export_result['content']:
            raise ValueError('File not present or file is empty')

        # Send pcap to the user.
        self.response.setHeader('Content-Type', 'application/vnd.tcpdump.pcap')
        self.response.write(dev.xapi.export_result['content'])
        logger.info('Done')


import numpy as np
import torch

def get_volume(shape):
    """
    An arbitrary volume form.
    """
    return np.sum(np.ones(shape))

def normalize(I):
    """
    Normalize a density
    """
    factor = get_volume(I.shape)/I.sum()
    return I*factor

from diffeopt.cometric import laplace

def get_random_diffeo(group, nb_steps=10, scale=1.):
    cometric = laplace.get_laplace_cometric(group, s=2)
    rm = np.random.randn(*group.zero().shape)
    rv = cometric(rm)
    vmx = rv.abs().max()
    shape_scale = (group.shape[0] + group.shape[1])/2
    rv_ = rv/nb_steps/shape_scale**2*scale*32
    current = group.element()
    for i in range(nb_steps):
        current = group.exponential(rv_).compose(current)
    return current


import streamlit as st
import pandas as pd
import numpy as np
from xgboost import XGBRegressor

st.set_page_config(page_title="AirBnB Price Predictor in New York City", page_icon=":house:", layout="wide")

st.markdown("
Predicting Airbnb Prices in New York City
", unsafe_allow_html=True)

st.sidebar.header("Enter your Airbnb Details")

def user_input():
    txt1 = st.sidebar.subheader("Enter location")
    latitude = st.sidebar.number_input('Latitude', 40.499790, 40.913060, 40.755942, step=1e-7, format="%.6f")
    longitude = st.sidebar.number_input('Longitude', -74.244420	, -73.712990, -73.977619, step=1e-7, format="%.6f")
    availability_365 = st.sidebar.number_input('Number of days available per year', 1, 365, 24)
    minimum_nights = st.sidebar.number_input('What is the minimum number of days a visitor must reserve for your listing ?', 1, 12, 2)
    host_activity = st.sidebar.number_input('How many active listings do you have ?', 0, 629, 4)
    reviews_per_month = st.sidebar.number_input('How many reviews per month do your listing receive ?', 0, 58, 5)
    number_of_reviews = st.sidebar.number_input('In total, how many reviews does your listing have? ', 0, 629, 4)
    time_since_last_review = st.sidebar.number_input('When was your last reviews (in days) ?', 0, 4000, 50)
    #neighbourhood_popularity = st.sidebar.selectbox('Neighbourhood popularity', 0.00, 10.00, 1.43)
    txt2 = st.sidebar.subheader("Choose 1 if your listing is located in one of these boroughs (leave 0 for others)")
    dummy_Bronx = st.sidebar.selectbox('Bronx', (0, 1))
    dummy_Brooklyn = st.sidebar.selectbox('Brooklyn', (0, 1))
    dummy_Manhattan = st.sidebar.selectbox('Manhattan', (0, 1))
    dummy_Queens = st.sidebar.selectbox('Queens', (0, 1))
    dummy_Staten_Island = st.sidebar.selectbox('Staten Island', (0, 1))
    txt3 = st.sidebar.subheader("Choose 1 if your listing is one of the following (leave 0 for others) : ")
    dummy_Entire_home_apt = st.sidebar.selectbox('Entire Home / Appartment', (0, 1))
    dummy_Private_room = st.sidebar.selectbox('Private Room', (0, 1))
    dummy_Shared_room = st.sidebar.selectbox('Share Room', (0, 1))
    txt4 = st.sidebar.subheader("How far (in kilometers) is your listing from : ?")
    distance_to_wall_street = st.sidebar.number_input('Wall Street', 0, 31, 23)
    distance_to_closest_station = st.sidebar.number_input('Closest Metro Station', 0, 31, 23)

    data = {'latitude':latitude,
            'longitude':longitude,
            'availability_365': availability_365,
            'minimum_nights': minimum_nights,
            'host_activity':  host_activity,
            'reviews_per_month': reviews_per_month,
            'number_of_reviews': number_of_reviews,
            'time_since_last_review': time_since_last_review,
            #'neighbourhood_popularity': neighbourhood_popularity,
            'dummy_Bronx': dummy_Bronx,
            'dummy_Brooklyn': dummy_Brooklyn,
            'dummy_Manhattan': dummy_Manhattan,
            'dummy_Queens': dummy_Queens,
            'dummy_Staten_Island': dummy_Staten_Island,
            'dummy_Entire_home_apt':  dummy_Entire_home_apt,
            'dummy_Private_room': dummy_Private_room,
            'dummy_Shared_room': dummy_Shared_room,
            'distance_to_wall_street': distance_to_wall_street,
            'distance_to_closest_station': distance_to_closest_station
    }
    appt_parametre = pd.DataFrame(data,index=[0])
    return appt_parametre

df = user_input()

model = XGBRegressor()
model.load_model("/Users/Koko/Desktop/xg_boost6.json")
y_pred = model.predict(df)

price = round(np.exp(y_pred[0]))

st.caption("
Price Per Night
", unsafe_allow_html=True)

m2 = st.metric("", "$"+f"{price:,}")
css='''
[data-testid="metric-container"] {
    width: fit-content;
    margin: auto;
}

[data-testid=m2] > div {
    width: fit-content;
    margin: auto;
}

[data-testid="metric-container"] label {
    width: fit-content;
    margin: auto;
}
'''

st.markdown(f'',unsafe_allow_html=True)

st.subheader("Property Location")
st.map(df)


import random
def hangman():
    words=["car","house","city","bridge","football","player","president"]
    word=words[random.randint(0,6)]
    wrong=0
    stages=["",
            "_____    ",
            "|        ",
            "|    |   ",
            "|    0   ",
            "|   /|\  ",
            "|   / \  ",
            "|_       "]
    rletters= list(word)
    board="_"*len(word)
    win = False
    print("Gra w wisielca")
    while wrong<=len(stages)-1:
        print("\n")
        msg="Odgadnij litere: "
        char = input(msg)
        if char in rletters:
            cind = rletters.index(char)
            board=board[:cind]+char+board[cind+1:]
            rletters[cind] = "$"
        else:
            wrong+=1
        print((" ".join(board)))
        e = wrong+1
        print("\n".join(stages[0:e]))
        if "_" not in board:
            print("Wygrales!")
            print(" ".join(board))
            win=True
            break
    if not win:
        print("\n".join(stages[0:wrong]))
        print("Przegrales! Miales odgadnac: {}.".format(word))
hangman()


class Node:
  def __init__(self, e=None, n=None, p=None):
    self.element = e
    self.next = n
    self.prev = p

class DoublyList:

  def __init__(self, a):
      self.head = None
      if type(a) == list:
          for i in a:
              if self.head == None:
                 self.head = Node(i)
                 cur_Node = self.head
              else:
                  cur_Node.next = Node(i)
                  cur_Node.next.prev = cur_Node
                  cur_Node = cur_Node.next

  def countNode(self):
      count = 0
      cur_Node = self.head
      while cur_Node != None:
          count += 1
          cur_Node = cur_Node.next
      return count

  def forwardprint(self):
    cur_node = self.head
    while cur_node != None:
        print(cur_node.element, end=" ")
        cur_node = cur_node.next
    print()

  def backwardprint(self):
    cur_node = self.head
    while cur_node.next != None:
        cur_node = cur_node.next
    while cur_node != None:
        print(cur_node.element, end=' ')
        cur_node = cur_node.prev
    print()

  def nodeAt(self, idx):
    curIndex = 0
    curNode = self.head
    while curIndex != idx and curNode != None:
        curIndex += 1
        curNode = curNode.next
    return curNode

  def indexOf(self, elem):
    curNode = self.head
    idx = 0 # 1  2
    while curNode.element != elem and curNode != None:
        idx += 1
        curNode = curNode.next
    if curNode.element == None:
        return -1
    else:
        return idx

  def insert(self, elem, idx):
      curNode = self.head
      curIdx = 0
      if idx == 0:
          curNode = self.head
          self.head = Node(elem, curNode)
          curNode.prev = self.head
          return
      while idx != curIdx and curNode != None:
          curIdx += 1
          prev = curNode
          curNode = curNode.next
      if curIdx < idx:
          return None
      else:
          prev.next = Node(elem, curNode, prev)
          if curNode != None:
           curNode.prev = prev.next

  def remove(self, idx):
    curNode = self.head
    curIdx = 0
    if idx == 0:
        val = self.head.element
        self.head = self.head.next
        self.head.prev = None
        return str(val)
    while idx != curIdx and curNode != None:
        curIdx += 1
        prev = curNode
        curNode = curNode.next
    if curIdx < idx:
        return None
    else:
        val = curNode.element
        prev.next = curNode.next
        if curNode.next != None:
          curNode.next.prev = prev
        return str(val)

print("///  Test 01  ///")
a1 = [10, 20, 30, 40]
h1 = DoublyList(a1) # Creates a linked list using the values from the array

h1.forwardprint() # This should print: 10,20,30,40.
h1.backwardprint() # This should print: 40,30,20,10.
print(h1.countNode()) # This should print: 4

print("///  Test 02  ///")
# returns the reference of the at the given index. For invalid idx return None.
myNode = h1.nodeAt(2)
print(myNode.element) # This should print: 30. In case of invalid index This will print "index error"

print("///  Test 03  ///")
# returns the index of the containing the given element. if the element does not exist in the List, return -1.
index = h1.indexOf(40)
print(index) # This should print: 3. In case of element that
#doesn't exists in the list this will print -1.

print("///  Test 04  ///")

a2 = [10, 20, 30, 40]
h2 = DoublyList(a2) # uses the  constructor
h2.forwardprint() # This should print: 10,20,30,40.

# inserts containing the given element at the given index. Check validity of index.
h2.insert(85,0)
h2.forwardprint() # This should print: 85,10,20,30,40.
h2.backwardprint() # This should print: 40,30,20,10,85.

print()
h2.insert(95,3)
h2.forwardprint() # This should print: 85,10,20,95,30,40.
h2.backwardprint() # This should print: 40,30,95,20,10,80.

print()
h2.insert(75,6)
h2.forwardprint() # This should print: 85,10,20,95,30,40,75.
h2.backwardprint() # This should print: 75,40,30,95,20,10,85.

print("///  Test 05  ///")
a3 = [10, 20, 30, 40, 50, 60, 70]
h3 = DoublyList(a3) # uses the constructor
h3.forwardprint() # This should print: 10,20,30,40,50,60,70.

# removes at the given index. returns element of the removed node. Check validity of index. return None if index is invalid.
print("Removed element: "+ h3.remove(0)) # This should print: Removed element: 10
h3.forwardprint() # This should print: 20,30,40,50,60,70.
h3.backwardprint() # This should print: 70,60,50,40,30,20.
print("Removed element: "+ h3.remove(3)) # This should print: Removed element: 50
h3.forwardprint() # This should print: 20,30,40,60,70.
h3.backwardprint() # This should print: 70,60,40,30,20.
print("Removed element: "+ h3.remove(4)) # This should print: Removed element: 70
h3.forwardprint() # This should print: 20,30,40,60.
h3.backwardprint() # This should print: 60,40,30,20.


import numpy as np

def matrix(Tin,w):
    M = Tin.copy()
    N = len(M)

    for i in range(N):
        for j in range(N):
            M[i,j] = w*M[i,j]*delta(i+1,j+1) + delta(i+1,j+2) + delta(i+2,j+1)
    return M

def delta(i, j):
    if i == j:
        return 1.0
    else:
        return 0.0


from fastapi import FastAPI
# from starlette.requests import Request
# from starlette.responses import Response

from app.auth.router import router as auth_router
from app.forum.router import router as forum_router
from app.profile.router import router as profile_router
from app.subjects.router import router as subject_router
# from app.database import SessionLocal

tags_metadata = [
    {
        "name": "forum",
        "description": "MetaBSUIR forum",
    },
    {
        "name": "authentication",
        "description": "MetaBSUIR authentication",
    },
    {
        "name": "profile",
        "description": "Members profile",
    },
    {
        "name": "subjects",
        "description": "Subjects profile",
    },
]

def get_application() -> FastAPI:
    application = FastAPI(
        title="MetaBSUIR",
        description="BSUIR meta-universe",
        version="0.1.0",
        openapi_tags=tags_metadata,
    )
    application.include_router(auth_router)
    application.include_router(forum_router)
    application.include_router(profile_router)
    application.include_router(subject_router)
    return application

app = get_application()


"""
    Undistort images in Human3.6M and save them alongside (in ".../imageSequence-undistorted/...").

    Usage: `python3 undistort-h36m.py <path/to/Human3.6M-root> <path/to/human36m-multiview-labels.npy> <num-processes>`
"""
# code from https://github.com/karfly/learnable-triangulation-pytorch
import torch
import numpy as np
import cv2
from tqdm import tqdm
from collections import defaultdict

import os, sys

h36m_root = os.path.expanduser('~/data/human36m/training/subject/processed')
number_of_processes = 8
labels_path = os.path.expanduser('~/data/human36m/training/subject/extra/human36m-multiview-labels-GTbboxes.npy')
labels = np.load(labels_path, allow_pickle=True).item()
# print("Dataset length:", len(labels['table']))

n_subjects = len(labels['subject_names'])
n_cameras = len(labels['camera_names'])

# First, prepare: compute distorted meshgrids
print("Computing distorted meshgrids")
meshgrids = np.empty((n_subjects, n_cameras), dtype=object)

class Camera:
    def __init__(self, R, t, K, dist=None, name=""):
        self.R = np.array(R).copy()
        assert self.R.shape == (3, 3)

        self.t = np.array(t).copy()
        assert self.t.size == 3
        self.t = self.t.reshape(3, 1)

        self.K = np.array(K).copy()
        assert self.K.shape == (3, 3)

        self.dist = dist
        if self.dist is not None:
            self.dist = np.array(self.dist).copy().flatten()

        self.name = name

    def update_after_crop(self, bbox):
        left, upper, right, lower = bbox

        cx, cy = self.K[0, 2], self.K[1, 2]

        new_cx = cx - left
        new_cy = cy - upper

        self.K[0, 2], self.K[1, 2] = new_cx, new_cy

    def update_after_resize(self, image_shape, new_image_shape):
        height, width = image_shape
        new_height, new_width = new_image_shape

        fx, fy, cx, cy = self.K[0, 0], self.K[1, 1], self.K[0, 2], self.K[1, 2]

        new_fx = fx * (new_width / width)
        new_fy = fy * (new_height / height)
        new_cx = cx * (new_width / width)
        new_cy = cy * (new_height / height)

        self.K[0, 0], self.K[1, 1], self.K[0, 2], self.K[1, 2] = new_fx, new_fy, new_cx, new_cy

    @property
    def projection(self):
        return self.K.dot(self.extrinsics)

    @property
    def extrinsics(self):
        return np.hstack([self.R, self.t])

def shot2sample(shot, labels):
    sample = defaultdict(list) # return value

    subject = labels['subject_names'][shot['subject_idx']]
    action = labels['action_names'][shot['action_idx']]
    frame_idx = shot['frame_idx']

    for camera_idx, camera_name in enumerate(labels['camera_names']):
        # load bounding box
        bbox = shot['bbox_by_camera_tlbr'][camera_idx][[1,0,3,2]] # TLBR to LTRB
        bbox_height = bbox[2] - bbox[0]
        if bbox_height == 0:
            # convention: if the bbox is empty, then this view is missing
            continue

        # load image
        image_path = os.path.join(
            os.path.expanduser('~/data/human36m/training/subject/processed'),
            subject, action, 'imageSequence',
            camera_name, 'img_%06d.jpg' % (frame_idx+1))
        assert os.path.isfile(image_path), '%s doesn\'t exist' % image_path
        image = cv2.imread(image_path)

        # load camera
        shot_camera = labels['cameras'][shot['subject_idx'], camera_idx]
        retval_camera = Camera(shot_camera['R'], shot_camera['t'], shot_camera['K'], shot_camera['dist'], camera_name)
        sample['cameras'].append(retval_camera)
        sample['images'].append(image)

    # save sample's index
    sample.default_factory = None
    return sample

for sample_idx in tqdm(range(len(labels['table']))):
    subject_idx = labels['table']['subject_idx'][sample_idx]

    if not meshgrids[subject_idx].any():
        bboxes = labels['table']['bbox_by_camera_tlbr'][sample_idx]

        if (bboxes[:, 2] - bboxes[:, 0]).min() > 0: # if == 0, then some camera is missing
            sample = shot2sample(labels['table'][sample_idx], labels)
            assert len(sample['images']) == n_cameras

            for camera_idx, (camera, image) in enumerate(zip(sample['cameras'], sample['images'])):
                h, w = image.shape[:2]

                fx, fy = camera.K[0, 0], camera.K[1, 1]
                cx, cy = camera.K[0, 2], camera.K[1, 2]

                grid_x = (np.arange(w, dtype=np.float32) - cx) / fx
                grid_y = (np.arange(h, dtype=np.float32) - cy) / fy
                meshgrid = np.stack(np.meshgrid(grid_x, grid_y), axis=2).reshape(-1, 2)

                # distort meshgrid points
                k = camera.dist[:3].copy(); k[2] = camera.dist[-1]
                p = camera.dist[2:4].copy()

                r2 = meshgrid[:, 0] ** 2 + meshgrid[:, 1] ** 2
                radial = meshgrid * (1 + k[0] * r2 + k[1] * r2**2 + k[2] * r2**3).reshape(-1, 1)
                tangential_1 = p.reshape(1, 2) * np.broadcast_to(meshgrid[:, 0:1] * meshgrid[:, 1:2], (len(meshgrid), 2))
                tangential_2 = p[::-1].reshape(1, 2) * (meshgrid**2 + np.broadcast_to(r2.reshape(-1, 1), (len(meshgrid), 2)))

                meshgrid = radial + tangential_1 + tangential_2

                # move back to screen coordinates
                meshgrid *= np.array([fx, fy]).reshape(1, 2)
                meshgrid += np.array([cx, cy]).reshape(1, 2)

                # cache (save) distortion maps
                meshgrids[subject_idx, camera_idx] = cv2.convertMaps(meshgrid.reshape((h, w, 2)), None, cv2.CV_16SC2)

# Now the main part: undistort images
def undistort_and_save(idx):
    sample = shot2sample(labels['table'][idx], labels)

    shot = labels['table'][idx]
    subject_idx = shot['subject_idx']
    action_idx  = shot['action_idx']
    frame_idx   = shot['frame_idx']

    subject = labels['subject_names'][subject_idx]
    action = labels['action_names'][action_idx]

    available_cameras = list(range(len(labels['action_names'])))
    for camera_idx, bbox in enumerate(shot['bbox_by_camera_tlbr']):
        if bbox[2] == bbox[0]: # bbox is empty, which means that this camera is missing
            available_cameras.remove(camera_idx)

    for camera_idx, image in zip(available_cameras, sample['images']):
        camera_name = labels['camera_names'][camera_idx]

        output_image_folder = os.path.join(
            h36m_root, subject, action, 'imageSequence-undistorted', camera_name)
        output_image_path = os.path.join(output_image_folder, 'img_%06d.jpg' % (frame_idx+1))
        os.makedirs(output_image_folder, exist_ok=True)

        meshgrid_int16 = meshgrids[subject_idx, camera_idx]
        image_undistorted = cv2.remap(image, *meshgrid_int16, cv2.INTER_CUBIC)

        cv2.imwrite(output_image_path, image_undistorted)

print(f"Undistorting images using {number_of_processes} parallel processes")
cv2.setNumThreads(1)
import multiprocessing

pool = multiprocessing.Pool(number_of_processes)
for _ in tqdm(pool.imap_unordered(
    undistort_and_save, range(len(labels['table'])), chunksize=10), total=len(labels['table'])):
    pass

pool.close()
pool.join()


import argparse
import pandas as pd
import os
from happytransformer import HappyTextToText
import progressbar
from multiprocessing import freeze_support
from params import testing_args

def parseArguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('process', help='evaluate a csv file or generate corrected sentences, eval/gen', default='generate')
    parser.add_argument('model_path', help='path to the model', type=str)
    parser.add_argument('-s', '--sentence', help='if you want to predict only one sentence, default=None', type=str, default=None)
    parser.add_argument('-f', '--file', help='if you want to multiple sentences in a file, default=None', type=str, default=None)
    args = parser.parse_args()
    return args

def prepare_data(df):
    df.columns = ['input', 'target']
    df['input'] = 'grammar: ' + df['input']
    return df

if __name__ == '__main__':
    freeze_support()
    args = parseArguments()

    model = HappyTextToText('t5', args.model_path)

    if args.process == 'eval':
        if args.file:
                testing_data = pd.read_csv(args.file)
                processed_data = prepare_data(testing_data)
                test_file = os.path.join(os.path.dirname(args.file),'prepared_'+os.path.basename(args.file))
                processed_data.to_csv(test_file, index=False)

                result = model.eval(test_file)
                print(result)
        else:
            print('CSV File is neccessary for the evaluation...')
            exit()

    if args.process == 'gen':
        if args.sentence:
            generated_text = model.generate_text('grammar: '+args.sentence, args=testing_args)
            print('\nCorrected sentence: '+generated_text.text)

        if args.file:
            read_file = open(args.file, 'r', encoding='utf8')
            write_filename = os.path.join(os.path.dirname(args.file), 'corrected_'+os.path.basename(args.file))
            write_file = open(write_filename, 'w', encoding='utf8')

            lines = read_file.readlines()
            text_list = []

            print('\nGenerating correction file...')
            bar = progressbar.ProgressBar(maxval=len(lines), widgets=[progressbar.Bar(), ' ', progressbar.SimpleProgress()])
            bar.start()
            sentence_count = 0
            for line in lines:
                line = 'grammar: ' + line.strip()
                output = model.generate_text(line, args=testing_args)
                text_list.append(line[9:] + ' --> ' +output.text.strip()+'\n')
                sentence_count += 1
                bar.update(sentence_count)
            write_file.writelines(text_list)
            bar.finish()

            read_file.close()
            write_file.close()
            print('File stored at '+ write_filename)

        if not args.sentence and not args.file:
            print('Either a sentence or a file is necessary...')
            exit()


import jieba.posseg as pseg
import codecs
from gensim import corpora
from gensim.summarization import bm25
import os
import re
from util import jsonutil, configutil, jiebautil
from util.linereader import LineReader
import json

DICTIONARY_PATH = configutil.config["sim"]["dic_path"]

def buildModel(jsonFile, fieldNames, query_str):
    # iterable 不能循环两次，所以创建两个变量
    t1 = jsonutil.iterCutFieldList(jsonFile, fieldNames)
    t2 = jsonutil.iterCutFieldList(jsonFile, fieldNames)

    # 建立单词索引字典
    dictionary = corpora.Dictionary(t1)
    dictionary.save(DICTIONARY_PATH)

    # 建立词袋模型.将词汇表示的文本，转换成用id表示
    corpus = [dictionary.doc2bow(text) for text in t2]
    print("词袋: %i " % len(corpus))

    bm25Model = bm25.BM25(corpus)

    # print("bm25 idf lens: %i " %len(bm25Model.f))

    average_idf = sum(map(lambda k: float(bm25Model.idf[k]), bm25Model.idf.keys())) / len(bm25Model.idf.keys())

    query = jiebautil.cutWords(query_str).split()
    query_bow = dictionary.doc2bow(query)

    scores = bm25Model.get_scores(query_bow, average_idf)
    # i = scores.index(max(scores))

    lineRead = LineReader(jsonFile)
    for i in range(5):
        score = max(scores)
        lineNum = scores.index(score) + 1
        s = lineRead.load(lineNum)
        j = json.loads(s)
        print(jsonutil.recursive_get(j, fieldNames[0]))

        del scores[lineNum-1]


#!/usr/bin/env python

import json

if __name__== "__main__":
   import argparse

   parser = argparse.ArgumentParser(description='Arguments.')

   parser.add_argument('-f','-file', dest='json_file',action='store',default='whitelist.json',
                       help='JSON file')

   args = parser.parse_args()

   with open(args.json_file, "r") as rf:
      val = json.load(rf)

   with open(args.json_file, "w") as wf:
      json.dump(val,wf,indent=4)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import pytest
from pypengine.skeleton import fib

__author__ = "Francesco Strozzi"
__copyright__ = "Francesco Strozzi"
__license__ = "none"

def test_fib():
    assert fib(1) == 1
    assert fib(2) == 1
    assert fib(7) == 13
    with pytest.raises(AssertionError):
        fib(-10)


import os, sys
import time
import argparse
import numpy as np

import tensorflow as tf

sys.path.append('../')
from src import helper as hp
from src import fixed_parameters as fpf

parser = argparse.ArgumentParser(description='Run experiment')
parser.add_argument('-i','--chunk_id', type=int, help='Id of the current data chunk', required=True)
parser.add_argument('-n','--n_chunk', type=int, help='Total number of chunks', required=True)
parser.add_argument('-r','--root', type=str, help='Root for models and data', required=True)
parser.add_argument('-m','--mode', type=str, help='If test, will run on a small data to test. Else choose deploy', required=True)

def do_chunkify(lst,n):
    """get n chunk of equal length except last to cover lst"""
    return [lst[i::n] for i in range(n)]

def one_hot_encode(tokenized_input, n_chars):
    output = np.zeros([len(tokenized_input), n_chars])
    for j, token in enumerate(tokenized_input):
        output[j, int(token)] = 1
    return output

def get_tok_smi(smi, token_indices, max_len_model, start_char, end_char, pad_char):

    token_list = hp.smi_tokenizer(smi)
    token_list = [start_char] + token_list + [end_char]
    padding = [pad_char]*(max_len_model - len(token_list))
    token_list.extend(padding)
    token_list = np.array([token_indices[x] for x in token_list])

    return token_list

def get_tok_prot(prot, token_indices, max_len_model, start_char, end_char, pad_char):

    token_list = list(prot)
    token_list = [start_char] + token_list + [end_char]
    padding = [pad_char]*(max_len_model - len(token_list))
    token_list.extend(padding)
    token_list = np.array([token_indices[x] for x in token_list])

    return token_list

# SMILES fixed parameters
s_pad_char = fpf.s_pad_char
s_start_char = fpf.s_start_char
s_end_char = fpf.s_end_char
s_indices_token = fpf.s_indices_token
s_token_indices = fpf.s_token_indices

# PROTEIN fixed parameters
p_pad_char = fpf.p_pad_char
p_start_char = fpf.p_start_char
p_end_char = fpf.p_end_char
p_indices_token = fpf.p_indices_token
p_token_indices = fpf.p_token_indices

if __name__ == '__main__':

    start = time.time()

    ####################################
    # get back parameters
    args = vars(parser.parse_args())

    verbose = True
    chunk_id = args['chunk_id']
    n_chunk = args['n_chunk']
    root = args['root']
    mode = args['mode']

    if mode not in ['test', 'deploy']:
        raise ValueError('Your mode does not exist. Pick: test or deploy')

    if mode=='test':
        savepath = f'{root}results/'
    else:
        savepath = f'{root}results/test/'
    os.makedirs(savepath, exist_ok=True)

    if verbose: print('\nSTART PREDICTIONS')
    ####################################

    ####################################
    if mode=='test':
        all_smi = hp.read_with_pd(f'{root}data_clean/small.txt')
    elif mode=='deploy':
        all_smi = hp.read_with_pd(f'{root}data_clean/all_smi_clean.txt')

    if n_chunk>1:
        data_chunkified = do_chunkify(all_smi, n_chunk)

        # -1 because the arg to the bash script will go,
        # eg for 10, from 1 to 10 included
        # so we want idx 0 to 9
        chunk_id = chunk_id-1
        all_smi = data_chunkified[chunk_id]
        print(f'N of data in the current chunk: {len(all_smi)}')

    n_data = len(all_smi)
    max_len_model = 100+2
    p_n_chars = len(p_token_indices)
    s_n_chars = len(s_token_indices)

    s_data = np.empty([n_data, max_len_model])

    for i,s in enumerate(all_smi):
        s_data[i] = get_tok_smi(s, s_token_indices, max_len_model, s_start_char, s_end_char, s_pad_char)

    # protein
    protein = hp.read_with_pd(f'{root}data_clean/rcsb_pdb_6LZE.txt')
    p_max_len_model = 1000+2
    p_data = np.empty([1, p_max_len_model])

    p_data[0] = get_tok_prot(protein, p_token_indices, p_max_len_model, p_start_char,
                             p_end_char, p_pad_char)
    ####################################

    ####################################
    # start prediction
    n_model = 5
    all_y_preds = {}

    # we have only one prot
    p = p_data[0]
    p = one_hot_encode(list(p), p_n_chars)
    p = np.expand_dims(p, axis=0)

    for i in range(1,n_model+1):
        print(f'\ncurrent model: {i}')

        path = f'{root}A02_5repeats/{i}/16/last.h5'
        model = tf.keras.models.load_model(path)

        l_start = time.time()
        log_every = 200
        for n in range(n_data):
            print('current data ', n)
            if n%log_every==0 and n!=0:
                l_end = time.time()
                l_time = l_end - l_start
                print(f'current data: {n}')
                print(f'{log_every} data done in: {l_time:.05} seconds')
                l_start = time.time()

            s = s_data[n]
            s = one_hot_encode(list(s), s_n_chars)
            s = np.expand_dims(s, axis=0)

            logit = model.predict([p,s])
            pred = tf.nn.sigmoid(logit)
            fpred = float(pred.numpy())

            if all_smi[n] in all_y_preds:
                all_y_preds[all_smi[n]].append(fpred)
            else:
                all_y_preds[all_smi[n]] = [fpred]

        hp.save_pkl(f'{savepath}chunk_{chunk_id}_model_{i}_all_y_preds.pkl', all_y_preds)

    end = time.time()
    print(f'PREDICTIONS DONE in {end - start:.05} seconds')
    ####################################


#PyGame Zero warmup on timelines
#Brygg Ullmer, Clemson University
#Begun 2022-01-23

t1, t2 = Actor('1860a'), Actor('1880a')

TITLE = "Timeline ex02a"
WIDTH = 1920; HEIGHT = 1080

t1.topleft= 0,   10
t2.topleft= 600, 10

def on_mouse_down(pos):
  animate(t1, tween='accel_decel', pos=pos, duration=0.5)

  #if t1.collidepoint(pos): updateT1(pos)
  #if t2.collidepoint(pos): updateT2(pos)

def draw():
  screen.clear()
  t1.draw(); t2.draw()

### end ###


# -*- coding: utf-8 -*-

import argparse
import numpy as np
import h5py
import os
import cv2

def prepare(args):
    h5_file = h5py.File(args.output_path, 'w')
    noise_patchs = list()
    source_patchs = list()

    for count, image_file in enumerate(os.listdir(args.images_dir)):
        if count < 6000:
            continue
        print(image_file)
        source_img = cv2.imread(os.path.join(args.images_dir, image_file))
        # source_img = cv2.resize(source_img, (256, 256))
        ret, noise_buf = cv2.imencode(".JPG", source_img, [int(cv2.IMWRITE_JPEG_QUALITY), 10])
        noise_img = cv2.imdecode(noise_buf, 1)
        source_img = source_img.transpose(2, 0, 1)
        noise_img = noise_img.transpose(2, 0, 1)
        # print(source_img.shape)

        # source_img = np.array(source_img)
        # noise_img = np.array(noise_img)
        # noise_ds.append(noise_img)
        # source_ds.append(source_img)
        for i in range(0, source_img.shape[1] - args.patch_size + 1, args.stride):
            for j in range(0, source_img.shape[2] - args.patch_size + 1, args.stride):
                noise_patchs.append(noise_img[:, i:i + args.patch_size, j:j + args.patch_size])
                source_patchs.append(source_img[:, i:i + args.patch_size, j:j + args.patch_size])
        if count > 6010 :
            break

    noise_ds = np.array(noise_patchs, dtype=np.uint8)
    source_ds = np.array(source_patchs, dtype=np.uint8)
    print(noise_ds.shape)
    h5_file.create_dataset('noise', data=noise_ds)
    h5_file.create_dataset('source', data=source_ds)
    h5_file.close()
    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--images-dir', type=str, required=True)
    parser.add_argument('--output-path', type=str, required=True)
    parser.add_argument('--patch-size', type=int, default=32)
    parser.add_argument('--stride', type=int, default=16)
    args = parser.parse_args()

    prepare(args)


import pandas as pd
import datetime
import torch
import torch.nn as nn
from statsmodels.tsa.stattools import adfuller
import numpy as np
import time

import random

import plotly.graph_objects as go

header = {'open time' : 1, 'open' : 2, 'high' : 3, 'low' : 4, 'close' : 5}

#load_data from file
def load_data(year, symbol, interval = "1m", normalise = True):
    if(type(year) != type([])):
        year = [year]

    if(type(symbol) != type([])):
        symbol = [symbol]

    data = []
    for s in symbol:
        frames = []
        for y in year:
            csv_data = pd.read_csv("data_{}/{}_{}.csv".format(interval, y, s))
            frames.append(csv_data)

        data_ = pd.concat(frames)

        #convert timestamp into month and day numbers
        data_['Hour'] = pd.to_datetime(data_["Open Time"], unit='ms').apply(lambda x: x.hour)
        data_['Day'] = pd.to_datetime(data_["Open Time"], unit='ms').apply(lambda x: x.day - 1)
        data_['Month'] = pd.to_datetime(data_["Open Time"], unit='ms').apply(lambda x: x.month - 1)
        data.append(data_)

    print("done")
    return data

def softmax(x, dim = 0):
    return np.exp(x) / np.sum(np.exp(x), axis=dim)

class Indexer():
    def __init__(self, r_bottom, r_top, batch_size, random = True, increment = 1):
        self.r_bottom = r_bottom
        self.r_top = r_top
        self.random = random
        self.increment = increment
        self.batch_size = batch_size
        self.indices = [0]
        self.losses = np.ones([r_top - r_bottom]) * 100
        self.next()

    def next(self, losses = None):
        if(self.random):
#             if (losses is not None):
#                 self.losses[np.array(self.indices) - self.r_bottom] = losses
#                 #new_indicies = self.losses.argsort()[-self.batch_size:][::-1] + self.r_bottom
#                 #self.indices = new_indicies

#                 new_indicies = np.arange(self.r_bottom, self.r_top)
#                 self.indices = np.random.choice(new_indicies , self.batch_size, p = softmax(self.losses ** (1/4), -1))
#                 return self.indices

            ####
            new_indices = []
            for b in range(self.batch_size):
                new_indices.append(random.randrange(self.r_bottom, self.r_top))
            self.indices = new_indices
        else:
#             new_indices = [self.indices[-1]]

#             for b in range(1, self.batch_size):
#                 i = new_indices[-1] + self.increment
#                 if(i >= self.r_top):
#                     new_indices.append((i - self.top) + self.r_bottom)
#                 else:
#                     new_indices.append(i)
#             self.indices = new_indices
            self.indices = np.arange(self.indices[-1] + self.increment, self.indices[-1] + self.increment + self.batch_size, self.increment)

        return self.indices

def get_batches(data_, in_seq_len, out_seq_len, con_cols, disc_cols, target_cols, one_hot_lens = [24, 31, 12], batch_size = 1, gpu = True, normalise = True, indexer = None, norm = None):
    data = data_.copy()

    given_indexer = True
    if indexer is None:
        given_indexer = False
        indexer = Indexer(1, data.shape[0] - (in_seq_len + out_seq_len + 1), batch_size)

    if normalise:
        if norm is None:
            norm = data
        data[con_cols] = (data[con_cols] - norm[con_cols].stack().mean()) / norm[con_cols].stack().std()

    #convert columns indices from dataframe to numpy darray
    con_cols = [data.columns.get_loc(x) for x in con_cols]
    disc_cols = [data.columns.get_loc(x) for x in disc_cols]
    target_cols = [data.columns.get_loc(x) for x in target_cols]

    if(not gpu):
        dtype = torch.FloatTensor
    else:
        dtype = torch.cuda.FloatTensor

    while True:
        #get batches
        n = np.array([pd.np.r_[i:(i + in_seq_len + out_seq_len)] for i in indexer.indices])
        batch_data = data.iloc[n.flatten()].values
        batch_data = torch.tensor(batch_data.reshape(batch_size ,in_seq_len + out_seq_len, data.shape[-1]))

        mask = ~torch.isnan(batch_data).any(2)
        batch_data[~mask] = 0

        #split up batch data
        in_seq_continuous = batch_data[:, 0:in_seq_len, con_cols]
        in_seq_discrete = batch_data[:, 0:in_seq_len, disc_cols]

        out_seq = batch_data[:,in_seq_len:in_seq_len + out_seq_len, disc_cols]
        target_seq = batch_data[:,in_seq_len:in_seq_len + out_seq_len, target_cols]

        yield in_seq_continuous, in_seq_discrete, out_seq, target_seq, mask

        if(not given_indexer):
            indexer.next()

def one_hot(x, dims, gpu = True):
    out = []
    batch_size = x.shape[0]
    seq_len = x.shape[1]

    if(not gpu):
        dtype = torch.FloatTensor
    else:
        dtype = torch.cuda.FloatTensor

    for i in range(0, x.shape[-1]):
        x_ = x[:,:,i].byte().cpu().long().unsqueeze(-1)
        o = torch.zeros([batch_size, seq_len, dims[i]]).long()

        o.scatter_(-1, x_,  1)
        out.append(o.type(dtype))
    return out


from pygel3d import hmesh, graph, gl_display as gl
from numpy import array
from numpy.linalg import norm
from scipy.spatial import KDTree
from time import time
from math import sqrt

t0 = time()
f = open("/Users/healpa/Documents/git/GEL_tree_geometries/data/point_clouds/WinterTree_pts_clean_filtered.off")
#f = open("/Users/healpa/Documents/git/GEL_tree_geometries/data/point_clouds/Quercus_rubra.txt")

lns = f.readlines()
g = graph.Graph()
pts = []
for l in lns[2:]:
    p = array([ float(num) for num in l.split() ])
    pts += [p]
    g.add_node(p)
print(f"Tree has {len(g.nodes())} nodes")
print(f"Time elapsed: {time()-t0:.5} seconds\n")

t0 = time()
pts = array(pts)
T = KDTree(pts)
print("Made KDTree")
print(f"Time elapsed: {time()-t0:.5} seconds\n")

t0 = time()
conn = 0
for i,j in T.query_pairs(r=0.025):
    g.connect_nodes(i,j)
    conn += 1
print(f"Connected {conn} nodes")
print(f"Time elapsed: {time()-t0:.5} seconds\n")

t0 = time()
graph.edge_contract(g, dist_thresh=0.01)
print("Contracted edges")
print(f"Time elapsed: {time()-t0:.5} seconds\n")

t0 = time()
graph.saturate(g, hops=10, rad=0.05, dist_frac=0.95)
print("Saturated graph")
print(f"Time elapsed: {time()-t0:.5} seconds\n")

#V = gl.Viewer()
#V.display(g, bg_col=[1,1,1])

t0 = time()
s = graph.LS_skeleton(g, True)
print("Made skeleton")
print(f"Time elapsed: {time()-t0:.5} seconds")

# t0 = time()
# s = graph.MSLS_skeleton(g, grow_thresh=128)
# print("Made skeleton")
# print(f"Time elapsed: {time()-t0:.5} seconds")

t0 = time()
sc = graph.color_detached_parts(s)
print("Colored detached parts of the skeleton")
print(f"Time elapsed: {time()-t0:.5} seconds")
V = gl.Viewer()
V.display(sc, bg_col=[1,1,1])
graph.save("tree_skel_col.graph", sc)
print("Saved graph")

# t0 = time()
# m = graph.to_mesh_iso(s, fudge=0.01, res=1024)
# print("Made iso mesh")
# print(f"Time elapsed: {time()-t0:.5} seconds")

# t0 = time()
# hmesh.quadric_simplify(m, 0.05)
# print("Simplified mesh")
# print(f"Time elapsed: {time()-t0:.5} seconds")
# V.display(m)
# hmesh.save("tree_iso_reduced.obj", m)


"""
Display a bar's height and name on top of it upon hovering
==========================================================

Using an event handler to change the annotation text and position.
"""

import string
import matplotlib.pyplot as plt
import mplcursors

fig, ax = plt.subplots()
ax.bar(range(9), range(1, 10), align="center")
labels = string.ascii_uppercase[:9]
ax.set(xticks=range(9), xticklabels=labels, title="Hover over a bar")

# With HoverMode.Transient, the annotation is removed as soon as the mouse
# leaves the artist.  Alternatively, one can use HoverMode.Persistent (or True)
# which keeps the annotation until another artist gets selected.
cursor = mplcursors.cursor(hover=mplcursors.HoverMode.Transient)
@cursor.connect("add")
def on_add(sel):
    x, y, width, height = sel.artist[sel.index].get_bbox().bounds
    sel.annotation.set(text=f"{x+width/2}: {height}",
                       position=(0, 20), anncoords="offset points")
    sel.annotation.xy = (x + width / 2, y + height)

plt.show()


from __future__ import unicode_literals

from djblets.testing.testcases import TestCase
from djblets.util.templatetags.djblets_js import json_dumps

class JSTagTests(TestCase):
    """Unit tests for djblets_js template tags."""
    def test_json_dumps_xss(self):
        """Testing json_dumps doesn't allow XSS injection"""
        # This is bug 3406.
        obj = {
            'xss': '</script><script>alert(1);</script>'
        }

        self.assertEqual(
            json_dumps(obj),
            '{"xss": "\\u003C/script\\u003E\\u003Cscript\\u003E'
            'alert(1);\\u003C/script\\u003E"}')


class Solution(object):
    def topKFrequent(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: List[int]
        """
        # mapy {}
        # check for thres hold passing
        # thres = []
        # increment mapy items or add item in a loop
        # return thres

        mapy = {}
        thres = {}

        for i in range(len(nums)):
            q = nums[i]
            # add items to amp for memory of maxes
            if q in mapy:
                mapy[q] += 1
            else:
                mapy[q] = 1

        # {1:3,2:2,3:1}

        r = sorted(mapy.items(), key=lambda x:x[1], reverse=True)[:k]
        print(r)

        return [x[0] for x in r]


import copy
import unittest

from diffusers import AutoencoderKL, Transformer2DModel
from mmengine.registry import MODELS
from mmengine.testing import RunnerTestCase
from mmengine.testing.runner_test_case import ToyModel
from torch import nn
from transformers import T5EncoderModel

from diffengine.engine.hooks import FastNormHook
from diffengine.models.editors import (
    SDDataPreprocessor,
    SDXLControlNetDataPreprocessor,
    SDXLDataPreprocessor,
    StableDiffusion,
    StableDiffusionXL,
    StableDiffusionXLControlNet,
)
from diffengine.models.losses import L2Loss
from diffengine.models.utils import TimeSteps, WhiteNoise

try:
    import apex
except ImportError:
    apex = None

class ToyModelPixArt(ToyModel):

    def __init__(self) -> None:
        super().__init__()
        height = 12
        width = 12

        model_kwargs = {
            "attention_bias": True,
            "cross_attention_dim": 32,
            "attention_head_dim": height * width,
            "num_attention_heads": 1,
            "num_vector_embeds": 12,
            "num_embeds_ada_norm": 12,
            "norm_num_groups": 32,
            "sample_size": width,
            "activation_fn": "geglu-approximate",
        }

        self.transformer = Transformer2DModel(**model_kwargs)
        self.text_encoder = T5EncoderModel.from_pretrained(
            "hf-internal-testing/tiny-random-t5")
        self.vae = AutoencoderKL(
            in_channels=4,
            out_channels=4,
            down_block_types=("DownEncoderBlock2D",),
            up_block_types=("UpDecoderBlock2D",),
            block_out_channels=(32,),
            layers_per_block=1,
            act_fn="silu",
            latent_channels=4,
            norm_num_groups=16,
            sample_size=16,
        )
        self.finetune_text_encoder = True

    @property
    def device(self):
        return next(self.parameters()).device

    def forward(self, *args, **kwargs):
        return super().forward(*args, **kwargs)

class TestFastNormHook(RunnerTestCase):

    def setUp(self) -> None:
        MODELS.register_module(name="StableDiffusion", module=StableDiffusion)
        MODELS.register_module(
            name="StableDiffusionXL", module=StableDiffusionXL)
        MODELS.register_module(
            name="SDDataPreprocessor", module=SDDataPreprocessor)
        MODELS.register_module(
            name="SDXLDataPreprocessor", module=SDXLDataPreprocessor)
        MODELS.register_module(
            name="StableDiffusionXLControlNet",
            module=StableDiffusionXLControlNet)
        MODELS.register_module(
            name="SDXLControlNetDataPreprocessor",
            module=SDXLControlNetDataPreprocessor)
        MODELS.register_module(name="L2Loss", module=L2Loss)
        MODELS.register_module(name="WhiteNoise", module=WhiteNoise)
        MODELS.register_module(name="TimeSteps", module=TimeSteps)
        MODELS.register_module(name="ToyModelPixArt", module=ToyModelPixArt)
        return super().setUp()

    def tearDown(self) -> None:
        MODELS.module_dict.pop("StableDiffusion")
        MODELS.module_dict.pop("StableDiffusionXL")
        MODELS.module_dict.pop("SDDataPreprocessor")
        MODELS.module_dict.pop("SDXLDataPreprocessor")
        MODELS.module_dict.pop("StableDiffusionXLControlNet")
        MODELS.module_dict.pop("SDXLControlNetDataPreprocessor")
        MODELS.module_dict.pop("L2Loss")
        MODELS.module_dict.pop("WhiteNoise")
        MODELS.module_dict.pop("TimeSteps")
        MODELS.module_dict.pop("ToyModelPixArt")
        return super().tearDown()

    @unittest.skipIf(apex is None, "apex is not installed")
    def test_init(self) -> None:
        FastNormHook()

    @unittest.skipIf(apex is None, "apex is not installed")
    def test_before_train(self) -> None:  # noqa
        from apex.contrib.group_norm import GroupNorm
        from apex.normalization import FusedLayerNorm

        cfg = copy.deepcopy(self.epoch_based_cfg)
        cfg.model.type = "StableDiffusion"
        cfg.model.model = "diffusers/tiny-stable-diffusion-torch"
        runner = self.build_runner(cfg)
        hook = FastNormHook(fuse_text_encoder_ln=True, fuse_gn=True)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, nn.GroupNorm)
        assert isinstance(
            runner.model.text_encoder.text_model.encoder.layers[
                0].layer_norm1, nn.LayerNorm)
        # replace norm
        hook.before_train(runner)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, GroupNorm)
        assert isinstance(
            runner.model.text_encoder.text_model.encoder.layers[
                0].layer_norm1, FusedLayerNorm)

        # Test StableDiffusionXL
        cfg = copy.deepcopy(self.epoch_based_cfg)
        cfg.model.type = "StableDiffusionXL"
        cfg.model.model = "hf-internal-testing/tiny-stable-diffusion-xl-pipe"
        runner = self.build_runner(cfg)
        hook = FastNormHook(fuse_text_encoder_ln=True, fuse_gn=True)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, nn.GroupNorm)
        assert isinstance(
            runner.model.text_encoder_one.text_model.encoder.layers[
                0].layer_norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.text_encoder_two.text_model.encoder.layers[
                0].layer_norm1, nn.LayerNorm)
        # replace norm
        hook.before_train(runner)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, GroupNorm)
        assert isinstance(
            runner.model.text_encoder_one.text_model.encoder.layers[
                0].layer_norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.text_encoder_two.text_model.encoder.layers[
                0].layer_norm1, FusedLayerNorm)

        # Test StableDiffusionXLControlNet
        cfg = copy.deepcopy(self.epoch_based_cfg)
        cfg.model.type = "StableDiffusionXLControlNet"
        cfg.model.model = "hf-internal-testing/tiny-stable-diffusion-xl-pipe"
        cfg.model.controlnet_model = "hf-internal-testing/tiny-controlnet-sdxl"
        runner = self.build_runner(cfg)
        hook = FastNormHook(fuse_text_encoder_ln=True, fuse_gn=True)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.controlnet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, nn.GroupNorm)
        assert isinstance(
            runner.model.controlnet.down_blocks[0].resnets[0].norm2,
            nn.GroupNorm)
        assert isinstance(
            runner.model.text_encoder_one.text_model.encoder.layers[
                0].layer_norm1, nn.LayerNorm)
        assert isinstance(
            runner.model.text_encoder_two.text_model.encoder.layers[
                0].layer_norm1, nn.LayerNorm)
        # replace norm
        hook.before_train(runner)
        assert isinstance(
            runner.model.unet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.controlnet.down_blocks[
                1].attentions[0].transformer_blocks[0].norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.unet.down_blocks[0].resnets[0].norm2, GroupNorm)
        assert isinstance(
            runner.model.controlnet.down_blocks[0].resnets[0].norm2,
            GroupNorm)
        assert isinstance(
            runner.model.text_encoder_one.text_model.encoder.layers[
                0].layer_norm1, FusedLayerNorm)
        assert isinstance(
            runner.model.text_encoder_two.text_model.encoder.layers[
                0].layer_norm1, FusedLayerNorm)

        # test PixArt
        cfg = copy.deepcopy(self.epoch_based_cfg)
        cfg.model.type = "ToyModelPixArt"
        runner = self.build_runner(cfg)
        hook = FastNormHook(fuse_text_encoder_ln=True, fuse_gn=True)
        assert isinstance(
            runner.model.transformer.transformer_blocks[
                0].norm1.norm, nn.LayerNorm)
        # replace norm
        hook.before_train(runner)
        assert isinstance(
            runner.model.transformer.transformer_blocks[
                0].norm1.norm, FusedLayerNorm)


class Book:
    def __init__(self, id, title, author, price, rating):
        self._id = id
        self._title = title
        self._author = author
        self._price = price
        self._rating = rating
        self._next = None
        self._previous = None

    def __str__(self):
        return f'Book [Title= {self._title}, Author= {self._author}, Rating= {self._rating}, Price= {self._price}]'

class Node:
    def __init__(self, value,next=None, previous=None):
        self._value=value
        self._next=next
        self._previous=previous

class LinkedList:
    def __init__(self, *args):
        self._first=None
        self._last = None
        self.append(*args)

    def append(self, *args):
        for value in args:
            self._append(value)

    def _append(self, value):
        if not isinstance(value, Book):
            value = Node(value)
        if self._first == None:
            self._first = value
            self._last = self._first
        else:
            self._last._next = value
            value._previous = self._last
            self._last = value

    #def info(self):
    def __str__(self):
        if self._first==None:
            return "LinkedList(empty)"
        str="LinkedList(\t"
        n=self._first
        while n:
            if isinstance(n, Node):
                str+=f'{n._value}\t'
            else:
                str+=f'{n._title}\t'
            n=n._next
        str+=")"
        return str

    #def size(self):
    def __len__(self):
        c=0
        n=self._first
        while n:
            c+=1
            n=n._next
        return c

    def __contains__(self, item):
        ptr = self._first
        while ptr:
            if ptr._value == item:
                return True
            ptr = ptr._next
        return False

    def __locate(self,index):
        if index>=len(self):
            raise IndexError(f'Invalid Index :{index}')

        n=self._first
        for i in range(index):
            n=n._next

        return n

    #def get(self,index):
    def __getitem__(self,index):
        n=self.__locate(index)
        return n._value  #if n else None

    #def set(self,index,value):
    def __setitem__(self,index,value):
        n=self.__locate(index)
        n._value=value

    def insert(self, index, value):
        y=self.__locate(index)
        x=y._previous
        if not isinstance(value, Book):
            value=Node(value,previous=x,next=y)

        if x:
            x._next=value
        else:
            self._first=value

        y._previous=value

    def remove(self, index):
        n=self.__locate(index)

        if self._first == self._last:
            self._first = None
            self._last = None
            return n._value

        x= n._previous
        y= n._next

        if x:
            x._next=y
        else:
            self._first=y

        if y:
            y._previous=x
        else:
            self._last = self._last._previous
            self._last._next = None
        return n._value

    def count(self, element):
        count = 0
        ptr = self._first
        while ptr:
            if ptr._value == element: count+=1
            ptr = ptr._next
        return count

    def find_primes(self):
        primes = LinkedList()
        n = self._first
        while n:
            if isinstance(n, Node) and is_prime(n._value):
                primes.append(n._value)
            n = n._next
        return primes

    def find_evens(self):
        evens = LinkedList()
        n = self._first
        while n:
            if isinstance(n, Node) and n._value%2 == 0:
                evens.append(n._value)
            n = n._next
        return evens

    def _book_by_id(self, id):
        ptr = self._first
        while ptr:
            if isinstance(ptr, Book) and ptr._id == id:
                return ptr
            ptr = ptr._next
        return None

    def _books_by_author(self, author):
        books = LinkedList()
        ptr = self._first
        while ptr:
            if isinstance(ptr, Book) and ptr._author == author:
                new_book = Book(ptr._id, ptr._title, ptr._author, ptr._price, ptr._rating)
                books.append(new_book)
            ptr = ptr._next
        return books

    def _books_in_rating_range(self, min, max):
        books = LinkedList()
        ptr = self._first
        while ptr:
            if isinstance(ptr, Book) and ptr._rating>= min and ptr._rating<= max:
                new_book = Book(ptr._id, ptr._title, ptr._author, ptr._price, ptr._rating)
                books.append(new_book)
            ptr = ptr._next
        return books

    def _books_in_price_range(self, min, max):
        books = LinkedList()
        ptr = self._first
        while ptr:
            if isinstance(ptr, Book) and ptr._price>= min and ptr._price<= max:
                new_book = Book(ptr._id, ptr._title, ptr._author, ptr._price, ptr._rating)
                books.append(new_book)
            ptr = ptr._next
        return books

def is_prime(val):
    if val<2: return False
    else:
        i = 2
        while i < val:
            if val%i == 0:
                return False
            i+=1
        return True

book1 = Book(12, 'harry', 'jk rowling', 1200, 8.2)
list = LinkedList(2, 3, book1, 1, 4)
print(list)
even = list.find_evens()
print(even)
print(list._book_by_id(12))

book2 = Book(54, 'Harry Potter', 'J.K. Rowling', 1200, 8.2)
book3 = Book(10, 'Harry Potter: prisoner of azkaban', 'J.K. Rowling', 1500, 8.7)
book4 = Book(99, 'some random book', 'some random author', 800, 7.3)

list.append(12, book4, 7, book3, book2, 9, 5)

print(list)

books = list._books_in_rating_range(7, 8)
print(books)


import boto3
import os
from multiprocessing import Process

def InvokeLambda(lambdaClient, function, accountId, region, userId):
    tempString = '{"accountId":"' + accountId + '","region":"' + region + '","userId":"' + userId + '"}'
    lambdaClient.invoke(
        FunctionName=function,
        InvocationType='Event',
        Payload=tempString
    )

def lambda_handler(event, context):

    # Get features from event
    features = event['features']

    # Initialize Clients
    lambdaClient = boto3.client('lambda')

    # create a list to keep all processes
    processes = []

    # For each account
    for feature in features:
        # For each region enabled within an account
        # for region in account['regions']:
        #     for feature in region['features']:
                #If feature is enabled, add to appropriate message queue.

        if (feature['name'] == 'ec2-start'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnEc2Start'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'ec2-stop'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnEc2Stop'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'ec2-reboot'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnEc2Reboot'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'ec2-backup-manager'):
           process = Process(target=InvokeLambda, args=(lambdaClient,os.environ['LambdaArnEc2BackupManager'], feature['accountId'], feature['regionName'], feature['userId']))
           processes.append(process)
        elif (feature['name'] == 'rds-start'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnRdsStart'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'rds-stop'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnRdsStop'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'rds-reboot'):
            process = Process(target=InvokeLambda, args=(lambdaClient, os.environ['LambdaArnRdsReboot'], feature['accountId'], feature['regionName'], feature['userId']))
            processes.append(process)
        elif (feature['name'] == 'rds-backup-manager'):
           process = Process(target=InvokeLambda, args=(lambdaClient,os.environ['LambdaArnRdsBackupManager'], feature['accountId'], feature['regionName'], feature['userId']))
           processes.append(process)
        #if (region['ChaosManager')
        #if (region[Ec2Resize])
        #if (region[RdsResize])
        #if (region[Ec2DynamicDns]
        #if (region[RdsDynamicDns]

    print ("Number of features to distribute: " + str(len(features)))

    # start all processes
    for process in processes:
        process.start()

    # make sure that all processes have finished
    for process in processes:
        process.join()