import json
import logging
import os
from typing import Dict, Optional
import torch
import requests
from torch import nn
from torch import Tensor

logger = logging.getLogger(__name__)

from huggingface_hub.constants import CONFIG_NAME, PYTORCH_WEIGHTS_NAME
from huggingface_hub.file_download import (
    cached_download,
    hf_hub_url,
)
from huggingface_hub.hf_api import HfApi, HfFolder
from huggingface_hub.repository import Repository

StateDict = Dict[str, Tensor]

class HFModelHub:
    @staticmethod
    def save_pretrained(
        model: nn.Module,
        save_directory: str,
        config: Optional[dict] = None,
        push_to_hub: bool = False,
        **kwargs,
    ):
        """
        Saving weights in local directory.

        Parameters:
            save_directory (:obj:`str`):
                Specify directory in which you want to save weights.
            config (:obj:`dict`, `optional`):
                specify config (must be dict) incase you want to save it.
            push_to_hub (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Set it to `True` in case you want to push your weights to huggingface_hub
            model_id (:obj:`str`, `optional`, defaults to :obj:`save_directory`):
                Repo name in huggingface_hub. If not specified, repo name will be same as `save_directory`
            kwargs (:obj:`Dict`, `optional`):
                kwargs will be passed to `push_to_hub`
        """

        os.makedirs(save_directory, exist_ok=True)

        # saving config
        if isinstance(config, dict):
            path = os.path.join(save_directory, CONFIG_NAME)
            with open(path, "w") as f:
                json.dump(config, f)

        # saving model weights
        path = os.path.join(save_directory, PYTORCH_WEIGHTS_NAME)
        torch.save(model.state_dict(), path)

        if push_to_hub:
            return HFModelHub.push_to_hub(save_directory, **kwargs)

    @staticmethod
    def from_pretrained(
        pretrained_model_name_or_path: Optional[str],
        strict: bool = True,
        map_location: Optional[str] = "cpu",
        force_download: bool = False,
        resume_download: bool = False,
        proxies: Dict = None,
        use_auth_token: Optional[str] = None,
        cache_dir: Optional[str] = None,
        local_files_only: bool = False,
    ) -> StateDict:
        r"""
        Instantiate a pretrained pytorch model from a pre-trained model configuration from huggingface-hub.
        The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated). To
        train the model, you should first set it back in training mode with ``model.train()``.

        Parameters:
            pretrained_model_name_or_path (:obj:`str` or :obj:`os.PathLike`, `optional`):
                Can be either:
                    - A string, the `model id` of a pretrained model hosted inside a model repo on huggingface.co.
                      Valid model ids can be located at the root-level, like ``bert-base-uncased``, or namespaced under
                      a user or organization name, like ``dbmdz/bert-base-german-cased``.
                    - You can add `revision` by appending `@` at the end of model_id simply like this: ``dbmdz/bert-base-german-cased@main``
                      Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id,
                      since we use a git-based system for storing models and other artifacts on huggingface.co, so ``revision`` can be any identifier allowed by git.
                    - A path to a `directory` containing model weights saved using
                      :func:`~transformers.PreTrainedModel.save_pretrained`, e.g., ``./my_model_directory/``.
                    - :obj:`None` if you are both providing the configuration and state dictionary (resp. with keyword
                      arguments ``config`` and ``state_dict``).
            cache_dir (:obj:`Union[str, os.PathLike]`, `optional`):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            force_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to force the (re-)download of the model weights and configuration files, overriding the
                cached versions if they exist.
            resume_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to delete incompletely received files. Will attempt to resume the download if such a
                file exists.
            proxies (:obj:`Dict[str, str], `optional`):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., :obj:`{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to only look at local files (i.e., do not try to download the model).
            use_auth_token (:obj:`str` or `bool`, `optional`):
                The token to use as HTTP bearer authorization for remote files. If :obj:`True`, will use the token
                generated when running :obj:`transformers-cli login` (stored in :obj:`~/.huggingface`).
            model_kwargs (:obj:`Dict`, `optional`)::
                model_kwargs will be passed to the model during initialization
        .. note::
            Passing :obj:`use_auth_token=True` is required when you want to use a private model.
        """

        model_id = pretrained_model_name_or_path
        map_location = torch.device(map_location)

        revision = None
        if len(model_id.split("@")) == 2:
            model_id, revision = model_id.split("@")

        if model_id in os.listdir() and CONFIG_NAME in os.listdir(model_id):
            config_file = os.path.join(model_id, CONFIG_NAME)
        else:
            try:
                config_url = hf_hub_url(
                    model_id, filename=CONFIG_NAME, revision=revision
                )
                config_file = cached_download(
                    config_url,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    local_files_only=local_files_only,
                    use_auth_token=use_auth_token,
                )
            except requests.exceptions.RequestException:
                logger.warning("config.json NOT FOUND in HuggingFace Hub")
                config_file = None

        if model_id in os.listdir():
            print("LOADING weights from local directory")
            model_file = os.path.join(model_id, PYTORCH_WEIGHTS_NAME)
        else:
            model_url = hf_hub_url(
                model_id, filename=PYTORCH_WEIGHTS_NAME, revision=revision
            )
            model_file = cached_download(
                model_url,
                cache_dir=cache_dir,
                force_download=force_download,
                proxies=proxies,
                resume_download=resume_download,
                local_files_only=local_files_only,
                use_auth_token=use_auth_token,
            )

            logger.debug(model_file)

        if config_file is not None:
            with open(config_file, "r", encoding="utf-8") as f:
                config = json.load(f)
                # we are not using config

        state_dict = torch.load(model_file, map_location=map_location)

        return state_dict

    @staticmethod
    def push_to_hub(
        save_directory: Optional[str],
        model_id: Optional[str] = None,
        repo_url: Optional[str] = None,
        commit_message: Optional[str] = "add model",
        organization: Optional[str] = None,
        private: bool = None,
    ) -> str:
        """
        Parameters:
            save_directory (:obj:`Union[str, os.PathLike]`):
                Directory having model weights & config.
            model_id (:obj:`str`, `optional`, defaults to :obj:`save_directory`):
                Repo name in huggingface_hub. If not specified, repo name will be same as `save_directory`
            repo_url (:obj:`str`, `optional`):
                Specify this in case you want to push to existing repo in hub.
            organization (:obj:`str`, `optional`):
                Organization in which you want to push your model.
            private (:obj:`bool`, `optional`):
                private: Whether the model repo should be private (requires a paid huggingface.co account)
            commit_message (:obj:`str`, `optional`, defaults to :obj:`add model`):
                Message to commit while pushing

        Returns:
            url to commit on remote repo.
        """
        if model_id is None:
            model_id = save_directory

        token = HfFolder.get_token()
        if repo_url is None:
            repo_url = HfApi().create_repo(
                model_id,
                token=token,
                organization=organization,
                private=private,
                repo_type=None,
                exist_ok=True,
            )
        repo = Repository(save_directory, clone_from=repo_url, use_auth_token=token)

        return repo.push_to_hub(commit_message=commit_message)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct  8 20:08:30 2018

@author: patrickmcfarlane

draft.py contains the Draft class that
enables API calls for draft related endpoints
"""

from .utils import api_call, parse_api_call, get_season_year

class Draft:
    """ The Draft class contains all resources needed to use the draft-
    related API calls. `stats.nba.com <https://stats.nba.com>`_
    has the following draft-related API endpoints:

        - **draftcombinestats**: Draft combine drill results and measurement \
        data for players.
        - **drafthistory**: Draft results detailing metadata of draftees \
        by season.

    The Draft class has the following required parameters:

        @param **headers** (*dict*): Dictionary of request header information
            required by the API. Specifically, the API requires you to declare
            the 'User-Agent' key of the request header dictionary. More information
            can be found `here <https://stackoverflow.com/questions/46781563/
            how-to-obtain-a-json-response-from-the-stats-nba-com-api>`_ and
            an example request header dictionary can be found in the __init__.py
            file in the tests folder of this module.

        @param **league_id** (*str*): LeagueID in the API. String of a \
            two-digit number corresponding to the league. '00' is the NBA, \
            '10' is the WNBA, '01' is the ABA, and '20' is the G-League. \
            Draft data is only available for the NBA.

        @param **season_year** (*str*): SeasonYear in the API. String of \
            a two-year season year in a YYYY-ZZ format, where the ZZ are the \
            last two digits of the following year. For example, '2017-18' is \
            a valid value of **season_year** and represents the 2017-18 NBA season. \
            'All Time' is a valid value for the 'draftcombinestats' endpoint.

    Attributes:

        **api_resp** (*dict*): JSON object of the API response. The API \
            response has three keys. The 'resource' key describes the type of \
            response returned (the endpoint in this instance). The 'parameters' \
            key describes the parameters provided in the API call. The \
            'resultSets' key contains the data returned in the API call.

        **data** (*dict*): A dictionary of response names. Each response \
            name is a key to a list of dictionaries containing the corresponding \
            data.
    """

    def __init__(self, headers, endpoint='draftcombinestats',
                 league_id='00', season_year=get_season_year("00")):

        # Controlling the parameters depending on the endpoint
        if endpoint in ['drafthistory']:
            params = {'LeagueID': league_id}
        else:
            params = {'LeagueID': league_id,
                      'SeasonYear': season_year}

        self.api_resp = api_call(endpoint=endpoint,
                                 params=params,
                                 headers=headers)

        # Storing the API response in a dictionary called data
        # The results can come in different formats, namely a
        # dictionary keyed by either resultSets or resultSet
        self.data = parse_api_call(self.api_resp)


import urllib
import re

def urlencodedquery(**kwargs):
    kw_copy = kwargs.copy()
    for key, val in kwargs.items():
        if val == "":
            del kw_copy[key]
    return urllib.parse.urlencode(kw_copy)

def urlcheck(url):
    """check if url is valid for agrimet api

    Arguments:
        url {string} -- http://www.usbr.gov/pn-bin/
    """
    match = True

    pattern = r"https?:\/\/www\.usbr\.gov\/?.*?$"

    regex = re.compile(pattern)

    if regex.search(url) is None:
        match = False

    return match

def urlformatdetect(url):

    pattern = r".*format=(.*).*"
    regex = re.compile(pattern)

    match = "Not Found"

    if urlcheck(url):

        regex_result = regex.search(url)

        if regex_result is not None:
            match = regex_result.group(1)

    return match


# --------------------------------------------------------------------------
# Name:        Climbing Stairs
# Author(s):   Phu Tran
# --------------------------------------------------------------------------
"""
    You are climbing a stair case. It takes n steps to reach to the top.

    Each time you can either climb 1 or 2 steps. In how many distinct ways can you climb to the top?

    Example 1:
      Input: 2
      Output: 2
      Explanation: There are two ways to climb to the top.
        1. 1 step + 1 step
        2. 2 steps

    Example 2:
      Input: 3
      Output: 3
      Explanation: There are three ways to climb to the top.
        1. 1 step + 1 step + 1 step
        2. 1 step + 2 steps
        3. 2 steps + 1 step

    Constraints:
      1. 1 <= n <= 45
"""

class Solution(object):
    def climbStairs(self, n):
        """
        :type n: int
        :rtype: int
        """
        # F[n] = F[n-1] + F[n-2]
        dp = (1, 1)
        for i in range(n-1):
            dp = (dp[1], dp[0] + dp[1])
        return dp[1]

        ###### Time Limit Exceeded
#         def DFS(n, curNumSteps, result):
#             if curNumSteps == n:
#                 result[0] += 1
#                 return

#             if curNumSteps > n:
#                 return

#             DFS(n, curNumSteps+1, result)
#             DFS(n, curNumSteps+2, result)

#         result = [0]
#         DFS(n, 0, result)

#         return result[0]

'''other faster methods (from other submissions)
##################################################
def climbStairs(self, n):
        a, b = 1, 1
        for i in range(n):
            a, b = b, a + b
        return a
##################################################
def climbStairs(self, n):
        """
        :type n: int
        :rtype: int
        """
        dp = []
        dp.append(1)
        dp.append(2)
        for i in range(2, n):
            dp.append(dp[i-1]+dp[i-2])
        return dp[n-1]
##################################################
def climbStairs(self, n):
        """
        :type n: int
        :rtype: int
        """
        ways = [0] * (n + 1)
        ways[0] = 1
        ways[1] = 1
        for i in range(2, n + 1):
            ways[i] = ways[i - 1] + ways[i - 2]
        return ways[n]
##################################################
##################################################
'''


import argparse

import numpy as np
# import pandas as pd

from datetime import datetime
from pathlib import Path

from cuml.decomposition import PCA
from dist_matrix.cuda_dist_matrix_full import dist_matrix as gpu_dist_matrix
from gensim.models.doc2vec import Doc2Vec

def main():
    # models_dir = Path(Path.cwd().parent, 'models')
    ap = argparse.ArgumentParser()

    ap.add_argument('-p', '--path', type=str, help='Size of features or columns')

    # ap.add_argument('-f', '--features', type=int,
    #                 default=10, help='Size of features or columns')
    # ap.add_argument('-r', '--rows', type=int,
    #                 default=100, help='Size of rows')

    args = ap.parse_args()

    model = Doc2Vec.load(str(Path(args.path)))
    data = model.dv.vectors.astype(np.float64)

    if data.shape[1] > 256:
        pca_float = PCA(n_components = 256)
        data = pca_float.fit_transform(data)
        print(sum(pca_float.explained_variance_ratio_))

    weights = np.ones(data.shape, dtype=np.float64)
    print('Vectors', data.shape)
    print('Weights', weights.shape)

    # n_features = args.features
    # n_rows = args.rows
    # U, V, U_weights, V_weights = np.random.rand(4, args.rows, args.features)
    # print('U: ', U.shape)
    # print('V: ', V.shape)

    start = datetime.now()
    # distances = gpu_dist_matrix(U=U, V=V, U_weights=U_weights, V_weights=V_weights, metric='wasserstein')
    distances = gpu_dist_matrix(data, V=data, U_weights=weights, V_weights=weights, metric='wasserstein')

    print(f'Took {str(datetime.now() - start)}')
    print(distances.shape)
    print(distances)

main()


import pytorch_lightning as pl
from argparse import ArgumentParser
from models import cctv_trainer
import pdb
import torch
import numpy as np
import os
import json
import torchnet.meter as meter

def main():
    parser = ArgumentParser()

    parser.add_argument('--video_root', default='', type=str, help='Root path of input videos')
    parser.add_argument('--batch_size', default=64, type=int, help='Batch Size')
    parser.add_argument('--n_threads', default=4, type=int, help='Number of threads for multi-thread loading')
    parser.add_argument('--learning_rate', default=0.001, type=float, help='learning rate')
    parser.add_argument('--checkpoint', type=str, help='path to checkpoint file')
    parser.add_argument('--data_type', default='test', type=str, help='test or val')

    parser = cctv_trainer.CCTVPipeTrainer.add_model_specific_args(parser)

    parser = pl.Trainer.add_argparse_args(parser)

    args = parser.parse_args()

    device = 'cuda:0'

    model = cctv_trainer.CCTVPipeTrainer.load_from_checkpoint(args.checkpoint)
    model = model.to(device)
    model.hparams.batch_size = args.batch_size
    model.hparams.n_threads = args.n_threads

    model.eval()

    data_type = args.data_type
    if data_type == 'val':
        test_loader = model.val_dataloader()
    else:
        test_loader = model.test_dataloader()
    total = 0

    output = {}

    mapmeter = meter.mAPMeter()
    mapmeter_norm = meter.mAPMeter()

    with torch.no_grad():
        idx_to_label = test_loader.dataset.idx_to_label
        all_results = {}
        all_results["results"] = {}
        for batch in test_loader:
            img, label, names, moments = batch
            img = img.to(device)
            out = model(img)
            out = torch.sigmoid(out)
            for idx, res in enumerate(out):
                nm = names[idx]
                moment = moments[idx].detach().cpu().item()
                res2 = res[:-1]
                for clsidx, score in enumerate(res2):
                    label = idx_to_label[clsidx]
                    res_obj = {'score': "%.3f" % score , 'moment': "%.3f" % moment, 'label': label, 'nominal_score': '%.3f' % res[-1]}
                    if nm not in all_results["results"].keys():
                        all_results["results"][nm] = []
                    all_results["results"][nm].append(res_obj)
            total += 1
            print("%d / %d" % (total * args.batch_size, len(test_loader.dataset)), flush=True)
    with open('%s_result_%s.json' % (args.data_type, os.path.basename(args.checkpoint)), 'w') as fp:
        json.dump(all_results, fp)

if __name__ == '__main__':
    main()


# Databricks notebook source
# MAGIC %md
# MAGIC ###1. Connect to Azure Data Lake using Access Key

# COMMAND ----------

spark.conf.set(
                  "fs.azure.account.key.*** Data Lake Name ***.dfs.core.windows.net",

                  "*** Data Lake Access Key ***"
              )

# COMMAND ----------

# MAGIC %md
# MAGIC ###2. List files in Data Lake

# COMMAND ----------

display(
          dbutils.fs.ls("abfss://*** Container Name ***@*** Data Lake Name ***.dfs.core.windows.net/*** Path ***")
       )

# COMMAND ----------

# MAGIC %md
# MAGIC ###3. Read file from Data Lake

# COMMAND ----------

inputFolderPath = "abfss://*** Container Name ***@*** Data Lake Name ***.dfs.core.windows.net/Raw/"

outputFolderPath = "abfss://*** Container Name ***@*** Data Lake Name ***.dfs.core.windows.net/Output/"

# COMMAND ----------

# Read TaxiRides csv file to create DataFrame

taxiRidesDF = (
                  spark
                    .read

                    .option("header", "true")
                    .option("inferSchema", "true")

                    .csv(inputFolderPath + "TaxiRides.csv")
              )

# Show DataFrame content
display( taxiRidesDF )

# COMMAND ----------

# MAGIC %md
# MAGIC ###4. Write data in Parquet format

# COMMAND ----------

(
    taxiRidesDF
        .write
        .mode("overwrite")

        .partitionBy("VendorId")

        .format("parquet")

        .save(outputFolderPath + "TaxiRides.parquet")
)

# COMMAND ----------

# MAGIC %md
# MAGIC ###5. Write data in Delta format

# COMMAND ----------

(
    taxiRidesDF
        .write
        .mode("overwrite")

        .partitionBy("VendorId")

        .format("delta")

        .save(outputFolderPath + "TaxiRides.delta")
)

# COMMAND ----------

# MAGIC %md Check output folder to see folders for Parquet and Delta
# MAGIC
# MAGIC Check for the presence of _delta_log subfolder in Delta folder

# COMMAND ----------

# MAGIC %md
# MAGIC ###6. Create Database

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC CREATE DATABASE IF NOT EXISTS TaxisDB

# COMMAND ----------

# MAGIC %md
# MAGIC ###7. Create Spark Table referencing Delta folder

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC -- Create table based on Delta data
# MAGIC
# MAGIC CREATE TABLE TaxisDB.TaxiRides
# MAGIC
# MAGIC   USING DELTA
# MAGIC
# MAGIC   LOCATION "abfss://*** Container Name ***@*** Data Lake Name ***.dfs.core.windows.net/Output/TaxiRides.delta"

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SELECT COUNT(*) FROM TaxisDB.TaxiRides

# COMMAND ----------

# MAGIC %md
# MAGIC ###8. Audit History of Delta Table
# MAGIC
# MAGIC This shows transaction log of Delta Table

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC DESCRIBE HISTORY TaxisDB.TaxiRides

# COMMAND ----------

# MAGIC %md
# MAGIC ###9. Options to Add Data to Delta Table

# COMMAND ----------

# MAGIC %md
# MAGIC ####Option 9.1: Insert command
# MAGIC
# MAGIC Use typical SQL Insert command to add data to table

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC INSERT INTO TaxisDB.TaxiRides
# MAGIC
# MAGIC (RideId, VendorId, PickupTime, DropTime, PickupLocationId, DropLocationId, CabNumber, DriverLicenseNumber, PassengerCount, TripDistance, RateCodeId, PaymentType, TotalAmount, FareAmount, Extra, MtaTax, TipAmount, TollsAmount, ImprovementSurcharge)
# MAGIC
# MAGIC VALUES (10000, 3, '2021-12-01T00:00:00.000Z', '2021-12-01T00:15:34.000Z', 170, 140, 'TAC399', '5131685', 1, 2.9, 1, 1, 15.3, 13.0, 0.5, 0.5, 1.0, 0.0, 0.3)

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SELECT COUNT(*) FROM TaxisDB.TaxiRides

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC DESCRIBE HISTORY TaxisDB.TaxiRides

# COMMAND ----------

# MAGIC %md
# MAGIC ####Option 9.2: Append a DataFrame
# MAGIC
# MAGIC Read data as a DataFrame and append to Delta Table using PySpark

# COMMAND ----------

# Extract new records from Data Lake

taxiRidesAppendDF = (
                        spark
                          .read
                          .option("header", "true")
                          .option("inferSchema", "true")

                          .csv(inputFolderPath + "TaxiRides_append.csv")
                    )

# Show DataFrame content
display( taxiRidesAppendDF )

# COMMAND ----------

# Append to data lake in delta format

(
    taxiRidesAppendDF
        .write

        .mode("append")

        .partitionBy("VendorId")
        .format("delta")
        .save(outputFolderPath + "TaxiRides.delta")
)

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC SELECT COUNT(*) FROM TaxisDB.TaxiRides

# COMMAND ----------

Develop a Batch Processing Solution/L28 - Delta Lake - Part 1/Delta Lake - Part 1.py


import psycopg2, json, csv
import re
import os
import status
import operator
import math
import numpy as np
from scipy.sparse import csr_matrix
import multiprocessing as mp
import itertools
from difflib import SequenceMatcher
from sklearn.feature_extraction.text import TfidfVectorizer
from threading import Thread
import threading
import time
import functools
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from random import shuffle
from common_utils import gen_csv_from_tuples, read_csv_list, make_query

import pickle
import matplotlib
matplotlib.use('Agg')
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt

class FeatureScore(object):

	def __init__(self, identifier, dataset_filename, cdf_filename,
		bin_matrix_filename, weight_matrix_filename,
		coincidence_score_filename, uniqueness_score_filename,
		coincidence_csv_filename, uniqueness_csv_filename, rarity_bound, pickle_file,
		user_removal=None, value_removal=None):
		self.filenames=dict()
		self.ident = identifier
		self.filenames['dataset_filename'] = dataset_filename
		self.filenames['cdf_filename'] = cdf_filename
		self.filenames['bin_matrix_filename'] = bin_matrix_filename
		self.filenames['weight_matrix_filename'] = weight_matrix_filename
		self.filenames['coincidence_score_filename'] = coincidence_score_filename
		self.filenames['uniqueness_score_filename'] = uniqueness_score_filename
		self.filenames['coincidence_csv_filename'] = coincidence_csv_filename
		self.filenames['uniqueness_csv_filename'] = uniqueness_csv_filename
		self.filenames['pickle_file'] = pickle_file
		self.filenames['promising_combinations'] = pickle_file[:-4] + "promising_combinations.pkl"
		self.filenames['bin_sparse_matrix'] = pickle_file[:-4] + "bin_sparse_matrix.pkl"
		self.filenames['weight_sparse_matrix'] = pickle_file[:-4] + "weight_sparse_matrix.pkl"
		self.rarity_bound = rarity_bound
		self.user_removal = user_removal
		self.value_removal = value_removal

	def identifier(self):
		return self.ident

	def store_clean_dataset(self, dictio_of_users, name = None):
		lst_users = [(key,) + tuple(values) for key, values in dictio_of_users.items()]
		if not name is None:
			gen_csv_from_tuples(name,
				["IdAuthor1", "IdAuthor2", "Score"], lst_users)
		else:
			gen_csv_from_tuples(self.filenames['dataset_filename'][:-4]+"_clean.csv",
				["IdAuthor1", "IdAuthor2", "Score"], lst_users)

	### This function generates a dictionary of the users associated to the values used by that user.
	def gen_dictio_of_users(self,lst):
		dictio = {}
		for i in lst:
			key = i[0]
			dictio[key] = []
			for j in i[1:]:
				dictio[key].append(j)
		return dictio

	### This function generates a dictionary of the values associated to the user.
	def gen_dictio_of_values(self,lst):
		dictio = {}
		for i in lst:
			key = i[0]
			for j in i[1:]:
				if j in dictio.keys():
					dictio[j].append(key)
				else:
					dictio[j] = [key]
		return dictio

	def get_upper_lower_bounds(self, dictionary, c=2.0):
		lst = [len(v) for k, v in dictionary.items()]
		v = np.array(lst)
		mean = np.mean(v)
		std = np.std(v)
		upper_bound = math.ceil(mean + (c * std))
		lower_bound = math.ceil(max((mean - (c * std)), 0))
		#upper_bound = math.ceil(mean + (c * std))
		#lower_bound = math.ceil(max((mean - (c * std)), 0))
		return upper_bound, lower_bound

	def get_mean_std(self, dictionary, c=1):
		lst = [len(v) for k, v in dictionary.items()]
		v = np.array(lst)
		mean = np.mean(v)
		std = np.std(v)
		return mean, std

	def remove_values_from_dictios(self, list_value, dictio_of_users, dictio_of_values):
		if len(list_value) == 0:
			return dictio_of_users, dictio_of_values
		elif len(list_value) == len(dictio_of_values):
			return {}, {}

		old_len_u, old_len_value = len(dictio_of_users), len(dictio_of_values)
		#print("values removed: ", len(list_value) )
		for value in list_value:
			for user in dictio_of_values[value]:
				dictio_of_users[user].remove(value)
			ret2 = dictio_of_values.pop(value, None)
			if ret2 is None:
				print("THERE IS AN ERROR: ", value)
		# Update the list of users
		users_removed = [k for k, v in dictio_of_users.items() if len(v) == 0]
		# Remove users which have no value.
		for user in users_removed:
			ret = dictio_of_users.pop(user, None)
			if ret is None:
				print("ERROR")
		row_format ="{:>15}" * 4
		print("-" * 15 * 4)
		print(row_format.format("Original values", "Removed values", "New values", "Percentage"))
		print(row_format.format("%d"% (old_len_value), "%d"%(len(list_value)), "%d"%(len(dictio_of_values)), "%f" %(len(dictio_of_values)/old_len_value)))
		print(row_format.format("Original Users", "Removed Users", "New Users", "Percentage"))
		print(row_format.format("%d"% (old_len_u), "%d"%(len(users_removed)), "%d"%(len(dictio_of_users)), "%f" %(len(dictio_of_users)/old_len_u)))
		print("-" * 15 * 4)
		return dictio_of_users, dictio_of_values

	### Removes users from the interchangeable dictionaries.
	def remove_users_from_dictios(self, list_users, dictio_of_users, dictio_of_values):
		if len(list_users) == 0:
			return dictio_of_users, dictio_of_values
		elif len(list_users) == len(dictio_of_values):
			return {}, {}

		old_len_u, old_len_value = len(dictio_of_users), len(dictio_of_values)
		# Remove users and values from the list
		for user in list_users:
			for value in dictio_of_users[user]:
				dictio_of_values[value].remove(user)
			ret2 = dictio_of_users.pop(user, None)

		# Update the list of users
		values_removed = [k for k, v in dictio_of_values.items() if v == []]

		# Remove values which are no longer used by a user.
		for value in values_removed:
			ret = dictio_of_values.pop(value, None)
			if ret is None:
				print("ERROR")
		row_format ="{:>15}" * 4
		print("-" * 15 * 4)
		print(row_format.format("Original values", "Removed values", "New values", "Percentage"))
		print(row_format.format("%d"% (old_len_value), "%d"%(len(values_removed)), "%d"%(len(dictio_of_values)), "%f" %(len(dictio_of_values)/old_len_value)))
		print(row_format.format("Original Users", "Removed Users", "New Users", "Percentage"))
		print(row_format.format("%d"% (old_len_u), "%d"%(len(list_users)), "%d"%(len(dictio_of_users)), "%f" %(len(dictio_of_users)/old_len_u)))
		print("-" * 15 * 4)
		#print(len(dictio_of_users)/old_len_u, len(dictio_of_values)/old_len_value,
			#len(dictio_of_users), len(dictio_of_values))
		return dictio_of_users, dictio_of_values

	def clean_dataset(self, dictio_of_users, dictio_of_values, user_removal=None, value_removal=None):
		# Remove values with 1 appearance.
		# print("[-] Getting user upper and lower bound...")
		# user_upper_bound, user_lower_bound = self.get_upper_lower_bounds(dictio_of_users, c=0.5)
		# user_mean, user_std = self.get_mean_std(dictio_of_users)
		# print("[-] Getting value upper and lower bound...")
		# value_upper_bound, value_lower_bound = self.get_upper_lower_bounds(dictio_of_values, c=0.5)
		# value_mean, value_std = self.get_mean_std(dictio_of_values)
		print("[-] Removing values that appear once...")
		oneapp = [k for k,v in dictio_of_values.items() if len(v) == 1]
		dictio_of_users, dictio_of_values = self.remove_values_from_dictios(oneapp, dictio_of_users, dictio_of_values)

		# We execute all user removal procedures specified
		if not user_removal is None:
			for procedure in user_removal:
				user_list = procedure(dictio_of_users, dictio_of_values)
				dictio_of_users, dictio_of_values = self.remove_users_from_dictios(user_list, dictio_of_users, dictio_of_values)
		# We execute all value removal procedures specified by the user
		if not value_removal is None:
			for procedure in value_removal:
				value_list = procedure(dictio_of_users, dictio_of_values)
				dictio_of_users, dictio_of_values = self.remove_values_from_dictios(value_list, dictio_of_users, dictio_of_values)
		self.store_clean_dataset(dictio_of_users)
		# print("[-] Removing values that appear more than %d times or less than %d times" % (value_upper_bound, value_lower_bound))
		# multivalue = [k for k,v in dictio_of_values.items() if len(v) > value_upper_bound or len(v) < value_lower_bound]
		# dictio_of_users, dictio_of_values = self.remove_values_from_dictios(multivalue, dictio_of_users, dictio_of_values)
		# print("[-] Removing users that have less than %d values or more than %d values" % (user_lower_bound, user_upper_bound))
		# multiuser = [k for k,v in dictio_of_users.items() if len(v) > user_upper_bound or len(v) < user_lower_bound]
		# dictio_of_users, dictio_of_values = self.remove_users_from_dictios(multiuser, dictio_of_users, dictio_of_values)
		# oneapp = [k for k,v in dictio_of_values.items() if len(v) == 1]
		# print("[-] Removing final single values that appear once...")
		# dictio_of_users, dictio_of_values = self.remove_values_from_dictios(oneapp, dictio_of_users, dictio_of_values)
		return dictio_of_users, dictio_of_values

	def clean_dataset_2(self, dictio_of_users, dictio_of_values, dictio_of_weights, rarity_bound):
		print("[-] Removing values with rarity lower than %d" % (rarity_bound))
		lst_remove = [value for value, rarity in dictio_of_weights.items() if rarity < rarity_bound]
		dictio_of_users, dictio_of_values = self.remove_values_from_dictios(lst_remove, dictio_of_users, dictio_of_values)
		#print("[-] Removing values that appear once...")
		#oneapp = [k for k,v in dictio_of_values.items() if len(v) == 1]
		#dictio_of_users, dictio_of_values = self.remove_values_from_dictios(oneapp, dictio_of_users, dictio_of_values)
		return dictio_of_users, dictio_of_values

	def clean_dataset_3(self, dictio_of_users, dictio_of_values, dictio_of_weights, rarity_bound):
		print("[-] Removing values with rarity lower than %d" % (rarity_bound))
		lst_remove = [value for value, rarity in dictio_of_weights.items() if rarity < rarity_bound]
		dictio_of_users, dictio_of_values = self.remove_values_from_dictios(lst_remove, dictio_of_users, dictio_of_values)
		for value in lst_remove:
			dictio_of_weights.pop(value, None)
		#print("[-] Removing values that appear once...")
		#oneapp = [k for k,v in dictio_of_values.items() if len(v) == 1]
		#dictio_of_users, dictio_of_values = self.remove_values_from_dictios(oneapp, dictio_of_users, dictio_of_values)
		return dictio_of_users, dictio_of_values, dictio_of_weights
	#Generates a clean dataset from the filename.
	def generate_clean_dataset(self, filename, do_rarity=True):
		tic = time.time()
		lst_users = read_csv_list(filename)[1:]

		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)
		if not "_clean" in filename:
			dictio_of_users, dictio_of_values = self.clean_dataset(dictio_of_users,dictio_of_values,
				user_removal=self.user_removal, value_removal=self.value_removal)

			if do_rarity:
				dictio_of_weights = self.gen_dictio_of_weigths (dictio_of_values)
				dictio_of_users, dictio_of_values = self.clean_dataset_2(dictio_of_users, dictio_of_values,
					dictio_of_weights, self.rarity_bound)

		dictio_of_weights = self.gen_dictio_of_weigths (dictio_of_values)
		print(len(dictio_of_weights), len(dictio_of_values))
		print("[+] Finished Dataset Cleanup in: %f" %(time.time() - tic))
		return dictio_of_users, dictio_of_values, dictio_of_weights

 	# Generates weights of the values depending on its rarity in the dataset
	def gen_dictio_of_weigths(self, dictio_of_values, num_elems=255):
		dictio_of_values = dict(dictio_of_values)
		lst_lengths = list(set([(len(v))  for k, v in dictio_of_values.items()]))
		lst_lengths = sorted(lst_lengths, key=lambda x: x, reverse=True)
		#print(num_elems, len(lst_lengths))
		if num_elems > len(lst_lengths):
			num_elems = len(lst_lengths)
		#print(num_elems)
		divisions = int(math.ceil(float(len(lst_lengths)) / float(num_elems)))
		len_rarity = {}
		for i in range(0, num_elems):
			start = i * divisions
			end = (i + 1)  * divisions
			for elem in lst_lengths[start:end]:
				len_rarity[elem] = (i + 1)

		#print(len_rarity)
		# We modify the dictio_of_values and return according to the rarity
		for key, values in dictio_of_values.items():
			dictio_of_values[key] = len_rarity[len(values)]

		return dictio_of_values

	def order_users(self, entry):
		if entry[0] > entry[1]:
			return entry[1], entry[0]
		else:
			return entry[0], entry[1]

	def get_promising_combinations(self, dictio_of_users, dictio_of_values):
		print("[-] Extracting promising combinations...")
		set_combinations = set()
		length = len(dictio_of_values.items())
		status.create_numbar(100, length)
		for ind, users in enumerate(dictio_of_values.items()):
			status.update_numbar(ind, length)
			users = sorted(users[1])
			combinations = list(itertools.combinations(users, 2))
			for combination in combinations:
				set_combinations.add(combination)
		status.end_numbar()
		print("[+] Extracted %d promising combinations" % (len(set_combinations)))
		return list(set_combinations)

	def gen_rarity_dist(self, dictio):
		rarity_num_dictio = {}
		for value, rarity in dictio.items():
			if rarity in rarity_num_dictio.keys():
				rarity_num_dictio[rarity] += 1
			else:
				rarity_num_dictio[rarity] = 1

		lst_dist = [(rarity,repetitions) for rarity,repetitions in rarity_num_dictio.items()]
		lst_dist = sorted(lst_dist, key=lambda x: x[0], reverse=False)
		return lst_dist

	def get_cdf(self, filename, graph_filename):
		tic = time.time()
		lst_users = read_csv_list(filename)[1:]
		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)
		dictio_of_users, dictio_of_values = self.clean_dataset(dictio_of_users,dictio_of_values,
			user_removal=self.user_removal, value_removal=self.value_removal)

	def generate_cdf(self, filename, graph_filename):
		tic = time.time()
		lst_users = read_csv_list(filename)[1:]

		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)
		dictio_of_users, dictio_of_values = self.clean_dataset(dictio_of_users,dictio_of_values,
			user_removal=self.user_removal, value_removal=self.value_removal)

		self.store_clean_dataset(dictio_of_users)

		dictio_of_weights = self.gen_dictio_of_weigths(dictio_of_values)
		#print(dictio_of_weights)
		lst_rarity = self.gen_rarity_dist(dictio_of_weights)
		#print(lst_rarity)
		#print(len(dictio_of_weights), len(dictio_of_values))

		X = np.array([float(x[0]) for x in lst_rarity])
		Y = np.array([float(x[1]) for x in lst_rarity])
		Y /= np.sum(Y)
		#print(X, Y)
		CY = np.cumsum(Y)
		percentage = 0.1
		plt.style.use('seaborn-darkgrid')
		palette = plt.get_cmap('Set1')
		for x,y,cy in zip(X,Y,CY):
			if y > percentage:
				plt.scatter(x, cy, color=palette(2), label="x(Y > %f) = %d" % (percentage, x))
		for x,y,cy in zip(X,Y,CY):
			if cy > percentage:
				plt.axvline(x=x, color=palette(2), linestyle='-.', label="x(CDF > %f) = %d" % (percentage, x))
				break
		plt.plot(X, Y, color=palette(0), label="Y")
		plt.plot(X, CY, color=palette(1), linestyle='--', label="CDF(Y)")
			#marker='o', markersize=3, markerfacecolor=palette(0))
		plt.xlabel('Rarity')
		plt.ylabel('Percentage of this rarity (%)')

				#plt.scatter('', xy=(x, cy), xytext=(0, 0), color='red' , textcoords='offset points')
		plt.legend()
		plt.savefig(graph_filename, format="PNG")
		plt.clf()
		print("[+] Finished Graph Generation Cleanup in: %f" %(time.time() - tic))

	def cdf(self):
		self.generate_cdf(self.filenames['dataset_filename'], self.filenames['cdf_filename'])

	def gen_binary_matrix_mem(self, dictio_of_values, dictio_of_users):
		tic = time.time()
		dictio_of_values = dict(dictio_of_values)
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		# Transform dictionary to indexes
		for indk, value in enumerate(dictio_of_values.keys()):
			dictio_of_values[value] = indk
		#Transform users to matrices.
		print("[-] ESTIMATED SIZE OF BINARY MATRIX: %f GB" % (num_users * num_values * 1 / (1024 ** 3)))
		matrix_map = np.zeros(shape=(num_users, num_values), dtype=np.uint8)
		status.create_numbar(100, num_users)
		for ind, user in enumerate(dictio_of_users.keys()):
			status.update_numbar(ind, num_users)
			base = np.zeros((num_values,1), dtype=np.uint8)
			for value in dictio_of_users[user]:
				base[dictio_of_values[value]] = 1
			base = np.squeeze(base)
			matrix_map[ind] = base[:]
		status.end_numbar()
		print("[+] Finished Binary Matrix Generation in: %f" %(time.time() - tic))
		return matrix_map

	def gen_binary_matrix(self, dictio_of_values, dictio_of_users, matrix_file):
		tic = time.time()
		dictio_of_values = dict(dictio_of_values)
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		# Transform dictionary to indexes
		for indk, value in enumerate(dictio_of_values.keys()):
			dictio_of_values[value] = indk
		#Transform users to matrices.
		print("[-] ESTIMATED SIZE OF BINARY MATRIX: %f GB" % (num_users * num_values * 1 / (1024 ** 3)))
		matrix_map = np.memmap(matrix_file, dtype=np.uint8, mode ='w+', shape=(num_users, num_values))
		status.create_numbar(100, num_users)
		for ind, user in enumerate(dictio_of_users.keys()):
			status.update_numbar(ind, num_users)
			base = np.zeros((num_values,1), dtype=np.uint8)
			for value in dictio_of_users[user]:
				base[dictio_of_values[value]] = 1
			base = np.squeeze(base)
			matrix_map[ind] = base[:]
		status.end_numbar()
		print("[-] Flushing binary matrix to memory")
		matrix_map.flush()
		print("[+] Finished Binary Matrix Generation in: %f" %(time.time() - tic))

	def gen_binary_sparse_matrix(self, dictio_of_values, dictio_of_users):
		print("[-] Generating Binary Sparse Matrix..")
		tic = time.time()
		dictio_of_values = dict(dictio_of_values)
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		value_indices = {k:indk for indk, k in enumerate(dictio_of_values.keys())}
		user_indices = {k:indk for indk, k in enumerate(dictio_of_users.keys())}
		rows = []
		cols = []
		data = []

		status.create_numbar(100, num_users)
		for ind, row in enumerate(dictio_of_users.items()):
			status.update_numbar(ind, num_users)
			user, values = row[0], row[1]
			for value in values:
				rows.append(user_indices[user])
				cols.append(value_indices[value])
				data.append(1)

		status.end_numbar()
		matrix = csr_matrix((data, (rows, cols)), shape=(num_users, num_values), dtype=np.uint8)
		print("[+] Finished Binary Sparse Matrix Generation in: %f" %(time.time() - tic))
		return matrix

	def gen_weight_matrix_mem(self, dictio_of_values, dictio_of_users, dictio_of_weights):
		tic = time.time()
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		dictio_of_values = dict(dictio_of_values)
		# Transform dictionary to indexes
		for indk, value in enumerate(dictio_of_values.keys()):
			dictio_of_values[value] = indk
		#Transform users to matrices.
		print("[-] ESTIMATED SIZE OF WEIGHTS MATRIX: %f GB" % (num_users * num_values * 1 / (1024 ** 3)))
		matrix_map = np.zeros(shape=(num_users, num_values), dtype=np.uint8)
		status.create_numbar(100, num_users)
		for ind, user in enumerate(dictio_of_users.keys()):
			status.update_numbar(ind, num_users)
			base = np.zeros((num_values,1), dtype=np.uint8)
			for value in dictio_of_users[user]:
				base[dictio_of_values[value]] = dictio_of_weights[value]
			base = np.squeeze(base)
			matrix_map[ind] = base[:]
		status.end_numbar()
		print("[+] Finished Weight Matrix Generation in: %f" %(time.time() - tic))
		return matrix_map

	def gen_weight_matrix(self, dictio_of_values, dictio_of_users, dictio_of_weights, matrix_file):
		tic = time.time()
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		dictio_of_values = dict(dictio_of_values)
		# Transform dictionary to indexes
		for indk, value in enumerate(dictio_of_values.keys()):
			dictio_of_values[value] = indk
		#Transform users to matrices.
		print("[-] ESTIMATED SIZE OF WEIGHTS MATRIX: %f GB" % (num_users * num_values * 1 / (1024 ** 3)))
		matrix_map = np.memmap(matrix_file, dtype=np.uint8, mode ='w+', shape=(num_users, num_values))
		status.create_numbar(100, num_users)
		for ind, user in enumerate(dictio_of_users.keys()):
			status.update_numbar(ind, num_users)
			base = np.zeros((num_values,1), dtype=np.uint8)
			for value in dictio_of_users[user]:
				base[dictio_of_values[value]] = dictio_of_weights[value]
			base = np.squeeze(base)
			matrix_map[ind] = base[:]
		status.end_numbar()
		print("[-] Flushing weights matrix to memory")
		matrix_map.flush()
		print("[+] Finished Weight Matrix Generation in: %f" %(time.time() - tic))

	def gen_weight_sparse_matrix(self, dictio_of_values, dictio_of_users, dictio_of_weights):
		print("[-] Generating Weight Sparse Matrix..")
		tic = time.time()
		dictio_of_values = dict(dictio_of_values)
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		value_indices = {k:indk for indk, k in enumerate(dictio_of_values.keys())}
		user_indices = {k:indk for indk, k in enumerate(dictio_of_users.keys())}
		rows = []
		cols = []
		data = []

		status.create_numbar(100, num_users)
		for ind, row in enumerate(dictio_of_users.items()):
			status.update_numbar(ind, num_users)
			user, values = row[0], row[1]
			for value in values:
				rows.append(user_indices[user])
				cols.append(value_indices[value])
				data.append(dictio_of_weights[value])

		status.end_numbar()
		matrix = csr_matrix((data, (rows, cols)), shape=(num_users, num_values), dtype=np.uint8)
		print("[+] Finished Weight Sparse Matrix Generation in: %f" %(time.time() - tic))
		return matrix

	def process_score(self, dictio_of_users, dictio_of_values, matrix_filename, score_filename):
		tic = time.time()
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)

		matrix_map = np.memmap(matrix_filename, dtype=np.uint8,  shape=(num_users, num_values))
		#matrix_map = np.array(matrix_map)

		print(matrix_map.shape)
		print("[-] ESTIMATED SIZE OF SCORE MATRIX: %f GB" % (num_users * num_users * 4 / (1024 ** 3)))
		status.create_numbar(100, num_users)
		#res = np.dot(matrix_map, matrix_map.T)
		#res2 = np.memmap('value_files/value_euc_score.dat', dtype=float ,mode ='w+', shape=(num_users, num_users))
		res3 = np.memmap(score_filename, dtype=np.uint32 ,mode ='w+', shape=(num_users, num_users))
		for i1 in range(num_users):
			status.update_numbar(i1, num_users)
			v1 = np.array(matrix_map[i1], dtype=np.uint32)
			#v1p = gen_vector_for_user(list(dictio_of_users.keys())[i1], dictio_of_users, dictio_of_values)
			for i2 in range(i1 + 1, num_users):
				v2 = np.array(matrix_map[i2], dtype=np.uint32)
				#print(v1)
				#euc_score = np.linalg.norm(v1-v2)
				dis_score = np.dot(v1, v2)
				#res2[i1][i2] = euc_score
				res3[i1][i2] = dis_score
				#res2[i2][i1] = score
		status.end_numbar()
		print("[-] Flushing uniqueness score matrix to memory")
		res3.flush()

		print("[+] Finished score calculation in: %f SECONDS" %(time.time() - tic))

	def process_pairs_of_users(self, dictio_of_users, dictio_of_values, lst_users, matrix, csv_filename):
		tic = time.time()
		if csv_filename is None:
			print("[WARN] No file provided, returning matrix")
		dictio_of_users = dict(dictio_of_users)
		for indk, user in enumerate(dictio_of_users.keys()):
			dictio_of_users[user] = indk
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		num_lst_users = len(lst_users)
		matrix_map = None
		if type(matrix) == str:
			matrix_map = np.memmap(matrix, dtype=np.uint8,  shape=(num_users, num_values))
		else:
			matrix_map = matrix
		#matrix_map = np.array(matrix_map)

		## ##print(matrix_map.shape)
		print("[-] ESTIMATED SIZE OF SCORE MATRIX: %f GB" % (num_users * num_users * 4 / (1024 ** 3)))
		#status.create_numbar(100, num_lst_users)
		#res = np.dot(matrix_map, matrix_map.T)
		#res2 = np.memmap('value_files/value_euc_score.dat', dtype=float ,mode ='w+', shape=(num_users, num_users))
		#res3 = np.memmap(coincidence_score_filename, dtype=np.uint32 ,mode ='w+', shape=(num_users, num_users))
		lst_res = list()
		for ind, pair in enumerate(lst_users):
			#status.update_numbar(ind, num_lst_users)
			user1, user2 = pair[0], pair[1]
			v1 = np.array(matrix_map[dictio_of_users[user1]], dtype=np.uint32)
			v2 = np.array(matrix_map[dictio_of_users[user2]], dtype=np.uint32)
			pair_score = np.dot(v1, v2)
			lst_res.append((user1, user2, pair_score))
		#status.end_numbar()
		#lst_res = sorted(lst_res, key=lambda x: str(x[2]) + x[0] + x[1], reverse=True)
		print("[+] Finished score calculation in: %f SECONDS" %(time.time() - tic))
		if csv_filename is None:
			return lst_res
		else:
			gen_csv_from_tuples(csv_filename , ["IdAuthor1", "IdAuthor2", "Score"], lst_res)

	def gen_vector_for_user(self, user_values, dictio_of_values, dictio_of_weights=None):
		num_values = len(dictio_of_values)
		base = np.zeros((num_values,1), dtype=np.uint32)
		if dictio_of_weights is None:
			for value in user_values:
				base[dictio_of_values[value]] = 1
		else:
			for value in user_values:
				base[dictio_of_values[value]] = dictio_of_weights[value]
		base = np.squeeze(base)
		return base

	def process_pairs_of_users_sparse_matrix(self, dictio_of_users, dictio_of_values,
		lst_users, sparse_matrix, csv_filename):
		tic = time.time()
		dictio_of_users = dict(dictio_of_users)
		value_indices = {k:indk for indk, k in enumerate(dictio_of_values.keys())}
		user_indices = {k:indk for indk, k in enumerate(dictio_of_users.keys())}
		num_users = len(dictio_of_users)
		num_values = len(dictio_of_values)
		num_lst_users = len(lst_users)

		print("[-] ESTIMATED SIZE OF SCORE MATRIX: %f GB" % (num_users * num_users * 4 / (1024 ** 3)))
		status.create_numbar(100, num_lst_users)
		#res = np.dot(matrix_map, matrix_map.T)
		#res2 = np.memmap('value_files/value_euc_score.dat', dtype=float ,mode ='w+', shape=(num_users, num_users))
		#res3 = np.memmap(coincidence_score_filename, dtype=np.uint32 ,mode ='w+', shape=(num_users, num_users))
		lst_res = list()
		for ind, pair in enumerate(lst_users):
			status.update_numbar(ind, num_lst_users)
			user1, user2 = pair[0], pair[1]
			v1 = np.array(sparse_matrix[user_indices[user1]].toarray(), dtype=np.uint32)
			v2 = np.array(sparse_matrix[user_indices[user2]].toarray(), dtype=np.uint32)
			pair_score = np.squeeze(np.dot(v1, v2.T))
			lst_res.append((user1, user2, pair_score))
		status.end_numbar()
		sortedl = sorted(lst_res, key=lambda x: str(x[2]) + x[0] + x[1], reverse=True)
		gen_csv_from_tuples(csv_filename , ["IdAuthor1", "IdAuthor2", "Score"], sortedl)
		print("[+] Finished score calculation in: %f SECONDS" %(time.time() - tic))

	def generate_processing_args(self, dictio_of_users, dictio_of_values,
		lst_users, bin_sparse_matrix, weight_sparse_matrix, csv_filename_bin, csv_filename_wei):
		self.processing_args = dict()
		self.processing_args['bin_sparse_matrix'] = bin_sparse_matrix
		self.processing_args['weight_sparse_matrix'] = weight_sparse_matrix
		#self.processing_args['lst_users'] = lst_users
		#self.processing_args['dictio_of_values'] = dictio_of_values
		#self.processing_args['dictio_of_users'] = dict(dictio_of_users)
		#self.processing_args['value_indices'] = {k:indk for indk, k in enumerate(dictio_of_values.keys())}
		self.processing_args['user_indices'] = {k:indk for indk, k in enumerate(dictio_of_users.keys())}
		#self.processing_args['num_users'] = len(dictio_of_users)
		#self.processing_args['num_values'] = len(dictio_of_values)
		#self.processing_args['num_lst_users'] = len(lst_users)

	def calculate_pair_of_users(self, pair):
		user1, user2 = pair[0], pair[1]

		bin_sparse_matrix = self.processing_args['bin_sparse_matrix']
		weight_sparse_matrix = self.processing_args['weight_sparse_matrix']
		user_indices = self.processing_args['user_indices']

		v1 = np.array(bin_sparse_matrix[user_indices[user1]].toarray(), dtype=np.uint32)
		v2 = np.array(bin_sparse_matrix[user_indices[user2]].toarray(), dtype=np.uint32)
		pair_score1 = np.squeeze(np.dot(v1, v2.T))

		v3 = np.array(weight_sparse_matrix[user_indices[user1]].toarray(), dtype=np.uint32)
		v4 = np.array(weight_sparse_matrix[user_indices[user2]].toarray(), dtype=np.uint32)
		pair_score2 = np.squeeze(np.dot(v3, v4.T))
		return (pair_score1, pair_score2)
		#return (0, pair_score2)
	def process_pairs_of_users_sparse_matrix_alt(self, dictio_of_users, dictio_of_values,
		lst_users, bin_sparse_matrix, weight_sparse_matrix, csv_filename_bin, csv_filename_wei):

		tic = time.time()
		self.generate_processing_args(dictio_of_users, dictio_of_values,
			lst_users, bin_sparse_matrix, weight_sparse_matrix, csv_filename_bin, csv_filename_wei)
		pool = mp.Pool(16)
		print("[-] Started processing selected users")
		interval_len = 100000
		intervals = math.ceil(len(lst_users) / interval_len)
		for i in range(intervals):
			toc = time.time()
			targets = lst_users[i * interval_len:(i + 1) * interval_len]
			print("[- -]Going for interval %d of %d" % (i + 1, intervals))
			print(i, intervals, i * interval_len, (i+1) * interval_len)
			lst_res = pool.map(self.calculate_pair_of_users, targets)
			lst_res1 = [(pair[0], pair[1], elem[0]) for pair, elem in zip(targets, lst_res)]
			gen_csv_from_tuples(csv_filename_bin + "_" + str(i) , ["IdAuthor1", "IdAuthor2", "Score"], lst_res1)
			lst_res1 = [(pair[0], pair[1], elem[1]) for pair, elem in zip(targets, lst_res)]
			gen_csv_from_tuples(csv_filename_wei + "_" + str(i) , ["IdAuthor1", "IdAuthor2", "Score"], lst_res1)
			print("[+ +]Finished for interval %d of %d in %d seconds" % (i + 1, intervals, time.time() - toc))

		pool.close()
		#lst_res = [(pair[0], pair[1], elem) for pair, elem in zip(lst_users, lst_res)]
		#print("[+] Finished processing selected users")
		#sortedl = sorted(lst_res, key=lambda x: str(x[2]) + x[0] + x[1], reverse=True)
		#gen_csv_from_tuples(csv_filename , ["IdAuthor1", "IdAuthor2", "Score"], sortedl)
		print("[+] Finished score calculation in: %f SECONDS" %(time.time() - tic))

	def gen_score_csv(self, dictio_of_users, dictio_of_values, score_filename, csv_filename):
		tic = time.time()
		print("[-] Generating scores csv...")
		num_users = len(dictio_of_users)
		lst_users = list(dictio_of_users.keys())
		lst = []
		res2 = np.memmap(score_filename, dtype=np.uint32, mode ='r', shape=(num_users, num_users))
		status.create_numbar(100, num_users)
		for i in range(num_users):
			status.update_numbar(i, num_users)
			for j in range(i + 1, num_users):
				if res2[i][j] > 0:
					lst.append((lst_users[i],lst_users[j],res2[i][j]))
		status.end_numbar()
		#print("Obtained all scores: %f" %(time.time() - tic))
		sortedl = sorted(lst, key=lambda x: str(x[2]) + x[0] + x[1], reverse=True)
		gen_csv_from_tuples(csv_filename , ["IdAuthor1", "IdAuthor2", "Score"], sortedl)
		print("[+] Finished score generation in: %f SECONDS" %(time.time() - tic))

	def pickle_object(self, obj, filename):
		with open(filename, 'wb') as f:
			pickle.dump(obj, f)

	def unpickle_object(self, filename):
		with open(filename, 'rb') as f:
			obj = pickle.load(f)
			return obj

	def generate_processing_info(self):
		dictio_of_users, dictio_of_values, dictio_of_weights = self.generate_clean_dataset(
			self.filenames['dataset_filename'])

		#self.store_clean_dataset(dictio_of_users)
		promising_combinations = None
		bin_sparse_matrix = None
		weight_sparse_matrix = None

		if os.path.exists(self.filenames['promising_combinations']):
			promising_combinations = self.unpickle_object(self.filenames['promising_combinations'])
		else:
			promising_combinations = self.get_promising_combinations(dictio_of_users, dictio_of_values)
			self.pickle_object(promising_combinations, self.filenames['promising_combinations'])

		print("Number of promissing combinations %d" % (len(promising_combinations)))
		if os.path.exists(self.filenames['bin_sparse_matrix']):
			bin_sparse_matrix = self.unpickle_object(self.filenames['bin_sparse_matrix'])
		else:
			bin_sparse_matrix = self.gen_binary_sparse_matrix(dictio_of_values, dictio_of_users)
			self.pickle_object(bin_sparse_matrix, self.filenames['bin_sparse_matrix'])

		if os.path.exists(self.filenames['weight_sparse_matrix']):
			weight_sparse_matrix = self.unpickle_object(self.filenames['weight_sparse_matrix'])
		else:
			weight_sparse_matrix = self.gen_weight_sparse_matrix(dictio_of_values, dictio_of_users, dictio_of_weights)
			self.pickle_object(weight_sparse_matrix, self.filenames['weight_sparse_matrix'])

		return dictio_of_users, dictio_of_values, dictio_of_weights, promising_combinations, bin_sparse_matrix, weight_sparse_matrix

	def get_filename_dir(self, path):
		directory = path[0:len(path)-path[::-1].find("/")]
		filename = path[len(path) - path[::-1].find("/"):]
		return directory, filename

	def join_all_results(self, origin_filename):
		directory, filename = self.get_filename_dir(origin_filename)
		print("[-] Joining all subfiles in same file")
		list_files = [directory + name for name in os.listdir(directory) if filename in name and not filename == name]
		print("[-] Total files for %s: %d" % (self.identifier(), len(list_files)))
		result_file = origin_filename
		f1 = open(result_file, 'w+', buffering=2)
		total_list = []
		status.create_numbar(100, len(list_files))
		first = True
		for ind, file in enumerate(list_files):
			status.update_numbar(ind, len(list_files))
			with open(file, 'r') as f2:
				line = f2.readline()
				if first:
					f1.write(line)
					f1.flush()
					first = False
				line = f2.readline()
				while line:
					f1.write(line)
					f1.flush()
					line = f2.readline()

			os.remove(file)
		status.end_numbar()
		f1.close()
		print("[+] Done joining all subfiles in same file")
		# print("[-] Removing files that are not needed")
		# status.create_numbar(100, len(list_files))
		# for ind, file in enumerate(list_files):
		# 	status.update_numbar(ind, len(list_files))
		# 	os.remove(file)
		# status.end_numbar()
		# print("[+] Done removing all subfiles")
		#print("[-] Sorting file before storing")
		#total_list = sorted(total_list, key=lambda x: str(x[2]) + x[0] + x[1], reverse=True)
		#print("[+] Done sorting the file, storing it to file...")
		#gen_csv_from_tuples(result_file , ["IdAuthor1", "IdAuthor2", "Score"], total_list)

	def get_sizes(self):
		dictio_of_users, dictio_of_values, dictio_of_weights = self.generate_clean_dataset(self.filenames['dataset_filename'])
		lu,lv = len(dictio_of_users), len(dictio_of_values)
		print("USERS: %d" %(lu))
		print("VALUES: %d" % (lv))
		return np.array([lu, lv]).reshape(1,-1)

	def do_all_sparse_matrix_link(self):
		print("[+] Generating processing information")
		dictio_of_users, dictio_of_values, dictio_of_weights, promising_combinations, bin_sparse_matrix, weight_sparse_matrix = self.generate_processing_info()

		print("Generating promising combinations v2")
		lst_users2 = [(x[0], x[1]) for x in read_csv_list("combined_results.csv")[1:]]
		self.pickle_object(lst_users2, "myname.pkl")
		promising_combinations = self.gen_intersection(promising_combinations, lst_users2)
		print("NEW PROMISING COMBINATIONS: %d" % (len(promising_combinations)))

		self.process_pairs_of_users_sparse_matrix_alt(dictio_of_users, dictio_of_values,
		promising_combinations, bin_sparse_matrix, weight_sparse_matrix,
		self.filenames['coincidence_csv_filename'], self.filenames['uniqueness_csv_filename'])
		# Joining all files into one
		self.join_files()
	def do_all_sparse_matrix(self):
		print("[+] Generating processing information")
		dictio_of_users, dictio_of_values, dictio_of_weights, promising_combinations, bin_sparse_matrix, weight_sparse_matrix = self.generate_processing_info()

		# print("Generating promising combinations v2")
		# lst_users2 = [(x[0], x[1]) for x in read_csv_list("combined_results.csv")[1:]]
		# self.pickle_object(lst_users2, "myname.pkl")
		# promising_combinations = self.gen_intersection(promising_combinations, lst_users2)
		print("NEW PROMISING COMBINATIONS: %d" % (len(promising_combinations)))

		self.process_pairs_of_users_sparse_matrix_alt(dictio_of_users, dictio_of_values,
		promising_combinations, bin_sparse_matrix, weight_sparse_matrix,
		self.filenames['coincidence_csv_filename'], self.filenames['uniqueness_csv_filename'])
		# Joining all files into one
		self.join_files()

	def do_all(self):
		dictio_of_users, dictio_of_values, dictio_of_weights = self.generate_clean_dataset(self.filenames['dataset_filename'])
		#self.store_clean_dataset(dictio_of_users)
		promising_combinations = self.get_promising_combinations(dictio_of_users, dictio_of_values)
		self.gen_binary_matrix(dictio_of_values, dictio_of_users, self.filenames['bin_matrix_filename'])
		self.process_pairs_of_users(dictio_of_users, dictio_of_values, promising_combinations,
			self.filenames['bin_matrix_filename'], self.filenames['coincidence_csv_filename'])
		os.remove(self.filenames['bin_matrix_filename'])
		#self.process_score(dictio_of_users, dictio_of_values, self.filenames['bin_matrix_filename'],
		#	self.filenames['coincidence_score_filename'])
		#self.gen_score_csv(dictio_of_users, dictio_of_values, self.filenames['coincidence_score_filename'],
		#	self.filenames['coincidence_csv_filename'])
		self.gen_weight_matrix(dictio_of_values, dictio_of_users, dictio_of_weights, self.filenames['weight_matrix_filename'])
		lst_users = read_csv_list(self.filenames['coincidence_csv_filename'])[1:]
		self.process_pairs_of_users(dictio_of_users, dictio_of_values, lst_users,
			self.filenames['weight_matrix_filename'], self.filenames['uniqueness_csv_filename'])
		os.remove(self.filenames['weight_matrix_filename'])

	def join_files(self):
		self.join_all_results(self.filenames['coincidence_csv_filename'])
		self.join_all_results(self.filenames['uniqueness_csv_filename'])

##################################################################################################
## FUNCTIONS FROM THIS POINT ARE USED TO ANALYZE THE DISTRIBUTION OF DATA BUT NOT AS PROCESSING ##
##################################################################################################
	def gen_intersection(self, lst_users1, lst_users2):
		return list(set(lst_users1).intersection(set(lst_users2)))

	def evaluate_reduction(self):
		tic = time.time()
		lst_users = read_csv_list(self.filenames['dataset_filename'])[1:]

		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)
		ulen1, vlen1 = len(dictio_of_users), len(dictio_of_values)
		umean, ustd = self.get_mean_std(dictio_of_users)
		vmean, vstd = self.get_mean_std(dictio_of_values)

		dictio_of_users, dictio_of_values, _= self.generate_clean_dataset(self.filenames['dataset_filename'])

		print("[+] Finished Dataset Cleanup in: %f" %(time.time() - tic))
		ulen2, vlen2 = len(dictio_of_users), len(dictio_of_values)
		return ulen1, ulen2, vlen1, vlen2, umean, ustd, vmean, vstd

	def gen_distribution_from_dictio(self, dictio):
		lst_dist = []
		v = np.array([len(v) for k, v in dictio.items()])
		maxnum = np.max(v)

		for i in range(1, maxnum):
			count = np.count_nonzero(v == i)
			if count > 0:
				lst_dist.append((i, count))
		return lst_dist

	def gen_optimal_reduction_csv(self, filename):
		tic = time.time()
		print("[-] Starting the optimal reduction calculation")
		lst_users = read_csv_list(self.filenames['dataset_filename'])[1:]
		lst_results = []
		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)

		dictio_of_users, dictio_of_values = self.clean_dataset(dictio_of_users,dictio_of_values,
			user_removal=self.user_removal, value_removal=self.value_removal)
		init_len_u, init_len_v = len(dictio_of_users), len(dictio_of_values)

		lst_results.append((0, init_len_v, init_len_u, 0, 0))
		dictio_of_weights = self.gen_dictio_of_weigths (dictio_of_values)
		lst_rarity = self.gen_rarity_dist(dictio_of_weights)
		for rarity, _ in lst_rarity:

			dictio_of_users, dictio_of_values, dictio_of_weights = self.clean_dataset_3(dictio_of_users, dictio_of_values,
				dictio_of_weights, rarity)
			post_len_u, post_len_v = len(dictio_of_users), len(dictio_of_values)
			lst_results.append((rarity, post_len_v, post_len_u, init_len_v - post_len_v, init_len_u - post_len_u))

		gen_csv_from_tuples(filename,
			["Rarity Removed", "Value Length", "User Length", "Values Removed", "Users Removed"],
			lst_results)
		print("[+] Finished Dataset Cleanup in: %f" %(time.time() - tic))
		return lst_results
	def gen_reduction_graphs(self,filename, graph_filename):
		tic = time.time()

		lst = read_csv_list(filename)[1:]
		X = np.array([int(x[0]) for x in lst])
		V = np.array([float(x[1]) for x in lst])
		U = np.array([float(x[2]) for x in lst])
		RV = np.array([float(x[3]) for x in lst])
		RU = np.array([float(x[4]) for x in lst])
		maxu = U[0]
		maxv = V[0]

		V /= maxv / 100
		U /= maxu / 100
		#print("U", U)
		#print("V", V)
		plt.style.use('seaborn-darkgrid')
		palette = plt.get_cmap('Set1')

		plt.plot(X, V, color=palette(0), label="Values (V)")
		plt.plot(X, U, color=palette(1), label="Users (U)")
		#plt.plot(X, CY, color=palette(1), linestyle='--', label="CDF(Y)")
			#marker='o', markersize=3, markerfacecolor=palette(0))
		plt.axvline(x=self.rarity_bound, color=palette(2), linestyle='-.',
			label="Rarity Cutoff (%d)"%(self.rarity_bound))
		plt.yticks(np.arange(30, 105, 5))#, rotation='vertical')
		plt.xlabel('Rarity', fontstyle = 'italic', fontsize=12.0)
		plt.ylabel('Removal Ratio', fontstyle = 'italic', fontsize=12.0)

				#plt.scatter('', xy=(x, cy), xytext=(0, 0), color='red' , textcoords='offset points')
		plt.legend(loc='best')
		plt.savefig(graph_filename, format="PDF", bbox_inches='tight')
		plt.clf()
		print("[+] Finished Graph Generation Cleanup in: %f" %(time.time() - tic))

	def get_coincidences_for_users(self, lst_users, dictio = None):
		dictio_of_users, dictio_of_values, dictio_of_weights = self.generate_clean_dataset(
			self.filenames['dataset_filename'])
		dictio = dict() if dictio is None else dictio
		for tup in lst_users:
			if (tup[0] in dictio_of_users.keys()) and (tup[1] in dictio_of_users.keys()):
				if not tup in dictio.keys():
					dictio[tup] = list()
				coin = set(dictio_of_users[tup[0]]).intersection(set(dictio_of_users[tup[1]]))
				for i in coin:
					dictio[tup].append(i)

		return dictio

	def get_optimal_reduction_graph(self, filename):
		tic = time.time()

		lst = read_csv_list(filename)[1:]
		X = np.array([int(x[0]) for x in lst])
		V = np.array([float(x[1]) for x in lst])
		U = np.array([float(x[2]) for x in lst])
		RV = np.array([float(x[3]) for x in lst])
		RU = np.array([float(x[4]) for x in lst])
		maxu = U[0]
		maxv = V[0]

		V /= maxv
		U /= maxu
		#print("U", U)
		#print("V", V)

		return X, V, U, self.rarity_bound

	def gen_optimal_reduction(self, filename, graph_filename):
		self.gen_optimal_reduction_csv(filename)
		self.gen_reduction_graphs(filename, graph_filename)

	def gen_distribution_csv(self, filename):
		tic = time.time()
		lst_users = read_csv_list(self.filenames['dataset_filename'])[1:]

		dictio_of_users = self.gen_dictio_of_users(lst_users)
		dictio_of_values = self.gen_dictio_of_values(lst_users)
		#dictio_of_users, dictio_of_values = self.clean_dataset(dictio_of_users,dictio_of_values,
			#user_removal=self.user_removal, value_removal=self.value_removal)

		user_dist = self.gen_distribution_from_dictio(dictio_of_users)
		gen_csv_from_tuples(filename+"_u", ["n", "num_users_with_n_elements"], user_dist)
		value_dist = self.gen_distribution_from_dictio(dictio_of_values)
		gen_csv_from_tuples(filename+"_v", ["n", "num_values_with_n_users"], value_dist)
		print("[+] Finished Distribution Extraction in: %f" %(time.time() - tic))


import torch
import torch.nn as nn
import numpy as np
import random
import floris.tools as wfct

__author__ = "Jens Bauer"
__copyright__ = "Copyright 2021, CNNwake"
__credits__ = ["Jens Bauer"]
__license__ = "MIT"
__version__ = "1.0"
__email__ = "jens.bauer20@imperial.ac.uk"
__status__ = "Development"

class Generator(nn.Module):
    """
    The class is the Neural Network that generates the flow field around a
    wind turbine. The network uses the pytorch framwork and uses fully
    connected and transpose convolutional layers.
    The methods of this class include the training of the network,
    testing of the accuracy and generaton of the training data.
    """

    def __init__(self, nr_input_var, nr_filter):
        """
        init method that generates the network architecture using pytroch's
        ConvTranspose2d and Sequential layers. The number of input varibles
        and size of the given network can be changed. The output size will not
        change and it set at 163 x 163 pixels.

        Args:
            nr_input_var (int): Nr. of inputs, usually 3 for u, ti and yaw
            nr_filter (int): Nr. filters used in deconv layers, more filters
                             means that the network will have more parameters
        """
        super(Generator, self).__init__()
        # linear layer
        self.FC_Layer = nn.Sequential(nn.Linear(in_features=nr_input_var,
                                                out_features=9))
        # Deconvolutional layer
        self.net = nn.Sequential(
            self.layer(1, nr_filter * 16, 4, 2, 1),
            self.layer(nr_filter * 16, nr_filter * 8, 4, 1, 1),
            self.layer(nr_filter * 8, nr_filter * 8, 4, 2, 1),
            self.layer(nr_filter * 8, nr_filter * 4, 4, 2, 1),
            self.layer(nr_filter * 4, nr_filter * 4, 3, 2, 1),
            nn.ConvTranspose2d(nr_filter * 4, 1, kernel_size=3,
                               stride=3, padding=1),
        )

    def layer(self, in_filters, out_filters, kernel_size, stride, padding):
        """
        One layer of the CNN which consits of ConvTranspose2d,
        a batchnorm and LRelu activation function.
        Function is used to define one layer of the network

        Args:
            in_filters (int): Nr. of filters in the previous layer
            out_filters (int): Nr. of output filters
            kernel_size (int): Size of the ConvTranspose2d layer
            stride (int): Stride of the ConvTranspose2d layer
            padding (int): Padding used in this layer

        Returns:
            nn.Sequential: Pytroch Sequential container that defines one layer
        """
        # One layer of the network uses:
        # Deconvolutional layer, then batch norm and leakyrelu
        # activation function
        single_layer = nn.Sequential(nn.ConvTranspose2d(in_filters,
                                                        out_filters,
                                                        kernel_size,
                                                        stride,
                                                        padding,
                                                        bias=False, ),
                                     nn.BatchNorm2d(out_filters),
                                     nn.LeakyReLU(0.2), )

        return single_layer

    def initialize_weights(self):
        """
        Initilize weights using a normal distribution with mean = 0,std2 = 0.02
        which has helped training. Loop over all modules, if module is
        convolutional layer or batchNorm then initialize weights.

        Args:
            model (torch model): Neural network model defined using Pytorch
        """
        # for ever layer in model
        for m in self.modules():
            # check if it deconvolutional ot batch nrom layer
            if isinstance(m, (nn.Conv2d, nn.BatchNorm2d)):
                # initialize weights using a normal distribution
                nn.init.normal_(m.weight.data, 0.0, 0.02)

    def forward(self, x):
        """
        Functions defines a forward pass though the network. Can be used for
        a single input or a batch of inputs

        Args:
            x (torch.tensor): input tensor, to be passed through the network

        Returns:
            flow_fields (torch.tensor): Output of network
        """
        # first the fully connected layer takes in the input, and outputs
        # 9 neurons which are reshaped into a 3x3 array
        x = self.FC_Layer(x).view(len(x), -1, 3, 3)
        # the Conv layers take in the 3x3 array and output a 163x163 array
        return self.net(x)

    @staticmethod
    def create_floris_dataset(
            size, image_size, u_range, ti_range, yaw_range,
            floris_init_path=".", curl=False):
        """
        Function to generate the dataset needed for training using FLORIS.
        The flowfield around a turbine is generated for a large range of wind
        speeds, turbulent intensities and yaw angles. The 2d array and
        correspoding init conditions are saved for training. The data is
        generated using a  Gaussian wake mode, please see:
        https://doi.org/10.1016/j.renene.2014.01.002.
        For more information about FLORIS see: https://github.com/NREL/floris.
        Function can be used to generated training, validation and test sets.

        Args:
            size (int): Size of the dataset
            image_size (int): Size of the flow field outputs that
                                        are generated, this depends on the
                                        Neural network used, should be 163.
            u_range (list): Bound of u values [u_min, u_max] used
            ti_range (list): Bound of TI values [TI_min, TI_max] used
            yaw_range (list): Bound of yaw angles [yaw_min, yaw_max] used
            floris_init_path (str, optional): Path to the FLORIS jason file.
                                              Defaults to ".".
            curl (bool, optional): If curl model should be used please set
                                   to True, see this for more information:
                                   https://doi.org/10.5194/wes-4-127-2019.
                                   Defaults to False.

        Returns:
            y (torch.tensor): Tensor of size (size, image_size, image_size)
            which includes all the generated flow fields. The flow fields
             are normalised to help training
            x (torch.tensor): Tensor of size (size, 1, 3) which includes the
            flow conditons of the correspoding flow field in the x tensor.
        """

        # sample u, ti and yawn angle from a uniform distribution
        u_list = [round(random.uniform(u_range[0], u_range[1]), 2) for
                  i in range(0, size)]
        ti_list = [round(random.uniform(ti_range[0], ti_range[1]), 2) for
                   i in range(0, size)]
        yawn_list = [round(random.uniform(yaw_range[0], yaw_range[1]), 2) for
                     i in range(0, size)]

        # initialize FLORIS model using the jason file
        if curl is False:
            floris_turbine = wfct.floris_interface.FlorisInterface(
                floris_init_path + "/FLORIS_input_gauss.json")
        else:
            floris_turbine = wfct.floris_interface.FlorisInterface(
                floris_init_path + "/FLORIS_input_curl.json")

        # initialize empty numpy array to store 2d arrays and
        # corresponding u, ti and yawn values
        y = np.zeros((size, image_size, image_size))
        x = np.zeros((size, 3))

        # create train examples
        print("generate FLORIS data")
        for _ in range(0, size):
            if _ % 500 == 0:
                print(f"{_}/{size}")
            # set wind speed, ti and yawn angle for FLORIS model
            floris_turbine.reinitialize_flow_field(
                wind_speed=u_list[_],
                turbulence_intensity=ti_list[_])
            floris_turbine.change_turbine([0], {'yaw_angle': yawn_list[_]})

            # calculate the wakefield
            floris_turbine.calculate_wake()
            # extract horizontal plane at hub height
            cut_plane = floris_turbine.get_hor_plane(
                height=90,
                x_resolution=image_size,
                y_resolution=image_size,
                x_bounds=[0, 3000],
                y_bounds=[-200, 200]).df.u.values.reshape(image_size,
                                                          image_size)
            # save the wind speed values of the plane at hub height and
            # the corresponding turbine stats
            y[_] = cut_plane
            x[_] = u_list[_], ti_list[_], yawn_list[_]

        # turn numpy array into a pytroch tensor
        x = torch.tensor(x, dtype=torch.float)
        # The wind speeds are normalised by dividing it by 12
        # i.e. every value will be between 0-1 which helps training
        y = torch.tensor(y, dtype=torch.float)/12

        return x, y

    def error(self, x_eval, y_eval, device, image_size=163, normalisation=12):
        r"""
        Calculate the average pixel wise percentage error of the model on
        a evaluation set. For error function is:
        error = 1/set_size *\sum_{n=0}^{set_size}(1/image_size**2 *
                \sum_{i=0}^{image_size**2}(100*abs(FLORIS_{n,i} - GAN_{n,i})/
                max(FLORIS_{n,i})))
        For a detailed explanation of this function please see the report in
        the ACSE9 repo.
        """
        error_list = []
        # Use model to predict the wakes for the given conditions in x
        model_predict = self.forward(x_eval.to(device))
        for n in range(0, len(x_eval)):
            # Calculate the mean error between CNNwake output and FLORIS
            # for a given flow field using the function given above
            pixel_error = np.sum(abs(
                    y_eval.detach().cpu().numpy()[n] -
                    model_predict.squeeze(1)[n].detach().cpu().numpy()) /
                    (torch.max(y_eval.detach()[n]).cpu().numpy()))
            # divide by number of pixels in array for an mean value
            pixel_error /= image_size * image_size
            error_list.append(pixel_error * 100)

        # return mean error
        return np.mean(error_list)

    def evaluate_model(self, set_size, u_range, ti_range, yaw_range,
                       image_size=163, device='cpu', normalisation=12,
                       florisjason_path="."):
        """
        Function to calculate a average pixel wise percentage error
        of the model using the error function. This functions generates
        a test set and evaluates the model on this unseen data to provide
        a test error.

        Args:
            set_size (int, optional): Nr. of samples to be used for testing.
            u_range (list): Bound of u values [u_min, u_max] used
            ti_range (list): Bound of TI values [TI_min, TI_max] used
            yaw_range (list): Bound of yaw angles [yaw_min, yaw_max] used
            image_size (int, optional): Size of the flow field.
             Defaults to 163.
            device (str): Device to store and run the neural network on,
             either cpu or cuda.
            normalisation (int, optional): The CNN output is between
             0 and 1 due to the  normalisation used, therefore it needs to
             be renormalised. Defaults to 12.
            florisjason_path (str, optional): Location of the FLORIS jason
             file. Defaults to ".".

        Returns:
           error (float): Error of model on test set
        """
        # Create a dataset to test the model on
        x_eval, y_eval = self.create_floris_dataset(
            set_size, image_size, u_range=u_range, ti_range=ti_range,
            yaw_range=yaw_range, floris_init_path=florisjason_path)

        # Use generated data set to calculate the error of CNNwake when
        # compared with the FLORIS output
        test_error = self.error(x_eval, y_eval, device,
                                image_size, normalisation=12)

        return test_error

    def epoch_training(self, criterion, optimizer, dataloader, device):
        """
        Trains the model for one epoch data provided by dataloader. The model
        will be updated after each batch and the function will return the
        train loss of the last batch

        Args:
            criterion (torch.nn.criterion): Loss function used to train model
            optimizer (torch.optim.Optimizer): Optimizer for gradient descent
            dataloader (torch.utils.DataLoader): Dataloader to store dataset
            device (str): Device on which model/data is stored, cpu or cuda

        Returns:
            training loss (float): Loss of training set defined by criterion
        """
        # For all training data in epoch
        for real_images, label in dataloader:
            # move data to device
            real_images = real_images.to(device)
            label = label.to(device)
            # images need to be in correct shape: batch_size x 1 x 1 x 3

            # compute reconstructions of flow-field using the CNN
            outputs = self.forward(label)

            # compute training reconstruction loss using the
            # loss function set
            train_loss = criterion(outputs, real_images)

            optimizer.zero_grad()  # Zero gradients of previous step
            train_loss.backward()  # compute accumulated gradients
            optimizer.step()  # Do optimizer step

        # return training loss
        return train_loss.item()

    def load_model(self, path='.', device='cpu'):
        """
        Function to load model from a pt file into this class.

        Args:
            path (str): path to saved model.
            device (torch.device): Device to load onto, cpu or cuda

        """
        # load the pretrained model
        self.load_state_dict(torch.load(path, map_location=device))

    def save_model(self, name='generator.pt'):
        """
        Function to save current model paramters so that it can
        be used again later. Needs to be saved with as .pt file

        Args:
            name (str): name of .pt file from which to load model
        """
        # Save current model parameters
        torch.save(self.state_dict(), name)


'''Convert pretrained VGG model to SSD.

VGG model download from PyTorch model zoo: https://download.pytorch.org/models/vgg16-397923af.pth
'''
import torch
from torchvision.models import vgg16
from model import SSD
from config import opt

vgg = torch.load('../checkpoints/vgg16-397923af.pth')

ssd = SSD(opt)
layer_indices = [0, 2, 5, 7, 10, 12, 14, 17, 19, 21]  # 这是因为只载入卷及层的权重，Relu, Maxpooing那些层是没有参数的

for layer_idx in layer_indices:
    ssd.base[layer_idx].weight.data = vgg['features.%d.weight' % layer_idx]
    ssd.base[layer_idx].bias.data = vgg['features.%d.bias' % layer_idx]

# [24,26,28]
ssd.conv5_1.weight.data = vgg['features.24.weight']
ssd.conv5_1.bias.data = vgg['features.24.bias']
ssd.conv5_2.weight.data = vgg['features.26.weight']
ssd.conv5_2.bias.data = vgg['features.26.bias']
ssd.conv5_3.weight.data = vgg['features.28.weight']
ssd.conv5_3.bias.data = vgg['features.28.bias']

torch.save(ssd.state_dict(), 'ssd.pth')  # state_dict() 仅保存和加载模型参数(推荐使用)


from page.models import Page
from django.contrib import admin
from django.db import models
from .models import Page, Carousel

class PageAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("title",)}
    list_display = (
        'pk',
        'title',
        'slug',
        'status',
        'updated_at',
    )
    list_filter = ('status',)
    list_editable = (
        'title',
        'status',
    )

class CarouselAdmin(admin.ModelAdmin):
    list_display = [
        'pk',
        'title',
        'cover_image',
        'status',
    ]
    list_filter = ['status', ]
    list_editable = list_filter

admin.site.register(Page, PageAdmin)
admin.site.register(Carousel, CarouselAdmin)

# M Db yapisi
# V View / Control
# T Template
# A Admin
# F Forms
# M Message
# S Session


#!/usr/bin/env python
# -*- coding: utf-8 -*-
## Project: Simple4All - November 2013 - www.simple4all.org
## Contact: Oliver Watts - owatts@staffmail.ed.ac.uk

import sys
import re
import os

from argparse import ArgumentParser

from util import *

def main_work():

    #################################################

    # ======== Get stuff from command line ==========

    a = ArgumentParser()
    a.add_argument('-treefile', required=True, help= "...")

    opts = a.parse_args()
    # ===============================================

    f = open(opts.treefile, 'r')
    data = f.read()
    f.close()
    trees = re.split('\n\s*\n', data)
    trees = [t for t in trees if t != '']

    ## first block is questions:
    questions = trees[0] + '\n\n'
    assert questions[:2] == 'QS'

    tree_dict = {}
    for tree in trees[1:]:
        stream = re.search('(?<=stream\[)[^\]]+(?=\])', tree).group()
        if ',' in stream:
            stream = stream.split(',')[0]
        if stream not in tree_dict:
            tree_dict[stream] = []
        tree_dict[stream].append(tree + '\n\n')

    for (stream, trees) in tree_dict.items():
        writelist([questions] + trees, opts.treefile+'_'+stream)

if __name__=="__main__":

    main_work()


import numpy as np
import wicksell_transform as wt
from scipy.optimize import minimize
from wickselluniform import cdf_uni
from scipy import stats
from collections.abc import Iterable

def _histogram_error(sample, bin_edges, freq, method):
    """
    Computes the error function between a tranformed histogram and a finite sample. This error can be KS test or
    negative log-likelihood.
    """
    hist = (freq, bin_edges)
    wh = wt.from_histogram(hist)
    if method.lower() == 'mde':
        return stats.kstest(sample, wh.cdf)[0]
    elif method.lower() == 'mle':
        return wh.nnlf((0, 1), sample)
    else:
        raise ValueError('Method can be either \'MDE\' or \'MLE\'.')

def _wicksell(r, lb, ub):
    """
    When a sphere of radius r is cut at random latitude, the probability of finding a disk with radius comprised
    between lb and ub is:

    .. math::
        \frac{1r}\left( \sqrt{r^2-lb^2} - \sqrt{r^2-ub^2} \right)

    References
    ----------
    Sahagian, D. and Proussevitch, A. (1998). doi: 10.1016/S0377-0273(98)00043-2

    """
    return (np.sqrt(r ** 2 - lb ** 2) - np.sqrt(r ** 2 - ub ** 2)) / r

def _wicksell_uniform(a, b, lb, ub):
    return cdf_uni(b, lb, ub) - cdf_uni(a, lb, ub)

def Saltykov(sample, bins=10):
    """
    Compute the unfolded histogram from a folded sample, using the Saltykov method [1,2]. This implementation of the
    Saltykov method uses a modified estimation of probabilities, taking advantage of cdf_uni function [3].

    Parameters
    ----------
    sample : Iterable
        Random values of apparent radii
    bins : int or Iterable
        If bins is int, the Saltykov method will use the specified numer of bins
        If bins is a series of increasing values, they will be used as bin edges.

    Returns
    -------
    tuple
        Results from the Saltykov method, as (frequencies, bin_edges). Note that frequencies are normalized so that:
        np.sum(frequencies * np.diff(bin_edges))=1.

    See also
    --------
    two_step_method: Apply the Saltykov method and fit a continuous distribution on the unfolded histogram

    References
    ----------
        .. [1] S.A. Saltikov (1967), DOI: 10.1007/978-3-642-88260-9_31
        .. [2] M.A. Lopez-Sanchez (2018), DOI: 10.21105/joss.00863
        .. [3] D. Depriester and R. Kubler (2019), DOI: 10.5566/ias.2133
    """
    freq, bin_edges = np.histogram(sample, bins=bins)
    bin_sizes = np.diff(bin_edges)
    freq = freq / np.sum(freq * bin_sizes)
    n_bins = len(freq)
    for i in reversed(range(0, n_bins)):
        p_i = _wicksell(bin_edges[i + 1], bin_edges[i], bin_edges[i + 1])
        for j in range(0, i):
            p_j = _wicksell_uniform(bin_edges[j], bin_edges[j + 1], bin_edges[i], bin_edges[i + 1])
            p_norm = p_j * freq[i] / p_i
            freq[j] = np.abs(freq[j] - p_norm)
    freq = freq / np.sum(freq * bin_sizes)
    return freq, bin_edges

def two_step_method(sample, distribution, bins=10, **kwargs):
    """Unfold the distribution by first fitting an histogram, then fit a continuous distribution on it. This is an
    extension of the two-step method, first proposed for lognormal distribution in [1], to any continuous distribution.

    Parameters
    ----------
    sample : Iterable
        Random values of apparent radii
    distribution : scipy.stats.rv_continuous
        Continuous distribution from scipy.stats
    bins : int or Iterable
        If bins is int, uses the specified number of bins
        If bins is a series of increasing values, they will be used as bin edges.
    **kwargs
        Extra argument. They are the same as for the fit() method for scipy.stats.rv_continuous. For example, if one
        wants to fix location to 0.0, use 'floc=0'.

    Returns
    -------
    theta : list
        Parameters of the distribution, resulting in the best consistency between the PDF and the unfolded histogram.
    hist : list
        Unfolded histogram. It is given by the pair (frequencies, bin_edges).

    See also
    --------
    fit_histogram: Unfold a sample to find the underlying histogram resulting in the best goodness-of-fit KS test.
    Saltykov: Perform the saltykov method and return the unfolded histogram

    References
    ----------
        [1] M. A. Lopez-Sanchez and S. Llana-Fúnez (2016). doi: 10.1016/j.jsg.2016.10.008
    """
    (freq, bin_edges), _ = fit_histogram(sample, bins=bins)
    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2

    # The hack is to take advantage of argument parser from fit() method, but the cost function must be adapted.
    def error_fun(args, x, _):    # moment_error, used in Method of Moments (MM), replaced by Least-square criterion
        return np.sum((distribution.pdf(x, *args) - freq) ** 2)
    moment_error_old = distribution._moment_error
    distribution._moment_error = error_fun     # Overwrites the default function used in MM
    theta = distribution.fit(bin_centers, **kwargs, method='MM')
    distribution._moment_error = moment_error_old     # Restore default function, just to be sure
    hist = (freq, bin_edges)
    return theta, hist

def fit_histogram(sample, rmin=0.0, rmax=None, bins=10, method='MDE'):
    """
    Unfold a sample to find the underlying histogram resulting in the best goodness-of-fit KS test.

    Parameters
    ----------
    sample : iterable
        The sample one wants to unfold. It consists in a list of value
    rmin : float
        Left-most location of bin edges. Default is 0.0.
    rmax : float
        Right-most location of bin edges. Default is max(sample)*4/pi.
    bins : int or Iterable
        Number of bins to use. If int, the function will be run with only this number of bins. If bins is a list, like
        (n_min, n_max), this function will run for each value ranging between n_min and n_max. The returned value will
        be that maximizing the likelihood. If bins is a list of increasing float values, they will be used as bin edges;
        in this case, rmin and rmax are omitted.
    method : str
        Use either Maximum Likelihood Estimation (MLE) as a fitting criterion, of Minimum Distance Estimation (MDE),
        through the Kolmogorov-Smirnov (KS) goodness-of-fit test. Default is MDE.

    Returns
    -------
    hist : tuple
       The unfolded histogram which minimizes the KS test.
    res : scipy.optimize._optimize.OptimizeResult
        Information about minimizing procedure. In particular, the final error value (KS statistics of negative log-
        likelihood, depending on the method) is provided by res.fun.

    """
    if rmax is None:
        rmax = max(sample) * 4 / np.pi
    if isinstance(bins, int):
        # e.g. bins=10
        bin_edges = np.linspace(rmin, rmax, bins + 1)
        return fit_histogram(sample, bins=bin_edges)
    elif isinstance(bins, Iterable):
        if len(bins) == 2 and isinstance(bins[0], int) and isinstance(bins[1], int):
            # e.g. bins=(10,20)
            cost = np.inf
            hist = ()
            res = ()
            for n in range(bins[0], bins[1] + 1):
                bin_edges = np.linspace(rmin, rmax, bins=n+1)
                hist_n, res_n = fit_histogram(sample, bins=bin_edges)
                if res_n.fun < cost:
                    hist, res = hist_n, res_n
                    cost = res_n.fun
            return hist, res
        else:
            # e.g. bins=(0., 1., 2., 3.)
            spacing = np.diff(bins)
            n_bins = len(bins) - 1

            def fun(x):
                return _histogram_error(sample, bins, x, method)

            def confun(x):
                return np.sum(x * spacing) - 1

            lb = np.zeros(n_bins)
            ub = np.ones(n_bins) * np.inf
            bounds = np.vstack((lb, ub)).T
            x_0, _ = Saltykov(sample, bins=bins)
            res = minimize(fun, x_0, bounds=bounds, constraints={'type': 'eq', 'fun': confun})
            freq = res.x
            hist = (freq, bins)
            return hist, res

def plot_histogram(ax, hist, *args, **kwargs):
    """
    Simple function to help plotting the resulting histogram.

    Parameters
    ----------
    ax : matplotlib.axes._subplots.AxesSubplot
        Handle to axis or figure where the histogram will go.
    hist : tuple
        Histogram details, so that hist == (frequencies, bin_edges)
    *args : tuple
        Extra value arguments passed to the ax.bar() function
    **kwargs
        Extra keyword arguments passed to the ax.bar() function

    Returns
    -------
     l : list
        A list of lines representing the plotted data.

    References
    ----------
    Inspired from https://stackoverflow.com/a/33372888/12056867
    """
    freq, bin_edges = hist
    return ax.bar(bin_edges[:-1], freq, width=np.diff(bin_edges), align='edge', *args, **kwargs)


from sympy import Symbol
from sympy.matrices import Matrix
import numpy as np
from qalcore.dwave.qubols.encodings import RealUnitQbitEncoding
from typing import Optional, Union, List, Callable, Dict, Tuple
from dwave.system import DWaveSampler , EmbeddingComposite
import neal
from dimod import ExactSolver

from .solution_vector import SolutionVector

class QUBOLS:

    def __init__(self, options: Optional[Union[Dict, None]] = None):
        """Linear Solver using QUBO

        Args:
            options: dictionary of options for solving the linear system
        """

        self.default_solve_options = {
            "sampler": neal.SimulatedAnnealingSampler(),
            "encoding": RealUnitQbitEncoding,
            "num_qbits": 11,
            "num_reads": 100,
            "verbose": False
        }
        self.options = self._validate_solve_options(options)
        self.sampler = self.options.pop('sampler')

    def _validate_solve_options(self, options: Union[Dict, None]) -> Dict:
        """validate the options used for the solve methods

        Args:
            options (Union[Dict, None]): options
        """
        valid_keys = self.default_solve_options.keys()

        if options is None:
            options = self.default_solve_options

        else:
            for k in options.keys():
                if k not in valid_keys:
                    raise ValueError(
                        "Option {k} not recognized, valid keys are {valid_keys}"
                    )
            for k in valid_keys:
                if k not in options.keys():
                    options[k] = self.default_solve_options[k]

        return options

    def solve(self,
              matrix: np.ndarray,
              vector: np.ndarray ):
        """Solve the linear system

        Args:
            sampler (_type_, optional): _description_. Defaults to neal.SimulatedAnnealingSampler().
            encoding (_type_, optional): _description_. Defaults to RealUnitQbitEncoding.
            nqbit (int, optional): _description_. Defaults to 10.

        Returns:
            _type_: _description_
        """

        self.A = matrix
        self.b = vector
        self.size = self.A.shape[0]

        sol = SolutionVector(size=self.size,
                             nqbit=self.options['num_qbits'],
                             encoding=self.options['encoding'])
        self.x = sol.create_polynom_vector()
        self.qubo_dict = self.create_qubo_matrix(self.x)

        self.sampleset = self.sampler.sample_qubo(self.qubo_dict, num_reads = self.options['num_reads'])
        self.lowest_sol = self.sampleset.lowest()
        return sol.decode_solution(self.lowest_sol.record[0][0])

    def create_qubo_matrix(self, x, prec=None):
        """Create the QUBO dictionary requried by dwave solvers
        to solve the linear system

        A x = b

        Args:
            Anp (np.array): matrix of the linear system
            bnp (np.array): righ hand side of the linear system
            x (sympy.Matrix): unknown

        Returns:
            _type_: _description_
        """
        A = Matrix(self.A)
        b = Matrix(self.b)

        polynom = x.T @ A.T @ A @ x - x.T @ A.T @ b - b.T@ A @ x + b.T @ b
        polynom = polynom[0]
        polynom = polynom.expand()
        polynom = polynom.as_ordered_terms()

        out = dict()

        for term in polynom:
            m = term.args
            if len(m) == 0:
                continue

            if len(m) == 2:
                varname = str(m[1])
                varname = varname.split("**")[0]
                key = (varname , varname)

            elif len(m) == 3:
                key = (str(m[1]),str(m[2]))

            if key not in out:
                out[key] = 0.0

            out[key] += m[0]

        if prec is None:
            return out

        elif prec is not None:
            nremoved = 0
            out_cpy = dict()
            for k, v in out.items():
                if np.abs(v)>prec:
                    out_cpy[k] = v
                else:
                    nremoved += 1
            print('Removed %d elements' %nremoved)
            return out_cpy


from django.contrib import admin
from django.urls import path

from vendor import views

urlpatterns = [
        path('',views.openLogin,name='vendor_main'),
    path('vendor_login_check/',views.vendor_login_check,name='vendor_login_check'),
    path('vendor_register/',views.vendor_register,name='vendor_register'),
    path('vendor_save/',views.vendor_save,name='vendor_save'),
    path('vendor_welcome/',views.vendor_welcome,name='vendor_welcome'),
]