'''
  # Tiny OpenAI ChatGPT and Whisper API Library
  - 2023/3     V0.1       By Charles Lai
  - 2023/4     V0.13
'''
__version__ = 0.13
__author__ = 'Charles Lai'

import requests, json

def get_delta(rdata):
  ''' Return stream mode delta text '''
  # len() > 13 by pass  b'' and b'data: [DONE]'
  if len(rdata) > 13:
    return json.loads(rdata.decode('utf-8')[5:]).get('choices', [{}])[0].get('delta', {}).get('content','')
  else:
    return ''

#
# HTTP_Post Function
#
def httpPost(debug, *args, **kw):
  ''' OpenAI API httpPost function
    debug: True/False - Display Debug info;
    *args/**kw: requests.post args;
  '''
  # get stream mode
  stream_mode = kw.get('json', {}).get('stream', False) == True
  kw['stream'] = stream_mode

  # init return value
  rdata = {}
  try:
    r = requests.post(*args, **kw)
  except Exception as x:
    if debug:
      if type(x) == requests.exceptions.SSLError:
        print('@ SSL Error, Please using: pip install certifi --upgrade ')
      else:
        print('@ Network Error!')
    # break return
    return rdata
  # Get Result
  if r.status_code == 200:
    try:
      if stream_mode:
        # return a iter
        rdata = ( get_delta(i) for i in r.iter_lines() )
      else:
        rdata = r.json()
    except:
      pass
  # status code not 200, print error info
  elif debug:
    # 400-File Problem, 401-API_KEY, 404-URL
    if r.status_code == 400:
      print('@ Error, Please Check File type(mp3/m4a/wav/... audio file !')
    elif r.status_code == 401:
      print('@ Error, Please Check API_Key !')
    elif r.status_code == 404:
      print('@ Error, Please Check API_URL !')
    else:
      print('@ Error, HTTP Status_code is: %d !' % r.status_code)
  # Return Result, stream_mode
  return rdata, stream_mode

#
# ChatGPT API
#
class ChatGPT():
  def __init__(self, API_Key='', Proxy='', Model='gpt-3.5-turbo', URL='https://api.openai.com/v1/chat/completions', Debug=False, stream=False):
    ''' ChatGPT API Access, args: API_Key, Proxy, Model, API_URL, Debug, stream '''
    self.API_Key = API_Key
    self.URL = URL
    self.Proxy = Proxy
    self.Model = Model
    self.Debug = Debug      # Debug info print
    self.stream = stream    # openAI stream mode
    self.Call_cnt = 0       # Total Call count
    self.Total_tokens = 0   # Total tokens count
    self.Hinfo = []         # Query History List

  def call(self, data):
    ''' Call ChatGPT '''
    headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.API_Key}" }
    # Call API
    rdata, stream_mode = httpPost(self.Debug, self.URL, headers = headers, proxies={"http": self.Proxy, "https": self.Proxy}, json = {"model": self.Model, "messages": data, "stream": self.stream})
    # add total_tokens, call times
    self.Call_cnt += 1
    if not stream_mode and rdata:
      self.Total_tokens += rdata.get('usage', {}).get('total_tokens', 0)
    # return result
    return rdata, stream_mode

  def get_text(self, rdata):
    ''' Return only message text, rdata = chatGPT return json '''
    if rdata:
      # get rdata['choices'][0]['message']['content']
      return rdata.get('choices', [{}])[0].get('message', {}).get('content', '').encode('utf8').decode()
    else:
      return ''

  def query(self, data, flag=False, hcnt=0, system='You are a helpful assistant.'):
    ''' Quick ask ChatGPT, data = Query string
      flag: Using Query History flag  (Using obj.Hinfo = [] clean history.)
      hcnt: History count, 0:all, >0 last n (etc: 3-last 3, 10-last 10)
      system: ChatGPT system role content
    '''
    if flag:
      # Using Query History [abs to avoid negative numbers, *2 for Q&A paired]
      qdata = [{"role": "system", "content": system}] + self.Hinfo[-abs(hcnt*2):] + [{"role": "user", "content": data}]
    else:
      qdata = [{"role": "system", "content": system}, {"role": "user","content": data}]
    # query openAI
    r, stream_mode = self.call(qdata)
    if not stream_mode:
      ret = self.get_text(r)
    else:
      # stream mode, print out and then return result
      ret = []
      for t in r:
        print(t, end='', flush=True)
        ret.append(t)
        # calc tokens (data is not available and only be estimated)
        self.Total_tokens += 1
      print()
      ret = ''.join(ret)
    # add Query + Answer to History list
    self.Hinfo.append({"role": "user", "content": data})
    self.Hinfo.append({"role": "assistant", "content": ret})
    return ret

  def translate(self, text, lang='simplified chinese'):
    ''' Translate text to Simple Chinese or other '''
    text = text.strip()
    if text:
      return self.query( f"Please help me to translate,`{text}` to {lang}, please return only translated content not include the origin text", False, 0, '')
    else:
      return ''

  def cHinfo(self):
    ''' Clean query History '''
    self.Hinfo = []

#
# Whisper API
#
class Whisper():
  def __init__(self,API_Key='', Proxy='', Model='whisper-1', URL=('https://api.openai.com/v1/audio/transcriptions','https://api.openai.com/v1/audio/translations'), Debug=False):
    # translations, transcriptions
    ''' Whisper API Access, args: API_Key, Type, Proxy, Model, API_URL, Debug '''
    self.API_Key = API_Key
    self.URL = URL
    self.Proxy = Proxy
    self.Model = Model
    self.Debug = Debug      # Debug info print
    self.Call_cnt = 0       # Total Call count
    self.Total_tokens = 0   # Total tokens count

  def call(self, file, Type=0):
    ''' Call Whisper
      file: file name
      Type: 0.transcribes in source language  1.transcribes into English
    '''
    # check type
    if Type > len(self.URL): Type = len(self.URL) - 1
    # # Call ChatGPT API
    try:
      rdata, _ = httpPost(self.Debug, self.URL[Type], headers = { "Authorization": f"Bearer {self.API_Key}" }, proxies = {"http": self.Proxy, "https": self.Proxy}, data = {'model': self.Model}, files = {'file': (file, open(file,'rb'))} )
    except FileNotFoundError:
      print('@ Error, File Not Found!')
      rdata = ''
    # get result
    if rdata:
      rdata = rdata.get('text', '')
      # add total_tokens, call times
      self.Total_tokens += len(rdata)
      self.Call_cnt += 1
    else:
      rdata = ''
    # Return result
    return rdata

#
# OpenAI embeddings
#
class Embedding():
  def __init__(self, API_Key='', Proxy='', Model='text-embedding-ada-002', URL='https://api.openai.com/v1/embeddings', Debug=False):
    ''' OpenAI embeddings API, args: API_Key, Proxy, Model, API_URL, Debug '''
    self.API_Key = API_Key
    self.URL = URL
    self.Proxy = Proxy
    self.Model = Model
    self.Debug = Debug      # Debug info print
    self.Call_cnt = 0       # Total Call count
    self.Total_tokens = 0   # Total tokens count

  def embed(self, data):
    ''' embedding data , data can be string or python list
      How to using return value?
      1. ret[0]['embedding']
      2. for v in ret:
          v.get('embedding')
    '''
    headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.API_Key}" }
    # Call API
    rdata, _ = httpPost(self.Debug, self.URL, headers = headers, proxies={"http": self.Proxy, "https": self.Proxy}, json = {"model": self.Model, "input": data})
    # add total_tokens, call times
    if rdata:
      self.Total_tokens += rdata.get('usage', {}).get('total_tokens', 0)
      self.Call_cnt += 1
    # return result
    return rdata.get('data', [])

#
# ChatGPT Query Demo
#
def QueryDemo():
  import sys
  if len(sys.argv) > 1:
    # get API_Key
    API_Key = sys.argv[1]
    # get Proxy
    if len(sys.argv) > 2:
      Proxy = sys.argv[2]
    else:
      Proxy = ''
    r = ChatGPT(API_Key, Proxy, Debug=True, stream=True)
    print('>>> ChatGPT Query <<<')
    while True:
      try:
        cmd = input('> ').strip()
        if cmd == '?':
          print(f'Call Time: {r.Call_cnt}, Total Tokens: {r.Total_tokens}.')
        elif cmd:
          print('= '*36)
          r.query(cmd, True, 6)
          print()
      except KeyboardInterrupt:
        print()
        break
  else:
    print('> Using python tinyOpenAI.py your_OpenAI_Key Proxy')

if __name__ == "__main__":
  QueryDemo()


#!/usr/bin/python3

"""
Q1.4
    Palindrome Permutation: Given a string, write a function to check if it is a permutation of a palindrome.
    A palindrome is a word or phrase that is the same forwards and backwards. A permutation is a rearrangement of letters.The palindrome does not need to be limited to just dictionary words.
"""

import unittest

def is_palindrome(s):
    return s == s[::-1]

def permutation_palindrome(s):
    penalty = 0
    charaCount = {i:0 for i in "abcdefghijklmnopqrstuvwxyz "}
    s = s.lower()
    for i in s:
        charaCount[i] += 1
    for k,v in charaCount.items():
        if k == ' ':
            continue
        if v % 2 != 0:
            penalty += 1
            if penalty > 1:
                return False
    return True

#TEST
class Test(unittest.TestCase):
    '''Test Cases'''
    data = [
        ('Tact Coa', True),
        ('jhsabckuj ahjsbckj', True),
        ('Able was I ere I saw Elba', True),
        ('So patient a nurse to nurse a patient so', False),
        ('Random Words', False),
        ('Not a Palindrome', False),
        ('no x in nixon', True),
        ('azAZ', True)]

    def test_pal_perm(self):
        for [test_string, expected] in self.data:
            actual = permutation_palindrome(test_string)
            self.assertEqual(actual, expected)

if __name__ == "__main__":
    unittest.main()


import subprocess
import threading
import time
import os
import shutil
import sys
import math
import nbformat

def get_lines(std_pipe):
    '''Generator that yields lines from a standard pipe as there are printed.'''
    for line in iter(std_pipe.readline, ''):
        yield line
    std_pipe.close()

def get_cell(seconds):
    '''Return a cell that waits for seconds and logs its execution time (from the end of the previous cell).'''
    cell = ["time.sleep(" + str(seconds) + ")\n",
            "t1 = time.time()\n",
            "log += str(t1 - t0) + '\\n'\n",
            "t0 = t1"]
    return cell

def get_ipynb(cell_nb, sleep_per_cell, result_dir):
    '''Return a notebook which consists of all the sleeping cells and the creation of a log file.'''
    cells = []
    cell = ["import uuid\n",
            "import time\n",
            "fname = str(uuid.uuid4()) + '.log'\n",
            "log = ''\n",
            "t0 = time.time()"]
    cells.append(cell)
    for i in range(cell_nb):
        cell = get_cell(sleep_per_cell)
        cells.append(cell)
    cell = ["with open('" + result_dir + "/' + fname, 'w') as f:\n",
            "    f.write(log)"]
    cells.append(cell)
    ipynb = nbformat.v4.new_notebook(
        metadata={
            'kernelspec': {
                'display_name': 'Python 3',
                'language': 'python',
                'name': 'python3'
            }
        },
        cells=[nbformat.v4.new_code_cell(cell) for cell in cells],
    )
    return ipynb

def test_performance(sleep_per_cell=0.1, cell_nb=100, kernel_nb=10, max_meantime_per_cell=0.5, ipynb_path='./sleep.ipynb'):
    '''Test for Voila serving several clients concurrently.
    Several clients connect to the Voila server URL at the same time, and each one gets a dedicated kernel.
    The served notebook consists of a number of cells that just wait for some time and log their execution
    time (from the end of the previous cell). At the end of the notebook, the results are written to a unique
    file.
    This allows for testing the performance of serving all the cells concurrently when all the clients are
    requesting them at the same time. Ideally, the time needed to serve a cell should be close to the cell
    sleeping time. When all clients receive the entire notebook, we kill them as well as the Voila server
    (there is a timeout to check that the clients don't wait too long). Then we compute the cell execution
    mean time. It is an error if it is much greater than the ideal value (sleep_per_cell), and we allow for
    it to be <= max_meantime_per_cell.

    Keyword arguments:
    sleep_per_cell -- number of seconds a cell waits for (default 0.1)
    cell_nb -- number of waiting cells in the notebook (default 100)
    kernel_nb -- number of kernels to launch in parallel (default 10)
    max_meantime_per_cell -- maximum tolerated mean execution time per cell (default 0.5)
    ipynb_path -- path to the saved notebook (default 'tests/notebooks/sleep.ipynb')
    '''

    ipynb_dname, ipynb_fname = os.path.split(ipynb_path)
    os.chdir(ipynb_dname)
    result_dir = ipynb_fname[:-6]

    if os.path.exists(result_dir):
        shutil.rmtree(result_dir)
    os.makedirs(result_dir)

    # generate notebook
    ipynb = get_ipynb(cell_nb, sleep_per_cell, result_dir)
    with open(ipynb_fname, 'w') as f:
        nbformat.write(ipynb, f)

    # launch Voila
    voila = subprocess.Popen(('voila --no-browser ' + ipynb_fname).split(), stderr=subprocess.PIPE, universal_newlines=True)

    # wait until server is ready
    for line in get_lines(voila.stderr):
        sys.stderr.write(line)
        if line.startswith('http://'):
            break

    # launch clients
    clients = [subprocess.Popen('curl http://127.0.0.1:8866 --output /dev/null --silent'.split()) for i in range(kernel_nb)]

    # wait until all kernels have started
    kernel_i = 0
    for line in get_lines(voila.stderr):
        sys.stderr.write(line)
        if 'Kernel started' in line:
            kernel_i += 1
            if kernel_i == kernel_nb:
                # continue printing Voila's stderr in the background
                def target():
                    for line in get_lines(voila.stderr):
                        sys.stderr.write(line)
                threading.Thread(target=target).start()
                break

    # wait for all notebooks to execute
    t0 = time.time()
    done = False
    timeout = False
    timeout_time = 2 * max_meantime_per_cell * cell_nb
    while not done:
        time.sleep(1)
        done = True
        for client in clients:
            if client.poll() is None:
                done = False
        if not done:
            if time.time() - t0 > timeout_time:
                done = True
                timeout = True

    # stop all clients first
    for client in clients:
        client.kill()
    # then stop Voila
    voila.kill()

    if timeout:
        raise TimeoutError

    # analyze data
    fnames = os.listdir(result_dir)
    data = []
    for fname in fnames:
        with open(result_dir + '/' + fname) as f:
            data += [float(d) for d in f.read().split()]

    meantime_per_cell = sum(data) / len(data)
    stddevtime_per_cell = math.sqrt(sum([(d - meantime_per_cell) ** 2 for d in data]) / (len(data) - 1))
    print('Mean time per cell', meantime_per_cell, end='')
    if meantime_per_cell > max_meantime_per_cell:
        print(' >', sleep_per_cell, '(maximum tolerated is', max_meantime_per_cell, ')')
    else:
        print(", should ideally be", sleep_per_cell, "but it's close enough!")
    return meantime_per_cell, stddevtime_per_cell

if __name__ == '__main__':
    test_performance()


# Definition for a binary tree node.
class TreeNode(object):
    """节点类"""
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Tree(object):

    # 建立二叉树是以层序遍历方式输入，节点不存在时以 'None' 表示
    def creatTree(self, nodeList):
        if nodeList[0] == None:
            return None
        head = TreeNode(nodeList[0])
        Nodes = [head]
        j = 1
        for node in Nodes:
            if node != None:
                node.left = (TreeNode(nodeList[j]) if nodeList[j] != None else None)
                Nodes.append(node.left)
                j += 1
                if j == len(nodeList):
                    return head
                node.right = (TreeNode(nodeList[j]) if nodeList[j] != None else None)
                j += 1
                Nodes.append(node.right)
                if j == len(nodeList):
                    return head

class Solution:

    def rob(self, root):
        """
        :type root: TreeNode
        :rtype: int
        """
        return max(self.tryRob(root))

    def tryRob(self, node):
        if node == None:
            return [0, 0]
        # result[0] is max value if not rob current node
        # result[1] is max value if rob current node
        result = [0] * 2
        left_val = self.tryRob(node.left)
        right_val = self.tryRob(node.right)
        result[0] = max(left_val[0], left_val[1]) + max(right_val[0], right_val[1])
        result[1] = left_val[0] + right_val[0] + node.val
        return result

if __name__ == "__main__":
    elements = [3, 4, 5, 1, 3, None, 1]
    tree = Tree()
    head = tree.creatTree(elements)
    solution = Solution()
    print("result:{}".format(solution.rob(head)))

Data Structure and Algorithm/LeetCode代码/houseRobber3_337.py


""" In the 20×20 grid below, four numbers along a diagonal line have been marked in red. The product of these numbers is 26 × 63 × 78 × 14 = 1788696. What is the greatest product of four adjacent numbers in the same direction (up, down, left, right, or diagonally) in the 20×20 grid? """

# Expected answer - 70600674

# Converting number.txt file into a matrix of numbers

import numpy as np

filename = "euler11.txt"

with open(filename,"r") as lines:
    array =[]
    for line in lines:
        array.append(line)

newArray =[]
for i in array:
    j = i.split(' ')
    k = [int(n) for n in j]
    newArray.append(k)
print(newArray)

matrix = np.asmatrix(newArray)
shape = np.shape(matrix)
rows, columns =  shape[0], shape[1]

def largestProduct(n, matrix):
    max = 0
    for i in range (0, rows):
        for j in range (0, columns-n):
            product = matrix[i,j] * matrix[i,j+1] * matrix[i,j+2] * matrix [i,j+3]
            if product >= max:
                max = product
    for i in range (0, rows-n):
        for j in range (0, columns):
            product = matrix[i+1,j] * matrix[i+2,j] * matrix[i+3,j] * matrix [i+4,j]
            if product >= max:
                max = product
    for i in range (0, rows-n):
        for j in range (0, columns-n):
            product = matrix[i,j] * matrix[i+1,j+1] * matrix[i+2,j+2] * matrix [i+3,j+3]
            if product >= max:
                max = product
    for i in range (0,rows-n):
        for j in range (columns-1,n+1,-1):
            product = matrix[i,j] * matrix[i+1,j-1] * matrix[i+2,j-2] * matrix [i+3,j-3]
            if product >= max:
                max = product
    return max

print(largestProduct(4, matrix))


from django.db import models
from user_management.models import MyUser
from django.urls import reverse
from django.contrib.postgres.fields import JSONField

from subscriber_management.models import List
from template_management.models import Template

import uuid as UUID

class Campaign(models.Model):
    """Campaign"""

    # Relationships
    author = models.ForeignKey(MyUser)
    email_list = models.ForeignKey(List)
    using_template = models.ForeignKey(Template, blank=True, null=True)
    # Core info
    name = models.CharField(max_length=100, blank=True)
    email_subject = models.CharField(max_length=150)
    email_from_name = models.CharField(max_length=100)
    email_reply_to_email = models.CharField(max_length=100)
    email_from_email = models.CharField(max_length=100, blank=True) # populated from email_list.from_email
    schedule_send = models.DateTimeField(blank=True, null=True)
    html_email_for_editing = models.TextField()
    html_email_for_sending = models.TextField()
    text_email = models.TextField(blank=True, default='')
    placeholders_value = JSONField(null=True)
    # Status
    is_sent = models.BooleanField(default=False)
    recipient_count = models.PositiveIntegerField(blank=True, null=True)
    is_draft = models.BooleanField(default=True)
    is_composed = models.BooleanField(default=False)
    unsubscribe_count = models.PositiveIntegerField(default=0)
    # Metadata
    uuid = models.UUIDField(default=UUID.uuid4, editable=False)
    created = models.DateTimeField(auto_now_add=True)
    edited = models.DateTimeField(auto_now=True)
    sent = models.DateTimeField(editable=False, null=True, blank=True)
    # Tracking relative to campaign
    utm_campaign = models.CharField(blank=True, max_length=140)
    utm_content = models.CharField(blank=True, max_length=140)
    utm_term = models.CharField(blank=True, max_length=100)

    def __str__(self):
        return self.email_subject

    def get_absolute_url(self):
        return reverse('campaign-choose-tmplt', kwargs={'pk': self.pk})


"""
Plots reservation wage against the discount factor

"""

import numpy as np
import matplotlib.pyplot as plt
from mccall_bellman_iteration import McCallModel, solve_mccall_model
from compute_reservation_wage import compute_reservation_wage

grid_size = 25
beta_vals = np.linspace(0.8, 0.99, grid_size)
w_bar_vals = np.empty_like(beta_vals)

mcm = McCallModel()

fig, ax = plt.subplots()

for i, beta in enumerate(beta_vals):
    mcm.beta = beta
    w_bar = compute_reservation_wage(mcm)
    w_bar_vals[i] = w_bar

ax.set_xlabel('discount factor')
ax.set_ylabel('reservation wage')
ax.set_xlim(beta_vals.min(), beta_vals.max())
txt = r'$\bar w$ as a function of $\beta$'
ax.plot(beta_vals, w_bar_vals, 'b-', lw=2, alpha=0.7, label=txt)
ax.legend(loc='upper left')
ax.grid()

plt.show()


#!/usr/bin/env python
# coding: utf-8

# # Vou fazer com o dataset da datacamp e com o do DataSprints

# ## Data Pre-processing and Visualization

# In[1]:

import pandas as pd
import numpy as np
import sklearn
loan_data = pd.read_csv('d:/dev/datacamp/Preparing for Machine Learning Interview Questions in Python/LoansTrainingSetReduced.csv')
pd.set_option("display.max_columns", 200)

# In[2]:

print(loan_data.head(3))

# ## Detecção de Outlier

# In[3]:

# Boxplots monovariados e multivariados
# Univariate and multivariate boxplots
# veja que aqui o x e y são os eixos do boxplot
# loan status é campo texto, que tem as categorias Chardeg Off e Fully Paid (este é o bloxpot multivariado)<br>

import matplotlib.pyplot as plt
import seaborn as sns

fig, ax =plt.subplots(1,2)
sns.boxplot(y=loan_data['Annual Income'], ax=ax[0])
sns.boxplot(x='Loan Status', y='Annual Income', data=loan_data, ax=ax[1])
plt.show()

# a caixa da esquerda é Annual Income (rendimento anual), até 02 estão as pessoas comuns e tem uns lá no alto que ganham muito.
# O box vai da faixa 0 até 1
# os status do empréstimo (loan status), realmente se dividem em Charged Off e Fully paid cobrado e pago

print(loan_data['Annual Income'].head())

# In[4]:

# Multivariate boxplot
# o monthly debt vai de 0 a 2000 são os pontos no boxplot e o Loan Status é charged off e fully paid
fig, ax =plt.subplots(1,2)
sns.boxplot(y=loan_data['Monthly Debt'], ax=ax[0])
sns.boxplot(x='Loan Status', y='Monthly Debt', data=loan_data, ax=ax[1])
plt.show()

# # Manipulando outliers
#
# Pense no seguinte cenário: a maioria dos pontos tem um forte relacionamento positivo, no entanto, devido a alguns outliers, a correlação de Pearson é negativa. Para isto usamos Winsorize
#
#
# *** Winsorize foi inventado por Charles P. Winsor (1895-1951). O efeito é o mesmo que o recorte no processamento de sinais.***
#
# Você aprendeu como visualizar valores discrepantes pode ser útil em uma entrevista de aprendizado de máquina. Outra maneira conveniente de lidar com valores discrepantes é calcular a pontuação Z que fornece um limite para os valores discrepantes aproximadamente +/- 3 desvios-padrão da média.
#
# Neste exercício, você usará o módulo scipy.stats para calcular o Z-score usando a função stats.zscore () e a função mstats.winsorize () para substituir outliers usando uma técnica chamada Winsorizing.
#
# Lembre-se da vídeo aula que os pontos acima e / ou abaixo de 1,5 vezes o IQR devem ser considerados possíveis outliers. Para a última etapa deste exercício, esse valor é 2120.
#
# Os pacotes relevantes foram importados para você, e as colunas numéricas e categóricas de loan_data foram subdivididas e salvas como numeric_cols e categoric_cols, respectivamente.
#
# -----
#
#
# You learned how visualizing outliers could come in handy in a machine learning interview. Another convenient way for handling outliers is by calculating the Z-score que fornece um limite para os valores discrepantes aproximadamente +/- 3 desvios-padrão da média.
#
# In this exercise, you will use the scipy.stats module to calculate the Z-score using the stats.zscore() function and the mstats.winsorize() function to replace outliers using a technique called Winsorizing.
#
# Recall from the video lesson that those points above and/or below 1.5 times the IQR should be suspected as possible outliers. For the last step in this exercise, that value is 2120.
#
# The relevant packages have been imported for you, and loan_data's numeric and categorical columns have been subset and saved as numeric_cols and categoric_cols, respectively.

# In[5]:

# Antes de rodar tem que definir uma listinha com nomes das colunas numericas:
# Subset numeric features: numeric_cols
numeric_cols = loan_data.select_dtypes(include=[np.number])

# In[6]:

from scipy import stats
categoric_cols = loan_data[[i for i in loan_data.columns if i not in numeric_cols.columns]]

# Print: before dropping
print("before dropping")
print(numeric_cols.mean())
print(numeric_cols.median())
print(numeric_cols.max())

#Create an index of rows to keep for absolute z-scores less than 3 on the numeric columns and
#use it to index and concatenate subsets.
# Create index of rows to keep
idx = (np.abs(stats.zscore(numeric_cols)) < 3).all(axis=1)

# Concatena subsets numericos e categoricos
ld_out_drop = pd.concat([numeric_cols.loc[idx], categoric_cols.loc[idx]], axis=1)

# Print: after dropping
print(ld_out_drop.mean())
print(ld_out_drop.median())
print(ld_out_drop.max())

# # winsorize

# deu o seguinte erro
# C:\Users\94275793668\AppData\Local\Continuum\anaconda3\lib\site-packages\ipykernel_launcher.py:14: RuntimeWarning: invalid value encountered in less, este erro também apareceu no caderno original
#
#

# In[7]:

from scipy.stats import mstats
# Print: Média do Monthly Debt Antes do winsorize
print((loan_data['Monthly Debt']).mean())
# print((loan_data['Monthly Debt']).median())
# print((loan_data['Monthly Debt']).max())

# faz Winsorize nas columnas numéricas
debt_win =  mstats.winsorize(loan_data['Monthly Debt'], limits=[0.05, 0.05])

# Converte para DataFrame, atribui nome de coluna novamente
debt_out = pd.DataFrame(debt_win, columns=['Monthly Debt'])

# Print: depois de aplicar winsorize
print(debt_out.mean())
# print(debt_out.median())
# print(debt_out.max())

# veja que melhorou bastante a média
# # Substituir pela mediana

# In[8]:

# Print: Antes de substituir com a mediana
print((loan_data['Monthly Debt']).mean())
# print((loan_data['Monthly Debt']).median())
# print((loan_data['Monthly Debt']).max())

# Ache a mediana
median = loan_data.loc[loan_data['Monthly Debt'] < 2120, 'Monthly Debt'].median()
loan_data['Monthly Debt'] = np.where(loan_data['Monthly Debt'] > 2120, median, loan_data['Monthly Debt'])

# Print: depois de substituir com a mediana
print((loan_data['Monthly Debt']).mean())
# print((loan_data['Monthly Debt']).median())
# print((loan_data['Monthly Debt']).max())

# Veja como melhorou a média, isto é porque tirei os outliers

# # Z-score standardization

# In[9]:

from sklearn.preprocessing import StandardScaler
# Subset features
numeric_cols = loan_data.select_dtypes(include=[np.number])
categoric_cols = loan_data.select_dtypes(include=[object])

# Instantiate
scaler = StandardScaler()

# Fit and transform, convert to DF
numeric_cols_scaled = scaler.fit_transform(numeric_cols)
numeric_cols_scaledDF = pd.DataFrame(numeric_cols_scaled, columns=numeric_cols.columns)

# Concatenate categoric columns to scaled numeric columns
final_DF = pd.concat([numeric_cols_scaledDF, categoric_cols], axis =1)
final_DF.head()

# In[ ]:


from art import *
from PIL import Image
import numpy
import encode_auxil as binary
import math
import os

# Plan is to import image into numpy array

global image

 # array[h,w,rgbo(3 for opactiy)]

image = input('Please enter the image path:')
prep_im = Image.open(image)
array = numpy.array(prep_im)
print('Height: ', array.shape[0])
print('Width: ', array.shape[1])
pixelCount = array.shape[0]*array.shape[1]

newArray = [1] * pixelCount

print(newArray)
i = 0
j = 0
# Grab image width and load into variable

# Iterate loop with length of width array

testArray = binary.toBinary("teststringteststringteststringteststringteststringteststringteststringteststringteststringteststring")

letter = binary.split(str(testArray[j]))

print(letter)

print(letter[0])

#print(testArray)

for h in range(array.shape[0]):
		for w in range(array.shape[1]):
			i = i + 1
			j = j + 1

			#print(array[h,w,3])


#用于生成居中字符
def add_spaces(string, num_spaces,type):
    num_spaces1 = num_spaces - len(string)
    num_spaces2 = num_spaces - 2 * len(string)
    spaces1 = ' ' * num_spaces1
    spaces2 = ' ' * num_spaces2
    if type=='英':
        return spaces1 + string + spaces1
    elif type=='中':
        return spaces2 + string + spaces1


from chose import *

class node:
    def __init__(self, name, airPlaneCode, timeDepart, timeArrival, prev):
        self.name = name                    # string
        self.airPlaneCode = airPlaneCode    # string
        self.timeDepart = timeDepart        # string
        self.timeArrival = timeArrival      # string
        self.prev = prev                    # string

nodes = {} # Create dictionary save node

def days_hours_minutes(td):
    return f'days = {td.days}, hours = {td.seconds//3600}, minutes = {(td.seconds//60)%60}'

# Recovery path
def path(start, end):
    result = []; temp = end
    while temp != start:
        result.append(nodes[temp].prev + " → " + nodes[temp].name + " " + flightName[nodes[temp].airPlaneCode] + ": " + nodes[temp].timeDepart + " " + nodes[temp].timeArrival)
        temp = nodes[temp].prev
    return list(reversed(result))

# Algorithm dijkstra (file, start, end)
def dijkstra(start, end, d, m, y, h, mn):
    start = start.strip().upper()
    end = end.strip().upper()
    d = d.strip()
    m = m.strip()
    y = y.strip()
    h = h.strip()
    mn = mn.strip()
    startTime = d + '/' + m + '/' + y + ' ' + h + ':' + mn

    V = list(flightRoute.keys())
    T = V.copy()

    if start not in V:
        return timeINFINITY, ['Start is not in the graph']
    if end not in V:
        return timeINFINITY, ['End is not in the graph']

    for name in V:
        nodes.update({name : node(name, None, timeStrInfinity, timeStrInfinity, None)})

    nodes[start].timeDepart = startTime
    nodes[start].timeArrival = startTime

    while True:
        i = min(T, key = lambda x: tInt(t(nodes[x].timeArrival)))
        T.remove(i)

        if end not in T or i == end:
            break

        for k in flightRoute[i]:
            if k not in T:  continue

            airPlaneCode, timeDepart, timeArrival = choseFlight(i, k, nodes[i].timeArrival)
            if tInt(t(nodes[k].timeArrival)) > tInt(t(timeArrival)):
                nodes[k].airPlaneCode = airPlaneCode
                nodes[k].timeDepart = timeDepart
                nodes[k].timeArrival = timeArrival
                nodes[k].prev = i

    # return length, path
    if nodes[end].timeArrival == timeStrInfinity:
        return timeINFINITY, ["The path does not exist"]
    else:
        return days_hours_minutes(t(nodes[end].timeArrival) - t(nodes[start].timeDepart)), path(start, end)


import os
import numpy as np
import bz2
import json
import pickle
from collections import defaultdict

class SegmentationAnalyser:
    def __init__(self, sample_name, segm_res_dir, model_view='psax'):
        dir = os.path.join(segm_res_dir, sample_name, model_view)
        segm_mask_path = os.path.join(dir, sample_name + '-frames.segment_pbz2')
        label_path = os.path.join(dir, sample_name + '-segmentation_label.json')
        label_path = os.path.join(os.path.dirname(label_path), sample_name + '-segmentation_label.json')
        with open(label_path, 'r') as file:
            self.labels = json.load(file)
        data = bz2.BZ2File(segm_mask_path, 'rb')
        self.segm_mask = np.asarray(pickle.load(data))
        self.w, self.h = self.segm_mask.shape[1:3]

    def extract_max_percentile_frames(self, percentile=90, min_exp=False):
        volume_to_frame_nr = defaultdict(list)  # initialise the dict with an empty list (to add frame ids)
        for frame_nr, segm_mask_frame in enumerate(self.segm_mask):
            rv_vol = np.count_nonzero(segm_mask_frame == self.labels['rv'])
            lv_vol = np.count_nonzero(segm_mask_frame == self.labels['lv'])
            volume_to_frame_nr[lv_vol + rv_vol].append(frame_nr)

        volume_list = np.asarray(list(volume_to_frame_nr))
        p = (100-percentile) if min_exp else percentile  # if find minp, reverse
        percentile = np.percentile(volume_list, p, interpolation="nearest")
        if min_exp:
            top_percentile_volumes = volume_list[volume_list <= percentile]
        else:
            top_percentile_volumes = volume_list[volume_list >= percentile]
        max_expansion_frames = []
        for top_p in top_percentile_volumes:
            top_frame = volume_to_frame_nr[top_p]
            max_expansion_frames.extend(top_frame)
        return max_expansion_frames

    def get_segm_mask(self):
        return self.segm_mask

model_dict = {
            'psax': {
                'label_dim': 4,
                'restore_path': 'psax_45_20_all_model.ckpt-9300'
            },
            'plax': {
                'label_dim': 7,
                'restore_path': 'plax_45_20_all_model.ckpt-9600'
            },
            'a4c': {
                'label_dim': 6,
                'restore_path': 'a4c_45_20_all_model.ckpt-9000'
            }
}

segmentation_labels = {
            'plax': {
                'lv': 1,  # left ventricle
                'ivs': 2,  # interventricular septum
                'la': 3,  # left atrium
                'rv': 4,  # right ventricle
                'pm': 5,  # papilliary muscle
                'aor': 6  # aortic root (ascending)
            },
            'psax': {
                'lv': 2,  # left ventricle
                'rv': 3,  # right ventricle
                'lv-o': 1  # left ventricle outer tissue
            },
            'a4c': {
                'lv': 2, # left ventricle
                'rv': 3, # right ventricle
                'la': 4, # left atrium
                'ra': 5, # right atrium
                'lv-o': 1 # left ventricle outer tissue
            }
}

our_view_to_segm_view = {
    'CV': 'a4c',
    'KAPAP': 'psax'
}


# 缩进相当于{} 规定了代码的层次

# 字符串的序号 正向递增序号和反向递减序号
# 索引：返回字符串中的单个字符 例如 TempStr[0](请) TempStr[-1](:)
# 切片：返回字符串的字串 <字符串>[M:N]返回的是索引[M,N)的一段字串 （即不取<字符串>[N]）

# 列表：使用[]表示，用,分割元素 使用in判断一个元素是否在列表中

# if elif else 条件的末尾要加:

# input()从控制台获得用户输入的函数 括号内可以输入提示信息，提示信息并不作为变量的一部分
# 在print()中，{}表示槽，后续变量填充到槽中，{:.2f}表示将变量C填充到这个位置时取小数点后两位
# 评估函数 eval(<字符串或字符串变量>)去掉参数最外层引号并执行余下语句 例：eval('print("HelloWorld")')

TempStr =input("请输入带有符号的温度值:")
if TempStr[-1] in ['F','f']:
    C=(eval(TempStr[0:-1])-32)/1.8
    print("转换后的温度是{:.2f}C".format(C))
elif TempStr[-1] in ['C','c']:
    F=1.8*eval(TempStr[0:-1])+32
    print("转换后的温度是{:.2f}F".format(F))
else:
    print("输入格式错误")


from prefect import task, Flow

import pandas as pd
import fsspec

url_pattern = 'https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/access/avhrr-only/{{ %Y%m }}/avhrr-only-v2.{{ %Y%m%d }}.nc'

date_range = {'start': '1981-09-01', 'end': '2019-08-11', 'freq': 'D'}
dates = pd.date_range(**date_range)
files = dates.strftime(url_pattern)

@task
def download(source):
    import wget

    wget.download(source)

    return target

@task
def combine(sources):
    import xarray as xr

    ds = xr.open_mfdataset(sources).chunk({'time': 5})

    return ds

@task
def publish(dataset, target):
    mapper = fsspec.get_mapper('gcs://pangeo-noaa/')
    dataset.to_zarr(mapper)

with Flow("Pangeo-Forge") as flow:
    sources = [download(f) for f in files]

    ds = combine(chunk)

    publish(ds, target_path)


#!/usr/bin/python3
"""class square that defining a square"""

class Square():
    """square class with d size and proper validation"""

    def __init__(self, size=0):
        if (type(size) is not int):
            raise TypeError("size must be in integer")
        elif (size < 0):
            raise ValueError("size must be greater than or equal to 0")
        self.__size = size


# import socket programming library
import socket, json, time
import sys, logging
import ra_mutex as ricart_agrawala

from colorama import Fore

# import thread module
from _thread import *
import threading

self_id = int(sys.argv[1])
self_port = int(sys.argv[2])
server_port = int(sys.argv[3])
num_simuations = int(sys.argv[4])
#print(f"{server_port=}")

print_lock = threading.Lock()

# thread function
#############################3
# thread for sending heartbeat

SERVER_HEART_BEAT_PORT = 1729

def heart_beat_sender_thread():
    #print("Started heartbeat thread, 5 secs interaval")
    client_socket_hb = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    client_socket_hb.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    msg = {
        "process_id": self_id,
        "port":self_port,
        "msg_type":"ZINDA",
        "time_stamp": time.time()
    }

    while True:
        msg["time_stamp"] = time.time()
        client_socket_hb.sendto(json.dumps(msg).encode(), ("localhost", int(SERVER_HEART_BEAT_PORT)))
        time.sleep(2)
    # data = server_socket.recv(1024)

client_heart_thread = threading.Thread(target=heart_beat_sender_thread)
client_heart_thread.start()

#############################################
def thread_for_accepting_connections():
    while True:
        logging.debug("Received connection")
        # data received from client
        receivingSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        data = receivingSocket.recv(4096)
        # data = c.recv(1024)
        #print(f"Request data={data.decode()}")
        with open("./log.txt", "a") as f:
            f.write(f"received request at {self_id} , {self_port}")
        #
        if not data:
            #print('Boye')

            # lock released on exit
            print_lock.release()
            break

        # reverse the given string from client
        data = data[::-1]

        # send back reversed string to client
        #c.send(data)

    # connection closed
    # c.close()

def thread_for_request(server_port):
    # establish connection with main server
    server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    # server_socket.connect(("localhost", server_port))

    msg = {
        "process_id": self_id,
        "port":self_port
    }
    server_socket.sendto(json.dumps(msg).encode(), ("localhost", int(server_port)))
    data = server_socket.recv(1024)

    data_json = json.loads(data.decode())
    #print(f"received in process {data_json=}")
    for key,val in data_json.items():
        message = f"REQUEST {key} time"
        sendingSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sendingSocket.sendto(str(message).encode(), ("localhost", int(val)))
        #print(message)
    # print(f"Received {data!r}")

    # ask main server for lisst of peers

    # establish connection with peers

    # send reqeusts to peers

    pass

def start_process_communication():

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    s.bind(("localhost", self_port))
    print("socket binded to port", self_port)

    # put the socket into listening mode
    # s.listen(5)
    # print("socket is listening")

    with open("config.txt", "a") as fp:
        lines = [f"\n{self_id}, {self_port}"]
        fp.writelines(lines)

    # talk to main server in  a separate thread.
    # start_new_thread(thread_for_request, (server_port,))

    # a forever loop until client wants to exit

    i_sim_count = 1
    while True:

    # for i in range(num_simuations):

        server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        msg = {
            "process_id": self_id,
            "port":self_port
        }
        server_socket.sendto(json.dumps(msg).encode(), ("localhost", int(server_port)))
        data = server_socket.recv(1024)

        data_json = json.loads(data.decode())

        localAddr = ("localhost", int(self_port))
        procPID = self_id
        procName = self_id
        remoteAddr = data_json
        numRemotes = len(data_json)

        if i_sim_count <= num_simuations:
            if i_sim_count == 1:
                time.sleep(5)
            ricart_agrawala.initialize_mutex(localAddr, procPID, procName, remoteAddr, numRemotes, s)
            # time.sleep(2)
            while(not ricart_agrawala.lock_mutex()): pass

            sleep_interval = 10 * (self_id + 1)
            time.sleep(sleep_interval)
            ricart_agrawala.release_mutex()
            #time.sleep(5)

            #print("sleeping for next request 10 second")
            time.sleep(10)
            #print("goint for next request after 10 seconds")

            i_sim_count += 1
            # break

            #print(f"{Fore.CYAN} NEXT ITERATION {i_sim_count} {Fore.RESET}\n\n")

            #print("-------------------------------------------------------------------------------------------------------------")

            #print("\n\n\n\n")

    s.close()

if __name__ == '__main__':
    #time.sleep(5)
    start_process_communication()


from flask import Blueprint, render_template, redirect, url_for
from myproject import db
from myproject.models import Department
from myproject.departments.forms import AddForm, DelForm

department_blueprint = Blueprint('department', __name__, template_folder='templates/departments')

@department_blueprint.route('/add', methods=['GET', 'POST'])
def add():
    form = AddForm()
    if form.validate_on_submit():
        # Grab info provided by user
        dept_name = form.name.data
        # Create Table instance
        new_dept = Department(dept_name)
        # DB interaction
        db.session.add(new_dept)
        db.session.commit()
        return redirect(url_for('department.list_depts'))

    else:
        return render_template('add.html', form=form)

@department_blueprint.route('/list')
def list_depts():
    all_departments = Department.query.all()
    return render_template('list.html', all_departments=all_departments)

@department_blueprint.route('/delete', methods=['GET', 'POST'])
def delete():
    form = DelForm()
    if form.validate_on_submit():
        dept_id = form.id.data
        dept = Department.query.get(dept_id)
        db.session.delete(dept)
        db.session.commit()
        return redirect(url_for('department.list_depts'))

    else:
        return render_template('delete.html', form=form)


import datetime
import multiprocessing
import threading
import time
import math as math
import numpy as np
import pandas as pd
import scipy.sparse as sps
from colorama import Fore, Style
from sklearn.preprocessing import normalize
import itertools as it
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--rating_file', type=str, default="data/competition/interactions.csv")
parser.add_argument('--target_users', type=str, default="data/competition/target_users.csv")
parser.add_argument('--rating_key', type=str, default='rating')
parser.add_argument('--user_item_sep', type=str, default=',')
parser.add_argument('--item_item_sep', type=str, default=' ')
parser.add_argument('--user_key', type=str, default='user_id')
parser.add_argument('--item_key', type=str, default='item_id')
parser.add_argument('--k', type=int, default=50)
parser.add_argument('--to_keep', type=int, default=50)
parser.add_argument('--shrink_factor', type=int, default=0)
parser.add_argument('--normalize', type=int, default=1)
parser.add_argument('--prediction_file', type=str, default="target/competition/"
                                                           + str(datetime.datetime.now()).replace(" ", "").
                                                           replace(".", "").replace(":", "") + "submission.csv")
parser.add_argument('--user_bias', type=int, default=1)
parser.add_argument('--item_bias', type=int, default=1)
parser.add_argument('--rec_length', type=int, default=5)
parser.add_argument('--verbosity_level', type=str, default="Info")
parser.add_argument('--number_of_cpu', type=int, default=4)
parser.add_argument('--urm_file', type=str, default=None)
parser.add_argument('--similarity_matrix_file', type=str, default=None)
parser.add_argument('--estimations_file', type=str, default=None)
parser.add_argument('--holdout_perc', type=int, default=0)
parser.add_argument('--seed', type=int, default=1234)
parser.add_argument('--weights_policy', type=str, default="man")
parser.add_argument('--recommend_on_pearson', type=int, default=1)
parser.add_argument('--rating_policy', type=str, default="boolean")
parser.add_argument('--help_me', type=int, default=19835)
parser.add_argument('--log_scale', type=int, default=0)
parser.add_argument('--timestamp_key', type=str, default="created_at")
args = parser.parse_args()

def hold_out_split(hold_out_percentage, interactions_file, user_key="user_id", item_key="item_id", rating_key="rating",
                   seed=1234):

    temp_dic, row_number, col_number, target_users, users, position_uid_dic, \
        uid_position_dic, position_iid_dic, iid_position_dic = data_importation(interactions_file, user_key=user_key,
                                                                                item_key=item_key,
                                                                                rating_key=rating_key,
                                                                                target_users_file=args.target_users)

    print(len(target_users))
    # set the random seed
    rng = np.random.RandomState(seed)
    #  shuffle data
    keys = list(temp_dic.keys())
    number_of_ratings = len(keys)
    acc_order = keys.copy()
    shuffled_access_order = rng.permutation(acc_order)
    train_size = int(number_of_ratings * hold_out_percentage/100)
    non_recommendable = set(check_expiration())
    train_dic = dict()
    removed_ratings_per_user = {}
    test_target_users = []
    for key in shuffled_access_order[:train_size]:
        key_tuple = key[0], key[1]
        train_dic[key_tuple] = temp_dic[key_tuple]
    for key in shuffled_access_order[train_size:]:
        user_id = position_uid_dic[key[0]]
        item_id = position_iid_dic[key[1]]
        if user_id not in removed_ratings_per_user.keys():
            removed_ratings_per_user[user_id] = []
        if item_id not in non_recommendable:
            removed_ratings_per_user[user_id].append(item_id)
            test_target_users.append(user_id)

    test_target_users = list(set(test_target_users))
    print(len(test_target_users))
    test_target_users = test_target_users[:len(target_users)]
    return train_dic, row_number, col_number, test_target_users, users, position_uid_dic, uid_position_dic, \
        position_iid_dic, iid_position_dic, removed_ratings_per_user

def check_expiration():
    # Returns a list in which all the items that are expired are stored
    # Import the item description file in a Pandas data frame
    filename = "data/competition/item_profile.csv"

    # This list contains the id's of all the expired items
    expired_ids = []
    with open(filename, 'r') as f:
        item_profiles_reader = pd.read_csv(filename, delimiter='\t')

    expired_items = item_profiles_reader[item_profiles_reader.active_during_test == 0]

    for i, row in expired_items.iterrows():
        expired_ids.append(row["id"])

    f.close()

    # return the list of invalid item_id
    return expired_ids

# TODO think about how to set the weights for the top pop suggestions (are they better or worse of an item similar
# but with a rating worse than your average rating?
def non_personalized_recommendation(rec_length, non_recommendable_set):
    # For reasons of time this function is absolutely data-set dependent.
    top_100_pops = [1053452, 2778525, 1244196, 1386412, 657183, 2791339, 536047, 2002097, 1092821, 784737, 1053542,
                    278589, 79531, 1928254, 1133414, 1162250, 1984327, 343377, 1742926, 1233470, 1140869, 830073,
                    460717, 1576126, 2532610, 1443706, 1201171, 2593483, 1056667, 1754395, 1237071, 1117449, 734196,
                    437245, 266412, 2371338, 823512, 2106311, 1953846, 2413494, 2796479, 1776330, 365608, 1165605,
                    2031981, 2402625, 1679143, 2487208, 315676, 1069281, 818215, 419011, 931519, 470426, 1695664,
                    2795800, 2313894, 1119495, 2091019, 2086041, 84304, 72465, 499178, 2156629, 906846, 468120,
                    1427250, 117018, 471520, 2466095, 1920047, 1830993, 2198329, 335428, 2512859, 1500071, 2037855,
                    434392, 951143, 972388, 1047625, 2350341, 2712481, 542469, 1123592, 152021, 1244787, 1899627,
                    625711, 1330328, 2462072, 1419444, 2590849, 1486097, 1788671, 2175889, 110711,
                    16356, 291669, 313851]

    to_recommend = []
    weight = -2
    counter = 0

    for item in top_100_pops:
        if item not in non_recommendable_set:
            to_recommend.append((item, weight))
            weight -= 1
            counter += 1
            if counter == rec_length:
                break

    for i in range(0, rec_length-counter):
        to_recommend.append((0, weight))
        weight -= 1

    return to_recommend

def estimate_quality(rec_dictionary, target_users, rec_length, test_set):
    dirty_map = 0.0
    number = 0
    for user in target_users:
        dirty_map += map_computer(rec_dictionary[user], pos_items=test_set[user], at=rec_length)
        number += 1
    clean_map = dirty_map / number

    return clean_map

def map_computer(ranked_list, pos_items, at=None):
    ranked_list = ranked_list[:at]
    is_relevant = np.in1d(ranked_list, pos_items, assume_unique=True)
    p_at_k = is_relevant * np.cumsum(is_relevant, dtype=np.float32) / (1 + np.arange(is_relevant.shape[0]))
    map_score = np.sum(p_at_k) / np.min([len(pos_items), len(ranked_list)])
    assert 0 <= map_score <= 1
    return map_score

def mapper(array, subject="the given input"):
    # returns a dictionary where each value is associated to position
    to_return = {}
    return_to = {}
    for i in range(len(array)):
        to_return[array[i]] = i
    print("Completed position-id association for ", subject)
    for key in to_return.keys():
        return_to[to_return[key]] = key
    print("Completed id-position association for ", subject)
    return to_return, return_to

def data_importation(interactions, target_users_file=None, user_key="user_id", item_key="item_id", rating_key="rating"):
    # Import the interaction file in a Pandas data frame
    with open(interactions, 'r') as f:
        interactions_reader = pd.read_csv(interactions, delimiter='\t')

    # If the policy is decremental we need to sort the interactions by timestamp in order to be sure to give
    # more value to the first interactions, and less to the further
    # NOTE here is given more importance to the oldest interactions... to change it set ascending to False in the sort
    # If the policy is decremental we need to sort the interactions by timestamp in order to be sure to give
    # more value to the first interactions, and less to the further
    if args.rating_policy == "decremental_a":
        interactions_reader.sort_values(args.timestamp_key, inplace=True)
    if args.rating_policy == "decremental_d":
        interactions_reader.sort_values(args.timestamp_key, inplace=True, ascending=False)

    # Here the program fills the array with all the users to be recommended
    # after this if/else target_users contains this information
    print("Listing the target users")
    interacting_users = interactions_reader[user_key].as_matrix()
    if target_users_file is None:
        target_users = interacting_users
        users = interacting_users
    else:
        with open(target_users_file, 'r') as t:
            target_users_reader = pd.read_csv(target_users_file, delimiter='\t')
        target_users = target_users_reader[user_key].as_matrix()
        users = list(set(np.hstack((target_users, interacting_users))))
        t.close()

    # Computes the size of the user rating matrix using a simple parallel splitting to increase the speed
    print("Defining the dimension of the URM and mapping")
    items = list(set(interactions_reader[item_key].as_matrix()))
    pool = multiprocessing.Pool()
    position_iid_dic_process = pool.apply_async(mapper, (items, "Items",))
    iid_position_dic, position_iid_dic = position_iid_dic_process.get()
    position_uid_dic_process = pool.apply_async(mapper, (users, "Users",))
    uid_position_dic, position_uid_dic = position_uid_dic_process.get()

    pool.close()
    pool.terminate()

    row_number = len(users)
    col_number = len(items)
    temp_dic = {}

    # Building a dictionary indexed by tuples (u_id,i_id) each one associated to the number of interactions
    # of the user u_id with the item i_id. The rating assigned to each user to an item is equal to the number
    # of interactions that he had with the item
    print("Building the temporary dictionary of interactions")
    for i, row in interactions_reader.iterrows():
        key = (uid_position_dic[row[user_key]], iid_position_dic[row[item_key]])
        if key not in temp_dic.keys():
            temp_dic[key] = 0
        if args.rating_policy == "sum":
            temp_dic[key] += row[rating_key]
        elif args.rating_policy == "max":
            temp_dic[key] = max(row[rating_key], temp_dic[key])
        elif args.rating_policy == "boolean":
            temp_dic[key] = 1
        elif args.rating_policy == "decremental_a" or args.rating_policy == "decremental_d":
            if temp_dic[key] == 0:
                temp_dic[key] = row[rating_key]
            else:
                temp_dic[key] += row[rating_key] * (1 / temp_dic[key])

    f.close()

    return temp_dic, row_number, col_number, target_users, users, position_uid_dic, \
        uid_position_dic, position_iid_dic, iid_position_dic

def write_recommendations(recommendations_dic, target_users, user_caption="user_id",
                          rec_item_caption="recommended_items",
                          user_items_sep=',', item_item_sep=' ',
                          prediction_file="predictions.csv"):
    # This function takes as input a dictionary composed by lists of tuples (item,estimated_rating). one for each
    # target user. User caption and rec_item_caption are the name to be printed in the header of the file,
    # user_items_sep and item_item_sep are the two separators respectively to divide the user_id from the
    # recommendations and each single recommended item_id. Prediction file is the name of the output file

    # Opening the output file
    output_file = open(prediction_file, "w+")
    dictionary_path = prediction_file.replace(".csv", "[DICTIONARY].txt")
    dictionary_file = open(dictionary_path, "w+")

    # Writing the header
    output_file.write(user_caption+user_items_sep+rec_item_caption+"\n")
    dictionary_file.write("{")
    # Writing the lines per each target user
    for user in target_users:
        # Each line starts with the id of the target user and a user_item_sep
        if user in recommendations_dic.keys():
            best_choices = recommendations_dic[user]
        else:
            best_choices = []
            print("ERROR")
        line = str(user) + user_items_sep
        dic_line = str(user) + ":["
        # Then we iterate on all the recommendations for that user adding them to the string
        for recommendation in best_choices:
            line += str(recommendation[0]) + item_item_sep
            dic_line += "("+str(recommendation[0])+","+str(recommendation[1])+"),"

        dic_line += "]"
        # Removes the comma after the last tuple
        dic_line.replace("),]", ")],")

        # At the end we write the line corresponding to that target user on the file
        output_file.write(line+"\n")
        dictionary_file.write(dic_line+"\n")

    # Last token at the end of the dictionary file
    dictionary_file.write("}")
    # Closing the resources
    output_file.close()
    dictionary_file.close()

    print("Output operations concluded")

def urm_computer(interactions, target_users_file=None, user_key="user_id", item_key="item_id", rating_key="rating",
                 _user_bias_=1, _item_bias_=1):
    # This functions returns the couple, users to recommend and user rating matrix

    if args.holdout_perc != 0:
        temp_dic, row_number, col_number, target_users, users, position_uid_dic, \
            uid_position_dic, position_iid_dic, iid_position_dic, \
            removed_positions = hold_out_split(hold_out_percentage=args.holdout_perc,
                                               interactions_file=interactions, user_key=user_key, item_key=item_key,
                                               rating_key=rating_key, seed=args.seed)
    else:
        (temp_dic, row_number, col_number, target_users, users,
         position_uid_dic, uid_position_dic, position_iid_dic, iid_position_dic) = \
            data_importation(interactions, target_users_file, user_key, item_key, rating_key)
        removed_positions = dict()

    # Converting dictionary values into integers and subtracting the user bias
    print("Computing the user bias for each user")

    # For each line summing all the votes, and counting the number
    totals = {}
    row_elements = {}

    for key in temp_dic.keys():
        value = int(temp_dic[key])
        if key[0] in totals.keys():
            totals[key[0]] += value
            row_elements[key[0]] += 1
        else:
            totals[key[0]] = value
            row_elements[key[0]] = 1

    # Computing the bias of each user
    user_average = {}
    for key in temp_dic.keys():
        user_average[key[0]] = totals[key[0]]/row_elements[key[0]]

    # For each column summing all the votes and counting the number
    print("Computing the item bias for each item")
    totals.clear()
    col_elements = {}

    for key in temp_dic.keys():
        value = int(temp_dic[key])
        if _user_bias_ == 1:
            value -= user_average[key[0]]
        if key[1] in totals.keys():
            totals[key[1]] += value
            col_elements[key[1]] += 1
        else:
            totals[key[1]] = value
            col_elements[key[1]] = 1

    # Computing the bias of each item
    item_average = {}
    for key in temp_dic.keys():
        item_average[key[1]] = totals[key[1]] / col_elements[key[1]]

    # Conversion and subtraction of the bias if requested
    data = []
    data_pearson = []
    row_ind = []
    col_ind = []

    # user rated items is a dictionary in which to every user_id is associated the list of the items (indicated by
    # column number) that he rated
    user_rated_items = {}
    item_rating_users = {}
    print("Converting values to the right format and subtracting the user bias...")
    for key in temp_dic.keys():
        value = int(temp_dic[key])
        data.append(value)
        if _user_bias_ == 1:
            value = value - user_average[key[0]]
        if _item_bias_ == 1:
            value = value - item_average[key[1]]
        if args.log_scale > 0:
            value = math.log(value, args.log_scale)
        if value == 0:
            value = 0.1
        data_pearson.append(value)
        row_ind.append(int(key[0]))
        col_ind.append(int(key[1]))
        key_user = position_uid_dic[key[0]]
        if key_user not in user_rated_items.keys():
            user_rated_items[key_user] = []
        if key[1] not in item_rating_users.keys():
            item_rating_users[key[1]] = []
        item_rating_users[key[1]].append((int(key[0])))
        user_rated_items[key_user].append((int(key[1])))

    tot = 0
    c = 0
    zeros = 0
    interactive_targets = 0
    for key in user_rated_items.keys():
        tot += len(user_rated_items[key])
        c += 1
        if key in target_users:
            interactive_targets += 1
        if len(user_rated_items[key]) == 0 and key in target_users:
            zeros += 1

    zeros += len(target_users)-interactive_targets
    print("in average every user evaluated ", str(tot/c)+" ", " items and ", str(zeros) +
          " of the target users evaluated no items.")

    # Create a sparse matrix from the interactions
    print("Creating the sparse matrix with the data")
    user_rating_matrix = sps.csc_matrix((data, (row_ind, col_ind)), shape=(row_number, col_number))
    pearson_urm = sps.csc_matrix((data_pearson, (row_ind, col_ind)), shape=(row_number, col_number))
    data.clear()
    row_ind.clear()
    col_ind.clear()

    if args.recommend_on_pearson == 1:
        user_rating_matrix = pearson_urm

    # returns the non-normalized User Rating Matrix
    return user_rating_matrix, target_users, users, position_uid_dic, uid_position_dic, position_iid_dic,\
        iid_position_dic, row_number, col_number, user_rated_items, item_rating_users, pearson_urm, removed_positions

def main():
    user_rating_matrix, target_users, users, position_uid_dic, uid_position_dic, position_iid_dic, \
        iid_position_dic, row_number, col_number, user_rated_items, item_rating_users, pearson_urm, removed_positions = \
        urm_computer(args.rating_file, args.target_users, user_key="user_id", item_key="item_id", rating_key="rating",
                     _user_bias_=0, _item_bias_=0)
    prediction_file = args.prediction_file
    rec_dictionary = {}
    expired_items = check_expiration()
    counter = 0
    summer = []
    non_rec_users = 0
    uri_keys = set(user_rated_items.keys())
    if args.normalize:
        # Normalization of the matrix. Row wise
        print("Starting the row-wise normalization")
        user_rating_matrix = normalize(pearson_urm, 'l2', 1, copy=False)
        print("Normalization successfully completed")
    transposed_urm = user_rating_matrix.transpose(copy=True)
    print(user_rating_matrix.shape)

    similarity_matrix = user_rating_matrix.dot(transposed_urm)
    """"
    pool = multiprocessing.Pool()
    tot = len(target_users)
    print(len(target_users))
    step = math.floor(tot / args.number_of_cpu)

    blocks = []
    for i in range(0, args.number_of_cpu):
        start = i * step
        end = (i + 1) * step
        if i == args.number_of_cpu-1:
            end = tot
        blocks.append(pool.apply_async(recommend, (similarity_matrix, user_rating_matrix, target_users[start:end],
                                                   position_iid_dic, uid_position_dic, user_rated_items,
                                                   expired_items, str(i), uri_keys,)))

    results = []
    # Fetching of the results from the workers who run the tasks
    for j in range(0, args.number_of_cpu):
        results.append(blocks[j].get())

    maxind = 0
    extra = 0
    for k in results:
        non_rec_users += k[1]
        extra += k[2]
        rec_dictionary.update(k[0])
        maxind = max(maxind, k[3])
"""""
    iter_counter = 0
    counter = 0
    extra = 0
    summer = []
    maxind = 0
    name = "ciccio"
    rec_dictionary = dict()
    for target in target_users:
        iter_counter += 1
        print(name + " : " + str(iter_counter))
        counter2 = 0
        if target not in uri_keys:
            user_rated_items[target] = []
            counter += 1
        non_recommendable_set = set(it.chain(expired_items, user_rated_items[target]))
        row_ind = uid_position_dic[target]
        row = similarity_matrix.getrow(row_ind).copy()
        size = row.shape[1]
        k_best_indices = np.argpartition(row.todense().tolist()[0], -args.k)[-args.k:]
        rows = k_best_indices.copy()
        data = []
        for i in k_best_indices:
            data.append(1)
        sparse_filter = sps.csc_matrix((data, (rows, k_best_indices)), shape=(size, size))
        filtered_row = row.dot(sparse_filter)
        line = filtered_row.dot(user_rating_matrix)
        line = line.todense().tolist()[0]
        k_best_matches = np.argpartition(line, -args.to_keep)[-args.to_keep:]
        possible_matches = non_personalized_recommendation(args.rec_length, non_recommendable_set)
        for i in k_best_matches:
            if position_iid_dic[i] not in non_recommendable_set:
                possible_matches.append((position_iid_dic[i], line[i]))
                counter2 += 1
                if i > maxind:
                    maxind = i
        if counter2 != 0:
            summer.append(counter2)
        if counter2 < 5:
            extra += 1
        possible_matches.sort(key=lambda tup: tup[1], reverse=True)
        rec_dictionary[target] = possible_matches[:args.rec_length]

    print(non_rec_users)
    print(extra)
    print(maxind)

    if args.holdout_perc != 0:
        real_map = estimate_quality(rec_dictionary, target_users,
                                    rec_length=args.rec_length, test_set=removed_positions)
        print("The estimated map should be: "+str(real_map))
        prediction_file = prediction_file.replace(".csv", "[TEST].csv")
        print(prediction_file)

    write_recommendations(rec_dictionary, target_users, prediction_file=prediction_file)

def recommend(similarity_matrix, user_rating_matrix, target_users, position_iid_dic, uid_position_dic, user_rated_items,
              expired_items, name, uri_keys):
    iter_counter = 0
    counter = 0
    extra = 0
    summer = []
    maxind = 0
    rec_dictionary = dict()
    for target in target_users:
        iter_counter += 1
        print(name + " : " + str(iter_counter))
        counter2 = 0
        if target not in uri_keys:
            user_rated_items[target] = []
            counter += 1
        non_recommendable_set = set(it.chain(expired_items, user_rated_items[target]))
        row_ind = uid_position_dic[target]
        row = similarity_matrix.getrow(row_ind).copy()
        size = row.shape[1]
        k_best_indices = np.argpartition(row.todense().tolist()[0], -args.k)[-args.k:]
        rows = k_best_indices.copy()
        data = []
        for i in k_best_indices:
            data.append(1)
        sparse_filter = sps.csc_matrix((data, (rows, k_best_indices)), shape=(size, size))
        filtered_row = row.dot(sparse_filter)
        line = filtered_row.dot(user_rating_matrix)
        line = line.todense().tolist()[0]
        k_best_matches = np.argpartition(line, -args.to_keep)[-args.to_keep:]
        possible_matches = non_personalized_recommendation(args.rec_length, non_recommendable_set)
        for i in k_best_matches:
            if position_iid_dic[i] not in non_recommendable_set:
                possible_matches.append((position_iid_dic[i], line[i]))
                counter2 += 1
                if i > maxind:
                    maxind = i
        if counter2 != 0:
            summer.append(counter2)
        if counter2 < 5:
            extra += 1
        possible_matches.sort(key=lambda tup: tup[1], reverse=True)
        rec_dictionary[target] = possible_matches[:args.rec_length]

    return rec_dictionary, counter, extra, maxind

main()


import math
import os
import itertools
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from sklearn.metrics import classification_report, confusion_matrix
import pandas as pd
import seaborn as sns
import pickle

import helper_utils as hu
# import water_w_regression as wwr
import config as cfg
logger = hu.setup_logger(cfg.logging)

def annot_max(x, y, type_m, ax=None):
    """
    Annotate maximum of graph and last point of graph

    Args:
        x (np array): X values of graph
        y (np array): Y values of graph

    Returns:
        None
    """
    if type_m == 'categorical':
        annotations = [(x[np.argmax(y)], max(y)), (x[-1], y[-1])]
    else:
        annotations = [(x[np.argmin(y)], min(y)), (x[-1], y[-1])]
    for (xmax, ymax) in annotations:
        text = "{:.2f}".format(ymax)
        if not ax:
            ax=plt.gca()
        ax.annotate(text, xy=(xmax, ymax))

def plot_history(history, save_path, type_m, verbose=1):
    """
    Visualize the history object

    Args:
        history (Keras history): History object
        save_path (str): Path where to save visualizations
        verbose (int): Use 0 to silence prints and 1 or higher to get more verbosity

    Returns:
        None
    """
    if verbose:
        logger.debug('history %s', history)
        logger.debug(history.history.items())
    for key, metric in history.history.items():
        key_n = key.replace('_', ' ').title()
        if key[:3] != 'val':
            plt.figure()
            plt.grid(which='both', axis='y')
            plt.plot(history.history[key])
            if key != 'lr':
                plt.plot(history.history['val_' + key])
                annot_max(np.arange(len(history.history['val_' + key])), history.history['val_' + key], type_m)
            plt.title('Model ' + key_n)
            plt.ylabel(key_n)
            plt.xlabel('Epoch')
            plt.legend(['train', 'validation'], loc='upper left')
            if key == 'loss':
                plt.ylim(0, 2)
            elif key == 'accuracy':
                plt.ylim(0, 1)
            #plt.show()
            #Save
            plt.tight_layout()
            plt.savefig(save_path + key)
            plt.close()
            #asd

def standard_hist_from_df(df, path, file_n, title_in, bins_count='auto',
                          minv=False, maxv=False, title_add='stats', xlabel='PC1', ylabel='count',
                          xlim=3.5, xticks=1):
    if minv or maxv:
        df = df[(df > minv) & (df < maxv)]
    if bins_count == 'auto':
        bins_count = min(max(int(len(df) / 10), 7), 50)
    df.hist(bins=bins_count)#, density=1)
    if xlim:
        plt.xlim(-xlim, xlim)
    # if xticks:
    #     plt.xticks(np.arange(int(df.min()), int(df.max()), step=xticks))
    # write len, mean std as header to plot
    title = str(title_in)
    if title_add == 'stats':
        title += f"\nn={len(df)}, mean={df.mean():.2f}, std={df.std():.2f}, skew={df.skew():.2f}, kurt={df.kurtosis():.2f}"
    plt.title(title)#, pad=20)
    #label x axis
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.savefig(path + ' ' + file_n + ' ' + str(title_in) + '.png')
    plt.close()

def plot_DBSCAN(df_clustered, path, file_n):
    # Plot the results
    plt.figure(figsize=(10, 10))
    plt.scatter(df_clustered['LONGNUM'], df_clustered['LATNUM'], c=df_clustered['clustered'], cmap='viridis')
    plt.title('DBSCAN Clustering')
    plt.xlabel('Longitude')
    plt.ylabel('Latitude')
    plt.colorbar(label='Cluster')
    plt.grid(True)
    plt.savefig(path + file_n + '.png')
    plt.close()

from itertools import cycle

def plot_dataframe(df, x_column, y_columns, run_path, file_name, title, label_reduce=False,
                   color_keywords=None, linestyle_keywords=None, xlabel='km', ylabel=False,
                   add_linestyle_to_color_keywords=False):
    """
    Plot selected columns from a DataFrame with the ability to customize the color and linestyle of the lines
    in the plot based on specified keywords.

    Parameters:
    df (pd.DataFrame): DataFrame containing the data to plot.
    x_column (str): Column name to use for x-axis.
    y_columns (list[str]): List of column names to plot on y-axis.
    run_path (str): The path where the plot will be saved.
    file_name (str): The filename of the saved plot.
    title (str): Title of the plot.
    label_reduce (str, optional): Substring to be removed from labels. Defaults to False.
    color_keywords (list[str], optional): List of keywords to be matched in labels for assigning colors.
                                          The color assigned is based on the longest match. Defaults to None.
    linestyle_keywords (list[str], optional): List of keywords to be matched in labels for assigning line styles.
                                              The linestyle assigned is based on the longest match. Defaults to None.
    xlabel (str, optional): Label for the x-axis. Defaults to 'km'.
    ylabel (str, optional): Label for the y-axis. If False, no y-label will be added. Defaults to False.
    add_linestyle_to_color_keywords (bool, optional): If True, color_keywords must be provided and each element of
                                                      color_keywords will be combined with each element of linestyle_keywords
                                                      to form a new set of keywords for assigning colors. Defaults to False.
    """

    # Create a new figure
    plt.figure()
    if len(y_columns) > 7:
        plt.figure(figsize=(15, 10))

    # Define color palette
    color_palette = cm.get_cmap('tab10', len(y_columns))

    # Create a cycle of colors
    color_cycle = cycle(range(len(y_columns)))
    if color_keywords and not add_linestyle_to_color_keywords:
        color_cycle = cycle(range(len(color_keywords)))

    # Create a dictionary to hold assigned colors for labels
    label_color_dict = {}

    # Create dictionaries for color and linestyle keywords if provided
    if add_linestyle_to_color_keywords:
        if not color_keywords:
            raise ValueError("If add_linestyle_to_color_keywords is True, color_keywords must be provided.")
        if not linestyle_keywords:
            raise ValueError("If add_linestyle_to_color_keywords is True, linestyle_keywords must be provided.")
        #this should generate a
        color_keywords = [c + ' ' + l for c in color_keywords for l in linestyle_keywords]

    color_keywords_dict = {keyword: next(color_cycle) for keyword in color_keywords} if color_keywords else {}
    linestyle_list = ['-', '--', ':', '-.']
    linestyle_keywords_dict = {keyword: linestyle_list[i % len(linestyle_list)] for i, keyword in
                               enumerate(linestyle_keywords)} if linestyle_keywords else {}
    # Loop over the columns and add each one to the plot
    for y_column in y_columns:
        #skip empty columns
        if df[y_column].isnull().all():
            continue
        label = y_column
        if label_reduce:
            label = label.replace(label_reduce, "")

        # Sort color keywords by length in descending order
        sorted_color_keywords = sorted(color_keywords_dict, key=len, reverse=True)
        # Assign color based on label, with longer matches prioritized
        color_index = next((color_keywords_dict[keyword] for keyword in sorted_color_keywords if keyword in label),
                           label_color_dict.get(label, next(color_cycle)))
        label_color_dict[label] = color_index

        # Sort linestyle keywords by length in descending order
        sorted_linestyle_keywords = sorted(linestyle_keywords_dict, key=len, reverse=True)
        # Assign linestyle based on label, with longer matches prioritized
        linestyle = next((linestyle_keywords_dict[keyword] for keyword in sorted_linestyle_keywords if keyword in label),
            '-')

        plt.plot(df[x_column], df[y_column], label=label, color=color_palette(color_index), linestyle=linestyle)

    # Add a legend to the plot
    plt.legend()
    if xlabel:
        plt.xlabel(xlabel)
    if ylabel:
        plt.ylabel(ylabel)
    plt.title(title)

    # Save the plot
    plt.savefig(run_path + file_name + ' ' + title + '.png')
    plt.close()

def plot_dataframe_old(df, x_column, y_columns, run_path, file_name, title, label_split_at_space=False, dashed_lines=False):
    """
    Plot selected columns from a DataFrame.

    Parameters:
    df (DataFrame): DataFrame containing the data to plot.
    x_column (str): Column name to use for x-axis.
    y_columns (list): List of column names to plot on y-axis.
    """

    # Create a new figure
    plt.figure()

    # Loop over the columns and add each one to the plot
    for i, y_column in enumerate(y_columns):
        label = y_column
        if label_split_at_space:
            label = label.rsplit(' ', label_split_at_space)[0]
        linestyle = '--' if dashed_lines and dashed_lines[i] else '-'
        plt.plot(df[x_column], df[y_column], label=label, linestyle=linestyle)

    # Add a legend to the plot
    plt.legend()
    plt.title(title)

    # Show the plot
    plt.savefig(run_path + file_name + title + '.png')
    plt.close()

# def histogram(df, stat, title, path, file_n, limit_above=120):
#     plt.figure()
#     if not type(max(df)) == str and max(df) >= limit_above:
#         bins = list(range(-4, 101, 5))
#         ax = sns.displot(df, stat=stat, bins=bins)
#     else:
#         ax = sns.displot(df, stat=stat)
#     if type(max(df)) == str:
#         ax.set_xticklabels(rotation=90)
#         plt.xlabel('')
#     plt.title(title)
#     # plt.show(bbox_inches='tight')
#     plt.savefig(path + file_n, bbox_inches='tight')
#     plt.close()

#
# # plot as scatter plot and compute a regression line, including alpha and beta
# def scatterplotRegression(df, run_path, file_name=''):
#     """Plots a Regressionplot with a text annotation containing additional information
#     Args:
#         df (pd.DataFrame): DataFrame consisting of the Actual and the Predicted Volatility Values
#         model_type (str): Name of the model
#         EPOCHS (int): the amount of epochs, the model was trained on
#         BATCH_SIZE (int): number of training examples
#         It does not return anything, but plots a regression plot and saves it
#     """
#     beta, alpha = np.polyfit(df.iloc[:,0], df.iloc[:,1], 1)
#     corr_value = df.corr(method="pearson")
#     pearson_corr = corr_value.iloc[1,0]
#     rmse = ((df.iloc[:, 0] - df.iloc[:, 1]) ** 2).mean() ** .5
#     parameter_text = "\n".join(
#         (
#             r"$pearson\ corr=%.2f$" % (pearson_corr,),
#             r"$slope: \beta=%.2f$" % (beta,),
#             r"$intercept: \alpha=%.2f$" % (alpha,),
#             r"$rmse: \alpha=%.2f$" % (rmse,),
#         )
#     )
#     logger.debug(file_name)
#     logger.debug(parameter_text)
#     plt.figure()
#     sns.set(rc={"figure.figsize": (15, 10)})
#     sns.regplot(
#         data=None,
#         x=df.iloc[:,0],
#         y=df.iloc[:,1],
#         x_ci="sd",
#         scatter=True,
#         line_kws={"color": "red"},
#     )
#     # add text annotation
#     # set the textbox color to purple
#     purple_rgb = (255.0 / 255.0, 2.0 / 255.0, 255.0 / 255.0)
#     plt.grid(True)
#     plt.annotate(
#         parameter_text,
#         xy=(0.03, 0.8),
#         xycoords="axes fraction",
#         ha="left",
#         fontsize=14,
#         # color=purple_rgb,
#         backgroundcolor="w",
#     )
#     plt.title('Scatterplot', fontsize=14)
#     plt.xlabel('Actual', fontsize=14)
#     plt.ylabel('Prediction', fontsize=14)
#     # finally save the chart to disk
#     plt.savefig(run_path + 'Scatterplot_' + file_name + ".png", dpi=300, bbox_inches="tight")
#     # plt.show()
#     plt.close()
#     return beta, alpha, pearson_corr, rmse

def scatterplotRegressionMultiInputs(df, run_path, file_name='', multidataset_col=False,
                                     error_metrics_on_test_data=False):
    """Plots a Regressionplot with a text annotation containing additional information
    Args:
        df (pd.DataFrame): DataFrame consisting of the Actual and the Predicted Volatility Values
        model_type (str): Name of the model
        EPOCHS (int): the amount of epochs, the model was trained on
        BATCH_SIZE (int): number of training examples
        It does not return anything, but plots a regression plot and saves it
    """
    df_l = []
    # colors = ['r', 'b', 'g', 'y', 'm', 'c']  # List of colors for each predicted column
    colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan']
    unq_l = []
    if multidataset_col:
        for unq in df[multidataset_col].unique():
            df_l.append(df[df[multidataset_col] == unq])
            unq_l.append(unq)
    else:
        df_l.append(df)
        unq_l.append('all')

    metrics_df = df
    if error_metrics_on_test_data:
        metrics_df == df[df[multidataset_col] == 'test']

    beta, alpha = np.polyfit(metrics_df.iloc[:,0], metrics_df.iloc[:,1], 1)
    corr_value = metrics_df.iloc[:,:2].corr(method="pearson")
    pearson_corr = corr_value.iloc[1,0]
    rmse = ((metrics_df.iloc[:, 0] - metrics_df.iloc[:, 1]) ** 2).mean() ** 0.5
    nrmse = rmse / metrics_df.iloc[:, 0].std()

    parameter_text = "\n".join(
        (
            r"$pearson\ corr=%.2f$" % (pearson_corr,),
            r"$\alpha=%.2f,\ beta=%.2f$" % (alpha, beta,),
            r"$(n)rmse=%.2f\:\ (%.2f)$" % (rmse, nrmse),
        )
    )
    logger.debug(file_name)
    logger.debug(parameter_text)
    if not multidataset_col:
        sns.set(rc={"figure.figsize": (5, 4)})
        plt.figure()
        plt.annotate(
            parameter_text,
            xy=(0.03, 0.8),
            xycoords="axes fraction",
            ha="left",
            fontsize=10,
            # color=purple_rgb,
            backgroundcolor="w",
        )
    else:
        sns.set(rc={"figure.figsize": (15, 10)})
        plt.figure()
        plt.annotate(
            parameter_text,
            xy=(0.03, 0.8),
            xycoords="axes fraction",
            ha="left",
            fontsize=14,
            # color=purple_rgb,
            backgroundcolor="w",
        )

    plt.grid(True)

    for i, df in enumerate(df_l):
        sns.regplot(
            data=df,
            x=df.columns[0],
            y=df.columns[1],
            x_ci="sd",
            scatter=True,
            color=colors[i],
            label=unq_l[i],
            # line_kws={"color": colors[i]},  # Use a different color for each predicted column
        )

    plt.title(f'Scatterplot (n={len(df)})', fontsize=14)
    plt.xlabel('Actual', fontsize=14)
    plt.ylabel('Prediction', fontsize=14)
    if multidataset_col:
        plt.legend(loc='lower right')
    # finally save the chart to disk
    plt.savefig(run_path + 'Scatterplot: ' + file_name + ".png", dpi=300, bbox_inches="tight")
    # plt.show()
    plt.close('all')
    return beta, alpha, pearson_corr, rmse, nrmse

def plot_CM(label_true, prediction_true, classes, save_path, mode='both', title='Confusion matrix', cmap=plt.cm.Blues):
    """
    Prints and plots the confusion matrix.

    Args:
        label_true (np array): The actual label/ground truth
        prediction_true (np array): The prediction
        classes (list): Label names (ordered by quantity)
        save_path (str): The path where to save the image
        mode (str): If you want a normalized or absolute or both CM ('normalized', 'quantities' or 'both')
        title (str): Title of the image
        cmap (plt.cm.Blues): Colors

    Returns:
        None
    """
    if mode == 'both':
        cm_l = ['quantities', 'normalized']
    elif mode == 'normalized':
        cm_l = ['normalized']
    else:
        cm_l = ['quantities']
    for i in cm_l:
        logger.debug('Creating CM', i)
        plt.figure()
        # plt.figure(figsize=(7, 4))
        cm = confusion_matrix(y_true=label_true, y_pred=prediction_true)
        tick_marks = np.arange(len(classes))
        if len(tick_marks) <= 3:
            plt.xticks(tick_marks, classes, wrap=True)
            plt.yticks(tick_marks, classes, rotation='vertical', wrap=True, verticalalignment='bottom',
                       horizontalalignment='center')
            plt.ylabel('True label', {'size': 'large'})
            plt.xlabel('Predicted label', {'size': 'large'})
        else:
            plt.xticks(tick_marks, classes, rotation='vertical', wrap=True, horizontalalignment='right',
                       verticalalignment='top')
                       # horizontalalignment='center')
            plt.yticks(tick_marks, classes, rotation='horizontal', wrap=True)\
                # , wrap=True, verticalalignment='bottom',
                #        horizontalalignment='center')

        thresh = 0.6
        if i == 'normalized':
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            cm = np.around(cm, decimals=2)
            logger.debug("Normalized confusion matrix")
        else:
            thresh = thresh * len(label_true)
            logger.debug('Confusion matrix, without normalization')

        logger.debug('threshold for CM', thresh)
        for k, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            plt.text(j, k, cm[k, j],
                horizontalalignment="center", size='large',
                color="white" if cm[k, j] > thresh else "black")
        plt.imshow(cm, interpolation='nearest', cmap=cmap)
        plt.colorbar()
        plt.tight_layout()
        plt.savefig(save_path + i)
        plt.show()
        plt.close()

def plot_CM_sns(label_true, prediction_true, classes, save_path, mode='normalized', title='Confusion matrix', cmap=plt.cm.Blues):
    """
    Prints and plots the confusion matrix.

    Args:
        label_true (np array): The actual label/ground truth
        prediction_true (np array): The prediction
        classes (list): Label names (ordered by quantity)
        save_path (str): The path where to save the image
        mode (str): If you want a normalized or absolute or both CM ('normalized', 'quantities' or 'both')
        title (str): Title of the image
        cmap (plt.cm.Blues): Colors

    Returns:
        None
    """
    if mode == 'both':
        cm_l = ['quantities', 'normalized']
    elif mode == 'normalized':
        cm_l = ['normalized']
    else:
        cm_l = ['quantities']
    for i in cm_l:
        logger.debug('Creating CM', i)
        plt.figure()
        cm = confusion_matrix(y_true=label_true, y_pred=prediction_true)

        if i == 'normalized':
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            cm = np.around(cm, decimals=2)
            logger.debug("Normalized confusion matrix")
        else:
            logger.debug('Confusion matrix, without normalization')

        # df_cm = pd.DataFrame(cm, index=classes,
        #                      columns=classes)
        # plt.figure(figsize=(10, 7))
        # sns.heatmap(cm, annot=True)
        #from nele
        heatmap = sns.heatmap(cm, vmin=0, vmax=1, annot=True, cmap="PiYG")
        # sns.set(font_scale=1)
        tick_marks = np.arange(len(classes))
        plt.xticks(tick_marks, classes, wrap=True)
        plt.yticks(tick_marks, classes, rotation='vertical', wrap=True, verticalalignment='bottom',
                   horizontalalignment='left')
        plt.ylabel('True label')
        plt.xlabel('Predicted label')

        plt.savefig(save_path + '_sns_' + i)
        plt.close()

def plot_images(images_arr, save_path=False, show=False, title=False):
    """
    Plots and saves images (usually the actual input image into the model)

    Args:
        images_arr (np array): Input image array
        save_path (str): The path where to save the image (None if it shall not be saved)
        show (bool): True if you want python to actually print the image
        title (str): Title of the image

    Returns:
        None
    """
    for nr, img in enumerate(images_arr):
        img = zero_one_normalization(img.copy())
        fig, ax = plt.subplots()
        #ax.imshow(img, interpolation='none')
        ax.imshow(img)
        plt.title(title)
        if save_path:
            #not sure if necessary
            # img = np.rint(img * 256)
            #savefig before show, otherwise img gets empty
            plt.savefig(save_path + str(nr))
        if show:
            plt.show()

def zero_one_normalization(data):
    """
    Returns 0,1 normalized data of arrays - used to normalize images for visualization

    Args:
        data (np array): Input data

    Returns:
        data (np array): Normalized data"""
    return (data - np.min(data)) / (np.max(data) - np.min(data))

def main():
    run_path = '/mnt/datadisk/data/Projects/water/trainH_XV/split__out_of_country_all_w_TIF/PCA_w_location_weighting_all2/996x996_c432_fillmean_m2.5_rlocal_channel_mean_clipvoutlier_normZ_f31213vgg19_wimagenet_unfl100_d0_lr0.0001_momentum0.9_optimizerSGD__m_vgg19_2/'

    names = []
    val_dfs = []
    test_dfs = []

    for file in hu.files_in_folder(run_path, sort=True):
        print(file)
        if 'val_df_' in file:
            val_dfs.append(pd.read_csv(file))
            names.append(file[-11:-4])
        elif 'test_df_' in file:
            test_dfs.append(pd.read_csv(file))
    print(names)
    print('val\n', val_dfs)
    print('test\n', test_dfs)
    # wwr.evaluate_final_dataset(test_dfs, val_dfs, run_path, names, cfg.split)

if __name__ == "__main__":
    main()


'''
Created on May 17, 2020

@author: User
'''
from rest_framework import serializers
from avatar.forms import UploadAvatarForm
from collections import OrderedDict
from rest_framework.fields import empty
from avatar.models import Avatar
from avatar.signals import avatar_updated

from user.models import Profile
from djbase.utils import FileUtil

from . import UserSerializer

class ProfileSerializer(serializers.ModelSerializer):
    user = serializers.IntegerField(source='user.id', read_only=True)
    first_name = serializers.CharField(
            source='user.first_name',
            required=False,
            read_only=True
        )
    last_name = serializers.CharField(
            source='user.last_name',
            required=False,
            read_only=True
        )
    email = serializers.EmailField(
            source='user.email',
            required=False,
            read_only=True
        )
    avatar = serializers.SerializerMethodField()

    class Meta:
        model  = Profile
        fields = '__all__'

    def __init__(self, instance=None, data=empty, **kwargs):
        initial_data = dict()
        if empty != data:
            initial_data = data
            self.user_serializer = self._init_user_serializer(
                    instance, initial_data
                )

        serializers.ModelSerializer.__init__(self, instance=instance, data=data, **kwargs)

    def _init_user_serializer(self, instance, initial_data):
        email = initial_data.get('email', instance.user.email)
        first_name = initial_data.get('first_name', instance.user.first_name)
        last_name = initial_data.get('last_name', instance.user.last_name)
        user_serializer = None
        if instance:
            user_data = dict(
                    id=instance.user.id, email=email,
                    first_name=first_name, last_name=last_name
                )
            user_serializer = UserSerializer(
                    data=user_data, instance=instance.user
                )
        return user_serializer

    def get_avatar(self, instance):
        avatar = instance.user.avatar_set.filter(primary=True).first()
        if avatar is None:
            return None
        return avatar.get_absolute_url()

    def validate_avatar(self):
        user = self.context.get('user')
        avatar = self.initial_data.get('avatar')
        if avatar is None or 'base64' not in avatar or user is None:
            return None
        avatar = FileUtil.get_file_from_data_url(avatar)[0]
        avatar_form = UploadAvatarForm(None, dict(avatar=avatar), user=user)

        if False == avatar_form.is_valid():
            errors = OrderedDict()
            form_errors = dict(self.signup_form.errors)
            for error_field in form_errors.keys():
                errors[error_field] = form_errors[error_field]
            raise serializers.ValidationError(errors)

        return avatar

    def validate_user(self):
        if not self.user_serializer:
            return None

        if False == self.user_serializer.is_valid():
            errors = OrderedDict()
            form_errors = dict(self.user_serializer.errors)
            for error_field in form_errors.keys():
                errors[error_field] = form_errors[error_field]
            raise serializers.ValidationError(errors)
        return self.user_serializer

    def validate(self, attrs):
        self.avatar = self.validate_avatar()
        self.validate_user()
        validated_data = serializers.ModelSerializer.validate(self, attrs)
        return validated_data

    def save_avatar(self, instance):
        keys = self.initial_data.keys()
        if 'avatar' in keys and \
            (self.avatar is not None or self.initial_data.get('avatar') is None):
            avatar = instance.user.avatar_set.filter(primary=True).first()
            if avatar is not None:
                avatar.primary = False
                avatar.save()
                return avatar_updated.send(
                        sender=Avatar, user=instance.user, avatar=avatar
                    )
        elif self.avatar is None:
            return None

        if self.avatar:
            avatar = Avatar(user=instance.user, primary=True)
            image_file = self.avatar
            avatar.avatar.save(image_file.name, image_file)
            avatar.save()
            avatar_updated.send(sender=Avatar, user=instance.user, avatar=avatar)
            return avatar

    def update(self, instance, validated_data):
        if self.user_serializer:
            self.user_serializer.save()
        return serializers.ModelSerializer.update(self, instance, validated_data)

    def save(self, **kwargs):
        profile = serializers.ModelSerializer.save(self, **kwargs)
        self.save_avatar(profile)
        return profile


import sys

from zeus.common.util.get_xt_config import *

def test_main():
    """
    test parse config file
    """
    config_file = sys.argv[1]
    ret_para = parse_xt_multi_case_paras(config_file)
    for para in ret_para:
        print(para)

def test_find_items():
    """
    test find key in dictionary
    """
    config_file = sys.argv[1]
    with open(config_file) as file_hander:
        yaml_obj = yaml.safe_load(file_hander)

    ret_obj = finditem(yaml_obj, "agent_config")
    print(ret_obj)

if __name__ == "__main__":
    test_main()
    # test_find_items()


import torch
from torchvision import datasets
from torchvision.transforms import ToTensor
from torch.utils.data import DataLoader
from loguru import logger
import torch.optim as optim
import torch.nn as nn
import sys
from pydantic import BaseModel
from pathlib import Path
from mltrainer import metrics, Trainer, TrainerSettings, ReportTypes

logger.add("/tmp/explainer.log")
logger.add("explainer.log")

class NeuralNetworkExplainer(nn.Module):
    def __init__(self) -> None:
        super().__init__()

        self.flatten = nn.Flatten()

        self.conv_layers = nn.Sequential(
            nn.Conv2d(1, 10, kernel_size=5),
            nn.MaxPool2d(2),
            nn.ReLU(),
            nn.Conv2d(10, 20, kernel_size=5),
            nn.Dropout(),
            nn.MaxPool2d(2),
            nn.ReLU(),
        )

        self.fc_layers = nn.Sequential(
            nn.Linear(320, 50),
            nn.ReLU(),
            nn.Dropout(),
            nn.Linear(50, 10),
            nn.Softmax(dim=1),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.conv_layers(x)
        x = self.flatten(x)
        out = self.fc_layers(x)
        return out

class ImageExplainerSettings(BaseModel):
    data_dir: Path = Path("data")
    epochs: int = 10
    modelname: Path = Path("trained_model.pt")
    imgpath: Path = Path("img")
    num_classes: int = 10

if __name__ == "__main__":
    logger.info("start image_explainer.py")
    presets = ImageExplainerSettings()

    logger.info(f"Importing {sys.argv[1]}")

    if sys.argv[1] == "mnist":
        dataset_train = datasets.MNIST(
            root=presets.data_dir,
            train=True,
            download=True,
            transform=ToTensor(),
        )

        dataset_test = datasets.MNIST(
            root=presets.data_dir,
            train=False,
            download=True,
            transform=ToTensor(),
        )
    else:
        dataset_train = datasets.FashionMNIST(
            root=presets.data_dir,
            train=True,
            download=True,
            transform=ToTensor(),
        )

        dataset_test = datasets.FashionMNIST(
            root=presets.data_dir,
            train=False,
            download=True,
            transform=ToTensor(),
        )
    device = "cuda" if torch.cuda.is_available() else "cpu"
    logger.info(f"Using {device} device")

    train_dataloader = DataLoader(dataset_train, batch_size=128, shuffle=True)
    test_dataloader = DataLoader(dataset_test, batch_size=128, shuffle=True)

    X, y = next(iter(train_dataloader))

    logger.info(
        f"Length trainset: {len(dataset_train)}, length testset: {len(dataset_test)}"
    )

    model = NeuralNetworkExplainer()

    loss_fn = torch.nn.CrossEntropyLoss()
    accuracy = metrics.Accuracy()

    log_dir = "models/explainer"

    logger.info(f"starting training for {presets.epochs} epochs")

    settings = TrainerSettings(
        epochs=presets.epochs,
        metrics=[accuracy],
        logdir=log_dir,
        train_steps=50,
        valid_steps=50,
        reporttypes=[ReportTypes.TENSORBOARD],
        optimizer_kwargs={"lr": 1e-2},
    )

    trainer = Trainer(
        model=model,
        settings=settings,
        loss_fn=loss_fn,
        optimizer=optim.Adam,
        traindataloader=train_dataloader,
        validdataloader=test_dataloader,
        scheduler=optim.lr_scheduler.ReduceLROnPlateau,
    )
    trainer.loop()

    torch.save(model, presets.modelname)
    logger.success("finished making model.py")