# coding:utf-8
import mpl_finance
# import tushare as ts
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import pymysql
from matplotlib import ticker
from matplotlib.pylab import date2num
import numpy as np
import datetime
import logging
import re
from multiprocessing import Pool
import json
import openpyxl

logging.basicConfig(level=logging.DEBUG, filename='comp_redu_210120.py.log', filemode='w',
                    format='%(asctime)s-%(levelname)5s: %(message)s')

def get_df_from_db(sql, db):
    cursor = db.cursor()  # 使用cursor()方法获取用于执行SQL语句的游标
    cursor.execute(sql)  # 执行SQL语句
    data = cursor.fetchall()
    # 下面为将获取的数据转化为dataframe格式
    columnDes = cursor.description  # 获取连接对象的描述信息
    columnNames = [columnDes[i][0] for i in range(len(columnDes))]  # 获取列名
    df = pd.DataFrame([list(i) for i in data], columns=columnNames)  # 得到的data为二维元组，逐行取出，转化为列表，再转化为df
    # df = df.set_index(keys=['trade_date'])
    df = df.sort_values(axis=0, ascending=True, by='trade_date', na_position='last')
    df.reset_index(inplace=True)
    # df = df.dropna(axis=0, how='any')
    # df.reset_index(inplace=True)
    df['trade_date2'] = df['trade_date'].copy()
    # print('trade_date2:',type(df['trade_date2'][0]))
    df['trade_date2'] = pd.to_datetime(df['trade_date2']).map(date2num)
    df['dates'] = np.arange(0, len(df))
    df['avg_10'] = df['close_price'].rolling(10).mean()
    df['avg_5'] = df['close_price'].rolling(5).mean()
    cursor.close()
    # print(df)
    # df['trade_date'] = date2num(df['trade_date'])
    print('df:', df[['avg_10', 'close_price']])
    return df
#计算龙虎榜热度
def com_redu1(db,date,delta = 30):
    cursor = db.cursor()
    end_date = datetime.datetime.strptime(date, "%Y-%m-%d")
    start_date = (end_date - datetime.timedelta(days=delta)).strftime("%Y-%m-%d")
    # sql = "select sum(30+jmrate)+count(jmrate)*1000 from longhu_info where stock_code = '{0}' and " \
    #       "trade_date >= '{1}' and trade_date <= '{2}'".format(s_code,start_date,date)
    sql = "select stock_code,sum(30+jmrate)+count(jmrate)*10000 from longhu_info where  " \
          "trade_date >= '{0}' and trade_date <= '{1}' group by stock_code".format(start_date,date)
    # print('sql:',sql)
    cursor.execute(sql)  # 执行SQL语句
    data = cursor.fetchall()
    cursor.close()
    #print('data:',data)
    return dict(data)
#计算涨停热度
def com_redu2(db,ids,h_tab,date,delta = 30):
    cursor = db.cursor()
    # end_date = datetime.datetime.strptime(date, "%Y-%m-%d")
    # start_date = (end_date - datetime.timedelta(days=delta)).strftime("%Y-%m-%d")
    # sql = "select sum(30+jmrate)+count(jmrate)*1000 from longhu_info where stock_code = '{0}' and " \
    #       "trade_date >= '{1}' and trade_date <= '{2}'".format(s_code,start_date,date)
    # sql = "SELECT count(increase)  FROM stock_history_trade{0} " \
    #       "where trade_date >= '{1}' and trade_date <= '{2}' " \
    #       "and  stock_id  = '{3}' and increase >= 9.75".format(h_tab, start_date, date,ids)
    sql = "SELECT count(increase)  FROM (select * from stock_history_trade{0} " \
          "where trade_date <= '{1}' and stock_id  = '{2}' order by trade_date DESC limit {3}) H " \
          "where  increase >= 9.75 ".format(h_tab, date,ids,delta)
    print('sql:',sql)
    cursor.execute(sql)  # 执行SQL语句
    zhangting_res = cursor.fetchall()
    print('zhangting_res:',zhangting_res)
    if len(zhangting_res) !=0:
        zhangting_count = zhangting_res[0][0]*10000
    else:
        zhangting_count = 0
    cursor.close()
    print('zhangting_count:',zhangting_count)
    return zhangting_count
#判断当日下穿5日均线是10日内极大值之后首次下穿
def com_redu_init(redu,df):
    if redu<10000:
        return 0
    max_value = df['close_price'][len(df)-11 : len(df) - 1].max()
    print('max_value:',max_value)
    index = df.query("close_price == {}".format(max_value)).index[-1]
    for i in range(index,len(df)):
        if df.loc[i,'close_price'] <=  df.loc[i,'avg_5']:
            return 0
    return 1
#判断前溯极大值是10日内最大值
def find_max(df):
    len_df = len(df)
    # increase 未统一，暂不使用
    if df.loc[len_df-1,'increase'] <= -7:
        return False
    val = 0
    for i in range(len_df-1,len_df-11,-1):
        val = df.loc[i,'close_price']
        print('val:',val,df.loc[i-1,'close_price'])
        if val > df.loc[i-1,'close_price']:
            break
    print('max_val:',df['close_price'][len_df-11:len_df-1].max(),val)
    if df['close_price'][len_df-11:len_df-1].max() > val:
        return False
    else:
        return True
def main(h_tab,date):
    if date == None:
        date = datetime.datetime.now().strftime('%Y-%m-%d')
    db = pymysql.connect("localhost", "root", "Zzl08382020", "stockdb")
    cursor = db.cursor()  # 使用cursor()方法获取用于执行SQL语句的游标
    sql = "select distinct  stock_id,stock_name from stock_history_trade{0}".format(h_tab)
    cursor.execute(sql)
    stock_id_list = cursor.fetchall()
    date_time = datetime.datetime.strptime(date, '%Y-%m-%d')
    start_t = (date_time - datetime.timedelta(days=90)).strftime('%Y-%m-%d')
    # stock_id_list = (('600316','洪都航空'),)
    for ids_tuple in stock_id_list:
        ids = ids_tuple[0]
        stock_name = ids_tuple[1]
        trade_code = re.sub('-', '', date[0:10]) + ids
        redu_dict = com_redu1(db, date, delta=20)
        redu_grade = com_redu2(db, ids, h_tab, date, delta=20)
        sql = "SELECT stock_id,trade_date,open_price,close_price,high_price,low_price,increase  FROM stock_history_trade{0} \
                where trade_date >= '{1}' and trade_date <= '{2}' and  stock_id  = '{3}'".format(h_tab, start_t, date,
                                                                                ids)
        df = get_df_from_db(sql, db)
        if len(df) < 20:
            continue
        avg_5 = df.loc[len(df) - 1, 'avg_5']
        if ids in redu_dict:
            redu_grade += redu_dict[ids]
        if redu_grade == 0:
            continue
        redu_5 = 0
        if df.loc[len(df) - 1, 'avg_5'] < df.loc[len(df) - 1,'close_price']:
            continue
        if df.loc[len(df) - 3, 'avg_5'] > df.loc[len(df) - 3, 'close_price'] and df.loc[len(df) - 2, 'avg_5'] > df.loc[len(df) - 2, 'close_price'] and df.loc[len(df) - 1, 'avg_5'] > df.loc[len(df) - 1, 'close_price']:
            redu_5 = redu_grade /10000
        if redu_5 == 0 and df.loc[len(df) - 1, 'avg_5'] >= df.loc[len(df) - 1,'close_price']:
            if find_max(df):#return bool
                redu_5 = redu_grade
        print('redu_grade:',redu_grade,redu_5)
        redu_init = com_redu_init(redu_grade, df)
        sql = "insert into com_redu(trade_code,trade_date,stock_id,stock_name,redu,redu_5,avg_5,h_table,redu_init) \
            values('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}','{8}') " \
              "ON DUPLICATE KEY UPDATE trade_code='{0}',trade_date='{1}',stock_id='{2}',stock_name='{3}'," \
              "redu ='{4}',redu_5 ='{5}',avg_5='{6}',h_table = '{7}',redu_init = '{8}'\
            ".format(trade_code, date, ids, stock_name, redu_grade, redu_5, float(avg_5), h_tab, redu_init)
        print('sql:', sql)
        cursor.execute(sql)
    try:
        db.commit()
        print('存储完成')
        logging.info('存储完成:id:{},name:{}'.format(ids, stock_name))
    except Exception as err:
        db.rollback()
        print('存储失败:', err)
        logging.error('存储失败:id:{},name:{}\n{}'.format(ids, stock_name, err))
    cursor.close()
def run(date):
    p = Pool(8)
    for i in range(1, 11):
        p.apply_async(main, args=(str(i),date,))
    #    p.apply_async(main, args=('1',date,))
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()
    print('All subprocesses done.')
def history_com(h_tab,start_date,end_date):
    db_h = pymysql.connect("localhost", "root", "Zzl08382020", "stockdb")
    cursor = db_h.cursor()
    sql = "select distinct(trade_date) from stock_history_trade1 where trade_date >= '{0}' and trade_date <= '{1}'".format(start_date,end_date)
    cursor.execute(sql)
    date_list = cursor.fetchall()
    print('date_list:',date_list)
    for date_t in date_list:
        date = date_t[0].strftime('%Y-%m-%d')
        print(date)
        main(h_tab,date)
def run_h(start_date,end_date):
    p = Pool(8)
    for i in range(1, 11):
        p.apply_async(history_com, args=(str(i),start_date,end_date))
    #    p.apply_async(main, args=('1',date,))
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()
    print('All subprocesses done.')
if __name__ == '__main__':
    date ='2021-02-24'#'2021-02-01' #'2021-01-20'
    run(date)

    # h_tab = '2'
    # main(h_tab, date)

    # history_com(h_tab,start_date='2020-01-01', end_date='2021-01-19')
    # run_h(start_date='2020-07-31', end_date='2021-02-22')


"""Practica1 Ejercicio 1 Alumno: Pablo Uriel Alvarez Grupo 29"""

"""Defino la funcion Max()"""
def max(num1,num2):
    if num1>=num2:
        return num1
    else:
        return num2

#Utilizo el asset para verificar
assert max(10,9) == 10
assert max(10,12) == 12


import requests
from lxml import etree
import sqlite3

header = {
    'Referer': 'http://xt.hnnjgzbt.com/gongshi.aspx',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
    'Upgrade-Insecure-Requests': '1',
    'Origin': 'http://xt.hnnjgzbt.com',
    'Host': 'xt.hnnjgzbt.com',
}

s = requests.session()
URL = 'http://xt.hnnjgzbt.com/gongshi.aspx'

def get_value():
    response = s.get('http://xt.hnnjgzbt.com/gongshi.aspx').text
    html = etree.HTML(response)
    value = html.xpath('//*[@id="__VIEWSTATE"]/@value')
    return value

def get_data(value):
    for i in range(1, 1005):
        form_data = {
            '__VIEWSTATE': value,
            '__EVENTTARGET': 'AspNetPager1',
            '__EVENTARGUMENT': str(i),
            #'txtname': '',
            #'txtpm': '',
            #有些参数无关紧要，下面这个参数是跳转到另一页时的页码
            #'AspNetPager1_input': i - 1
        }
        response = s.post(URL,data=form_data).text
        print('正在打印第{}'.format(i))

        html = etree.HTML(response)
        div = html.xpath('//*[@id="form1"]//div[@class="contert_table"]')[0]
        row_tags = div.xpath('//table[@align="center"]//tr')

        for tr in row_tags[1:]:
            # 注意加上点，在此标签下
            column_tags = tr.xpath('.//td')
            item = []
            for td in column_tags:
                #print(td.text.strip())
                item.append(td.text.strip())

            c.execute(sql, item)

if __name__ == "__main__":

    conn = sqlite3.connect('test.db')
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE LEIWO
             (NAME          TEXT,
              ADDRESS       TEXT,
              PINMU         TEXT,
              FENDANG       TEXT,
              XINGHAO       TEXT,
              CHANGJIA      TEXT,
              JINGXIAO      TEXT,
              SHULIANG      INT,
              BUTIE1        INT,
              BUTIE2        INT,
              BUTIE3        INT,
              JIAGE         INT,
              ZHUANGTAI     INT);''')

        sql = "INSERT INTO LEIWO VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)"
        get_data(get_value())
        conn.commit()
    finally:
        c.close()
        conn.close()


import pytest

from environments.identities.traits.models import Trait

@pytest.mark.parametrize(
    "value, expected_data",
    (
        (1, {"value_type": "int", "integer_value": 1}),
        (0, {"value_type": "int", "integer_value": 0}),
        ("my_string", {"value_type": "unicode", "string_value": "my_string"}),
        (True, {"value_type": "bool", "boolean_value": True}),
        (False, {"value_type": "bool", "boolean_value": False}),
        (123.4, {"value_type": "float", "float_value": 123.4}),
    ),
)
def test_generate_trait_value_data_for_value(value, expected_data):
    assert Trait.generate_trait_value_data(value) == expected_data

@pytest.mark.parametrize(
    "deserialized_data, expected_data",
    (
        ({"type": "int", "value": 1}, {"value_type": "int", "integer_value": 1}),
        ({"type": "int", "value": 0}, {"value_type": "int", "integer_value": 0}),
        (
            {"type": "unicode", "value": "my_string"},
            {"value_type": "unicode", "string_value": "my_string"},
        ),
        (
            {"type": "bool", "value": True},
            {"value_type": "bool", "boolean_value": True},
        ),
        (
            {"type": "bool", "value": False},
            {"value_type": "bool", "boolean_value": False},
        ),
        (
            {"type": "float", "value": 123.4},
            {"value_type": "float", "float_value": 123.4},
        ),
    ),
)
def test_generate_trait_value_data_for_deserialized_data(
    deserialized_data, expected_data
):
    assert Trait.generate_trait_value_data(deserialized_data) == expected_data


import requests
from nksama import bot
from pyrogram import filters
from pyrogram.types.bots_and_keyboards.inline_keyboard_button import InlineKeyboardButton
from pyrogram.types.bots_and_keyboards.inline_keyboard_markup import InlineKeyboardMarkup
from nksama.plugins.helpers import call_back_in_filter

@bot.on_callback_query(call_back_in_filter('quotek'))
def callback_quotek(_,query):
    if query.data.split(":")[1] == "change":
#         query.message.delete()
        kk = requests.get('https://animechan.vercel.app/api/random').json()
        anime = kk['anime']
        quote = kk['quote']
        character = kk['character']
        caption = f"""
**Anime:** `{anime}`
**Character:** `{character}`
**Quote:** `{quote}`"""
        query.message.edit(caption , reply_markup=InlineKeyboardMarkup([
            [InlineKeyboardButton("Change" , callback_data="quotek:change")],
        ]))

@bot.on_message(filters.command('quote'))
def quote(_,message):
    kk = requests.get('https://animechan.vercel.app/api/random').json()
    anime = kk['anime']
    quote = kk['quote']
    character = kk['character']
    caption = f"""
**Anime:** `{anime}`
**Character:** `{character}`
**Quote:** `{quote}`"""
    bot.send_message(message.chat.id , caption , reply_markup=InlineKeyboardMarkup([
        [InlineKeyboardButton("Change" , callback_data="quotek:change")]
    ]))


#%%
# Etem Kaya 12-Apr-2019
# Iris Data Set Project.

# Histogram of Iris dataset Sepal length and width

# import pandas, seaborn and matplotlib libraries
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# import iris dataset with Pandas
f = pd.read_csv("iris.csv")

# split 'Setosa, Versicolor and Virginica' dataset from the entire Iris dataset
Setosa = f.loc[f["species"]=="setosa"]
Versicolor = f.loc[f["species"]=="versicolor"]
Virginica = f.loc[f["species"]=="virginica"]

# set style and background colour using seaborn library
sns.set(style="darkgrid")

# Histogram of 'Setosa, Versicolor and Virginica' Sepal Length
sns.distplot(Setosa["sepal_length"], color= "green")
sns.distplot(Versicolor["sepal_length"], color= "blue")
sns.distplot(Virginica["sepal_length"], color= "orange")
plt.title("Histogram - Sepal Length of 'Setosa, Versicolor and Virginica'")
plt.xlabel('Sepal_Length' + "\nSetosa, Versicolor and Virginica")
plt.ylabel('count')
plt.show()

# Histogram of 'Setosa, Versicolor and Virginica' Sepal Width
sns.distplot(Setosa["sepal_width"], color= "green")
sns.distplot(Versicolor["sepal_width"], color= "blue")
sns.distplot(Virginica["sepal_width"], color= "orange")
plt.title("Histogram - Sepal Width of 'Setosa, Versicolor and Virginica'")
plt.xlabel('Sepal_Width' + "\nSetosa, Versicolor and Virginica")
plt.ylabel('count')
plt.show()


import logging
import time
from contextlib import contextmanager

from .slack import slack_notify

@contextmanager
def timer(name: str, log: bool = True):
    t0 = time.time()
    msg = f"[{name}] start"
    print(msg)
    # slack_notify(msg)

    if log:
        logging.info(msg)
    yield

    msg = f"[{name}] done in {time.time() - t0:.2f} s"
    print(msg)
    # slack_notify(msg)

    if log:
        logging.info(msg)


import os, sys, glob
import numpy as np

All = []

for sFiles in glob.glob("./0.AllPeaks/*narrowPeak"):
    infile = open(sFiles,"r")
    for sLine in infile:
        #print(sLine)
        All.append(sLine)

Index = list(np.random.choice(np.arange(0,len(All)), 40000, replace=False))

outfile = open("./1.InputFile/ARFSimulated.GEM_events.narrowPeak","w")

Pos = []
Selected = []

for i in Index:
    Line = All[i]
    List = Line.strip().split("\t")
    sPos = "\t".join(List[0:3])
    if sPos not in Pos:
        Pos.append(sPos)
        Selected.append(Line)

for j in range(0,37840):
    outfile.write(Selected[j])


from flask import Flask
from flask_restful import Api
from redisConfig import Redis

import os

from ml_model.mlModel import Network
from register.register import Consul
import torch

redisHost = os.environ['REDIS_HOST']
redisPort = os.environ['REDIS_PORT']

redis = Redis(redisHost, redisPort)

consulHost = os.environ['CONSUL_HOST']
consulPort = os.environ['CONSUL_PORT']

serviceName = os.environ['SERVICE_NAME']
serviceHost = os.environ['SERVICE_HOST']
servicePort = int(os.environ['SERVICE_PORT'])

model = Network()
model.load_state_dict(torch.load('./ml_model/model_param/model_states.pth'))

def create_app():
    from controller import register_routes

    app = Flask(__name__)

    api = Api(app)
    register_routes(api, app)

    consul_client = Consul(consulHost, consulPort)
    consul_client.RegisterService(serviceName, serviceHost, servicePort)

    return app

app = create_app()

if __name__ == '__main__':
    app.run(debug=True)


import re

import setuptools
import unittest

def my_test_suite():
    test_loader = unittest.TestLoader()
    test_suite = test_loader.discover("testing", pattern="test_*.py")
    return test_suite

with open("readme.md", "r") as fh:
    long_description = fh.read()

_version_regex = (
    r"^__version__ = ('|\")((?:[0-9]+\.)*[0-9]+(?:\.?([a-z]+)(?:\.?[0-9])?)?)\1$"
)

with open("antispam/__init__.py") as stream:
    match = re.search(_version_regex, stream.read(), re.MULTILINE)

version = match.group(2)

setuptools.setup(
    name="Discord Anti-Spam",
    version=version,
    author="Skelmis",
    author_email="ethan@koldfusion.xyz",
    description="An easy to use package for anti-spam features in discord.py.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/Skelmis/DPY-Anti-Spam",
    packages=setuptools.find_packages(),
    install_requires=["fuzzywuzzy>=0.18", "discord.py>=1"],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
        "Development Status :: 4 - Beta",
    ],
    python_requires=">=3.6",
    test_suite="setup.my_test_suite",
)


# API testing student management system
# 1.Add a new Student (POST)
# 2.Fetch student details(GET)
# 3.Update Student details(PUT)
# 4.Delete Student details(DELETE)
import pytest
import requests
import json
import jsonpath

#Post method used to add the data to the Server
# @pytest.mark.skip
def test_add_student_data():
    # API_URL = "http://www.thetestingworldapi.com/api/studentsDetails"
    # file_read =open("C:/Hanamanta_Data/Api Testing/sudent_details.json",'r')
    # json_load = json.loads(file_read.read())
    # response_api = requests.post(API_URL,json_load)
    # print(response_api.text)
    url = "http://www.thetestingworldapi.com/api/studentsDetails"
    # open the file in read mode
    file = open("C:/Hanamanta_Data/Api Testing/sudent_details.json",'r')
    # Load the json into Jsonformat file.read
    json_load = json.loads(file.read())
    # Now send the request
    response_add = requests.post(url,json_load)
    print(response_add.text)
    print(response_add.status_code)

#Use Get method : Used to fetch the data from the server
# @pytest.mark.skip
def test_get_student_data():
    api_url = "http://www.thetestingworldapi.com/api/studentsDetails/522442"
    response = requests.get(api_url)
    print(response.text)
    # json_res = json.loads(response.text)
    json_response = response.json()
    id = jsonpath.jsonpath(json_response,'data.id')
    print(id[0])
    assert id[0] == 522442  #522441

#Put method to update the student data
# @pytest.mark.skip
def test_update_student_data():
    url = "http://www.thetestingworldapi.com/api/studentsDetails/522386"
    file = open('C:/Hanamanta_Data/Api Testing/update3.json','r')
    json_read = json.loads(file.read())
    response = requests.put(url,json_read)
    print(response.text)

# @pytest.mark.skip
def test_delete_student():
    url = "http://www.thetestingworldapi.com/api/studentsDetails/522386"
    response = requests.delete(url)
    print(response.text)

# @pytest.mark.skip
def test_get_student_data_delete():
    api_url = "http://www.thetestingworldapi.com/api/studentsDetails/522386"
    response = requests.get(api_url)
    print(response.text)
    # json_res = json.loads(response.text)
    json_response = response.json()
    id = jsonpath.jsonpath(json_response,'data.id')
    print(id[0])
    assert id[0] == 522386


import os
import tarfile
import urllib
import shutil
import datetime
import requests
from pandas import DataFrame, ExcelWriter
import time
import logging
from collections import defaultdict
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class JobPage():
    filename = os.path.abspath('../../UItest_Dashboard/Automation_Results/report.log.')
    logging.basicConfig(filename=filename, level=logging.INFO)

    def __init__(self, driver):
        self.driver = driver

    def click_first_posted_job(self):
        logging.info("Clicking first job")
        time.sleep(5)
        self.driver.find_element_by_xpath("//h3[contains(text(),'Posted Jobs')]/../ul/li").click()

    def click_ratings_Button(self):
        logging.info("Choosing Good fit option..")
        time.sleep(6)
        self.driver.find_element_by_xpath("//*[@data-control-name='rating_facet_toggle']").click()
        time.sleep(6)
        self.driver.find_element_by_xpath("//*[@data-control-name='rating_facet_toggle']/span/../..//li[1]").click()
        time.sleep(2)
        self.driver.find_element_by_xpath(
            "//*[@data-control-name='rating_facet_toggle']/span/../..//li[3]").click()
        time.sleep(2)
        self.driver.find_element_by_xpath("//*[@data-control-name='rating_facet_toggle']/span/../..//div//button[@data-control-name='rating_facet_toggle_show_results']").click()
        time.sleep(2)

    def export_data_to_excel(self, employee_dict):
        now = str(datetime.datetime.now())[:19]
        now = now.replace(":", "_")
        myDF = DataFrame(employee_dict)
        writer = ExcelWriter(os.path.abspath("..//..//UItest_Dashboard//Automation_Results//Employee_details-" +str(now)+ ".xlsx"))
        myDF.to_excel(writer)
        writer.save()

    def click_More_tab(self):
        time.sleep(4)
        self.driver.find_element_by_xpath("//*[@data-control-name='hiring_applicant_rate']//span/../../../../div[3]//span[1]").click()

    def fetch_candidate_details(self):
        logging.info("Extracting candidates details ..")
        time.sleep(8)
        self.driver.find_element_by_xpath("//h4/span[text()='Messaging']").click()
        candidate_dictionary = defaultdict(list)
        if self.driver.find_elements_by_xpath("//li[contains(@class,'artdeco-pagination__indicator')]"):
            pages = self.driver.find_elements_by_xpath("//li[contains(@class,'artdeco-pagination__indicator')]")
            totalpages = len(pages)+2
            logging.info("Total Pages are :"+str(totalpages))
            try:
                for page in range(2,totalpages):
                    #self.driver.refresh()
                    vars = 0
                    vare = 100
                    for index in range(1,26) :
                        candidate = self.driver.find_element_by_xpath('//li[contains(@class,"hiring-applicants__list-item")]['+str(index)+']')
                        ActionChains(self.driver).move_to_element(candidate).perform()
                        candidate.click()
                        time.sleep(1)
                        try:
                            element = WebDriverWait(self.driver, 5).until(
                                EC.presence_of_element_located((By.XPATH, "//button/span[text()='Good fit']"))
                            )
                            rate_as_validation_check = element.text
                            if "Good fit" in rate_as_validation_check:
                                self.driver.find_element_by_xpath(
                                    "//*[@data-control-name='hiring_applicant_message']/../div[3]//span").click()
                                time.sleep(3)
                                emp_details_email = self.driver.find_element_by_xpath(
                                     "//div[@class='artdeco-dropdown__content-inner']/ul/li[2]/a/div/span[2]").text
                                emp_details_phone = self.driver.find_element_by_xpath(
                                    " //div[@class='artdeco-dropdown__content-inner']/ul/li[3]/div/div/span[2]").text
                                candidate_dictionary['EmailId'].append(emp_details_email)
                                candidate_dictionary['PhoneNumber'].append(emp_details_phone)
                                time.sleep(3)
                                try:
                                    ActionChains(self.driver).move_to_element(self.driver.find_element_by_xpath("//a[text()='Download']")).perform()
                                    self.driver.find_element_by_xpath("//a[text()='Download']").click()
                                except Exception as e:
                                    ActionChains(self.driver).move_to_element(self.driver.find_element_by_xpath('//a[contains(@class,"ui-attachment__download-button")]')).perform()
                                    self.driver.find_element_by_xpath('//a[contains(@class,"ui-attachment__download-button")]').click()
                                time.sleep(3)

                        except Exception as e:
                            logging.error(str(e))
                        script = "window.scrollTo("+ str(vars) +","+ str(vare) + ")"
                        self.driver.execute_script(script)

                        vars = int(vars)+100
                        vare = int(vare)+100

                    self.driver.find_element_by_xpath("//button[@aria-label='Page "+str(page)+"']").click()
                    time.sleep(5)
            except Exception as e:
                logging.error(str(e))
        else:
            vars = 0
            vare = 100
            no_of_applicants = self.driver.find_element_by_xpath("//div[@class='hiring-applicants__left-column ']//span").text
            no_of_applicants = str(no_of_applicants).split("(")
            no_of_applicants=int(((no_of_applicants[0:2])[1].split(" "))[0])
            for index in range(1, no_of_applicants+1):
                candidate = self.driver.find_element_by_xpath(
                    '//li[contains(@class,"hiring-applicants__list-item")][' + str(index) + ']')
                ActionChains(self.driver).move_to_element(candidate).perform()
                candidate.click()
                time.sleep(1)
                try:
                    element = WebDriverWait(self.driver, 5).until(
                        EC.presence_of_element_located((By.XPATH, "//button/span[text()='Good fit']"))
                    )
                    rate_as_validation_check = element.text
                    if "Good fit" in rate_as_validation_check:

                        self.driver.find_element_by_xpath(
                            "//*[@data-control-name='hiring_applicant_message']/../div[3]//span").click()
                        time.sleep(3)
                        emp_details_email = self.driver.find_element_by_xpath(
                            "//div[@class='artdeco-dropdown__content-inner']/ul/li[2]/a/div/span[2]").text
                        emp_details_phone = self.driver.find_element_by_xpath(
                            " //div[@class='artdeco-dropdown__content-inner']/ul/li[3]/div/div/span[2]").text
                        candidate_dictionary['EmailId'].append(emp_details_email)
                        candidate_dictionary['PhoneNumber'].append(emp_details_phone)
                        time.sleep(3)
                        try:
                            ActionChains(self.driver).move_to_element(
                                self.driver.find_element_by_xpath("//a[text()='Download']")).perform()
                            self.driver.find_element_by_xpath("//a[text()='Download']").click()
                        except Exception as e:
                            ActionChains(self.driver).move_to_element(self.driver.find_element_by_xpath(
                                '//a[contains(@class,"ui-attachment__download-button")]')).perform()
                            self.driver.find_element_by_xpath(
                                '//a[contains(@class,"ui-attachment__download-button")]').click()
                        time.sleep(3)

                except Exception as e:
                    logging.error(str(e))
                script = "window.scrollTo(" + str(vars) + "," + str(vare) + ")"
                self.driver.execute_script(script)

                vars = int(vars) + 100
                vare = int(vare) + 100

                #self.driver.find_element_by_xpath("//button[@aria-label='Page " + str(page) + "']").click()
                time.sleep(5)
            # except Exception as e:
            #    logging.error(str(e))

        return candidate_dictionary

    def fetching_candidates_details(self):
        logging.info("Extracting candidates details ..")
        candidate_dictionary = defaultdict(list)
        time.sleep(3)
        self.driver.maximize_window()
        self.driver.find_element_by_xpath(
            "//*[@class='msg-overlay-bubble-header__controls display-flex']//*[@type='chevron-down-icon']").click()
        time.sleep(3)
        list_candidates = self.driver.find_element_by_xpath("//*[@class='hiring-applicants__list-container']/ul")
        count_of_child = len(list_candidates.find_elements_by_xpath("./li"))
        for i in range(1,count_of_child):
            self.driver.find_element_by_xpath(
                "//ul[@class='artdeco-list']/li[%d]" % (i,)).click()
            rate_as_validation_check= self.driver.find_element_by_xpath("//*[@data-control-name='hiring_applicant_rate']//span").text
            if "Good fit" in rate_as_validation_check:
                try:
                    time.sleep(3)
                    self.driver.find_element_by_xpath(
                        "//*[@data-control-name='hiring_applicant_rate']//span/../../../../div[3]//span[1]").click()
                    emp_details = self.driver.find_element_by_xpath(
                        "//*[@data-control-name='hiring_applicant_rate']//span/../../../../div[3]/div").text
                    candidate_details_list = str(
                        emp_details).split("\n")
                    candidate_dictionary['EmailId'].append(candidate_details_list[2])
                    candidate_dictionary['PhoneNumber'].append(candidate_details_list[5])
                    #url =self.driver.current_url
                    self.fetch_data_from_profile()
                except IndexError:
                    print("Handling Index error")
        return candidate_dictionary

    def fetch_data_from_profile(self):
        time.sleep(2)
        try:
            if self.driver.find_element_by_xpath("//*[@data-control-name='hiring_applicant_resume_pdf_download']") :
                self.driver.find_element_by_xpath("//*[@data-control-name='hiring_applicant_resume_pdf_download']").click()
            else:
                self.driver.execute_script("window.scrollTo(0, 1000)")
                element=self.driver.find_element_by_xpath("//*[contains(@class, 'ui-attachment__download-button display-flex justify-center align-items-center')][last()]").click()
                element.location_once_scrolled_into_view
        except NoSuchElementException:
            pass


from argparse import ArgumentParser
from socket import socket, AF_INET, SOCK_DGRAM
import time

import constants
from packet import packet
import log

logger = log.configure_receiver_logger("receiver", info_stdout=constants.PRINT_INFO)

class Receiver(object):

    def __init__(self, hostname: str, ack_port: int, data_port: int, filename: str):
        """

        Args:
            hostname: The hostname of the network emulator to connect to.
            ack_port: The port to send ack messages to on the emulator.
            data_port: The port the emulator will send data packets to the receiver via.
            filename: The name of the file to save data into.
        """
        self.hostname = hostname
        self.ack_port = ack_port
        self.data_port = data_port
        self.filename = filename
        self.seq_num = 0

    def send_ack(self, seq_num: int):
        """ Sends an ACK packet for a sequence number.

        Args:
            seq_num: Sequence number of the packet to mention in the ACK.
        """
        socket(AF_INET, SOCK_DGRAM).sendto(packet.create_ack(seq_num).get_udp_data(),
                                           (self.hostname, self.ack_port))

    def send_EOT(self, seq_num):
        """ Sends an EOT packet.
        """
        socket(AF_INET, SOCK_DGRAM).sendto(packet.create_eot(seq_num).get_udp_data(),
                                           (self.hostname, self.ack_port))

    def handle_message(self, socket) -> packet:
        """ Handles the receiving of packets, sending acks and storing data locally.

        Args:
            socket: The socket to receive data from.

        Returns:
            The parsed packet object of the most recent message.
        """
        message, _ = socket.recvfrom(constants.PACKET_DATA_SIZE)
        try:
            p = packet.parse_udp_data(message)
        except Exception as e:
            logger.log(f"ERROR: {e}")
            return None
        logger.log(f"Received packet with no: {p.seq_num}."
                   f"Looking for {(self.seq_num + 1) % constants.MODULO_RANGE}")

        if p.type == constants.TYPE_EOT:
            logger.log("Received EOT.")
            return p

        elif p.type == constants.TYPE_ACK:
            # Should not happened
            logger.log("[ERROR] Received ACK.")
            return p

        # Else data message
        logger.arrival(p.seq_num)
        if p.seq_num == (self.seq_num + 1) % constants.MODULO_RANGE:
            # Expected, next packet
            with open(self.filename, "a") as f:
                f.write(p.data)
            self.send_ack(self.seq_num)
            logger.log(f"Sending ACK for good packet with no: {self.seq_num + 1}")
            self.seq_num = p.seq_num
        else:
            self.send_ack(self.seq_num)
            logger.log(f"Sending ACK for bad packet with no: {self.seq_num}")
        return p

    def run(self):
        """ Main thread for running a receiver.
        """
        self.seq_num = 0
        self.data = ""

        # Setup UDP port for receiving data
        data_socket = socket(AF_INET, SOCK_DGRAM)
        data_socket.bind((self.hostname, self.data_port))

        # do-while handling packets until it receives EOT
        packet = self.handle_message(data_socket)
        while packet.type != constants.TYPE_EOT:
            new_packet = self.handle_message(data_socket)
            if new_packet:
                packet = new_packet
            time.sleep(constants.PROCESS_WAIT)

        # Send EOT back
        self.send_EOT(packet.seq_num)

def main():
    # Parse arguments
    parser = ArgumentParser(description='Receiver')
    parser.add_argument("hostname", type=str,
                        help="The hostname of the network emulator to connect to.")
    parser.add_argument("ack_port", type=int,
                        help="The port to send ack messages to on the emulator.")
    parser.add_argument("data_port", type=int,
                        help="The port the emulator will send data packets to the receiver via.")
    parser.add_argument("filename", type=str,
                        help="The name of the file to save data into.")
    args = parser.parse_args()

    # Run Receiver
    receiver = Receiver(args.hostname, args.ack_port, args.data_port, args.filename)
    receiver.run()

if __name__ == "__main__":
    main()


import json
import os
from math import sqrt

os.chdir('../log_input')
batchfileDir = os.getcwd()
batch = []
with open(os.path.join(batchfileDir, 'batch_log.json')) as f:
    for line in f:
        batch.append(json.loads(line))

D = int(batch[0]["D"])
T = int(batch[0]["T"])

network = []
eventbatch = batch[1:len(batch)]

for event in eventbatch:
    if event['event_type'] == 'befriend':
        network.append({event['id1']: event['id2']})
        network.append({event['id2']: event['id1']})

    if event['event_type'] == 'unfriend':
        network.remove({event['id1']: event['id2']})
        network.remove({event['id2']: event['id1']})

os.chdir('../log_input')
streamfileDir = os.getcwd()
stream = []
with open(os.path.join(streamfileDir, 'stream_log.json')) as g:
    for line in g:
        stream.append(json.loads(line))

flagged_purchases = []
for activity in stream:
    if activity['event_type'] == 'befriend':
        network.append({activity['id1']: activity['id2']})
        network.append({activity['id2']: activity['id1']})
    if activity['event_type'] == 'unfriend':
        network.remove({activity['id1']: activity['id2']})
        network.remove({activity['id2']: activity['id1']})
    if activity['event_type'] == 'purchase':
        person = activity['id']
        personlist = [person]
        friendlist = []
        for i in range(D):
            for friends in network:
                for eachperson in personlist:
                    if eachperson in friends:
                        if friends[eachperson] != person and friends[eachperson] not in personlist:
                            friendlist.append(friends[eachperson])
                            personlist.append(friends[eachperson])

        friendpurchase = []
        for everyfriend in friendlist:
            for event in eventbatch:
                if event['event_type'] != 'purchase': continue
                if event['id'] == everyfriend:
                    friendpurchase.append(event)

        if len(friendpurchase) < 2:
            flagged_purchases = flagged_purchases

        if len(friendpurchase) >= 2:
            if len(friendpurchase) < T:
                amountlist = []
                for everypurchase in friendpurchase:
                    amountlist.append(eval(everypurchase['amount']))
                mean = sum(amountlist) / len(amountlist)
                y = 0.0
                for xn in amountlist:
                    y = y + (xn - mean) ** 2
                sd = sqrt(y / len(amountlist))

                if eval(activity['amount']) > mean + (3 * sd):
                    showmean = round(mean, 2)
                    showsd = round(sd, 2)
                    activity['mean'] = '%.2f' % showmean
                    activity['sd'] = str(showsd)
                    flagged_purchases.append(activity)

            if len(friendpurchase) >= T:
                callist = friendpurchase[len(friendpurchase)-T:len(friendpurchase)]
                amountlist = []
                for everypurchase in callist:
                    amountlist.append(eval(callist['amount']))
                mean = sum(amountlist) / len(amountlist)
                y = 0.0
                for xn in amountlist:
                    y = y + (xn - mean) ** 2
                sd = sqrt(y / len(amountlist))
                if eval(activity['amount']) > mean + (3 * sd):
                    showmean = round(mean, 2)
                    showsd = round(sd, 2)
                    activity['mean'] = '%.2f' % showmean
                    activity['sd'] = str(showsd)
                    flagged_purchases.append(activity)

    eventbatch.append(activity)

print(flagged_purchases)


ownerclass = 'AppDelegateBase'
ownerimport = 'AppDelegateBase.h'
edition = args.get('edition', 'se')

result = Menu("")
appMenu = result.addMenu("dupeGuru")
fileMenu = result.addMenu("File")
editMenu = result.addMenu("Edit")
actionMenu = result.addMenu("Actions")
owner.columnsMenu = result.addMenu("Columns")
modeMenu = result.addMenu("Mode")
windowMenu = result.addMenu("Window")
helpMenu = result.addMenu("Help")

appMenu.addItem("About dupeGuru", Action(owner, 'showAboutBox'))
appMenu.addItem("Check for update...", Action(owner.updater, 'checkForUpdates:'))
appMenu.addSeparator()
appMenu.addItem("Preferences...", Action(owner, 'showPreferencesPanel'), 'cmd+,')
appMenu.addSeparator()
NSApp.servicesMenu = appMenu.addMenu("Services")
appMenu.addSeparator()
appMenu.addItem("Hide dupeGuru", Action(NSApp, 'hide:'), 'cmd+h')
appMenu.addItem("Hide Others", Action(NSApp, 'hideOtherApplications:'), 'cmd+alt+h')
appMenu.addItem("Show All", Action(NSApp, 'unhideAllApplications:'))
appMenu.addSeparator()
appMenu.addItem("Quit dupeGuru", Action(NSApp, 'terminate:'), 'cmd+q')

fileMenu.addItem("Load Results...", Action(None, 'loadResults'), 'cmd+o')
owner.recentResultsMenu = fileMenu.addMenu("Load Recent Results")
fileMenu.addItem("Save Results...", Action(None, 'saveResults'), 'cmd+s')
fileMenu.addItem("Export Results to XHTML", Action(owner.model, 'exportToXHTML'), 'cmd+shift+e')
fileMenu.addItem("Export Results to CSV", Action(owner.model, 'exportToCSV'))
if edition == 'pe':
    fileMenu.addItem("Clear Picture Cache", Action(owner, 'clearPictureCache'), 'cmd+shift+p')
elif edition == 'me':
    fileMenu.addItem("Remove Dead Tracks in iTunes", Action(owner, 'removeDeadTracks'))

editMenu.addItem("Mark All", Action(None, 'markAll'), 'cmd+a')
editMenu.addItem("Mark None", Action(None, 'markNone'), 'cmd+shift+a')
editMenu.addItem("Invert Marking", Action(None, 'markInvert'), 'cmd+alt+a')
editMenu.addItem("Mark Selected", Action(None, 'markSelected'), 'ctrl+cmd+a')
editMenu.addSeparator()
editMenu.addItem("Cut", Action(None, 'cut:'), 'cmd+x')
editMenu.addItem("Copy", Action(None, 'copy:'), 'cmd+c')
editMenu.addItem("Paste", Action(None, 'paste:'), 'cmd+v')
editMenu.addSeparator()
editMenu.addItem("Filter Results...", Action(None, 'focusOnFilterField'), 'cmd+alt+f')

actionMenu.addItem("Start Duplicate Scan", Action(owner, 'startScanning'), 'cmd+d')
actionMenu.addSeparator()
actionMenu.addItem("Send Marked to Trash...", Action(None, 'trashMarked'), 'cmd+t')
actionMenu.addItem("Move Marked to...", Action(None, 'moveMarked'), 'cmd+m')
actionMenu.addItem("Copy Marked to...", Action(None, 'copyMarked'), 'cmd+alt+m')
actionMenu.addItem("Remove Marked from Results", Action(None, 'removeMarked'), 'cmd+r')
actionMenu.addItem("Re-Prioritize Results...", Action(None, 'reprioritizeResults'))
actionMenu.addSeparator()
actionMenu.addItem("Remove Selected from Results", Action(None, 'removeSelected'), 'cmd+backspace')
actionMenu.addItem("Add Selected to Ignore List", Action(None, 'ignoreSelected'), 'cmd+g')
actionMenu.addItem("Make Selected into Reference", Action(None, 'switchSelected'), 'cmd+arrowup')
actionMenu.addSeparator()
actionMenu.addItem("Open Selected with Default Application", Action(None, 'openSelected'), 'cmd+return')
actionMenu.addItem("Reveal Selected in Finder", Action(None, 'revealSelected'), 'cmd+alt+return')
actionMenu.addItem("Invoke Custom Command", Action(None, 'invokeCustomCommand'), 'cmd+shift+c')
actionMenu.addItem("Rename Selected", Action(None, 'renameSelected'), 'enter')

modeMenu.addItem("Show Dupes Only", Action(None, 'togglePowerMarker'), 'cmd+1')
modeMenu.addItem("Show Delta Values", Action(None, 'toggleDelta'), 'cmd+2')

windowMenu.addItem("Results Window", Action(owner, 'showResultWindow'))
windowMenu.addItem("Folder Selection Window", Action(owner, 'showDirectoryWindow'))
windowMenu.addItem("Ignore List", Action(owner, 'showIgnoreList'))
windowMenu.addItem("Details Panel", Action(None, 'toggleDetailsPanel'), 'cmd+i')
windowMenu.addItem("Quick Look", Action(None, 'toggleQuicklookPanel'), 'cmd+l')
windowMenu.addSeparator()
windowMenu.addItem("Minimize", Action(None, 'performMinimize:'))
windowMenu.addItem("Zoom", Action(None, 'performZoom:'))
windowMenu.addItem("Close Window", Action(None, 'performClose:'), 'cmd+w')
windowMenu.addSeparator()
windowMenu.addItem("Bring All to Front", Action(None, 'arrangeInFront:'))

helpMenu.addItem("dupeGuru Help", Action(owner, 'openHelp'), 'cmd+?')
helpMenu.addItem("dupeGuru Website", Action(owner, 'openWebsite'))


# User-based collaborative filtering recommender class
# Implementation of an user based recommedation filter
# Team: rcmmndrs
# Paul Opitz

from scipy.sparse import *
from scipy import *
import pickle
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.decomposition import TruncatedSVD
from recommenders.recommender_base import RecommenderBase
from pathlib import Path
import logging
import sys

class RecommenderUserBased(RecommenderBase):

    def __init__(self):
        super().__init__()
        self.name = 'User-based'

    def setup(self, input_dataframe):
        users_reduced = Path('cache/users_reduced.npy')
        if not users_reduced.exists():
            logging.info(self.name + ': ' + 'Starting training.')
            self.fit(input_dataframe)
            logging.info(self.name + ': ' + 'Ended training.')
        else:
            logging.info(self.name + ': ' + 'Already trained.')

        self.is_setup = True

    def fit(self, input_dataframe):
        # read training set
        training = input_dataframe[['user', 'track']]

        tracks = training['track'].unique()
        tracks_size = len(tracks)

        users = training['user'].unique()
        users_size = len(users)

        users_sparse = csr_matrix( (users_size,tracks_size), dtype=int16 )

        track_id_dict = {}
        id_track_dict = {}
        for i,track in enumerate(tracks):
            track_id_dict[track] = i
            id_track_dict[i] = track

        user_id_dict = {}
        id_user_dict = {}
        for i,user in enumerate(users):
            user_id_dict[user] = i
            id_user_dict[i] = user

        with open('cache/track_to_id.pickle', 'wb') as handle:
            pickle.dump(track_id_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
        with open('cache/id_to_track.pickle', 'wb') as handle:
            pickle.dump(id_track_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

        with open('cache/user_to_id.pickle', 'wb') as handle:
            pickle.dump(user_id_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
        with open('cache/id_to_user.pickle', 'wb') as handle:
            pickle.dump(id_user_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

        # Create user dict
        user_dict = {}
        groups = training.groupby('user')
        for i, group in groups:
            user_id = group['user'].iloc[0]
            track_ids = set(group['track'])
            user_dict[user_id] = track_ids

        # Generate user item matrix
        count = 0
        for user, tracks in user_dict.items():
            track_ids = [track_id_dict[track] for track in tracks]
            user_id = user_id_dict[user]
            sys.stdout.write("User: " + str(count) + "\r")
            sys.stdout.flush()
            count += 1
            track_ids = track_ids[:1000]
            for track_id in track_ids:
                #print(track_id_dict[track_id])
                row = array([user_id for i in track_ids])
                col = array(track_ids)
                data = array([1 for i in track_ids])
                acc = csr_matrix( (data,(row,col)), shape=(users_size,tracks_size), dtype=int16 )
                users_sparse += acc

        # Apply SVD (dimensionality reduction to 300)
        svd = TruncatedSVD(n_components=50, algorithm='arpack')
        svd.fit(users_sparse)
        users_reduced = svd.transform(users_sparse)

        with open('cache/users_svd.pickle', 'wb') as handle:
            pickle.dump(svd, handle, protocol=pickle.HIGHEST_PROTOCOL)
        np.save('cache/users_reduced.npy', users_reduced)
        save_npz('cache/users_sparse.npz', users_sparse)

    def recommend_by_tracks(self, tracks, n):
        tracks = list(tracks)[:1000]

        users_sparse = load_npz('cache/users_sparse.npz')
        users_reduced = np.load('cache/users_reduced.npy')

        with open('cache/track_to_id.pickle', 'rb') as handle:
            track_id_dict = pickle.load(handle)
        with open('cache/id_to_track.pickle', 'rb') as handle:
            id_track_dict = pickle.load(handle)
        with open('cache/users_svd.pickle', 'rb') as handle:
            svd = pickle.load(handle)

        tracks_size = users_sparse.shape[1]

        track_ids = [track_id_dict[track] for track in tracks if track in track_id_dict.keys()]

        row = array([0] * len(track_ids))
        col = array(track_ids)
        data = array([1 for i in track_ids])

        user_sparse = csr_matrix((data, (row, col)), shape=(1, tracks_size), dtype=int16)
        user_reduced = svd.transform(user_sparse)

        user_sim = cosine_similarity(user_reduced, users_reduced)

        indices = user_sim[0].nonzero()[0].tolist()
        values = user_sim[0].data.tolist()
        zipped = list(zip(indices, values))
        zipped.sort(key=lambda item: item[1], reverse=True)
        # take best 25 users
        user_ids = [i[0] for i in zipped[:25]]
        # add up
        result_sparse = csr_matrix((1, tracks_size), dtype=int8)
        for user_id in user_ids:
            row_sparse = users_sparse.getrow(user_id)
            result_sparse += row_sparse

        indices = result_sparse.nonzero()[1].tolist()
        values = result_sparse.data.tolist()

        tracks = [id_track_dict[i] for i in indices]
        zipped = list(zip(tracks, values))
        filtered = list(filter(lambda item: item[0] not in track_ids, zipped))
        filtered.sort(key=lambda item: item[1], reverse=True)
        result = filtered[:n]

        return result


# -*- coding: utf-8 -*-

import sys

from PyQt5.QtWidgets import  QApplication, QMainWindow

from PyQt5.QtWidgets import  QFileDialog,QMessageBox

from PyQt5.QtCore import  pyqtSlot,QDir,QFile,QIODevice,QTextStream

##from PyQt5.QtGui import

##from PyQt5.QtSql import

##from PyQt5.QtMultimedia import

##from PyQt5.QtMultimediaWidgets import

from ui_MainWindow import Ui_MainWindow

class QmyMainWindow(QMainWindow):
   def __init__(self, parent=None):
      super().__init__(parent)   #调用父类构造函数，创建窗体
      self.ui=Ui_MainWindow()    #创建UI对象
      self.ui.setupUi(self)      #构造UI界面

      self.setCentralWidget(self.ui.textEdit)

##  ==============自定义功能函数========================
   def __openByIODevice(self,fileName):  ##用QFile打开文件
      fileDevice=QFile(fileName)
      if not fileDevice.exists():  #判断文件是否存在
         return False

      if not fileDevice.open(QIODevice.ReadOnly | QIODevice.Text):
         return False

###整个文件一次性读取的方式，可行
##      qtBytes=fileDevice.readAll() #返回QByteArray类型
##      pyBytes=bytes(qtBytes.data())  # 将QByteArray转换为bytes类型
##      text=pyBytes.decode("utf-8")  #用utf-8编码为字符串
##      self.ui.textEdit.setPlainText(text)

##  逐行读取方式，可行
      try:
         self.ui.textEdit.clear()
         while not fileDevice.atEnd():
            qtBytes = fileDevice.readLine()  # 返回QByteArray类型
            pyBytes=bytes(qtBytes.data())    # QByteArray转换为bytes类型
            lineStr=pyBytes.decode("utf-8")  #bytes转换为str型
            lineStr=lineStr.strip()          #去除结尾增加的空行
            self.ui.textEdit.appendPlainText(lineStr)
      finally:
         fileDevice.close()

      return True

   def __saveByIODevice(self,fileName):  ##用QFile保存文件
      fileDevice=QFile(fileName)
      if not fileDevice.open(QIODevice.WriteOnly | QIODevice.Text):
         return False

      try:
         text=self.ui.textEdit.toPlainText()  #返回str类型
         strBytes=text.encode("utf-8")        # str转换为bytes类型
         fileDevice.write(strBytes)           #写入文件
      finally:
         fileDevice.close()

      return  True

   def __openByStream(self,fileName):  ##用QTextStream打开文件
      fileDevice=QFile(fileName)
      if not fileDevice.exists():   #判断文件是否存在
         return False

      if not fileDevice.open(QIODevice.ReadOnly | QIODevice.Text):
         return False

      try:
         fileStream=QTextStream(fileDevice)
         fileStream.setAutoDetectUnicode(True)  #自动检测Unicode
         fileStream.setCodec("utf-8")  #必须设置编码,否则不能正常显示汉字

   # 一次性全部读出
   ##   text=fileStream.readAll()  #读取出来就是str
   ##   self.ui.textEdit.setPlainText(text)

   #逐行读取方式
         self.ui.textEdit.clear()
         while not fileStream.atEnd():
            lineStr=fileStream.readLine()    #读取文件的一行,读取出来就是str
            self.ui.textEdit.appendPlainText(lineStr)    #添加到文本框显示
      finally:
         fileDevice.close() #关闭文件
      return  True

   def __saveByStream(self,fileName):  ##用 QTextStream 保存文件
      fileDevice=QFile(fileName)
      if not fileDevice.open(QIODevice.WriteOnly | QIODevice.Text):
         return False

      try:
         fileStream=QTextStream(fileDevice)  #用文本流读取文件
         fileStream.setAutoDetectUnicode(True)  #自动检测Unicode
         fileStream.setCodec("utf-8")  #必须设置编码,否则不能正常显示汉字

         text=self.ui.textEdit.toPlainText()  #返回是str类型
         fileStream<<text   #使用流操作符
   ##         fileStream<<"\n***************在尾部添加的第1行"
      finally:
         fileDevice.close()
      return  True

##  ==============event处理函数==========================

##  ==========由connectSlotsByName()自动连接的槽函数============

   @pyqtSlot()   ##用QFile 打开文件
   def on_actQFile_Open_triggered(self):
      curPath=QDir.currentPath()    #获取系统当前目录
      title="打开一个文件"    #对话框标题
      filt="程序文件(*.h *.cpp *.py);;文本文件(*.txt);;所有文件(*.*)"   #文件过滤器
      fileName,flt=QFileDialog.getOpenFileName(self,title,curPath,filt)
      if (fileName == ""):
         return

      if self.__openByIODevice(fileName):
         self.ui.statusBar.showMessage(fileName)
      else:
         QMessageBox.critical(self,"错误","打开文件失败")

   @pyqtSlot()   ##用QFile 另存文件
   def on_actQFile_Save_triggered(self):
      curPath=QDir.currentPath()    #获取系统当前目录
      title="另存为一个文件"        #对话框标题
      filt="Python程序(*.py);; C++程序(*.h *.cpp);;文本文件(*.txt);;所有文件(*.*)"  #文件过滤器
      fileName,flt=QFileDialog.getSaveFileName(self,title,curPath,filt)
      if (fileName==""):
         return

      if self.__saveByIODevice(fileName):
         self.ui.statusBar.showMessage(fileName)
      else:
         QMessageBox.critical(self,"错误","保存文件失败")

   @pyqtSlot()   ##用QTextStream 打开文件
   def on_actStream_Open_triggered(self):
      curPath=QDir.currentPath() #获取系统当前目录
      title="打开一个文件"       #对话框标题
      filt="程序文件(*.h *.cpp *.py);;文本文件(*.txt);;所有文件(*.*)"   #文件过滤器
      fileName,flt=QFileDialog.getOpenFileName(self,title,curPath,filt)
      if (fileName == ""):
         return

      if self.__openByStream(fileName):
         self.ui.statusBar.showMessage(fileName)
      else:
         QMessageBox.critical(self,"错误","打开文件失败")

   @pyqtSlot()   ##用QTextStream 另存文件
   def on_actStream_Save_triggered(self):
      curPath=QDir.currentPath()  #获取系统当前目录
      title="另存为一个文件"      #对话框标题
      filt="Python程序(*.py);; C++程序(*.h *.cpp);;所有文件(*.*)"   #文件过滤器
      fileName,flt=QFileDialog.getSaveFileName(self,title,curPath,filt)
      if (fileName==""):
         return

      if self.__saveByStream(fileName):
         self.ui.statusBar.showMessage(fileName)
      else:
         QMessageBox.critical(self,"错误","保存文件失败")

   @pyqtSlot()   ##用 python 的file()打开文件
   def on_actPY_Open_triggered(self):
##      curPath=os.getcwd()   #获取系统当前目录
      curPath=QDir.currentPath()    #获取系统当前目录
      title="打开一个文件"          #对话框标题
      filt="程序文件(*.h *.cpp *.py);;所有文件(*.*)"    #文件过滤器
      fileName,flt=QFileDialog.getOpenFileName(self,title,curPath,filt)
      if (fileName == ""):
         return

      self.ui.textEdit.clear()
   ##      aFile=codecs.open(fileName,encoding='utf-8')#可用
   ##      aFile=codecs.open(fileName,'r', "utf-8")  #打开文件，用utf-8码
      fileDevice=open(fileName,mode='r', encoding='utf-8')  #打开文件

      try:
         for eachLine in fileDevice:  #每次读取一行
            lineStr=eachLine.strip()  #必须用strip()去掉末尾的'\0'
            self.ui.textEdit.appendPlainText(lineStr)
         self.ui.statusBar.showMessage(fileName)
      finally:
         fileDevice.close()

   @pyqtSlot()   ##用 python的file() 保存文件
   def on_actPY_Save_triggered(self):
   ##      curPath=os.getcwd()   #获取系统当前目录
      curPath=QDir.currentPath()    #获取系统当前目录
      title="另存为一个文件"         #对话框标题
      filt="Python程序(*.py);; C++程序(*.h *.cpp);;所有文件(*.*)"   #文件过滤器
      fileName,flt=QFileDialog.getSaveFileName(self,title,curPath,filt)
      if (fileName==""):
         return

      text=self.ui.textEdit.toPlainText()
      fileDevice=open(fileName,mode='w', encoding='utf-8')
      try:
         fileDevice.write(text)
         self.ui.statusBar.showMessage(fileName)
      finally:
         fileDevice.close()

##  =============自定义槽函数===============================

##  ============窗体测试程序 ================================
if  __name__ == "__main__":        #用于当前窗体测试
   app = QApplication(sys.argv)    #创建GUI应用程序
   form=QmyMainWindow()            #创建窗体
   form.show()
   sys.exit(app.exec_())


# -*- coding:utf-8 -*-

import configparser
import os

from performance_autotest.customexception import CustomError

class Config(object):
    __instance = None

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = super().__new__(cls)
        else:
            raise CustomError("Config 类只能含有一个实例, "
                              "使用类方法 get_instance 获取实例")

        return cls.__instance

    def __init__(self):
        self.conf = configparser.ConfigParser()
        config_path = os.path.dirname(__file__)
        if os.path.exists(config_path+"\\conf\\config.ini"):
            self.conf.read(config_path+"\\conf\\config.ini", encoding="GBK")
        else:
            raise CustomError("配置文件不存在")

    def reload_all_value(self):
        sections = self.conf.sections()
        for section in sections:
            items = self.conf.items(section)
            for item in items:
                if not self.set_default_value(item):
                    self.__setattr__(item[0], item[1].strip())

    @classmethod
    def get_instance(cls):
        if not cls.__instance:
            cls.__instance = Config()
            cls.__instance.reload_all_value()
        return cls.__instance

    def set_default_value(self, section_item):
        if not section_item[1] == "":
            return False

        if section_item[0] == "nmon_path":
            self.__setattr__(section_item[0], ".")
        elif section_item[0] == "nmon_acquisition_interval":
            self.__setattr__(section_item[0], "1")
        elif section_item[0] == "download_local_path":
            raise CustomError("存放监控文件路径不能为空")
        elif section_item[0] == "remote_host_num":
            raise CustomError("后台服务器数量不能为空")
        elif section_item[0] == "report_name":
            self.__setattr__(section_item[0], "report.html")
        elif section_item[0] == "run_in_win":
            self.__setattr__(section_item[0], "1")
        else:
            self.__setattr__(section_item[0], "")

        return True


from django.contrib.auth import authenticate, login
from django.contrib.auth import logout
from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse
import json,requests
from movierec import myconfig

# Create your views here.
def index(request):
    if not request.user.is_authenticated:
        return render(request, 'movierec/preview.html')
    else:
        return render(request,'bookrec/index.html')

from recombee_api_client.api_client import RecombeeClient
from recombee_api_client.api_requests import *

client = RecombeeClient(myconfig.book_db,myconfig.btoken)

def detail(request):
    if not request.user.is_authenticated:
        return render(request, 'movierec/login.html')
    else:
        user = request.user
        isbn_id= request.GET.get("isbn_id")

        result = client.send(GetItemValues(isbn_id))
        result = formatRes(result)
        return render(request, 'bookrec/detail.html', {'res': result})

# @login_required
# @transaction.atomic
#need to add pages
def update_profile(request):
    if not request.user.is_authenticated:
        return render(request, 'movierec/preview.html')
    if request.method == 'POST':
        user_form = UserForm(request.POST, instance=request.user)
        profile_form = ProfileForm(request.POST, instance=request.user.profile)
        if user_form.is_valid() and profile_form.is_valid():
            user_form.save()
            profile_form.save()
            messages.success(request, _('Your profile was successfully updated!'))
            return redirect('settings:profile')
        else:
            messages.error(request, _('Please correct the error below.'))
    else:
        user_form = UserForm(instance=request.user)
        profile_form = ProfileForm(instance=request.user.profile)
    return render(request, 'movierec/profile.html', {
        'user_form': user_form,
        'profile_form': profile_form
    })

def search_books(request):
    if request.method== "POST":
        query = request.POST.get("q")
        if query:
            result = client.send(ListItems(filter="\""+query+"\" in 'Book-Title'" , return_properties = True, included_properties = ['Book-Title','Book-Author']))
            data = {"Search": result}
            data = json.dumps(data)

            return render(request, 'bookrec/search_result.html', {'result':data })
        else:
            return render(request, 'bookrec/search_result.html')

def recommend_books(request):
    if request.method== "POST":
        uid = request.POST.get("uid")
        if uid:
            recommended= client.send(RecommendItemsToUser(10, 10,return_properties = True,included_properties = ['Book-Title','Book-Author','Image-URL-L']))
            return render(request, 'bookrec/recom_result.html', {'result':json.dumps(recommended) })
        else:
            return render(request, 'bookrec/recom_result.html')

def rate_books(request):
    if request.method == "POST":
        my_uid=request.POST.get("uid")
        my_isbn=request.POST.get("isbn")
        my_rating=float(request.POST.get("rating"))

        client.send(AddRating(my_uid,my_isbn,(my_rating-5)/5,cascade_create= True))
    return render(request, 'bookrec/recom_result.html')

def related_books(request):
    if request.method== "POST":
        uid = request.POST.get("uid")
        isbn = request.POST.get("isbn")
        if uid:
            recommended= client.send(RecommendItemsToItem(isbn ,uid ,5,return_properties=True))
            data = {"related":[]}
            for item in recommended["recomms"]:
                data["related"].append(item)
            return render(request, 'bookrec/related_result.html', {'result':data})
        else:
            return render(request, 'bookrec/related_result.html')

def formatRes(res):
    formated={}
    for k in res:
        if '-' in k:
            formated[k.replace('-','')]=res[k]
        else:
            formated[k]=res[k]
    return formated


# Adam Peter, Lab 6
# Solomon Chan, Lab 6
# We certify that we worked cooperatively on this programming
#   assignment, according to the rules for pair programming.
import goody
from collections import defaultdict
import random

def read_corpus(order_stas, file:'open file') -> dict:
    corpusdict = defaultdict(list)
    g = goody.read_file_values(f)
    prereadwords = []
    for x in range(eval(order_stas)):
        prereadwords.append(next(g))
    for i in g:
        if i not in corpusdict[tuple(prereadwords)]:
            corpusdict[tuple(prereadwords)].append(i)
        prereadwords = prereadwords[1:]
        prereadwords.append(i)
    return dict(corpusdict)

def print_corpus(d:dict)->None:
    print('Corpus')
    min = 10000
    max = -1
    for i in sorted(d.keys()):
        if len(d[i]) < min:
            min = len(d[i])
        if len(d[i]) > max:
            max = len(d[i])
        print(i, 'can be followed by any of ',d[i])
    print('min/max = {}/{}'.format(min, max))

def produce_text(d:dict, l:list, num:int)-> [str]:
    n_words = l
    new_words= []
    for i in range(num):
        w = random.choice(d[tuple(n_words)])
        n_words.remove(n_words[0])
        n_words.append(w)
        new_words.append(w)
    return new_words

if __name__ == '__main__':
    order = input('Enter order statistic: ')
    with open(input('Enter file to process: ')) as f:
        corpusdict = read_corpus(order, f)
        print_corpus(corpusdict)
        print()
        print("Enter", order,"words to start with")
        words = []
        for i in range(eval(order)):
            words.append(input("Enter word {}: ".format(i+1)))

        word_num = input("Enter # of words to generate: ")
        print("Random text = ", produce_text(corpusdict, words, eval(word_num)))


def buble(sort):
	#sort = list(array)
	for ii in range(0,len(sort)):
		for jj in range(0,len(sort)):
			if sort[ii] < sort[jj]:
				sort[ii],sort[jj] = sort[jj],sort[ii]
	return sort

if __name__ == '__main__':
	import random
	array = []
	for i in range(0,19):
		array.append(random.randint(1,100))
	print(array)
	print(buble(array))


'''
Created on Feb 20, 2018

@author: Evan Kountouris
'''

import pandas
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier, export_graphviz
from sklearn import metrics
import matplotlib.pyplot as plt
import seaborn

if __name__ == '__main__':

    #open csv, into data frame
    csv1 = "flat_records_wthHeader.csv"
    df = pandas.read_csv(csv1)

    # display stats
#     ax1 = df.groupby("education_level").size().plot(kind="barh",figsize=(8,5))
    ax2 = df.groupby("occupation").size().plot(kind="barh",figsize=(6,4))
#     ax3 = df.groupby("race").size().plot(kind="barh",figsize=(8,5))
    g = seaborn.factorplot("over_50k", col="marital_status", col_wrap=4,data=df,kind="count", size=3, aspect=0.8)
    plt.show()

    #delete columns to be ignored from analysis
    del df["id"]
    del df["education_level"]

    #split into training and test data (majority training)
    train, test = train_test_split(df, test_size=0.20)

    #create X vectors and Y outputs for training and testing
    y_train = train.pop("over_50k")
    y_test = test.pop("over_50k")
    x_train = pandas.get_dummies(train)
    x_test = pandas.get_dummies(test)

    # ensure any missing headers are included in the test data
    if not x_train.columns.equals(x_test.columns):
        missing = set(x_train.columns) - set(x_test.columns)
        for c in missing:
            x_test[c] = 0
        x_test = x_test[x_train.columns]

    # Build Decision tree on training data, to depth of 4 to combat overfitting
    model = DecisionTreeClassifier(max_depth=3)
    model.fit(x_train, y_train)

    # the tree can be visualised by using graphviz, transforming the dot file into an image
    # the dot file is created within this code, but must be transformed to image outside
    export_graphviz(model,feature_names=x_train.columns)

    # accuracy can be measured by predicting the test data set labels using our built model
    accuracy = metrics.mean_absolute_error(y_test,model.predict(x_test))
    print("Decision tree is created.")
    print("Accuracy of model applied to test data:",accuracy)


#!/usr/bin/env python3
#
# Copyright 2021 Graviti. Licensed under MIT License.
#

from itertools import zip_longest

import numpy as np
import pytest

from .. import Box2D, Polyline2D, Vector2D

_POLYLINE_INFO_1 = [
    {
        "vector": np.array([0, 1], dtype=np.float32),
        "last_time": 0,
        "time": 0.5,
        "point": np.array([1, 1], dtype=np.float32),
        "index": 0,
    },
    {
        "vector": np.array([1, 0], dtype=np.float32),
        "last_time": 0.5,
        "time": 1.0,
        "point": np.array([1, 2], dtype=np.float32),
        "index": 1,
    },
]
_POLYLINE_INFO_2 = [
    {
        "vector": np.array([-9, 12], dtype=np.float32),
        "last_time": 0,
        "time": 0.75,
        "point": np.array([10, -9], dtype=np.float32),
        "index": 0,
    },
    {
        "vector": np.array([3, 4], dtype=np.float32),
        "last_time": 0.75,
        "time": 1.0,
        "point": np.array([1, 3], dtype=np.float32),
        "index": 1,
    },
]
_POLYLINE_DATA = [{"x": 1, "y": 1}, {"x": 1, "y": 2}, {"x": 2, "y": 2}]

class TestPolyline2D:
    def test_init(self):
        sequence = [[1, 1], [1, 2], [2, 2]]
        assert Polyline2D() == Polyline2D([])
        assert Polyline2D(sequence) == Polyline2D([Vector2D(1, 1), Vector2D(1, 2), Vector2D(2, 2)])

    def test_eq(self):
        polyline_1 = Polyline2D([[1, 2], [2, 3], [2, 2]])
        polyline_2 = Polyline2D([[1, 2], [2, 3], [2, 2]])
        polyline_3 = Polyline2D([[1, 2], [3, 4], [2, 2]])
        assert (polyline_1 == polyline_2) == True
        assert (polyline_1 == polyline_3) == False

    def test_distance(self):
        sequence_1 = [1, 2]
        sequence_2 = [4, 6]
        assert Polyline2D._distance(sequence_1, sequence_2) == 5.0

    def test_get_polyline_info(self):
        sequence = [[1, 1], [1, 2], [2, 2]]
        polyline_info = Polyline2D._get_polyline_info(sequence)

        for info1, info2 in zip_longest(polyline_info, _POLYLINE_INFO_1):
            assert info1["last_time"] == info2["last_time"]
            assert info1["time"] == info2["time"]
            assert info1["index"] == info2["index"]
            np.testing.assert_array_equal(info1["vector"], info2["vector"])
            np.testing.assert_array_equal(info1["point"], info2["point"])

    def test_insert_point(self):
        inserted_point = Polyline2D._insert_point(_POLYLINE_INFO_1[0], _POLYLINE_INFO_2[0])
        assert inserted_point["index"] == 1
        np.testing.assert_array_equal(inserted_point["point"], np.array([4, -1]))

    def test_insert_points(self):
        info1 = iter(_POLYLINE_INFO_1)
        info2 = iter(_POLYLINE_INFO_2)
        inserted_points_1 = Polyline2D._insert_points(info1, info2)
        inserted_points_2 = (
            [{"index": 2, "point": np.array([1.5, 2.0])}],
            [{"index": 1, "point": np.array([4.0, -1.0])}],
        )
        for points_1, points_2 in zip_longest(inserted_points_1, inserted_points_2):
            for point_1, point_2 in zip_longest(points_1, points_2):
                assert point_1["index"] == point_2["index"]
                np.testing.assert_array_equal(point_1["point"], point_2["point"])

    def test_max_distance_in_point_pairs(self):
        polyline_1 = np.array([[1, 1], [1, 2], [2, 2]])
        polyline_2 = np.array([[4, 5], [2, 1], [3, 3]])
        polyline_3 = np.array([[4, 5], [2, 1]])

        with pytest.raises(AssertionError):
            Polyline2D._max_distance_in_point_pairs(polyline_1, polyline_3)
        assert Polyline2D._max_distance_in_point_pairs(polyline_1, polyline_2) == 5.0

    def test_uniform_frechet_distance(self):
        polyline_1 = [[1, 1], [1, 2], [2, 2]]
        polyline_2 = [[4, 5], [2, 1], [3, 3]]
        assert Polyline2D.uniform_frechet_distance(polyline_1, polyline_2) == 3.605551275463989

    def test_similarity(self):
        polyline_1 = [[1, 1], [1, 2], [2, 2]]
        polyline_2 = [[4, 5], [2, 1], [3, 3]]
        assert Polyline2D.similarity(polyline_1, polyline_2) == 0.2788897449072022

    def test_loads(self):
        polyline = Polyline2D.loads(_POLYLINE_DATA)
        assert polyline == Polyline2D([Vector2D(1, 1), Vector2D(1, 2), Vector2D(2, 2)])

    def test_dumps(self):
        polyline = Polyline2D([[1, 1], [1, 2], [2, 2]])
        assert polyline.dumps() == _POLYLINE_DATA

    def test_bounds(self):
        polyline = Polyline2D([[1, 1], [1, 2], [2, 2]])
        assert polyline.bounds() == Box2D(1, 1, 2, 2)


import pandas
import numpy as np
import random

from .recommender_system_base import RecommenderSystemBase

class HashTable:
    def __init__(self, hash_size, inp_dimensions):
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_table = dict()
        self.projections = np.random.randn(self.hash_size, inp_dimensions)

    def generate_hash(self, inp_vector):
        bools = (np.dot(inp_vector, self.projections.T) > 0).astype('int')
        return ''.join(bools.astype('str'))

    def __setitem__(self, inp_vec, label):
        #print("generating hash value for hash table generation")
        hash_value = self.generate_hash(inp_vec)
        #print("completed generating hash value for hash table generation")
        self.hash_table[hash_value] = self.hash_table.get(hash_value, list()) + [label]
        #print("completed Setting values to hash table")
    def __getitem__(self, inp_vec):
        #print("generating hash value for searching hash table")
        hash_value = self.generate_hash(inp_vec)
        return self.hash_table.get(hash_value, [])

class LSH:
    def __init__(self, num_tables, hash_size, inp_dimensions):
        self.num_tables = num_tables
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_tables = list()
        for i in range(self.num_tables):
            self.hash_tables.append(HashTable(self.hash_size, self.inp_dimensions))

    def __setitem__(self, inp_vec, label):
        #print("generating hash tables")
        for table in self.hash_tables:
            table[inp_vec] = label
        #print("hash table generation complete")

    def __getitem__(self, inp_vec):
        results = list()
        for table in self.hash_tables:
            #print("looping through hash tables")
            results.extend(table[inp_vec])
            #print("suggestions from table ",results)

        return list(set(results))

class ItemItemRecommenderSystem(RecommenderSystemBase):
    """
    Attributes
    ----------

    Methods
    -------
    compute_movie_embeddings
        Computes the movie embeddings.

    recommend_similar_movies
        Recommends the k most similar of the movie with the id 'movie_id'.

    recommend_movies_to_user
        Given a user with a watch history, it recommends the k movies that he will most likely watch.

    get_movies_embeddings
        Returns the embedding of the movies with movie_id in movie_ids.

    Notes
    -----
    - You can add other attributes and methods to this class.
    - In the constructor parameters, you can add other datasets if you need them.

    Examples
    --------
    >>> rec_sys = ItemItemRecommenderSystem(**kwargs)
    >>> ...
    >>> rec_sys.recommend_similar_movies(movie_id='the_promise-das_versprechen-en-1995', k=10)
    ...
    >>> rec_sys.recommend_movies_to_user(user_id=25, k=10)
    ...
    >>> movie_embeddings = rec_sys.get_movies_embeddings(movie_ids)
    >>> visualize_embeddings(movie_embeddings)
    ...
    """

    def __init__(self, ratings_dataframe: pandas.DataFrame, movies_metadata_dataframe: pandas.DataFrame,
                 keywords_dataframe: pandas.DataFrame, credits_dataframe: pandas.DataFrame) -> None:
        """Sets the movie_embeddings attribute.

        Parameters
        ----------
        ratings_dataframe : pandas.DataFrame
            The movie ratings of users.
        movies_metadata_dataframe : pandas.DataFrame
            The movies metadata.
        keywords_dataframe : pandas.DataFrame
            The movies keywords.
        credits_dataframe : pandas.DataFrame
            The movies credits.
        """
        print("starting init")
        super().__init__(ratings_dataframe, movies_metadata_dataframe, keywords_dataframe, credits_dataframe)
        self.movie_embeddings = self.make_embeddings(self.movies_dataframe, 'movie')
        self.user_embeddings = self.make_embeddings(self.movies_dataframe, 'user')
        print("ending init")

    def make_embeddings(self, merged_movie_data, emb_type):
        print("make embedding")

        if emb_type == 'movie':
            ratings_matrix = merged_movie_data.pivot_table(columns='userId',index='movie_id',values='rating')
        else:
            ratings_matrix = merged_movie_data.pivot_table(columns='movie_id',index='userId',values='rating')

        ratings_matrix.dropna(axis=1, how='all', inplace=True)
        ratings_matrix.fillna( 0, inplace = True )
        print("ending embedding")
        return ratings_matrix

    def getJaccardSim(self, movie1,movie2):
        movie1=''.join((movie1 > 3).astype(int).astype('str'))
        movie2=''.join((movie2 > 3).astype(int).astype('str'))
        N = 0
        D = 0
        for i in range(len(movie1)):
            sum_ = int(movie1[i]) + int(movie2[i])
            if(sum_ >= 1):
                flag = 1
                D += 1
                if(sum_ == 2):
                    N += 1
        if D == 0:
            return 0
        return(float(N)/D)

    def getCosineSim(self, movie1,movie2):
        return np.dot(movie1,movie2)/(np.linalg.norm(movie1)*np.linalg.norm(movie2))

    def recommend_movies_to_user(self, user_id: int, k: int, algo) -> pandas.DataFrame:
        """Given a user with a watch history, it recommends the k movies that he will most likely watch.

        user_favourite_movies = the set of movies that the user watched and liked.

        If len(user_favourite_movies) = 0:
            Recommend k random movies from the set of highly rated movies in the dataset.
            These k movies should be chosen randomly. So if the function is executed 2 times, it should
            return different results.

        If k < len(user_favourite_movies):
            Select a random set of movies from the user_favourite_movies set and recommend a movie for each item.

        If k > len(user_favourite_movies):
            Select n movies for each movie the user liked.
            Example :
                k = 10 and len(user_favourite_movies) = 1
                    Recommend 10 movies that are similar to the movie the user watched.
                k = 10 and len(user_favourite_movies) = 3
                    Recommend:
                        3 movies that are similar the 1st movie the user liked.
                        3 movies that are similar the 2nd movie the user liked.
                        4 movies that are similar the 3rd movie the user liked.

        Parameters
        ----------
        user_id : int
            The id of the user
        k : int
            The number of movies to recommend

        Returns
        -------
        pandas.DataFrame
            A subset of the movies_dataframe with the k movies that the user may like.
        """

        from scipy.sparse import csr_matrix
        embeddings_sparse = csr_matrix(self.movie_embeddings.values)

        from sklearn.neighbors import NearestNeighbors

        user_favourite_movies = self.movies_dataframe[self.movies_dataframe.userId == user_id][self.movies_dataframe.rating >= 3].movie_id.tolist()
        #print("favorite movies",user_favourite_movies)

        if len(user_favourite_movies) == 0:
            return self.movies_dataframe[self.movies_dataframe.rating >= 4].sample(k)

        elif algo == 'KNN':

            if k < len(user_favourite_movies):
                user_favourite_movies = random.sample(user_favourite_movies, k)
                model = NearestNeighbors(n_neighbors=k,algorithm='brute',metric='cosine')
                model.fit(embeddings_sparse)

                movie_embeddings = self.get_movies_embeddings(user_favourite_movies)
                distances,suggestions=model.kneighbors(movie_embeddings.values)

                movies = []
                distance = []
                for i in user_favourite_movies:
                    movie_embeddings = self.get_movies_embeddings(i)
                    distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1, -1),2)
                    distances= distances.flatten()
                    suggestions= suggestions.flatten()

                    for i in range(1,len(suggestions)):
                        movie_id=self.movie_embeddings.index[suggestions[i]]
                        movies.append(movie_id)
                        distance.append(distances[i])

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

            elif k > len(user_favourite_movies):
                n = len(user_favourite_movies)
                q = k//n
                r = k%n
                k_values = []
                for _ in range(n):
                    k_values.append(q)
                k_values[-1] += r

                movies = []
                distance = []

                model = NearestNeighbors(n_neighbors=k_values[-1],algorithm='brute',metric='cosine')

                model.fit(embeddings_sparse)

                for idx,i in enumerate(k_values):
                    movie_embeddings = self.get_movies_embeddings(user_favourite_movies[idx])
                    distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1, -1),i+1)
                    distances= distances.flatten()
                    suggestions= suggestions.flatten()

                    for i in range(1,len(suggestions)):
                        movie_id=self.movie_embeddings.index[suggestions[i]]
                        movies.append(movie_id)
                        distance.append(distances[i])

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

        else:
            if k < len(user_favourite_movies):
                user_favourite_movies = random.sample(user_favourite_movies, k)
                movies = []
                for user_fav in user_favourite_movies:
                    res = self.recommend_similar_movies(user_fav, 1, algo)
                    movies.append(res.movie_id.values[0])
                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

            elif k > len(user_favourite_movies):
                n = len(user_favourite_movies)
                q = k//n
                r = k%n
                k_values = []
                for _ in range(n):
                    k_values.append(q)
                k_values[-1] += r

                movies = []
                for i in range(n):
                    res = self.recommend_similar_movies(user_favourite_movies[i], k_values[i], algo)
                    movies.extend(res.movie_id.tolist())

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

    def recommend_similar_movies(self, movie_id: str, k: int, algo) -> pandas.DataFrame:
        """Recommends the k most similar movies of the movie with the id 'movie_id'.

        Parameters
        ----------
        movie_id : str
            The id of the movie.
        k : int
            The number of similar movies to recommend.

        Returns
        -------
        pandas.DataFrame
            A subset of the movies_dataframe with the k similar movies of the target movie (movie_id).
        """

        if algo == 'knn':

            from scipy.sparse import csr_matrix
            embeddings_sparse = csr_matrix(self.movie_embeddings)

            from sklearn.neighbors import NearestNeighbors
            model = NearestNeighbors(n_neighbors=k,algorithm='brute',metric='cosine')

            model.fit(embeddings_sparse)

            #condition = self.movies_dataframe['movie_id']==movie_id
            #idVal= self.movies_dataframe[condition].drop_duplicates(subset=['movie_id'])['movieId']
            #print("Movie id", idVal)
            movie_embeddings = self.get_movies_embeddings(movie_id)
            distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1,-1),k+1)
            suggestions= suggestions.flatten()

            print(suggestions)
            movies = []
            for i in range(1,len(suggestions)):
                movies.append(self.movie_embeddings.index[suggestions[i]])

            return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

        else:
            nusers = self.movie_embeddings.columns
            nmovies = self.movie_embeddings.index

            hash_table = LSH(num_tables=20,hash_size=10, inp_dimensions=len(nusers))

            for i in range(len(nmovies)):
                hash_table[self.movie_embeddings.loc[nmovies[i]]]=nmovies[i]

            inp_vec=self.movie_embeddings.loc[movie_id]
            # print("Movie_id" ,nmovies[movie_id])
            similar_movies = hash_table[inp_vec]
            cos_sim_values =[]
            jac_sim_values=[]

            for a in similar_movies:
                if a== movie_id:
                    continue
                out_vec = self.movie_embeddings.loc[a]
                cos_sim_values.append(self.getCosineSim(inp_vec,out_vec))
                jac_sim_values.append(self.getJaccardSim(inp_vec,out_vec))

            if algo == 'LSH-C':
                ranked_cos_sim = np.argsort(np.array(cos_sim_values))
                movies_id_cos = ranked_cos_sim[::-1][:k]
                cos_sugg = []
                for i in range(0,k):
                    movie_sugg_cos = similar_movies[movies_id_cos[i]]
                    cos_sugg.append(self.movies_dataframe[self.movies_dataframe["movie_id"]==str(movie_sugg_cos)]["movie_id"].values[0])
                return self.movies_dataframe.loc[self.movies_dataframe["movie_id"].isin(cos_sugg)].drop_duplicates(subset=['movie_id'])

            elif algo == 'LSH-J':
                ranked_jac_sim = np.argsort(np.array(jac_sim_values))
                movies_id_jac = ranked_jac_sim[::-1][:k]
                jac_sugg = []
                for i in range(0,k):
                    movie_sugg_jac= similar_movies[movies_id_jac[i]]
                    jac_sugg.append(self.movies_dataframe[self.movies_dataframe["movie_id"]==str(movie_sugg_jac)]["movie_id"].values[0])
                return self.movies_dataframe.loc[self.movies_dataframe["movie_id"].isin(jac_sugg)].drop_duplicates(subset=['movie_id'])

    def get_movies_embeddings(self, movie_ids: [str]) -> pandas.DataFrame:
        """Returns the embedding of the movies with movie_id in movie_ids.

        Parameters
        ----------
        movie_ids : [str]
            List of the movies movie_id.

        Returns
        -------
        pandas.DataFrame
            The embeddings of the movies with movie_id in movie_ids.
        """

        return self.movie_embeddings.loc[movie_ids,:]