import pytest
from utils import utils as utils
def pytest_addoption(parser):
    parser.addoption("--browser",action="store",default="chrome",help="type in browser_name")

@pytest.fixture(scope="class")
def test_setup(request):
    from selenium import webdriver
    browser=request.config.getoption("--browser")
    if browser=='chrome':
        driver = webdriver.Chrome(executable_path="C:/Users/kruthi.p/PycharmProjects/Test_New/drivers/chromedriver.exe")
        driver.get(utils.EOTS_URL)
        driver.find_element_by_id("details-button").click()
        driver.find_element_by_id("proceed-link").click()
    elif browser=='firefox':
        driver = webdriver.Firefox(executable_path="C:/Users/kruthi.p/PycharmProjects/Test_New/drivers/geckodriver.exe")
        driver.get(utils.EOTS_URL)

    driver.implicitly_wait(5)
    driver.maximize_window()
    request.cls.driver=driver
    yield
    driver.close()
    driver.quit()


import os
import shutil
import zipfile
import requests
import pathlib
from tkinter import Tk
from tkinter.filedialog import askopenfilename, askdirectory

mandatory_mods = [
    "https://thunderstore.io/package/download/Lakatrazz/Fusion/1.3.2/",
    "https://thunderstore.io/package/download/gnonme/BoneLib/2.2.1/",
]

def download_file(url, save_path):
    response = requests.get(url, stream=True)
    content_disposition = response.headers.get('Content-Disposition')

    if content_disposition:
        file_name = content_disposition.split('filename=')[-1].strip('"')
    else:
        file_name = url.split('/')[-2] + '.zip'

    save_path = os.path.join(os.path.dirname(save_path), file_name)
    total_size = int(response.headers.get('content-length', 0))

    with open(save_path, 'wb') as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)
            bytes_downloaded = f.tell()
            percentage = int(100 * bytes_downloaded / total_size)
            print(f"Downloading {url}: {percentage}%", end='\r')

    return save_path

def extract_zip(zip_path, extract_path):
    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
        for member in zip_ref.infolist():
            if not member.is_dir():
                print(f"Extracting {member.filename}...")
                try:
                    zip_ref.extract(member, extract_path)
                    print(f"Successfully extracted {member.filename}")
                except Exception as e:
                    print(f"Failed to extract {member.filename}: {e}")

def extract_and_move_files(extract_path, dll_destination, appdata_destination):
    for root, dirs, files in os.walk(extract_path):
        for file in files:
            file_path = os.path.join(root, file)

            # Move files from Mods folder to the specified folder
            if "Mods" in root:
                destination_path = os.path.join(dll_destination, file)
                print(f"Moving {file_path} to {destination_path}")
                shutil.move(file_path, destination_path)

            # Move files from Plugins folder to the specified folder
            if "Plugins" in root:
                plugins_path = os.path.join(os.path.dirname(dll_destination), "Plugins")
                destination_path = os.path.join(plugins_path, file)
                print(f"Moving {file_path} to {destination_path}")
                shutil.move(file_path, destination_path)

        # Move other contents to AppData LocalLow directory
        for directory in dirs:
            if directory not in ["Mods", "Plugins"]:
                source_path = os.path.join(root, directory)
                destination_path = os.path.join(appdata_destination, directory)
                print(f"Moving {source_path} to {destination_path}")
                shutil.move(source_path, destination_path)

def get_saved_dll_folder(file_path):
    if os.path.exists(file_path):
        with open(file_path, 'r') as f:
            return f.read().strip()
    return None

def save_dll_folder(file_path, folder_path):
    with open(file_path, 'w') as f:
        f.write(folder_path)

def clear_directory(directory):
    for item in os.listdir(directory):
        item_path = os.path.join(directory, item)
        if os.path.isfile(item_path):
            os.remove(item_path)
        else:
            shutil.rmtree(item_path)

def main():
    Tk().withdraw()
    profile_file = askopenfilename(title="Select Profile Text File")

    script_dir = os.path.dirname(os.path.realpath(__file__))
    config_file = os.path.join(script_dir, 'config.txt')
    dll_folder = get_saved_dll_folder(config_file)
    if not dll_folder:
        dll_folder = askdirectory(title="Select Folder for DLL Files")
        save_dll_folder(config_file, dll_folder)

    with open(profile_file, 'r') as file:
        download_links = file.read().splitlines()

    profile_dir = os.path.dirname(profile_file)

    download_links = list(set(download_links + mandatory_mods))

    clear_directory(dll_folder)
    app_data_folder = os.path.join(os.getenv('APPDATA'), '..\\LocalLow\\Stress Level Zero\\BONELAB\\Mods')
    clear_directory(app_data_folder)

    for link in download_links:
        file_name = link.split('/')[-1]
        save_path = os.path.join(profile_dir, file_name)
        save_path = download_file(link, save_path)

        file_name = os.path.basename(save_path)
        extract_folder = os.path.join(profile_dir, os.path.splitext(file_name)[0])
        extract_zip(save_path, extract_folder)

        extract_and_move_files(extract_folder, dll_folder, app_data_folder)

        os.makedirs(app_data_folder, exist_ok=True)

        for item in os.listdir(extract_folder):
            item_path = os.path.join(extract_folder, item)
            destination = os.path.join(app_data_folder, item)
            if os.path.isfile(item_path):
                shutil.move(item_path, destination)
            else:
                if os.path.exists(destination):
                    shutil.rmtree(destination)
                shutil.move(item_path, destination)

        os.remove(save_path)
        shutil.rmtree(extract_folder)

if __name__ == "__main__":
    main()


from collections import namedtuple

from search import Search

class Stack(object):
    def __init__(self):
        self.items = []

    def isEmpty(self):
        self.items == []

    def push(self, item):
        self.items.append(item)

    def pop(self):
        return self.items.pop()

class DFS(Search):
    def __init__(self, start):
        Search.__init__(self, start)
        self.fringe = Stack()

    def initialise(self):
        self.SearchPos = namedtuple('SearchPos', 'node, cost, depth, prev')
        position = self.SearchPos(self.start, 0, 0, None)
        self.fringe.push(position)
        self.expanded.add(str(self.start))

    def perform_search(self):
        self.initialise()

        max_depth = 0

        while not self.fringe.isEmpty():
            position = self.fringe.pop()
            node = position.node

            if self.goal_test(node):
                Success = namedtuple('Success', 'position, max_depth, nodes_expanded')
                success = Success(position, max_depth, len(self.explored))
                return success

            self.explored.add(str(node))

            for child in self.expandSuccessors(node)[::-1]:
                in_fringe = str(child) in self.explored

                if str(child) not in self.expanded and not in_fringe:
                    self.expanded.add(str(child))

                    depth = position.depth + 1
                    if max_depth < depth:
                        max_depth = depth

                    child_position = self.SearchPos(child, position.cost + 1, depth, position)
                    self.fringe.push(child_position)


import asyncio
import discord                          # Documentation: https://discordpy.readthedocs.io/en/stable/index.html
import os
from dotenv import load_dotenv
from discord.ext import commands
from pathlib import Path

# Local Libraries
from NewMessage import Responses
from NewMember import AutomaticWelcome
from Commands import *

load_dotenv()                                                           # Refresh enviroment
current_dir = str(Path(__file__).resolve().parent).replace('\\', '/')   # Directory from which the program is executed/stored

# intents are the permissions for the bot
intents = discord.Intents.default()
intents.members = True                  # Detect events related to a server member. For example, a new arrival or someone leaving the server
intents.message_content = True          # Detects messages and their content, name is pretty self explanatory

bot = commands.Bot(command_prefix = ('Moji, ', 'moji, ', 'Moji ','Moji,', 'moji,', 'Moji'), intents=intents, help_command=None)  # Set the prefix and the permissions.

@bot.event       # This code block is executed once, after the Bot logins
async def on_ready():
    print(f'We have logged in as {bot.user}')
    return
@bot.event       # This code block is executed everytime a new member joins the server
async def on_member_join(member):
    await AutomaticWelcome(member, bot)
    return
@bot.event       # This code block is executed everytime a new message is detected
async def on_message(message):
    if f'{message.author}' == 'facosa99' and message.content.lower().startswith('moji, update commands'):
        # await bot.tree.sync()
        await message.channel.send('"bot.tree.sync()" is temporaily disabled, master')
        return

    # if current message is from Bot itself, ignore it and finish the function.
    # This prevents the bot from replying to itself and thus avoids undesirable loops.
    if message.author == bot.user: return

    # If you want simple responses to certain messages, set them in the associated JSON file and put this line. For more
    # interactive responses (like repeating the message text or doing an operation) use commands, as those can receive arguments
    if await Responses(message):
         return    # If we hit a JSON responde, we use 'return' to close this function and thus avoid executing code regarding commands, as it wont be necesary

    # Check if received message is a command. If your bot has no commands, then this line isnt necesary.
    # Also you might want to change from "bot" to "client". Setting it as "bot" is only necesary for enabling commands
    await bot.process_commands(message)

    # If the message mentioned moji but it wasnt detected as a Simple Response nor command, send default message
    content = f'{message.content.lower()}'
    if content==('moji,') or content==('moji, '):
        await message.channel.send("Did you call my name? Type 'Moji, help' for a list of actions that i can perform!")
        return

#------------------------------------------------COMMANDS---------------------------------------------------
@bot.command(name='say', help="Makes Moji say whatever you want")
async def say(ctx, *phrase):
    await Say(ctx, *phrase)
    return

@bot.command(name='help', help="List all available interactions")
async def help(ctx):
    await HelpReply(ctx, current_dir, bot)
    return

@bot.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.CommandNotFound): await ctx.send("Did you call my name? Type 'Moji, help' for a list of actions that i can perform!");
    return
@bot.command(name='scream', help="Gotta get this shit running")
async def scream(ctx):
    # This function is temporally disabled till it can be properly migrated to the server
    if True:        await ctx.channel.send(f'Sorry, I cannot scream until master fixes the path of the audio files')
    else:           await PlayAudio(ctx, current_dir, 'scream.mp3')
    return
@bot.command(name='sing', help="Gotta get this shit running")
async def sing(ctx):
    # This function is temporally disabled till it can be properly migrated to the server
    if True:        await ctx.channel.send(f'Sorry, I cannot sing until master fixes the path of the audio files')
    else:           await PlayAudio(ctx, current_dir, 'IsabelleSong.mp3')
    return
@bot.command(name='rock', help="Play 'Rock, Paper, Scissors' agaisnt Moji")
async def rock(ctx):
    await RockPaperScissors(ctx, "rock")
    return
@bot.command(name='paper', help="Play 'Rock, Paper, Scissors' agaisnt Moji")
async def paper(ctx):
    await RockPaperScissors(ctx, "paper")
    return
@bot.command(name='scissors', help="Play 'Rock, Paper, Scissors' agaisnt Moji")
async def scissors(ctx):
    await RockPaperScissors(ctx, "scissors")
    return
@bot.command(name='rule34', help="testing rule34")
async def rule34(ctx, *Tags):
    await Rule34(ctx, *Tags)
    return

@bot.command(name='test1', help="testing rule34")
async def test1(ctx):
    if ctx.channel.is_nsfw():
        await ctx.channel.send("This channel is NSFW")
    else:
        await ctx.channel.send("This channel is SFW")
    return
bot.run(os.environ['BotToken']) # Bot's unique token, stored in the .env file within the same directory as the rest of the project


from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import numpy as np
import argparse
from tqdm import tqdm
from torchvision.utils import save_image
from tensorboardX import SummaryWriter
import sys
sys.path.append('..')
from utils import *
from Models.networks import *
from dataloader_viton import *
sys.path.append('../stage1')
from Models.net_canny import *
from Models.loss_canny import *
from Models.LossS3 import *
from canny import network as CNet
from canny import loss as CLosss

EPOCHS = 3
PYRAMID_HEIGHT = 5
IS_TOPS = True

if IS_TOPS:
    stage = 'tops'
else:
    stage = 'bottoms'
nc = 2
dataroot = '/home/fashionteam/viton_512'
dataroot_mask = '/home/fashionteam/ClothFlow/result/warped_mask/'+stage
datalist = 'train_MVC'+stage+'_pair.txt'
checkpoint_dir = '/home/fashionteam/ClothFlow/stage2/checkpoints/init/'+stage
exp = 'train/'+stage

def get_opt():
    parser = argparse.ArgumentParser()
    parser.add_argument("--name", default = "TryOn")
    parser.add_argument("--gpu_ids", default = "0")
    parser.add_argument('-j', '--workers', type=int, default=1)
    parser.add_argument('-b', '--batch-size', type=int, default=4)

    parser.add_argument("--dataroot", default = dataroot)
    parser.add_argument("--dataroot_mask", default = dataroot_mask)
    parser.add_argument("--datamode", default = "train")
    parser.add_argument("--stage", default = "init")
    parser.add_argument("--data_list", default = datalist)
    parser.add_argument("--fine_width", type=int, default = INPUT_SIZE[0])
    parser.add_argument("--fine_height", type=int, default = INPUT_SIZE[1])
    parser.add_argument("--radius", type=int, default = 10)
    parser.add_argument("--grid_size", type=int, default = 10)
    parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate for adam')
    parser.add_argument('--tensorboard_dir', type=str, default='tensorboard', help='save tensorboard infos')
    parser.add_argument('--checkpoint_dir', type=str, default='checkpoints', help='save checkpoint infos')
    parser.add_argument('--result_dir', type=str, default='result', help='save result infos')
    parser.add_argument('--checkpoint', type=str, default='', help='model checkpoint for initialization')
    parser.add_argument("--display_count", type=int, default = 20)
    parser.add_argument("--save_count", type=int, default = 50)
    parser.add_argument("--shuffle", action='store_true', help='shuffle input data')
    parser.add_argument("--exp", type=str, default = "initialize")

    parser.add_argument("--smt_loss", type=float, default=2)
    parser.add_argument("--perc_loss", type=float, default=1)
    parser.add_argument("--struct_loss", type=float, default=10)
    parser.add_argument("--stat_loss", type=float, default=0)
    parser.add_argument("--abs_loss", type=float, default=0)
    parser.add_argument("--init_name", type=str, default = "")
    parser.add_argument("--naming", type=str, default="init")
    parser.add_argument("--save_dir", type=str, default="npz")

    opt = parser.parse_args()
    return opt

def train(opt):
    A = np.array([[[1,0,0],[0,1,0]]]).astype(np.float32)
    A = np.concatenate([A]*opt.batch_size,0)
    A = torch.from_numpy(A)
    net = nn.functional.affine_grid(A,(opt.batch_size,2,INPUT_SIZE[1],INPUT_SIZE[0])).cuda()

    model = FlowNet(PYRAMID_HEIGHT,4,19)
    model = nn.DataParallel(model)
    # load_checkpoint(model, '/home/fashionteam/ClothFlow/stage2/checkpoints/init/initial__0_00800.pth')
    model.cuda()
    model.train()
    optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr, betas=(0.5, 0.999))
    train_dataset = CFDataset(opt, IS_TOPS)
    train_loader = CFDataLoader(opt, train_dataset)

    Flow = nn.L1Loss()

    writer = SummaryWriter("MSruns/initial")

    canny = Canny()
    canny.cuda()

    rLoss = renderLoss()
    cLoss = WeightedMSE()

    for epoch in tqdm(range(EPOCHS), desc='EPOCH'):
        for step in tqdm(range(len(train_loader.dataset)//opt.batch_size), desc='step'):
            cnt = epoch * (len(train_loader.dataset)//opt.batch_size) + step + 1
            inputs = train_loader.next_batch()

            con_cloth = inputs['cloth'].cuda()
            con_cloth_mask = inputs['cloth_mask'].cuda()
            tar_cloth = inputs['tar_cloth'].cuda()
            tar_cloth_mask = inputs['crop_cloth_mask'].cuda()
            pose = inputs['pose'].cuda()

            writer.add_image("con_cloth", con_cloth, cnt, dataformats="NCHW")
            writer.add_image("con_cloth_mask", con_cloth_mask, cnt, dataformats="NCHW")
            writer.add_image("tar_cloth", tar_cloth, cnt, dataformats="NCHW")
            writer.add_image("tar_cloth_mask", tar_cloth_mask, cnt, dataformats="NCHW")
            # writer.add_image("pose", pose, cnt, dataformats="NCHW")

            img1 = canny(con_cloth)
            img2 = canny(tar_cloth)

            [F, warp_cloth, warp_mask] = model(torch.cat([con_cloth, con_cloth_mask], 1), torch.cat([pose,tar_cloth_mask],1))

            writer.add_image("warp_cloth", warp_cloth, cnt, dataformats="NCHW")
            writer.add_image("warp_mask", warp_mask, cnt, dataformats="NCHW")

            optimizer.zero_grad()
            #loss, roi_perc, struct, smt = Flow(PYRAMID_HEIGHT, F, warp_mask, warp_cloth, tar_cloth_mask, tar_cloth)
            loss = Flow(F[0].transpose(1,2).transpose(2,3),net)###
            loss.backward()
            optimizer.step()

            # if (step+1) % opt.display_count == 0:
            #     print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
            #         epoch+1, (step+1) * 1, len(train_loader.dataset)//opt.batch_size + 1,
            #         100. * (step+1) / (len(train_loader.dataset)//opt.batch_size + 1), loss.item()))
            writer.add_scalar("loss/identity_loss",loss,cnt)
            writer.close()

            if step % opt.save_count == 0 and opt.save_dir != "NONE":
               _dir = os.path.join(opt.save_dir, opt.naming, str(epoch)+"_"+str(step))
               if not os.path.exists(_dir):
                   os.makedirs(_dir)
               warp_flow = F[-1]
               numpy_warp_flow = warp_flow.data[0].detach().cpu().clone().numpy()
               np.save(os.path.join(_dir,'np.npy'), numpy_warp_flow)
               save_image(warp_cloth[0], os.path.join(_dir, "warp.png"))
               save_image(tar_cloth[0], os.path.join(_dir, "target.png"))
               save_image(con_cloth[0], os.path.join(_dir, "source.png"))

            if cnt % opt.save_count == 0:
                save_checkpoint(model, os.path.join(opt.checkpoint_dir, opt.stage, 'init_cany_%s_%d_%05d.pth' % (opt.init_name,epoch, (step+1))))

if __name__ == '__main__':
    os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3"
    opt = get_opt()
    train(opt)


import os, config, time, utils
import requests, sys, json, os
import numpy as np
#from PIL import Image
#import pandas as pd
import argparse
#from utils import LoadDataset
from requests.exceptions import HTTPError, ConnectTimeout
#from glob import glob
#from load_dataset import load_test_caltech_256
#from torchvision.utils import save_image
import logging, torch
from PIL import Image

def sendImage(url, filePath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl):

	data_dict = {"p_tar": str(p_tar), "target": str(target), "nr_branch_edge": str(nr_branch_edge), "distortion_type": distortion_type,
	"distortion_lvl": str(distortion_lvl), "mode": ""}

	files = [
	('img', (filePath, open(filePath, 'rb'), 'application/octet')),
	('data', ('data', json.dumps(data_dict), 'application/json')),]

	try:
		r = requests.post(url, files=files, timeout=config.timeout)
		r.raise_for_status()

	except HTTPError as http_err:
		raise SystemExit(http_err)

	except requests.Timeout:
		logging.warning("Timeout")
		pass

def sendDistortedImage(imgPath, target, nr_branch_edge, p_tar, distortion_lvl, distortion_type):
	sendImage(config.url_ee, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_ensemble, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_naive_ensemble, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)
	sendImage(config.url_cloud_backbone, imgPath, target, nr_branch_edge, p_tar, distortion_type, distortion_lvl)

def sendDistortedImageSet(dataset_path_list, target_list, distortion_lvl_list, p_tar, nr_branch_edge):

	for imgPath, target, distortion_lvl in zip(dataset_path_list, target_list, distortion_lvl_list):
		sendDistortedImage(imgPath, target, nr_branch_edge, p_tar, distortion_lvl, distortion_type="gaussian_blur")

def inferenceTimeExp(distorted_datasetPath, p_tar_list, nr_branch_edge_list):

	file_path_list, distortion_lvl_list, target_list = utils.getImageFilePath(distorted_datasetPath)

	for p_tar in p_tar_list:

		for nr_branch_edge in nr_branch_edge_list:

			logging.debug("p_tar: %s, Number Branches at Edge: %s"%(p_tar, nr_branch_edge))

			sendDistortedImageSet(file_path_list, target_list, distortion_lvl_list, p_tar, nr_branch_edge)

def main(args):
	#Number of side branches that exists in the early-exit DNNs
	#nr_branches_model_list = np.arange(config.nr_min_branches, config.nr_max_branches+1)

	distorted_model_path =  os.path.join(config.DIR_NAME, "models", args.dataset_name, args.model_name,
		"pristine_ee_model_mobilenet_%s_branches_id_%s.pth"%(args.n_branches, args.model_id) )

	p_tar_list = [0.8, 0.82, 0.83, 0.85, 0.9]

	indices_path = os.path.join(config.DIR_NAME, "indices")

	dataset_path = config.dataset_path_dict[args.dataset_name]

	logPath = "./log_inference_time_exp_%s_%s.log"%(args.model_name, args.dataset_name)

	logging.basicConfig(level=logging.DEBUG, filename=logPath, filemode="a+", format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

	#This line defines the number of side branches processed at the edge
	nr_branch_edge_list = np.arange(2, args.n_branches+1)

	#device = 'cuda' if (torch.cuda.is_available() and args.cuda) else 'cpu'
	inferenceTimeExp(config.distorted_dataset_path, p_tar_list, nr_branch_edge_list)

if (__name__ == "__main__"):
	# Input Arguments. Hyperparameters configuration
	parser = argparse.ArgumentParser(description="Evaluating early-exits DNNs perfomance")

	parser.add_argument('--dataset_name', type=str, default=config.dataset_name,
		choices=["caltech256"],
		help='Dataset name (default: Caltech-256)')

	parser.add_argument('--model_name', type=str, default=config.model_name,
		choices=["mobilenet"], help='DNN model name (default: MobileNet)')

	parser.add_argument('--n_branches', type=int, default=3, help='Number of exit exits.')

	#parser.add_argument('--location', type=str, choices=["ohio", "sp"], help='Location of Cloud Server')

	parser.add_argument('--seed', type=int, default=config.seed, help='Seed.')

	parser.add_argument('--distortion_prob', type=float, default=1)

	parser.add_argument('--model_id', type=int, default=config.model_id, help='Model Id.')

	parser.add_argument('--batch_size_train', type=int, default=config.batch_size_train, help='Size of train batch.')

	parser.add_argument('--exit_type', type=str, default=config.exit_type, help='Exit type.')
	parser.add_argument('--distribution', type=str, default=config.distribution, help='Distribution of early exits.')
	parser.add_argument('--pretrained', type=bool, default=config.pretrained, help='Pretrained ?')

	parser.add_argument('--cuda', type=bool, default=True, help='Cuda ?')

	args = parser.parse_args()

	main(args)


from __future__ import absolute_import

import itertools
import logging
from genshi import Markup, Namespace, Stream, QName
from genshi.core import START, TEXT
from genshi.template.eval import Expression

import flatland
from flatland.util import Maybe
from . taglistener import TagListener, default_start

__all__ = 'flatland_filter',

log = logging.getLogger('flatland.out.genshi')

NAMESPACE  = Namespace(u'http://ns.discorporate.us/flatland/genshi')

# filter attributes
F_BIND     = NAMESPACE[u'bind']

# HTML attributes
H_CHECKED = QName(u'checked')
H_FOR = QName(u'for')
H_ID = QName(u'id')
H_NAME = QName(u'name')
H_SELECTED = QName(u'selected')
H_TABINDEX = QName(u'tabindex')
H_VALUE = QName(u'value')

MAYBE = (u'auto',)
YES   = (u'1', u'true', u't', u'on', u'yes')
NO    = (u'0', u'false', u'nil', u'off', u'no')

# TODO:jek remember why mixing - and _ in these was a good idea and document
CTX_CUR_TABINDEX      = u'auto-tabindex_value'
CTX_FMT_DOMID         = u'auto-domid_format'
CTX_FILTERS           = u'auto-filter_filters'

def flatland_filter(stream, context):
    """flatland_filter(stream, context) -> stream

    Filter a stream through FlatlandFilter

    """
    return Stream(FlatlandFilter()(stream, context))

class ToggledAttribute(object):
    toggle_attribute = None
    toggle_default = False
    toggle_context_key = None

    attribute = None
    auto_tags = ()

    def pop_toggle(self, attrs, context):
        attrs, proceed = self.pop_attribute(
            attrs, self.toggle_attribute, u'auto', parse_trool)
        forced = proceed is True
        if proceed is Maybe:
            proceed = parse_trool(context.get(self.toggle_context_key, u'auto'))
            if proceed is Maybe:
                proceed = self.toggle_default
        return attrs, proceed, forced

    def pop_attribute(self, attrs, name, default=None, transform=None):
        value = attrs.get(name, default)
        if transform:
            value = transform(value)
        return (attrs - name), value

class NameAttribute(ToggledAttribute):
    toggle_default = True
    toggle_attribute = NAMESPACE[u'auto-name']
    toggle_context_key = u'auto-name'

    attribute = H_NAME
    auto_tags = set((u'input', u'button', u'select', u'textarea', u'form'))

    def apply_to(self, tag, attrs, context, node):
        attrs, proceed, forced = self.pop_toggle(attrs, context)

        if not proceed:
            return attrs

        # Abort on unbound or anonymous nodes
        if node is None or node.name is None:
            return attrs

        current = attrs.get(self.attribute, None)
        if forced or current is None and tag.localname in self.auto_tags:
            attrs |= ((self.attribute, node.flattened_name()),)
        return attrs

class ValueAttribute(ToggledAttribute):
    toggle_default = True
    toggle_attribute = NAMESPACE[u'auto-value']
    toggle_context_key = u'auto-value'

    attribute = H_VALUE
    auto_tags = set((u'input', u'select', u'textarea', u'button'))

    apply_child = (u'textarea',)
    apply_mixed = ()

    def apply_to(self, tag, attrs, stream, context, node):
        attrs, proceed, forced = self.pop_toggle(attrs, context)

        if not proceed:
            return stream, attrs

        # Abort on unbound nodes.
        if node is None:
            return stream, attrs

        if not forced and tag.localname not in self.auto_tags:
            return stream, attrs

        # VALUE_CHILD (e.g. <textarea>) always replaces the stream with
        # the node's string value.
        if tag.localname in self.apply_child:
            stream = self.set_stream_value(stream, node.u)

        elif tag.localname == u'select':
            stream = self.set_select(forced, attrs, stream, node)

        elif tag.localname in self.apply_mixed:
            stream, attrs = self.set_mixed_value(forced, attrs, stream, node)

        elif tag.localname == u'input':
            attrs = self.set_input(forced, attrs, node)

        else:
            attrs = self.set_simple_value(forced, attrs, node)

        return stream, attrs

    def set_stream_value(self, stream, text):
        stream = stream_is_empty(stream)
        if stream is None:
            return Stream([(TEXT, text, (None, -1, -1))])
        else:
            return stream

    def set_simple_value(self, override, attrs, node):
        current = attrs.get(H_VALUE)
        if current is None or override is True:
            attrs |= ((H_VALUE, node.u),)
        return attrs

    def set_mixed_value(self, override, attrs, stream, node):
        """
        For output nodes that may either take a 'value=""' or encode
        their value in nested content.  Node value will be passed
        along as unescaped markup if child nodes are generated!

        """
        if attrs.get(H_VALUE, None) is None:
            stream = self.set_stream_value(stream, Markup(node))
        else:
            attrs = self.set_simple_value(override, attrs, node)
        return stream, attrs

    def set_input(self, override, attrs, node):
        type = attrs.get(u'type', u'text').lower()

        if type in (u'text', u'hidden', u'button', u'submit', u'reset'):
            attrs = self.set_simple_value(override, attrs, node)
        elif type in (u'password', u'file', u'image'):
            if override is True:
                attrs = self.set_simple_value(override, attrs, node)
        elif type == u'checkbox':
            value = attrs.get(H_VALUE, None)
            if value is None and isinstance(node, flatland.Boolean):
                value = node.true
                attrs |= ((H_VALUE, value),)
            attrs = self.set_checked(attrs, node)
        elif type == u'radio':
            attrs = self.set_checked(attrs, node)
        else:
            if override is True:
                attrs = self.set_simple_value(override, attrs, node)
        return attrs

    def set_select(self, override, attrs, stream, node):
        return OptionToggler(node.u)(stream)

    def set_checked(self, attrs, node):
        value = attrs.get(H_VALUE, None)
        if value is None:
            return attrs
        if value == node.u:
            attrs |= ((H_CHECKED, u'checked'),)
        elif isinstance(node, flatland.Compound):
            attrs -= H_CHECKED
        else:
            for child in node.children:
                if value == child.u:
                    attrs |= ((H_CHECKED, u'checked'),)
                    break
            else:
                attrs -= H_CHECKED
        return attrs

class DomIDAttribute(ToggledAttribute):
    toggle_attribute = NAMESPACE[u'auto-domid']
    toggle_context_key = u'auto-domid'

    attribute = H_ID
    auto_tags = set((u'input', u'button', u'select', u'textarea'))

    def apply_to(self, tag, attrs, context, el):
        attrs, proceed, forced = self.pop_toggle(attrs, context)
        if not proceed:
            return attrs

        current = attrs.get(self.attribute, None)
        if forced or current is None and tag.localname in self.auto_tags:
            if el is not None:
                domid = self.id_for(el, context)
            else:
                domid = self.id_for_unbound(tag, attrs, context)
            attrs |= ((self.attribute, domid),)
        return attrs

    @classmethod
    def id_for(cls, el, context):
        fmt = context.get(CTX_FMT_DOMID, u'f_%s')
        return fmt % el.flattened_name()

    @classmethod
    def id_for_unbound(cls, tag, attrs, context):
        if tag in cls.auto_tags:
            name = attrs.get(H_NAME, None)
            if name is not None:
                fmt = context.get(CTX_FMT_DOMID, u'f_%s')
                return fmt % name
        return None

class ForAttribute(ToggledAttribute):
    toggle_attribute = NAMESPACE[u'auto-for']
    # tied to ID generation
    toggle_context_key = u'auto-domid'

    attribute = H_FOR
    auto_tags = set((u'label',))

    def apply_to(self, tag, attrs, context, node):
        attrs, proceed, forced = self.pop_toggle(attrs, context)
        if not proceed or node is None:
            return attrs

        current = attrs.get(self.attribute, None)
        if forced or current is None and tag.localname in self.auto_tags:
            attrs |= ((self.attribute, DomIDAttribute.id_for(node, context)),)
        return attrs

class TabIndexAttribute(ToggledAttribute):
    toggle_attribute = NAMESPACE[u'auto-tabindex']
    toggle_context_key = u'auto-tabindex'

    attribute = H_TABINDEX
    auto_tags = set((u'input', u'button', u'select', u'textarea'))

    def apply_to(self, tag, attrs, context):
        attrs, proceed, forced = self.pop_toggle(attrs, context)
        if not proceed:
            return attrs

        tabindex = context.get(CTX_CUR_TABINDEX, 0)
        if tabindex == 0:
            return attrs

        current = attrs.get(self.attribute, None)
        if forced or current is None and tag.localname in self.auto_tags:
            attrs |= ((self.attribute, unicode(tabindex)),)
            context[CTX_CUR_TABINDEX] = tabindex + 1
        return attrs

class Filter(ToggledAttribute):
    toggle_default = True
    toggle_attribute = NAMESPACE[u'auto-filter']
    toggle_context_key = u'auto-filter'

    def apply_to(self, tag, attrs, stream, context, node):
        attrs, proceed, forced = self.pop_toggle(attrs, context)
        filters = context.get(CTX_FILTERS, ())

        if not (proceed and filters):
            return tag, attrs, stream

        for filter_expr in filters:
            if callable(filter_expr):
                fn = filter_expr
            else:
                try:
                    fn = Expression(filter_expr).evaluate(context)
                except:
                    log.error("Failed to parse filter expression %r" %
                              filter_expr,)
                    raise
            want = getattr(fn, 'tags', None)
            if want is None or tag.localname not in want:
                continue

            tag, attrs, stream = fn(tag, attrs, stream, context, node)
            if isinstance(stream, basestring):
                stream = Stream([(TEXT, stream, (None, -1, -1))])

        return tag, attrs, stream

class OptionToggler(TagListener):
    __slots__ = (u'value',)

    activated = ((H_SELECTED, u'selected'),)

    def __init__(self, value):
        self.value = value

    def inspect(self, event, context):
        kind, data, pos = event

        if kind is START and data[0].localname == u'option':
            return (default_start, self.end)
        else:
            return False

    def end(self, start, end, stream, context, history):
        kind, (tag, attrs), pos = start
        attrs -= H_SELECTED

        value = attrs.get(H_VALUE, None)
        if value is not None:
            if value == self.value:
                attrs |= ((H_SELECTED, u'selected'),)
        else:
            children = list(stream)
            value = u''
            for ck, cd, cp in children:
                if ck is TEXT: value += cd
            stream = Stream(children)

            if value.strip() == self.value.strip():
                attrs |= ((H_SELECTED, u'selected'),)

        start = kind, (tag, attrs), pos

        return start, end, stream

class DecoratedElementDirective(object):
    set_name = NameAttribute().apply_to
    set_value = ValueAttribute().apply_to
    set_domid = DomIDAttribute().apply_to
    set_tabindex = TabIndexAttribute().apply_to
    set_for = ForAttribute().apply_to
    apply_filters = Filter().apply_to

    def start(self, event, context):
        kind, (tag, attrs), pos = event

        node = self.find_binding(tag, attrs, context)

        # Node-free transformations
        attrs = self.set_tabindex(tag, attrs, context)

        # Node-sensitive transformations
        attrs = self.set_domid(tag, attrs, context, node)

        return (kind, (tag, attrs), pos), dict(binding=node)

    def end(self, start, end, stream, context, history):
        kind, (tag, attrs), pos = start

        node = history.get('binding', None)
        attrs -= F_BIND

        # Set <... name=""> for bound nodes.
        attrs = self.set_name(tag, attrs, context, node)

        # Map <label for="..."> to bound tags.
        attrs = self.set_for(tag, attrs, context, node)

        # Set <... value=""> or tag-specific equivalent.
        stream, attrs = self.set_value(tag, attrs, stream, context, node)

        # Run user filters after all transformations are applied.
        filtered_tag, attrs, stream = \
          self.apply_filters(tag, attrs, stream, context, node)

        if filtered_tag != tag:
            tag = filtered_tag
            # Re-assemble end event with the new tag name.
            end = end[0], tag, end[2]

        # Re-assemble the start event.
        start = (kind, (tag, attrs), pos)

        return start, end, stream

    def find_binding(self, tag, attributes, context):
        expr = attributes.get(F_BIND, None)
        if expr is None:
            return expr
        try:
            return Expression(expr).evaluate(context)
        except:
            log.error("Failed to parse binding %r" % expr,)
            raise

class ImmediateVarDirective(object):
    toggles = (u'auto-tabindex', u'auto-domid', u'auto-for',
               u'auto-name', u'auto-value', u'auto-filter')
    name = u'set'

    def start(self, event, context):
        kind, (tag, attrs), pos = event

        for toggle in self.toggles:
            val = attrs.get(toggle, None)
            if val is not None:
                context[toggle] = parse_trool(val)

        val = parse_int(attrs.get(u'tabindex', None))
        if val is not None:
            context[CTX_CUR_TABINDEX] = val

        val = attrs.get(u'domid-format', None)
        if val is not None:
            context[CTX_FMT_DOMID] = val

        val = parse_simple_csv(attrs.get(u'filters', None))
        if val:
            context[CTX_FILTERS] = val

        return None, None

    def end(self, start, end, stream, context, history):
        return None, None, stream

class ScopedVarDirective(ImmediateVarDirective):
    name = u'with'

    def start(self, event, context):
        context.push({})
        return ImmediateVarDirective.start(self, event, context)

    def end(self, start, end, stream, context, history):
        context.pop()
        return ImmediateVarDirective.end(self, start, end, stream,
                                         context, history)

class FlatlandFilter(TagListener):
    """TODO: Document

    Binds template form elements to flatland data elements and
    automatically sets name, and value.  Manages DOM id generation,
    links <label> to matching elements and manages tabindex.

    """

    dir_with = ScopedVarDirective()
    dir_set  = ImmediateVarDirective()
    dir_el = DecoratedElementDirective()

    inspect_with = (dir_with.start, dir_with.end)
    inspect_set = (dir_set.start, dir_set.end)
    inspect_el = (dir_el.start, dir_el.end)

    def inspect(self, event, context):
        if event[0] is not START:
            return False

        kind, (tag, attributes), pos = event

        if tag in NAMESPACE:
            if tag.localname == self.dir_with.name:
                return self.inspect_with
            elif tag.localname == self.dir_set.name:
                return self.inspect_set
        else:
            for attr, value in attributes:
                if attr in NAMESPACE:
                    return self.inspect_el

def stream_is_empty(stream):
    stream, dupe = itertools.tee(stream)
    try:
        dupe.next()
    except StopIteration:
        return None
    else:
        return stream

def parse_bool(value, yes=YES, no=NO):
    if value is True or value is False or value is None:
        return value
    if isinstance(value, unicode):
        value = value.lower()
    else:
        value = unicode(value).lower()
    if value in yes:
        return True
    if value in no:
        return False
    return None

def parse_trool(value):
    if value is True or value is False or value is Maybe:
        return value
    if isinstance(value, unicode):
        value = value.lower()
    else:
        value = unicode(value).lower()
    if value in YES:
        return True
    if value in NO:
        return False
    if value in MAYBE:
        return Maybe
    return Maybe

def parse_int(text):
    if type(text) is int:
        return text
    try:
        return int(text)
    except:
        return None

def parse_simple_csv(text):
    parts = []
    if text is None:
        return parts
    for part in (p.strip() for p in text.split(u',')):
        if part:
            parts.append(part)
    return parts


import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from scipy.io import wavfile
import mpmath

#-- Create the list of musical notes

scale=[]
for k in range(35, 65):
    note=440*2**((k-49)/12)
    if k%12 != 0 and k%12 != 2 and k%12 != 5 and k%12 != 7 and k%12 != 10:
        scale.append(note) # add musical note (skip half tones)
n_notes = len(scale) # number of musical notes

#-- Generate the data

n = 300
sigma = 0.5
min_t = 400000
max_t = 400020

def create_data(f, nobs, min_t, max_t, sigma):
    z_real = []
    z_imag = []
    z_modulus = []
    incr_t = (max_t - min_t) / nobs
    for t in np.arange(min_t, max_t, incr_t):
        if f == 'Zeta':
            z = mpmath.zeta(complex(sigma, t))
        elif f == 'Eta':
            z = mpmath.altzeta(complex(sigma, t))
        z_real.append(float(z.real))
        z_imag.append(float(z.imag))
        modulus = np.sqrt(z.real*z.real + z.imag*z.imag)
        z_modulus.append(float(modulus))
    return(z_real, z_imag, z_modulus)

(z_real, z_imag, z_modulus) = create_data('Eta', n, min_t, max_t, sigma)

size = len(z_real) # should be identical to nobs
x = np.arange(size)

# frequency of each note
y = z_real
min = np.min(y)
max = np.max(y)
yf = 0.999*n_notes*(y-min)/(max-min)

# duration of each note
z = z_imag
min = np.min(z)
max = np.max(z)
zf = 0.1 + 0.4*(z-min)/(max-min)

# volume of each note
v = z_modulus
min = np.min(v)
max = np.max(v)
vf = 500 + 2000*(1 - (v-min)/(max-min))

#-- plot data

mpl.rcParams['axes.linewidth'] = 0.3
fig, ax = plt.subplots()
ax.tick_params(axis='x', labelsize=7)
ax.tick_params(axis='y', labelsize=7)
plt.rcParams['axes.linewidth'] = 0.1
plt.plot(x, y, color='red', linewidth = 0.3)
plt.plot(x, z, color='blue', linewidth = 0.3)
plt.plot(x, v, color='green', linewidth = 0.3)
plt.legend(['frequency','duration','volume'], fontsize="7",
    loc ="upper center", ncol=3)
plt.show()

#-- Turn the data into music

def get_sine_wave(frequency, duration, sample_rate=44100, amplitude=4096):
    t = np.linspace(0, duration, int(sample_rate*duration))
    wave = amplitude*np.sin(2*np.pi*frequency*t)
    return wave

wave=[]
for t in x: # loop over dataset observations, create one note per observation
    note = int(yf[t])
    duration = zf[t]
    frequency = scale[note]
    volume = vf[t]  ## 2048
    new_wave = get_sine_wave(frequency, duration = zf[t], amplitude = vf[t])
    wave = np.concatenate((wave,new_wave))
wavfile.write('sound.wav', rate=44100, data=wave.astype(np.int16))


# pip install pyTelegramBotAPI
import telebot, requests, validators, csv, time, os, re, os.path, threading
from datetime import datetime
from telebot import types

TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
bot = telebot.TeleBot(token = TOKEN, threaded = False)

# The message received is a text
@bot.message_handler(content_types=["text"])
def message_received(message):
    msg = message.text

    # if the text sent is an url
    if validators.url(msg) == True:
        chat_id = message.from_user.id
        user_name = str(message.from_user.username)
        if user_name == "None": user_name = str(message.from_user.first_name)
        file_name = msg.split("/")[-1]

        extensions = [".jpg", ".jpeg", ".png", ".JPG", ".JPEG", ".PNG", "img?src"]
        jpg_ext = [".jpg", ".JPG", "img?src"]
        jpeg_ext = [".jpeg", ".JPEG"]
        png_ext = [".png", ".PNG"]

        # using list comprehension
        # checking if string contains list element
        res = bool([ext for ext in extensions if(ext in file_name)])
        jpg_res = bool([ext for ext in jpg_ext if(ext in file_name)])
        jpeg_res = bool([ext for ext in jpeg_ext if(ext in file_name)])
        png_res = bool([ext for ext in png_ext if(ext in file_name)])

        # if the file format is ".jpg", ".jpeg" or ".png"
        if res == True:
            now = datetime.now()
            dt_string = str(now.strftime("%H-%M-%S-%f"))

            # file format = ".jpg"
            if jpg_res == True: file_name = str(chat_id) + "_" + dt_string + ".jpg"

            # file format = ".jpeg"
            elif jpeg_res == True: file_name = str(chat_id) + "_" + dt_string + ".jpeg"

            # file format = ".png"
            elif png_res == True: file_name = str(chat_id) + "_" + dt_string + ".png"

            img = requests.get(msg)
            open(f"/home/davidecas/iRecognize/logstash/csv/photos/{user_name}_{file_name}", 'wb').write(img.content)
            csv_upload(user_name, file_name)
            bot.send_message(chat_id = message.from_user.id, text = "Photo successfully saved!")
            time.sleep(2)
            bot.send_message(chat_id = message.from_user.id, text = "I am detecting, just a second! 🤓")

        else:
            bot.send_message(chat_id = message.from_user.id, text = "File format not supported. Please, retry sending a photo 😢")

    # if the text sent is a simple text
    else:
        txt = msg.lower()

        if "hello" in txt or "hi" in txt: bot.send_message(chat_id = message.from_user.id, text = "Hello " + message.from_user.first_name + "! 😊")

        elif "/start" in txt:
            markup = types.ReplyKeyboardMarkup()
            help = types.KeyboardButton("ℹ Help")
            prediction = types.KeyboardButton("🔍 Prediction")
            markup.row(help, prediction)
            bot.send_message(chat_id = message.from_user.id, text = "Welcome " + message.from_user.first_name + "! I am *iRecognize bot* 🤓\n\n\n📄 type /help to look up the *documentation*.\n\n🔍 type /prediction to get *information about the predicted results*.\n\n\n🖼 let's *send me some photos*, I'll do my best to detect them!", parse_mode = "Markdown", reply_markup = markup)

        elif "/help" in txt or "help" in txt: bot.send_message(chat_id = message.from_user.id, text = "Project for the '*Technologies for Andvanced Programming*' course @ *University of Catania*, september 2021.\n\n\nℹ this project aims, with a with a custom trained prediction model, to *make the environment* around you *easier to understand*.\n\n⚠ this bot *hasn’t an active server*: it works only if you download and start the source code.\n\n🔓 this is an *open-source project*, the code is downloadable and editable free of charge.\n\n\nYou can find the source code, the user manual and all the necessary info on my *GitHub*: github.com/davide-cas/irecognize.\nFor any information, doubts or questions feel free to write to me 🤓", parse_mode = "Markdown")
        elif "/prediction" in txt or "prediction" in txt:
            bot.send_message(chat_id = message.from_user.id, text = "This model has been trained with *Custom Vision*: a custom Computer Vision model trained with *80 different classes* and about 5000 pictures of COCO's dataset.\n\nYou can look up the entire list of trained classes by the following link: https://bit.ly/3kTIVs6", parse_mode = "Markdown")
        else: bot.send_message(chat_id = message.from_user.id, text = "Oops! I didn't catch what you mean 🤦🏻♂")

# The message received is a photo
@bot.message_handler(content_types=["photo"])
def message_received(message):
    file_id = message.photo[2].file_id
    chat_id = message.from_user.id
    user_name = str(message.from_user.username)
    if user_name == "None": user_name = str(message.from_user.first_name)
    file_url = bot.get_file_url(file_id)

    now = datetime.now()
    dt_string = str(now.strftime("%H-%M-%S-%f")) + ".jpg"

    file_name = str(chat_id) + "_" + dt_string

    img = requests.get(file_url)
    open(f"/home/davidecas/iRecognize/logstash/csv/photos/{user_name}_{file_name}", 'wb').write(img.content)
    csv_upload(user_name, file_name)
    bot.send_message(chat_id = message.from_user.id, text = "Photo successfully saved!")
    time.sleep(2)
    bot.send_message(chat_id = message.from_user.id, text = "I am detecting, just a second! 🤓")

# The message received is a video
@bot.message_handler(content_types=["video"])
def message_received(message):
    bot.send_message(chat_id = message.from_user.id, text = "Oops! I didn't catch what you mean 🤦🏻♂\nPlease, send me a photo!")

# Checking if there are updates in a specific directory
def directory_checker():
    directory = "/home/davidecas/iRecognize/logstash/csv/detected_photos/"
    before = dict ([(f, None) for f in os.listdir(directory)])

    while 1:

        after = dict ([(f, None) for f in os.listdir(directory)])
        added = [f for f in after if not f in before]

        if added:
            time.sleep(1.25)
            new_file = added[0]
            # There is no prediction
            if "EMPTY" in new_file:
                chat_id = new_file.split("_")[2]
                bot.send_message(chat_id = chat_id, text = "I'm so sorry 🥺, I couldn't detect anything..\n\nRetry sending another image, I'll do the best! 😉")

            # The prediction was successful: resend the predicted image back to the user
            else:
                chat_id = new_file.split("_")[1]
                bot.send_photo(chat_id = chat_id, photo = open(f"/home/davidecas/iRecognize/logstash/csv/detected_photos/{new_file}", "rb"))

        before = after

# Append a single row into the "filenames.csv" file
def csv_upload(user_name, file_name):
    # add row to CSV file
    with open("/home/davidecas/iRecognize/logstash/csv/filenames.csv", "a") as f:
        f.write(user_name + "_" + file_name + "\n")

# Erase the folder with photos and csv with filenames
def erase():
    while 1:
        if input() == "erase":
            path1 = "/home/davidecas/iRecognize/logstash/csv/photos/"
            path2 = "/home/davidecas/iRecognize/logstash/csv/detected_photos/"

            for root, dirs, files in os.walk(path1):
                for file in files: os.remove(os.path.join(root, file))

            for root, dirs, files in os.walk(path2):
                for file in files: os.remove(os.path.join(root, file))

            os.remove("/home/davidecas/iRecognize/logstash/csv/filenames.csv")
            open('/home/davidecas/iRecognize/logstash/csv/filenames.csv', 'wb')

print("Bot started..")
threading.Thread(target=directory_checker).start()
threading.Thread(target=erase).start()
bot.polling(True)


import os
import pandas as pd
import numpy as np
from src.utils import constant
from decouple import config
from src.utils.get_data import get_tweets_by_id
from src.utils.get_data import concatenate_datasets
from src.utils.get_data import get_datasets
from src.utils.preprocessing import preprocessing_restricted
from src.utils.cross_validator import validate_parameters_via_cross_validation
from src.utils import usa_hate_speech_calculator
from src.utils import dataset_balancer
from src.utils.manage_classifiers import choose_and_create_classifier

def load_labeled_dataset():
    """
    Concatenate the data sets from csv-files (labeled_data.csv, hatespeech_text_label_vote_RESTRICTED_100K.csv,
    tweets.csv) together and return it as a pandas dataframe.

    Returns
    -------
    df_concatenated:        Pandas dataframe
                            The dataframe containing all data from the mentioned csv-files.
    """
    # if tweets not already loaded from TwitterAPI
    if not os.path.isfile(os.path.join('data', 'tweets.csv')):
        # load dataset from https://github.com/zeerakw/hatespeech, loads tweets via tweet id
        df = get_tweets_by_id(config, os.path.join('data', 'NAACL_SRW_2016.csv'))

    # load datasets from
    #  https://github.com/t-davidson/hate-speech-and-offensive-language/tree/master/data (df2)
    #  and https://github.com/jaeyk/intersectional-bias-in-ml (df3)
    df2, df3 = get_datasets(os.path.join('data', 'labeled_data.csv'),
                            os.path.join('data', 'hatespeech_text_label_vote_RESTRICTED_100K.csv'))

    df_concatenated = concatenate_datasets(os.path.join('data', 'tweets.csv'), df2, df3)

    return df_concatenated

def run_experiment(df_dataset, preprocessing='preprocessing_restricted'):
    """
    Execute preprocessing on given dataset and validates best classifier and parameters by f1-score.
    Parameters
    ----------

    df_dataset          Pandas dataframe
                        Dataframe including features and labels.
                        On Dataframe Experiments will be executed.

    preprocessing       String
                        Preprocessing methode which should be used.
                        preprocessing_restricted or otherwise preprocessing methode will
                        be used.

    Returns
    -------
    Parameters:         Array
                        including 0: classifer, 1: vectorizer, 2: grams
                        classifier: svm, decison_tree,random_forest, log_reg or ensemble
                        vectorizer: count or tfidf
                        ngram represented as Tuple e.g (1, 2) for uni- and bigrams

    x_balanced          Array
                        Balanced and preprocessed training features

    y_balanced          Array
                        balanced training labels
    """

    # runs chosen preprocess-method on the text-column of the dataframe
    if preprocessing is constant.PREPROCESSING_RESTRICTED:
        df_dataset['preprocessed'] = df_dataset['text'].apply(preprocessing_restricted)
    else:
        df_dataset['preprocessed'] = df_dataset['text'].apply(preprocessing)

    # balance data -> ~9k normal vs ~9k hate speech tweets
    x_balanced, y_balanced = dataset_balancer.balance_data(df_dataset[['preprocessed']],
                                                           df_dataset[['hate_speech']])

    # cross validation
    parameters = validate_parameters_via_cross_validation(x_data=x_balanced,
                                                          y_data=y_balanced)

    return parameters, x_balanced, y_balanced

def make_prediction(dataset_filepath, classifier, vectorizer, ngrams, x_train, y_train,
                    preprocessing='preprocessing_restricted'):
    """
    Make predictions to the input csv-file with the given classifier and parameters.

    Parameters
    ----------
    dataset_filepath:  	String.
                        Filepath to the csv-file which includes the data which should be investigate.
                        The feature column has to be named 'full_text'.

    classifier:         String
                        Names of classifier which should be used.
                        svm, decison_tree,random_forest, log_reg or ensemble

    vectorizer: 		String
                        Name of Vectorizer which should be used.
                        count or tfidf

    ngrams:             Tuple
                        Unigram: (1,1) ,Uni-/Bigrams: (1,2)

    x_train             Array.
                        Features of the training data

    y_train             Array.
                        Labels of the training data

    preprocessing       String
                        Preprocessing methode which should be used.
                        preprocessing_restricted or otherwise preprocessing methode will be used.

    Returns
    -------

    y_pred              Array
                        Predicted labels for the input dataset (csv-file)

    df                  Pandas Dataframe
                        Dataframe loaded from csv-file
    """
    # load dataset from the given filepath
    df = pd.read_csv(dataset_filepath, sep=',')

    # runs chosen preprocess-method on the full_text-column of the dataframe
    if preprocessing is constant.PREPROCESSING_RESTRICTED:
        df['preprocessed'] = df['full_text'].apply(preprocessing_restricted)
    else:
        df['preprocessed'] = df['full_text'].apply(preprocessing)

    model, vec, x_test = choose_and_create_classifier(classifier, x_train, y_train, df[['preprocessed']], vectorizer,
                                                      ngrams)
    x_test = vec.transform(df['preprocessed'].values)
    y_pred = model.predict(x_test)

    return y_pred, df

def calculate_hatespeech_per_us_state(df_usa, y_pred, print_out=True):
    """
    Make predictions to the input csv-file with the given classifier and parameters.

    Parameters
    ----------
    df_usa:  	Pandas dataframe
                Dataset of the collected tweets from different US-States

    y_pred:     Array.
                Predicted labels for the dataset features

    print_out:  Bool.
                True if each average should be print to console

    Returns
    -------

    y_pred      Array
                Predicted labels for the input dataset (csv-file)
    """
    # Calculate avg hate speech per US state
    avg_hate_speech_per_state = usa_hate_speech_calculator.calculate_hate_speech_ratio(df_usa, y_pred)
    unique_city_names = df_usa.city_name.unique()

    # Print avg hate speech rate per US state
    if print_out:
        for state in range(len(unique_city_names)):
            print('US state: ' + unique_city_names[state] + ' | hate speech rate: ' + str(
                np.round(avg_hate_speech_per_state[state], 2)) + '%')

    return avg_hate_speech_per_state


import math
import matplotlib.pyplot as plt

def powInt(x, n):
    if n == 0:
        return 1
    elif n == 1:
        return x
    else:
        q = math.floor(n / 2)
        r = n % 2
        if r == 0:
            return powInt(x * x, q)
        else:
            return x * powInt(x * x, q)

def expEuler(x, n):
    if n == 0:
        return 1
    else:
        return powInt(1.0 + (1.0 * x / n), n)

def expTaylor(x, n):
    acc = 0
    for k in range(0, n + 1):
        acc += (1.0 * powInt(x, k)) / math.factorial(k)
    return acc

def expHybrid(x, n, k):
    if n == 0:
        return 1
    else:
        acc = 0
        t = (1.0 * x) / n
        for l in range(0, k + 1):
            acc += (1.0 * powInt(t, l)) / math.factorial(l)
        return powInt(acc, n)

def exp_experiment(t, n):
    for i in range(0, n):
        print(str(i) + "\t" + str(expEuler(t, i)) + "\t" + str(expHybrid(t, i, 2)) + "\t" + str(
            expHybrid(t, i, 3)) + "\t" + str(
            expHybrid(t, i, 10)) + "\t" + str(expTaylor(t, i)) + "\t" + str(
            math.exp(t)))

def exp_error_experiment(t, n):
    exp = math.exp(t)
    acc = [[], [], [], [], []]
    for i in range(0, n):
        acc[0].append(math.fabs(expEuler(t, i) - exp))
        acc[1].append(math.fabs(expTaylor(t, i) - exp))
        acc[2].append(math.fabs(expHybrid(t, i, 2) - exp))
        acc[3].append(math.fabs(expHybrid(t, i, 3) - exp))
        acc[4].append(math.fabs(expHybrid(t, i, 10) - exp))
        print(str(i) + "\t" + str(acc[0][i]) + "\t" + str(acc[1][i]) + "\t" + str(acc[2][i]) + "\t" + str(
            acc[3][i]) + "\t" + str(acc[4][i]))
    plt.plot(acc[0], 'r', label='euler')
    plt.plot(acc[1], 'g', label='taylor')
    plt.plot(acc[2], 'b', label='hybrid2')
    plt.plot(acc[3], 'c', label='hybrid3')
    plt.plot(acc[4], 'y', label='hybrid10')
    plt.xlabel('iterations')
    plt.ylabel('error')
    plt.legend()
    plt.show()

# exp_experiment(100, 150)
exp_error_experiment(10, 150)


#!/usr/bin/env python3
import gym
from gym import spaces

import numpy as np
import asyncio
import time
import sys
import os
import math

from mavsdk import System
from mavsdk import (OffboardError, PositionNedYaw, ActuatorControl, AttitudeRate)

import pygazebo
from pygazebo import Manager
from pygazebo import Publisher
from pygazebo.msg import world_control_pb2
from pygazebo.msg.world_control_pb2 import WorldControl
from pygazebo.msg.world_stats_pb2 import WorldStatistics

threshold = 0.5

def lat_lon_to_coords(cords):
    radius_of_earth = 6371000
    rad_lat = math.radians(cords[0])
    rad_lon = math.radians(cords[1])
    x = radius_of_earth * math.cos(rad_lat) * math.cos(rad_lon)
    y = radius_of_earth * math.cos(rad_lat) * math.sin(rad_lon)
    z = cords[2]
    return x, y, z

async def init_env():

	global loop, drone, manager, pub_world_control, home_pos, desired

	drone = System()
	await drone.connect(system_address="udp://:14550")   ## connect to mavsdk
	async for state in drone.core.connection_state():
		if state.is_connected:
			break
	await asyncio.sleep(1)

	print('-- Connecting to Gazebo')
	manager = await pygazebo.connect(('localhost', 11345))   ## connect to pygazebo
	await asyncio.sleep(1)
	print('-- Connected')
	pub_world_control = await manager.advertise('/gazebo/default/world_control','gazebo.msgs.WorldControl')
	await asyncio.sleep(1)

	async for home_pos in drone.telemetry.home():  ## get absolute home position
		home_pos = np.array([home_pos.latitude_deg, home_pos.longitude_deg, home_pos.absolute_altitude_m])
		home_pos=np.array(lat_lon_to_coords(home_pos))
		break

	asyncio.ensure_future(get_lin_pos())  ## initiate linear position stream
	asyncio.ensure_future(get_lin_vel())  ## initiate linear velocity stream

	async for is_armed in drone.telemetry.armed():  ## check arm status
		break

	if not is_armed:  ## if not armed, arm and change to OFFBOARD mode
		print("-- Arming")
		await drone.action.arm()
		await drone.offboard.set_position_ned(PositionNedYaw(0.0, 0.0, 0.0, 0.0))
		print("-- Starting offboard")
		try:
			await drone.offboard.start()
		except OffboardError as error:
			print(f"Starting offboard mode failed with error code: {error._result.result}")
			print("-- Disarming")
			await drone.action.disarm()

async def reset_async(reset_pos):

    unpause_msg = WorldControl()
    unpause_msg.pause = False
    await asyncio.sleep(0.001)
    await pub_world_control.publish(unpause_msg)

    print('-- Resetting position')

    await drone.offboard.set_position_ned(PositionNedYaw(reset_pos[0],reset_pos[1],-reset_pos[2],reset_pos[3]))
    while True:
        await asyncio.sleep(0.1)
        # ob = [lin_pos[2], lin_vel[2]]
        # if ( ( np.abs(ob[0] + reset_pos[2]) < 0.5 ) and np.abs( ob[1] < 0.05 ) ):
        #     await asyncio.sleep(1)
        #     break
        if np.abs(np.linalg.norm(lin_pos - reset_pos[0:3])) < 0.5 and np.abs(np.linalg.norm(lin_vel)) < 0.05 :
            await asyncio.sleep(1)
            break
    print('-- Position reset')

    pause_msg = WorldControl()
    pause_msg.pause = True
    await asyncio.sleep(0.001)
    await pub_world_control.publish(pause_msg)

async def step_async(action):

    step_msg = WorldControl()
    step_msg.step = True
    # await asyncio.sleep(0.1)
    await pub_world_control.publish(step_msg)  ## perform pone step in gazebo
    ### for attitude contorl
    action = [0,0,0,action]
    ### for attitude contorl
    await drone.offboard.set_attitude_rate(AttitudeRate(action[0],action[1],action[2],action[3]))   ## publish action in deg/s, thrust [0:1]

async def get_lin_pos():  ## in m
	async for position in drone.telemetry.position():
		global lin_pos
		glob_pos = np.array([position.latitude_deg, position.longitude_deg, position.absolute_altitude_m])
		lin_pos = np.array(lat_lon_to_coords(glob_pos)) - home_pos

async def get_lin_vel():  ## in m/s
	async for vel in drone.telemetry.ground_speed_ned():
		global lin_vel
		lin_vel = np.array([vel.velocity_north_m_s, vel.velocity_east_m_s, vel.velocity_down_m_s])

async def get_ang_pos(): ## in rad
    async for ang_pos in drone.telemetry.attitude_euler():
        return [ang_pos.roll_deg, ang_pos.pitch_deg, ang_pos.yaw_deg]

async def get_ang_vel():  ## in rad/s
    async for ang_vel in drone.telemetry.attitude_angular_velocity_body():
        return np.array([ang_vel.roll_rad_s, ang_vel.pitch_rad_s, ang_vel.yaw_rad_s])

async def asyland():
    print('-- Landing')
    unpause_msg = WorldControl()
    unpause_msg.pause = False
    await asyncio.sleep(0.5)
    await pub_world_control.publish(unpause_msg)
    await asyncio.sleep(0.5)
    await drone.action.land()

class gymPX4(gym.Env):

    def __init__(self):
        self._start_sim()

        self.observation_space = spaces.Box( np.array([1,-2.0]), np.array([20.0,2.0]), dtype=np.float32)
        self.action_space = spaces.Box(0, 1, shape=(1,), dtype=np.float32)

    def _start_sim(self):
        global loop
        loop = asyncio.get_event_loop()
        loop.run_until_complete(init_env())

    def reset(self):
        global steps, desired

        desired = np.random.randint(4,20)
        initial = np.random.randint(2,20)
        reset_pos=[0,0,initial,0]
        print('Initial: ', initial, 'Desired: ', desired)

        steps = 0

        loop.run_until_complete(reset_async(reset_pos))

        observation = [desired - lin_pos[2], lin_vel[2]]

        return observation  # reward, done, info can't be included

    def step(self, action):
        global steps,  desired
        loop.run_until_complete(step_async(action))
        reward = -np.abs( desired - lin_pos[2] ) - 2*np.abs( lin_vel[2] )
        ob = [desired - lin_pos[2], lin_vel[2]]

        done = False
        info = 'running'

        if  np.abs(lin_pos[0]) > 5 or np.abs(lin_pos[1]) > 5 or np.abs(lin_pos[2]) > 25 or np.abs(lin_pos[2]) < 0.5 :
            done = True
            info = 'out of bounds'
        if steps > 100000 :
            done = True
            info = 'limit time steps'
        if  np.abs(ob[0]) < 0.2 and np.abs(ob[1] < 0.2 ):
            done = True
            info = 'sim success'

        return ob, reward, done, info

    def render(self):
        pass

    def close (self):
        pass

    def land(self):
        loop.run_until_complete(asyland())


import logging
from fn_microsoft_sentinel.lib.function_common import PACKAGE_NAME, SentinelProfiles
from fn_microsoft_sentinel.lib.sentinel_common import SentinelAPI
from fn_microsoft_sentinel.lib.resilient_common import ResilientCommon
from resilient_circuits import ResilientComponent, function, handler, StatusMessage, FunctionResult, FunctionError
from resilient_lib import ResultPayload, validate_fields

class FunctionComponent(ResilientComponent):
    """Component that implements Resilient function 'sentinel_get_incident_comments''"""

    def __init__(self, opts):
        """constructor provides access to the configuration options"""
        super(FunctionComponent, self).__init__(opts)
        self.options = opts.get(PACKAGE_NAME, {})
        self.sentinel_profiles = SentinelProfiles(opts, self.options)

    @handler("reload")
    def _reload(self, event, opts):
        """Configuration options have changed, save new values"""
        self.options = opts.get(PACKAGE_NAME, {})
        self.sentinel_profiles = SentinelProfiles(opts, self.options)

    @function("sentinel_get_incident_comments")
    def _sentinel_get_incident_comments_function(self, event, *args, **kwargs):
        """Function: Get Comments from a Sentinel Incident"""
        try:
            validate_fields(
                ["sentinel_profile", "sentinel_incident_id"],
                kwargs
            )

            yield StatusMessage("Starting 'sentinel_get_incident_comments'")

            rc = ResultPayload(PACKAGE_NAME, **kwargs)

            # Get the function parameters:
            incident_id = kwargs.get("incident_id")  # int
            sentinel_incident_id = kwargs.get("sentinel_incident_id")  # text
            sentinel_profile = kwargs.get("sentinel_profile")  # text

            log = logging.getLogger(__name__)
            log.info("incident_id: %s", incident_id)
            log.info("sentinel_incident_id: %s", sentinel_incident_id)
            log.info("sentinel_profile: %s", sentinel_profile)

            sentinel_api = SentinelAPI(self.options['tenant_id'],
                                       self.options['client_id'],
                                       self.options['app_secret'],
                                       self.opts, self.options)

            resilient_api = ResilientCommon(self.rest_client())

            profile_data = self.sentinel_profiles.get_profile(sentinel_profile)
            result, status, reason = sentinel_api.get_comments(profile_data, sentinel_incident_id)

            new_comments = []
            if status:
                new_comments = resilient_api.filter_resilient_comments(incident_id, result['value'])

            yield StatusMessage("Finished 'sentinel_get_incident_comments'")

            results = rc.done(status, { "value": new_comments }, reason=reason)

            # Produce a FunctionResult with the results
            yield FunctionResult(results)
        except Exception:
            yield FunctionError()


# -*- coding: utf-8 -*-
'''

@Author: Isabella
@Create time: 2018/6/13 14:57

'''
import requests
from bs4 import BeautifulSoup
import pymongo
import time

def save_to_mongo(result):
    try:
        db[MONGO_Collection].insert(result)
        print("存储成功")
    except Exception:
        print("存储失败")

def get_html(i):
    headers = {
        'x-requested-with': 'XMLHttpRequest',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
        'cookie':'_ga=GA1.2.1332905533.1528701127; route=6098c46d36cc5825a33759f7c165d408; _gid=GA1.2.1710498787.1528795610; Hm_lvt_17521045a35cb741c321d016e40c7f95=1528701129,1528795610,1528874985; JOSESSIONID=2D1A9F3D1488671C95B8BAD861CAD404-n1; Hm_lpvt_17521045a35cb741c321d016e40c7f95=1528884226'
    }
    rurl = 'https://search.jobmd.cn/s?wd='+ keyword +'&locations=420000&t='+str(i)
    r = requests.get(rurl,headers=headers)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text,'lxml')
    return soup

def get_detail(url):
    rd = requests.get(url)
    rd.encoding = 'utf-8'
    rdsoup = BeautifulSoup(rd.text,'lxml')
    return rdsoup

def get_page_num():
    soup = get_html(i=1)
    sum = soup.find('span', class_='num').text
    return sum

def get_jobs():
    soup = get_html(i)
    form = soup.find('div',class_ = 'list-box mt10 list-box-B')
    try:
        divs = form.select('.list-dl')
        for div in divs:
            # 基础信息
            jobname = div.find('a',class_='item-jobs-name').text
            hosname = div.find('a',class_='item-ent-name').text
            jobloc = div.find('span',class_='job-loc').text
            jobcate = div.find('span', class_='job-cate').text
            jobedu = div.find('span',class_='job-edu').text
            workyear = div.find('span',class_='job-work').text
            jobcontents = div.find('div',class_='job-class-content')
            posttime = jobcontents.find('span', class_='job-class-meta').text

            # 职位详情
            jobhref = div.find('a',class_='show-more')['href']
            rdsoup = get_detail(jobhref)
            items = rdsoup.find('div', class_='box-info_base')
            salary = items.find('span').text.strip().replace(',','')
            try:
                salarys = salary.split('-')
                salary_low = int(salarys[0])
                salary_high = int(salarys[1])
            except:
                salary_low = salary
                salary_high = None
                print('薪资为面议')
            jd = rdsoup.find('div', class_='work-content_box-detail c-box').text.strip()

            # 公司详情
            hosinfo = rdsoup.find('div',class_='work-hospital_info')
            hosinfos = hosinfo.find_all('span')
            industry = hosinfos[0].text
            hoslevel = hosinfos[1].text
            stuffcount = hosinfos[2].text

            # 创建dict。因为mongodb基于js
            dic = {
                '职位':jobname.strip(),
                '薪资最低':salary_low,
                '薪资最高':salary_high,
                '地址':jobloc.strip()[6:],
                '专业要求':jobcate.strip()[6:],
                '学历':jobedu.strip()[5:],
                '工作年限':workyear.strip()[5:-1],
                '招聘开始时间':posttime.replace("\n","")[0:10],
                '招聘截止时间':posttime.replace("\n","")[11:21],
                '职位链接': jobhref.strip(),
                '公司': hosname.strip(),
                '具体行业':industry.strip(),
                '医院类型':hoslevel.strip()[0:4],
                '医院级别':hoslevel.strip()[5:7],
                '公司人数':stuffcount.strip(),
                '职位详情': jd.replace('\n', ' '),
            }
            print(dic)
            save_to_mongo(dic)
    except Exception as err:
        print(err)
        print('可能是因为该岗位无数据')

def txt2list(file):
    with open(file,encoding='utf-8') as f:
        content = [line.strip() for line in f]
        return content

if __name__ == "__main__":
    worktime = time.strftime("%Y%m%d", time.localtime())

    SERVICE_ARGS = ['--load-images=false', '--disk-cache=true']
    client = pymongo.MongoClient('localhost', 27017)
    db = client['dingxiang']

    for keyword in txt2list('list.txt'):
        MONGO_Collection = worktime + '-' + keyword + '-' + '湖北'
        pagenum = int(get_page_num())//20+1
        for i in range(1,pagenum+1):
            get_jobs()


#   @author: Michael Wagner
#   @organization: TU Dresden
#   @contact: michael.wagner@tu-dresden.de

#   @author: Michael Wagner
#   @organization: TU Dresden
#   @contact: michael.wagner@tu-dresden.de

import datetime as dt

class Product:
    """
    Product class contains all relevant technical data for download of RadolanRW, RadvorRQ, RadolanRV, IconD2, IconD2EPS
    files (from the German Weather Service - DWD).
    """
    def __init__(self,
                 update_time: dt.timedelta = None,
                 base_time: dt.timedelta = dt.timedelta(minutes=0),
                 url_folder: str = None,
                 url_time_subfolder: str = None,
                 url_subfolder: str = None,
                 file_prefix: str = None,
                 file_mid: str = None,
                 file_mid_aux: list = None,
                 file_suffix: str = None,
                 file_extension: str = None):
        """
        Initialize Product class.

        :param update_time: update time at DWD
        :type update_time: datetime.timedelta, optional
        :param base_time: the product's base time, e.g. 50 min for RadolanRW or 0 min for IconD2
        :type base_time: datetime.timedelta, optional
        :param url_folder: url folder before the first time dependent part of the file url, e.g.
            https://opendata.dwd.de/weather/nwp/icon-d2/grib/ in case of IconD2
        :type url_folder: str, optional
        :param url_time_subfolder: datetime.strftime descriptor for the time dependent url folder part, e.g. '%H' for
            IconD2
        :type url_time_subfolder: str, optional
        :param url_subfolder: url folder after a potential time dependent url folder part (requires url_time_subfolder),
            e.g. 'tot_prec' for IconD2
        :type url_subfolder: str, optional
        :param file_prefix: file prefix, e.g. 'raa01-rw_10000-' for RadolanRW data
        :type file_prefix: str, optional
        :param file_mid: datetime.strftime descriptor for the time dependent file name part, e.g. '%y%m%d%H%M' for
            RadolanRW
        :type file_mid: str, optional
        :param file_mid_aux: auxiliary part after time dependent file name part as a list of strings, e.g.
            ['_000', '_060', '_120'] for RadvorRQ
        :type file_mid_aux: list, optional
        :param file_suffix: file suffix after middle part, e.g. '-dwd---bin' for recent part of RadolanRW
        :type file_suffix: str, optional
        :param file_extension: extension of file, e.g. `.gz` for RadolanRW or `.grib2.bz2` for IconD2
        :type file_extension: str, optional
        """
        self.update_time = update_time
        self.base_time = base_time
        self.url_folder = url_folder
        self.url_time_subfolder = url_time_subfolder
        self.url_subfolder = url_subfolder
        self.file_prefix = file_prefix
        self.file_mid = file_mid
        self.file_mid_aux = file_mid_aux
        self.file_suffix = file_suffix
        self.file_extension = file_extension


# Run this Dash app with `python app.py`
# visit http://127.0.0.1:8050/ in web browser.

####################################### Import dependencies and other files #####################################################
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import plotly.express as px
import plotly.graph_objects as go
import plotly.figure_factory as ff
import pandas as pd
import os, mediacloud.api
from io import BytesIO
from dotenv import load_dotenv
import sys
import csv
from datetime import date
from SubDirectory import dataprocessing

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

#Variable to switch to offline testing without requesting new
OFFLINE = False

####################################### Set variables & import data #####################################################
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

#define app colors
colors = {
    'background': '#f7f7f5',
    'text': '#2a2a2a'
}

#define colors for different media sources
media_colors = {
    "South China Morning Post": "#EFB938",
    "New York Times": "#647CAA",
    "BBC": "#7BB48D",
    "Peoples Daily": "#C0003A",
    "english.people.com.cn": "#C0003A"
}

# gets data from csv document which is updated by the dataprocessing file continously
# this makes it possible to both use online + offline, as well as update the data, eg. change sources /timespan

if OFFLINE:
    print("OFFLINE MODE")
    story_df= pd.read_csv('story-list.csv',parse_dates=True, squeeze=True)
    word_df= pd.read_csv('word_df.csv',parse_dates=True, squeeze=True)
    orgs_df= pd.read_csv('orgs_df.csv',parse_dates=True, squeeze=True)

    print(story_df.groupby('media_id')['subjectivity'].mean())

else:
    print("ONLINE MODE")
    #story_df= pd.read_csv('story-list.csv',parse_dates=True, squeeze=True)

    story_df= dataprocessing.getData(dataprocessing.media_dict)
    print(story_df.groupby('media_id')['subjectivity'].mean())

    #calculate most used words (for some reason does not want to work in dataprocessing file)
    word_df= pd.DataFrame()
    for i in dataprocessing.media_dict:
        query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement" OR assembly OR demonstration* OR “human chain” OR rally) and media_id:{i}'
        df= dataprocessing.get_wordcount(query,i)
        df['media_name'] = dataprocessing.media_dict[i]
        word_df = word_df.append(df)
    word_df.to_csv('word_df.csv')

    orgs_df= pd.DataFrame()
    for j in dataprocessing.media_dict:
        query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement" OR assembly OR demonstration* OR “human chain” OR rally) and media_id:{j}'
        df= dataprocessing.get_orgscount(query,j)
        df['media_name'] = dataprocessing.media_dict[j]
        orgs_df = orgs_df.append(df)
    orgs_df.to_csv('orgs_df.csv')

####################################### Create Visualisations & other dashboard elements ####################################

# create a scatterplot
fig = px.scatter(story_df, x="publish_date",y="polarity",
                 #size="subjectivity",
                 color="media_name", color_discrete_map= media_colors, hover_name="title",
                )
fig.update_layout(hovermode="x unified")

# Chart to compare coverage amount of different news sources

fig2 = px.histogram(story_df, x="publish_date", marginal="rug",color="media_name", hover_name= 'title', hover_data={'publish_date':False}, color_discrete_map= media_colors, barmode='group')
fig2.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        hovermode="x unified",
        yaxis=dict(
        title='Articles published',
        ),
        xaxis=dict(
        title='Date',
        )
    )

fig3= px.scatter(story_df, x="publish_date", color="polarity", color_continuous_scale= px.colors.diverging.balance, y="media_name", hover_name="title")
fig3.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Media Outlet',
        ),
        xaxis=dict(
        title='Date',
        )
)
fig3.update_traces(marker_opacity=1, marker_symbol= 42, marker_size= 50)

fig4 =px.bar(word_df, x="count", y= "term", barmode="group", color= "media_name", color_discrete_map= media_colors,)
fig4.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        height=600,
        yaxis=dict(
        title='Term',
        categoryorder= 'total descending'
        ),
        xaxis=dict(
        title='Count',
        )
    )

fig5 =px.bar(orgs_df, x="count", y= "description", color= "media_name", color_discrete_map= media_colors, width=30)

# Text to go with the visualisations
# also make this responsive in app callback?
markdown_text = f'''
### Development of the Hong Kong protests in the News

How much news content does each media source dedicate to the topic of the Hong Kong protests? \n
The following graph compares the number of news articles published about the topic by different news sources. \n
While the number of stories alone does not say much, it is interesting to compare what time each news media picked up \n
a news event or to explore when the number of articles spiked and what the headlines were around that time. \n
The ratio to overall stories of each source reveals the attention a media source gave to a topic: &nbsp
\n
**The percentage of a sources' news articles that mentioned the Hong Kong protests:**   \n
{dataprocessing.media_dict.get(1, 0)} : **{dataprocessing.ratio_dict.get(1,0):.2%}** \n
{dataprocessing.media_dict.get(39590,0)} : **{dataprocessing.ratio_dict.get(39590,0):.2%}** \n
{dataprocessing.media_dict.get(1094, 0)} : **{dataprocessing.ratio_dict.get(1094,0):.2%}** \n
{dataprocessing.media_dict.get(65173,0)} : **{dataprocessing.ratio_dict.get(65173,0):.2%}** \n
'''

# create small tables from text? or visualise the numbers in a better way?

markdown_text2 = f'''
### Sentiment and Polarity of Article Headlines

Since news headlines set the tone for an article and reach an even larger audience than the article text itself, it is vital that they are neutral and objective. \n
Looking at headlines that are subjective or overly positive/negative in tone can reveal biases of news sources. \n
The calculated polarity values range from from -1 (negative tone) to +1 (positive tone), the subjectivity values from 0 (objective) to 1 (subjective). \n
**Average polarity values for the headlines of each news source:**   \n
{dataprocessing.media_dict.get(1, 0)} : **{story_df[story_df['media_name']=='New York Times']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(39590,0)} : **{story_df[story_df['media_name']=='BBC']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(1094, 0)} : **{story_df[story_df['media_name']=='South China Morning Post']['polarity'].mean():.2}** \n

{dataprocessing.media_dict.get(65173,0)} : **{story_df[story_df['media_name']=='english.people.com.cn']['polarity'].mean():.2}** \n
'''

markdown_text3 = f'''
### Taking a closer look at the most used words

Analysing the most used words can show each news source's perspective, as well as which aspects of the news events they focus on. \n
Instances of news bias can be revealed for example through different labels chosen for the same entity or event.
'''

#display data table
def generate_table(dataframe, max_rows=10):
    return html.Table([
        html.Thead(
            html.Tr([html.Th(col) for col in dataframe.columns])
        ),
        html.Tbody([
            html.Tr([
                html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
            ]) for i in range(min(len(dataframe), max_rows))
        ])
    ])

####################################### Set app layout + HTML #####################################################

app.layout = html.Div(style={'backgroundColor': colors['background'], 'font-family': "Verdana"}, children=[   html.H1(
        children='News Explorer',
        style={
            'textAlign': 'center',
            'color': colors['text'],
            'letter-spacing': '5px',
            'font-weight': 'lighter',
            'padding-top': '1em',
        }
    ),

    #display subtitle
    html.Div(children='visualising digital news coverage', style={
        'textAlign': 'center',
        'color': '#5a5a5a',
        'margin-bottom': '3em'
    }),

    #Dropdown menu to select multiple news sources
    html.Label('Select a News Sources to explore'),
    dcc.Dropdown(
        id='dropdown',
        options=[
            {'label': 'New York Times (US)', 'value': 1},
            {'label': 'South China Morning Post (Hong Kong)', 'value': 39590},
            {'label': 'BBC (UK)', 'value': 1094},
            {'label': 'Peoples Daily (China)', 'value': 65173},
        ],
        value=[1, 39590, 1747,65173],
        multi=True,
        style= {
            'color': colors['text'],
        }
    ),

    #Date Picker to change time range
    html.Label('Pick a date range '),
    dcc.DatePickerRange(
        id='my-date-picker-range',
        min_date_allowed=min(story_df['publish_date']),
        max_date_allowed=max(story_df['publish_date']),
        initial_visible_month=date(2019, 1, 1),
        end_date=date(2019, 12, 31),
        start_date=min(story_df['publish_date']),
        clearable= False,
    ),

    #display markdown text
    dcc.Markdown(
        id="text",
        children=markdown_text,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='bar-chart',
        figure=fig2
    ),

    #display markdown text
    dcc.Markdown(
        id="text2",
        children=markdown_text2,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='rug-chart',
        figure=fig3
    ),

     # display graph
    dcc.Graph(
        id='scatter-plot',
        figure=fig
    ),

    #display markdown text
    dcc.Markdown(
        id="text3",
        children=markdown_text3,
        style={'backgroundColor': '#f0f0f0', 'margin' : '42px', 'padding': '8px'}
    ),

    # display graph
    dcc.Graph(
        id='word-bars',
        figure=fig4
    ),

    # # display graph
    # dcc.Graph(
    #     id='fifth-graph',
    #     figure=fig5
    # ),

    # # display table
    # html.Div(children=[
    # html.H4(children='All Stories'),
    # generate_table(story_df)
    # ],
    # style={
    #         'textAlign': 'center',
    #         'color': colors['text']
    #     })
])
####################################### Callback functions #####################################################

#Callback for dropdown menu (media sources) and date range picker -> Updates graphs
@app.callback(
    Output('bar-chart', 'figure'),
    Output('scatter-plot', 'figure'),
    Output('rug-chart', 'figure'),
    Output('word-bars', 'figure'),
    Input('dropdown', 'value'),
    [Input('my-date-picker-range', 'start_date'),
    Input('my-date-picker-range', 'end_date')])

def update_figure(selected_media, start_date, end_date):
    time_df = story_df.loc[(story_df['publish_date'] >= start_date) & (story_df['publish_date'] < end_date)]
    filtered_df= time_df[time_df.media_id.isin(selected_media)]
    filtered_word_df= word_df[word_df.media_id.isin(selected_media)]

    fig = px.scatter(filtered_df, x="publish_date",y="polarity",
                 marginal_y="rug", opacity= 0.6,
                 color="media_name", color_discrete_map= media_colors, hover_name="title",
                )

    fig.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Polarity',
        )
    )

    fig.update_xaxes(
        dtick="M1",
        title= "Date",
        tickformat="%b\n%Y",
        ticklabelmode="period"
        )

    fig.update_traces(marker_opacity=0.8)

    fig2 = px.histogram(filtered_df, x="publish_date", marginal="rug",color="media_name", hover_name= 'title', hover_data={'publish_date':False}, color_discrete_map= media_colors, barmode= 'group')

    fig2.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        hovermode="x unified",
        yaxis=dict(
        title='Articles published',
        ),
        xaxis=dict(
        title='Date',
        )
    )

    fig3= px.scatter(filtered_df, x="publish_date", color="subjectivity", #color_continuous_scale= px.colors.diverging.balance,
    color_continuous_scale= px.colors.sequential.matter, y="media_name", hover_name="title")

    fig3.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        yaxis=dict(
        title='Media Outlet',
        ),
        xaxis=dict(
        title='Date',
        )
    )
    fig3.update_traces(marker_opacity=1, marker_symbol= 42, marker_size= 50)

    #new word counts

    # calculating new word counts -> works, but takes too long, slows overall performance
    # would only work if the word_df is differently structured, so that it can be filtered by time

    # word_df= pd.DataFrame()
    # for i in selected_media:
    #     df= pd.DataFrame()
    #     query = f'"Hong Kong" AND (protest* OR unrest OR "Anti-Extradition" OR "democracy movement*" OR "extradition") and media_id:{i}'
    #     df= dataprocessing.get_wordcount(query,i)
    #     #df['media_name'] = dataprocessing.media_dict[i]
    #     word_df = word_df.append(df)

    fig4 =px.bar(filtered_word_df, x="count", y= "term", barmode="group", color= "media_name", color_discrete_map= media_colors,)
    fig4.update_layout(
        plot_bgcolor=colors['background'],
        paper_bgcolor=colors['background'],
        font_color=colors['text'],
        height=800,
        yaxis=dict(
        title='Term',
        tickmode='linear', # displays every label instead of only every other,
        categoryorder= 'total ascending'
        ),
        xaxis=dict(
        title='Count',
        )
    )

    return fig2, fig, fig3, fig4

if __name__ == '__main__':
    app.run_server(debug=True)


import os
import subprocess
from time import sleep

import pyautogui
from window import Window

class ScreenRecorder(Window):
    def __init__(self):
        super().__init__(["simplescreenrecorder"], ["simplescreenrecorder"])

    def start_record(self):
        if self.running:
            pyautogui.hotkey("winleft", "r")

    def stop_record(self):
        if self.running:
            pyautogui.hotkey("winleft", "r")

    def save_rec(self):
        self.bring_to_forground()
        pyautogui.press("tab", 14, interval=0.1)
        pyautogui.press("space", interval=0.1)
        pyautogui.press("tab", 1, interval=0.1)
        pyautogui.press("space", interval=0.1)

if __name__ == '__main__':
    sr = ScreenRecorder()
    sr.start()
    sr.bring_to_forground()
    sleep(1)
    sr.start_record()
    sleep(3)
    sr.stop_record()
    sleep(1)
    sr.bring_to_forground()
    sleep(5)


# Solve the ODE for a conducting ring spinning in a magnetic field.
import numpy as np
import scipy.integrate
import matplotlib.pyplot as plt

# This function evaluates the derivatives for the equation
# d^2 theta/dt^2 = - * sin(theta) -q* d theta/dt + Fsin(k*t)
# We work in the transformed variables y[0] = theta, y[1] = d(theta)/dt
k=0
F=0
q=0

def derivatives(y,t,q,f,k):
    return [y[1], -np.sin(y[0]) - q*y[1] + F*np.sin(k*t)]
# Main code starts here
t=np.linspace(0.0, 100.0, 10000)
y0=[0.01, 0.0]

y=scipy.integrate.odeint(derivatives,y0,t,args=(q,k,F))
for i in range(len(y)):
    print("{:8.4g} {:8.4g} {:8.4g}".format(t[i],y[i,0],y[i,1]))

#plotting
plt.plot(t, 0.5*y[:, 1]**2 + (1-np.cos(y[:, 0])), 'b', label='energy(t)')
plt.plot(t, 0.5*(0.01*np.sin(t))**2  + (1-np.cos(0.01*np.cos(t))), 'g', label='theoretical solution(t)')
plt.legend(loc='best')
plt.xlabel('t')
plt.xlim(0,100)
plt.grid()
plt.show()
plt.savefig('pendulum1.pdf')


import concurrent.futures
import functools
import pathlib
import re
import sys
import tarfile

import pandas as pd
import tqdm

sys.path.insert(0, '../src/')
import utils  # noqa:E402

def get_subfiles(archive_file_path, n_drugs):
    """
    Find all score, interaction, and log files within a `.tar.gz` archive.
    Each subfile is represented as a list.

    For OFFSIDES, files are [drug, bootstrap_iteration, file_type,
    subfile_name, archive_path].

    For TWOSIDES, files are [drug_1, drug_2, file_type, subfile_name,
    archive_path]
    """
    file_locations = list()
    try:
        tar = tarfile.open(archive_file_path, mode='r:gz')
        subfiles = tar.getnames()
    except tarfile.ReadError:
        return None
    except EOFError:
        return None

    if n_drugs == 1:
        for subfile in subfiles:
            if 'interaction' in subfile:
                drug = re.match(r'(?:interactions__)([0-9]+)(?:\.npy)', subfile)
                if not drug:
                    raise ValueError(f'{archive_file_path.name} contained {subfile} not matched')
                drug = drug.group(1)
                bootstrap = None
                file_type = 'interaction'
            else:
                drug, bootstrap = utils.extract_indices(subfile)
                file_type = re.match('^[a-z]+(?=_.+)', subfile).group()
            file_locations.append([drug, bootstrap, file_type, subfile,
                                   archive_file_path.name])
    elif n_drugs == 2:
        for subfile in subfiles:
            file_name_match = re.match(r'([a-z]+)(?:.*?__)([0-9]+)(?:_)([0-9]+)(?=\.npy)', subfile)
            if not file_name_match:
                raise ValueError(f'{archive_file_path.name} contained {subfile} not matched')
            file_type, drug_1, drug_2 = file_name_match.groups()
            drug_1, drug_2 = int(drug_1), int(drug_2)
            file_locations.append([drug_1, drug_2, file_type, subfile,
                                   archive_file_path.name])
    return file_locations

def compute_file_map(n_drugs, archives_path):
    """Compute a file map for an entire directory of archives"""
    get_subfiles_partial = functools.partial(get_subfiles, n_drugs=n_drugs)

    archive_files = list(archives_path.glob('scores_*.tgz'))
    with concurrent.futures.ProcessPoolExecutor() as executor:
        file_locations = list(tqdm.tqdm(
            executor.map(get_subfiles_partial, archive_files),
            total=len(archive_files)
        ))

    # Flatten list of lists of tuples to a list of tuples
    flattened_file_locations = [l for l in file_locations if l is not None]
    flattened_file_locations = [i for l in flattened_file_locations for i in l]

    # First two column names depend on the number of drugs
    n_drug_to_column_names = {
        1: ('drug', 'bootstrap'),
        2: ('drug_index_1', 'drug_index_2'),
    }
    column_names = [*n_drug_to_column_names[n_drugs], 'file_type', 'file_name',
                    'archive_file']

    # Instantiate DataFrame and save to disk immediately
    return pd.DataFrame(flattened_file_locations, columns=column_names)

def compute_all_filemaps():
    # Path to where the `.tgz` archives are stored
    archives_path = pathlib.Path('/data/archives/')

    # Path where the file maps will be saved
    meta_path = pathlib.Path('/data/meta')

    # Compute and save OFFSIDES file map
    offsides_file_map = compute_file_map(1, archives_path.joinpath('1/'))
    offsides_file_map.to_csv(meta_path.joinpath('file_map_offsides.csv'),
                             index=False)

    # Compute and save TWOSIDES file map
    twosides_file_map = compute_file_map(2, archives_path.joinpath('2/'))
    twosides_file_map.to_csv(meta_path.joinpath('file_map_twosides.csv'),
                             index=False)

if __name__ == "__main__":
    compute_all_filemaps()