import math

import torch
import torch.nn as nn
import numpy as np
from utils.diffusion_util import GLD

class DualDiffusion(nn.Module):
    def __init__(self, config, device, target_dim=96):
        super().__init__()
        self.device = device
        self.target_dim = target_dim

        self.emb_time_dim = config["model"]["timeemb"]
        self.emb_feature_dim = config["model"]["featureemb"]
        self.is_unconditional = config["model"]["is_unconditional"]
        self.finger_feature_embedding = nn.Linear(3, 3) # 3->3

        self.emb_total_dim = self.emb_time_dim + self.emb_feature_dim
        if not self.is_unconditional:
            self.emb_total_dim += 1  # for conditional mask
        self.embed_layer = nn.Embedding(
            num_embeddings=self.target_dim, embedding_dim=self.emb_feature_dim
        )

        config_diff = config["diffusion"]
        config_diff["side_dim"] = self.emb_total_dim

        input_dim = 1 if self.is_unconditional else 2
        self.diffmodel = GLD(config_diff, input_dim)

        # parameters for diffusion models
        self.num_steps = config_diff["num_steps"]
        if config_diff["schedule"] == "quad":
            self.beta = np.linspace(
                config_diff["beta_start"] ** 0.5, config_diff["beta_end"] ** 0.5, self.num_steps
            ) ** 2
        elif config_diff["schedule"] == "linear":
            self.beta = np.linspace(
                config_diff["beta_start"], config_diff["beta_end"], self.num_steps
            )
        elif config_diff["schedule"] == "cosine":
            self.beta = self.betas_for_alpha_bar(
                self.num_steps,
                lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2,
            )

        self.alpha_hat = 1 - self.beta
        self.alpha = np.cumprod(self.alpha_hat)
        self.alpha_torch = torch.tensor(self.alpha).float().to(self.device).unsqueeze(1).unsqueeze(1)

    def betas_for_alpha_bar(self, num_diffusion_timesteps, alpha_bar, max_beta=0.5):
        # """
        # Create a beta schedule that discretizes the given alpha_t_bar function,
        # which defines the cumulative product of (1-beta) over time from t = [0,1].
        # :param num_diffusion_timesteps: the number of betas to produce.
        # :param alpha_bar: a lambda that takes an argument t from 0 to 1 and
        #                   produces the cumulative product of (1-beta) up to that
        #                   part of the diffusion process.
        # :param max_beta: the maximum beta to use; use values lower than 1 to
        #                  prevent singularities.
        # """
        betas = []
        for i in range(num_diffusion_timesteps):
            t1 = i / num_diffusion_timesteps
            t2 = (i + 1) / num_diffusion_timesteps
            betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta))
        return np.array(betas)

    def time_embedding(self, pos, d_model=128):
        pe = torch.zeros(pos.shape[0], pos.shape[1], d_model).to(self.device)
        position = pos.unsqueeze(2)
        div_term = 1 / torch.pow(
            10000.0, torch.arange(0, d_model, 2).to(self.device) / d_model
        )
        pe[:, :, 0::2] = torch.sin(position * div_term)
        pe[:, :, 1::2] = torch.cos(position * div_term)
        return pe

    def get_side_info(self, observed_tp, cond_mask):
        B, K, L = cond_mask.shape

        time_embed = self.time_embedding(observed_tp, self.emb_time_dim)  # (B,L,emb)
        time_embed = time_embed.unsqueeze(2).expand(-1, -1, K, -1)
        feature_embed = self.embed_layer(
            torch.arange(self.target_dim).to(self.device)
        )  # (K,emb)
        feature_embed = feature_embed.unsqueeze(0).unsqueeze(0).expand(B, L, -1, -1)

        side_info = torch.cat([time_embed, feature_embed], dim=-1)  # (B,L,K,*)
        side_info = side_info.permute(0, 3, 2, 1)  # (B,*,K,L)

        if not self.is_unconditional:
            side_mask = cond_mask.unsqueeze(1)  # (B,1,K,L)
            side_info = torch.cat([side_info, side_mask], dim=1)

        return side_info

    def calc_loss_valid(
            self, observed_data, cond_mask, side_info, is_train,observed_data_latent
    ):
        loss_sum = 0
        for t in range(self.num_steps):  # calculate loss for all t
            loss = self.calc_loss(
                observed_data, cond_mask, side_info, is_train,observed_data_latent, set_t=t
            )
            loss_sum += loss.detach()
        return loss_sum / self.num_steps

    def calc_loss(
            self, observed_data, cond_mask, side_info, is_train, observed_data_latent, set_t=-1
    ):
        B, K, L = observed_data.shape
        if is_train != 1:  # for validation
            t = (torch.ones(B) * set_t).long().to(self.device)
        else:
            t = torch.randint(0, self.num_steps, [B]).to(self.device)
        current_alpha = self.alpha_torch[t].to(self.device)  # (B,1,1)
        noise = torch.randn_like(observed_data).to(self.device)
        noisy_data = (current_alpha ** 0.5) * observed_data + (1.0 - current_alpha) ** 0.5 * noise

        total_input = self.set_input_to_diffmodel(noisy_data, observed_data, cond_mask)

        predicted = self.diffmodel(total_input, side_info, t)  # (B,K,L)

        target_mask = 1 - cond_mask
        residual = (noise - predicted) * target_mask
        num_eval = target_mask.sum()
        loss = (residual ** 2).sum() / (num_eval if num_eval > 0 else 1)

        ####################################

        noise_latent = torch.randn_like(observed_data_latent).to(self.device)
        noisy_data_latent = (current_alpha ** 0.5) * observed_data_latent + (1.0 - current_alpha) ** 0.5 * noise_latent

        total_input_latent = self.set_input_to_diffmodel(noisy_data_latent, observed_data_latent, cond_mask)

        predicted_latent = self.diffmodel(total_input_latent, side_info, t)  # (B,K,L)

        target_mask = 1 - cond_mask
        residual_latent = (noise_latent - predicted_latent) * target_mask
        num_eval = target_mask.sum()
        loss_latent = (residual_latent ** 2).sum() / (num_eval if num_eval > 0 else 1)
        return loss+loss_latent

    def set_input_to_diffmodel(self, noisy_data, observed_data, cond_mask):
        if self.is_unconditional:
            total_input = noisy_data.unsqueeze(1)  # (B,1,K,L)
        else:
            cond_obs = (cond_mask * observed_data).unsqueeze(1)
            noisy_target = ((1 - cond_mask) * noisy_data).unsqueeze(1)
            total_input = torch.cat([cond_obs, noisy_target], dim=1)  # (B,2,K,L)

        return total_input

    def impute(self, observed_data, cond_mask, side_info, n_samples):
        B, K, L = observed_data.shape

        imputed_samples = torch.zeros(B, n_samples, K, L).to(self.device)

        for i in range(n_samples):
            # generate noisy observation for unconditional model
            if self.is_unconditional:
                noisy_obs = observed_data
                noisy_cond_history = []
                for t in range(self.num_steps):
                    noise = torch.randn_like(noisy_obs)
                    noisy_obs = (self.alpha_hat[t] ** 0.5) * noisy_obs + self.beta[t] ** 0.5 * noise
                    noisy_cond_history.append(noisy_obs * cond_mask)

            current_sample = torch.randn_like(observed_data)

            for t in range(self.num_steps - 1, -1, -1):
                if self.is_unconditional:
                    diff_input = cond_mask * noisy_cond_history[t] + (1.0 - cond_mask) * current_sample
                    diff_input = diff_input.unsqueeze(1)  # (B,1,K,L)
                else:
                    cond_obs = (cond_mask * observed_data).unsqueeze(1)
                    noisy_target = ((1 - cond_mask) * current_sample).unsqueeze(1)
                    diff_input = torch.cat([cond_obs, noisy_target], dim=1)  # (B,2,K,L)
                predicted = self.diffmodel(diff_input, side_info, torch.tensor([t]).to(self.device))

                coeff1 = 1 / self.alpha_hat[t] ** 0.5
                coeff2 = (1 - self.alpha_hat[t]) / (1 - self.alpha[t]) ** 0.5
                current_sample = coeff1 * (current_sample - coeff2 * predicted)

                if t > 0:
                    noise = torch.randn_like(current_sample)
                    sigma = (
                                    (1.0 - self.alpha[t - 1]) / (1.0 - self.alpha[t]) * self.beta[t]
                            ) ** 0.5
                    current_sample += sigma * noise

            imputed_samples[:, i] = (current_sample * (1 - cond_mask) + observed_data * cond_mask).detach()
        return imputed_samples

    def forward(self, batch, is_train=1):
        (observed_data,observed_tp,gt_mask,observed_data_latent) = self.process_data(batch)
        cond_mask = gt_mask

        side_info = self.get_side_info(observed_tp, cond_mask)

        loss_func = self.calc_loss if is_train == 1 else self.calc_loss_valid

        return loss_func(observed_data, cond_mask, side_info, is_train, observed_data_latent)

    def evaluate(self, batch, n_samples):
        (
            observed_data,
            observed_tp,
            gt_mask,
            observed_data_latent
        ) = self.process_data(batch)
        # print(np.shape(observed_data))
        # print(np.shape(observed_data))
        # print(np.shape(observed_data))
        pose_t = batch["pose"].to(self.device).float()
        batch_size, total_input_n , joints_n = pose_t.size()
        temp_input_n=10
        temp_output_n = total_input_n-temp_input_n

        with torch.no_grad():
            pose_t_output = pose_t[:, temp_input_n:, :]
            pose_t_output = pose_t_output.view(-1, temp_output_n, 21, 3).cpu().numpy()
            cond_mask = gt_mask
            target_mask = 1 - cond_mask

            side_info = self.get_side_info(observed_tp, cond_mask)

            samples = self.impute(observed_data, cond_mask, side_info, n_samples)
            # samples= samples.permute(0, 2, 1)
            samples_latent = self.impute(observed_data_latent, cond_mask, side_info, n_samples)
            # samples_latent = samples_latent.permute(0, 2, 1)
            samples_mean = np.mean(samples.cpu().numpy(), axis=1)
            samples_latent_mean = np.mean(samples_latent.cpu().numpy(), axis=1)
            samples_mean_output = samples_mean[:,:,temp_input_n:]
            samples_mean_output = samples_mean_output.transpose(0, 2, 1).reshape(-1, temp_output_n, 21, 3)
            # print(np.shape(samples_mean_output))
            samples_latent_mean_output = samples_latent_mean[:, :, temp_input_n:]
            samples_latent_mean_output = samples_latent_mean_output.transpose(0, 2, 1).reshape(-1, temp_output_n, 21, 3)

            best_samples = samples
            # print(np.shape(pose_t_output), np.shape(samples_mean_output))
            if self.mpjpe_error(torch.from_numpy(samples_mean_output),torch.from_numpy(pose_t_output)) > self.mpjpe_error(torch.from_numpy(samples_latent_mean_output),torch.from_numpy(pose_t_output)):
                best_samples = samples_latent
            samples =best_samples
            # trenorm_pose.view(-1, output_n, args.joints, 3)
        # print(224, np.shape(samples),'post truth',np.shape(pose_t),'mean',np.shape(samples_mean)) #torch.Size([20, 5, 63, 15]) post truth torch.Size([20, 15, 63]) mean (20, 63, 15)

        return samples, observed_data, target_mask, observed_tp

    def process_data(self, batch):
        latent_finger_63 = self.finger_features_process_data(batch)
        pose_latent = latent_finger_63 # [32, 15, 66]
        pose = batch["pose"].to(self.device).float() # [32, 15, 66]
        tp = batch["timepoints"].to(self.device).float()
        mask = batch["mask"].to(self.device).float()
        # print(216, np.shape(pose_t),np.shape(pose), np.shape(tp), np.shape(mask))

        pose_latent = pose_latent.permute(0, 2, 1)
        pose = pose.permute(0, 2, 1)
        mask = mask.permute(0, 2, 1)

        return (
            pose,
            tp,
            mask,
            pose_latent
        )
    def finger_features_process_data(self,batch):
        jt_idx = [[0,1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20]]
        pose = batch["pose"].to(self.device).float()
        batch_sise, input_feature_n, dimension_joints = pose.size()
        reshape_pose = pose.reshape(batch_sise, input_feature_n, 21, 3)
        finger = []
        for i in range(0,5):
            temp_reshape_pose = self.finger_feature_embedding(reshape_pose[:, :, jt_idx[i], :])
            finger.append(temp_reshape_pose) #[32, 15, 4, 3]

        latent_finger = torch.cat([finger[0], finger[1],finger[2],finger[3],finger[4]], dim=2)
        reshaped_latent_finger = torch.reshape(latent_finger, (batch_sise, input_feature_n, -1))
        # latent_finger_63 = self.fc1(reshaped_latent_finger)
        latent_finger_63 = reshaped_latent_finger

        # print(243, np.shape(latent_finger_63)) #[32, 15, 63]
        return  latent_finger_63

    def mpjpe_error(self, batch_imp, batch_gt):
        batch_imp = batch_imp.contiguous().view(-1, 3)
        batch_gt = batch_gt.contiguous().view(-1, 3)

        return torch.mean(torch.norm(batch_gt - batch_imp, 2, 1))

# import torch
# from thop import profile
#
# if __name__ == '__main__':
#     config = {
#         'train':
#             {
#                 'epochs': 100,
#                 'batch_size': 32,
#                 'batch_size_test': 32,
#                 'lr': 1.0e-3
#             },
#         'diffusion':
#             {
#                 'layers': 12,
#                 'channels': 64,
#                 'nheads': 8,
#                 'diffusion_embedding_dim': 128,
#                 'beta_start': 0.0001,
#                 'beta_end': 0.5,
#                 'num_steps': 50,
#                 'schedule': "cosine"
#             },
#         'model':
#             {
#                 'is_unconditional': 0,
#                 'timeemb': 128,
#                 'featureemb': 16
#             }
#     }
#     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#     print('Using device: %s' % device)
#
#     model = ModelMain(config, device, target_dim=(21 * 3)).to(device)
#     torch.Size([32, 15, 63])
#     torch.Size([32, 15])
#     torch.Size([32, 15, 63])
#
#     pose = torch.zeros(32,15,63).to(device)
#     mask=  torch.zeros(32,15,63).to(device)
#     timepoints = torch.zeros(32,15).to(device)
#     input_n= 75
#     s = {
#         "pose":pose[:, :input_n + 5],
#         "mask": mask[:, :input_n + 5],
#         "timepoints": timepoints[:, :input_n + 5]
#     }
#     print(271, np.shape(pose[:, :input_n + 5]))
#
#     output = model.evaluate(s, 5)
#     samples, _, eval_impute, _ = output
#     print(274, np.shape(samples))
# # #############################################
#     input_tensor = torch.randn(1, 10, 63)  # Example input tensor
#
#     flops, params = profile(model, inputs=(s,))
#     print(f"FLOPs: {flops}, Parameters: {params}")
#     total_params = sum(p.numel() for p in model.parameters())
#     print(f"Total Parameters: {total_params}")


import discord, datetime, time
from ..admin.managecommands import perms
import json
from discord.utils import get
from pymongo import MongoClient, collation
from discord.ext import commands, tasks
import time
import os
import pymongo as pm
import random

class Userinfo(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

# Userinfo

    @commands.command(pass_context=True, aliases= ['userinfo', 'profile'])
    @commands.check(perms)
    async def user(self, ctx, member:discord.Member = None):
        if member is None:

            roles_list = ' | '.join(map(str, ctx.author.roles))

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.add_field(name="ID", value=ctx.author.id, inline=False)
            embed.add_field(name="Nickname", value=ctx.author.nick, inline=False)
            embed.add_field(name="Highest role", value=ctx.author.top_role, inline=False)
            embed.add_field(name="Roles", value=roles_list, inline=False)
            embed.set_image(url=ctx.author.avatar_url)
            await ctx.reply(embed=embed)

        else:

            roles_list = ' | '.join(map(str, member.roles))

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.add_field(name="ID", value=member.id, inline=False)
            embed.add_field(name="Nickname", value=member.nick, inline=False)
            embed.add_field(name="Highest role", value=member.top_role, inline=False)
            embed.add_field(name="Roles", value=roles_list, inline=False)
            embed.set_image(url=member.avatar_url)
            await ctx.reply(embed=embed)

    @commands.command(pass_context=True, aliases=['av'])
    @commands.check(perms)
    async def avatar(self, ctx, member:discord.Member = None):
        if member is None:

            embed=discord.Embed(title=ctx.author, color=random.randint(0, 0xFFFFFF))
            embed.set_image(url=ctx.author.avatar_url)
            await ctx.reply(embed=embed)

        else:

            embed=discord.Embed(title=member, color=random.randint(0, 0xFFFFFF))
            embed.set_image(url=member.avatar_url)
            await ctx.reply(embed=embed)

def setup(bot):
    bot.add_cog(Userinfo(bot))


import os.path
import json
from util import is_macos, is_darkmode

class Preferences:
    pref_file = os.path.join(os.getcwd(), "prefs", "config.json")
    defaults = os.path.join(os.getcwd(), "prefs", "defaults.json")
    copy_list = [
        'report_sheet',
        'format_unfinished',
        'unfinished_color',
        'class_portal',
        'protected_set',
        'debug'
    ]

    def __init__(self):
        with open(Preferences.defaults, "r+") as df:
            self.defaults = json.load(df)
        if not os.path.isfile(Preferences.pref_file):
            with open(Preferences.pref_file, 'w+') as pf:
                build_prefs = self.get_default_theme()
                for pref in Preferences.copy_list:
                    build_prefs[pref] = self.defaults[pref]
                json.dump(build_prefs, pf)
        with open(Preferences.pref_file, "r+") as pf:
            self.prefs = json.load(pf)
        self.pref_file = Preferences.pref_file

    def has_pref(self, key):
        return key in self.prefs

    def get_pref(self, key):
        if self.has_pref(key): return self.prefs[key]
        return None

    def update_pref(self, key, value):
        self.prefs[key] = value
        self.save_prefs()

    def save_prefs(self):
        with open(Preferences.pref_file, "w+") as pf:
            json.dump(self.prefs, pf)

    def get_default_pref(self, key):
        if key in self.defaults: return self.defaults[key]
        return None

    def get_default_theme(self):
        return self.defaults['dark_theme'] if is_darkmode() else self.defaults['light_theme']

prefs = Preferences()


from urllib.request import urlopen, HTTPError
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from utils.parser import ParserHTML
from utils.data import saveDataOnFile, readDataFromFile

URL_SEED = 'https://pt.wikipedia.org/'
TODO_LIST = [URL_SEED]
DONE_LIST = []
BLACKLIST = set()
MAX_LINKS_VISITED = 100

parserHTML = ParserHTML()

def getInternalLinksFrom(url):
    regexInternalWikiLinks = r"^(/wiki/)(.*)?$"
    html = urlopen(url)
    parserHTML.loadPage(BeautifulSoup, html.read())
    return parserHTML.getTagA(regexInternalWikiLinks)

def isLinkNotAlreadyKnow(pulledLink):
    if 'href' in pulledLink.attrs:
        newURL = urljoin(URL_SEED, pulledLink.attrs['href'])
        if newURL not in DONE_LIST and newURL not in TODO_LIST:
            return True, newURL
    return False, None

def scrapper():
    counterVisitedLinks = 0

    while TODO_LIST:
        nextLinkToExplore = TODO_LIST.pop(0)
        if nextLinkToExplore not in DONE_LIST:
            try:
                if nextLinkToExplore in BLACKLIST:
                    continue

                newLinksList = getInternalLinksFrom(nextLinkToExplore)
                DONE_LIST.append(nextLinkToExplore)

                print(f'done: {nextLinkToExplore} - internal links: {len(newLinksList)}')

                for newLinkItem in newLinksList:
                    isNotAlreadyKnow, newUrl = isLinkNotAlreadyKnow(newLinkItem)

                    if isNotAlreadyKnow:
                        TODO_LIST.append(newUrl)
            except HTTPError as e:
                print(f'done: {nextLinkToExplore} - error: {e.errno}')
                BLACKLIST.add(nextLinkToExplore)

            counterVisitedLinks += 1

        if counterVisitedLinks == MAX_LINKS_VISITED:
            saveDataOnFile('TODO_LIST.txt', TODO_LIST)
            saveDataOnFile('DONE_LIST.txt', DONE_LIST)
            saveDataOnFile('BLACKLIST.txt', BLACKLIST)
            counterVisitedLinks = 0
            print('--> Data lists saved on files')

def start():
    global TODO_LIST
    global DONE_LIST
    global BLACKLIST

    TODO_LIST = readDataFromFile('TODO_LIST.txt') or TODO_LIST
    DONE_LIST = readDataFromFile('DONE_LIST.txt') or DONE_LIST
    BLACKLIST = readDataFromFile('BLACKLIST.txt') or BLACKLIST

    scrapper()

if __name__ == '__main__':
    try:
        print('--> Scraper running...')
        start()
    except KeyboardInterrupt:
        saveDataOnFile('TODO_LIST.txt', TODO_LIST)
        saveDataOnFile('DONE_LIST.txt', DONE_LIST)
        saveDataOnFile('BLACKLIST.txt', BLACKLIST)
        print('Bye bye!!')


import numpy as np
from random import random
from PIL import Image, ImageOps

import torch
import torch.utils.data as data
import torchvision

from scipy.spatial.distance import cdist

class BaselineDataset(data.Dataset):
    def __init__(self, fls, transforms=None):

        self.fls = fls
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss

    def __len__(self):
        return len(self.fls)

class CuMixloader(data.Dataset):

    def __init__(self, fls, clss, doms, dict_domain, transforms=None):

        self.fls = fls
        self.clss = clss
        self.domains = doms
        self.dict_domain = dict_domain
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss, self.dict_domain[sample_domain]

    def __len__(self):
        return len(self.fls)

class SAKELoader(data.Dataset):
    def __init__(self, fls, cid_mask, transforms=None):

        self.fls = fls
        self.cid_mask = cid_mask
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample, clss, self.cid_mask[clss]

    def __len__(self):
        return len(self.fls)

class SAKELoader_with_domainlabel(data.Dataset):
    def __init__(self, fls, cid_mask=None, transforms=None):

        self.fls = fls
        self.cid_mask = cid_mask
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])
        self.transforms = transforms

    def __getitem__(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
            domain_label = np.array([0])
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')
            domain_label = np.array([1])

        clss = self.clss[item]

        if self.transforms is not None:
            sample = self.transforms(sample)

        if self.cid_mask is not None:
            return sample, clss, self.cid_mask[clss], domain_label
        else:
            return sample, clss, domain_label

    def __len__(self):
        return len(self.fls)

class Doodle2Search_Loader(data.Dataset):
    def __init__(self, fls_sketch, fls_image, semantic_vec, tr_classes, dict_clss, transforms=None):

        self.fls_sketch = fls_sketch
        self.fls_image = fls_image

        self.cls_sketch = np.array([f.split('/')[-2] for f in self.fls_sketch])
        self.cls_image = np.array([f.split('/')[-2] for f in self.fls_image])

        self.tr_classes = tr_classes
        self.dict_clss = dict_clss

        self.semantic_vec = semantic_vec
        # self.sim_matrix = np.exp(-np.square(cdist(self.semantic_vec, self.semantic_vec, 'euclidean'))/0.1)
        cls_euc = cdist(self.semantic_vec, self.semantic_vec, 'euclidean')
        cls_euc_scaled = cls_euc/np.expand_dims(np.max(cls_euc, axis=1), axis=1)
        self.sim_matrix = np.exp(-cls_euc_scaled)

        self.transforms = transforms

    def __getitem__(self, item):

        sketch = ImageOps.invert(Image.open(self.fls_sketch[item])).convert(mode='RGB')
        sketch_cls = self.cls_sketch[item]
        sketch_cls_numeric = self.dict_clss.get(sketch_cls)

        w2v = torch.FloatTensor(self.semantic_vec[sketch_cls_numeric, :])

        # Find negative sample
        possible_classes = self.tr_classes[self.tr_classes!=sketch_cls]
        sim = self.sim_matrix[sketch_cls_numeric, :]
        sim = np.array([sim[self.dict_clss.get(x)] for x in possible_classes])

        # norm = np.linalg.norm(sim, ord=1) # Similarity to probability
        # sim = sim/norm
        sim /= np.sum(sim)

        image_neg_cls = np.random.choice(possible_classes, 1, p=sim)[0]
        image_neg = Image.open(np.random.choice(self.fls_image[np.where(self.cls_image==image_neg_cls)[0]], 1)[0]).convert(mode='RGB')

        image_pos = Image.open(np.random.choice(self.fls_image[np.where(self.cls_image==sketch_cls)[0]], 1)[0]).convert(mode='RGB')

        if self.transforms is not None:
            sketch = self.transforms(sketch)
            image_pos = self.transforms(image_pos)
            image_neg = self.transforms(image_neg)

        return sketch, image_pos, image_neg, w2v

    def __len__(self):
        return len(self.fls_sketch)

class JigsawDataset(data.Dataset):
    def __init__(self, fls, transforms=None, jig_classes=30, bias_whole_image=0.9):

        self.fls = fls
        self.clss = np.array([f.split('/')[-2] for f in fls])
        self.domains = np.array([f.split('/')[-3] for f in fls])

        self.permutations = self.__retrieve_permutations(jig_classes)
        self.grid_size = 3
        self.bias_whole_image = bias_whole_image # biases the training procedure to show the whole image more often

        self._image_transformer = transforms['image']
        self._augment_tile = transforms['tile']

        def make_grid(x):
            return torchvision.utils.make_grid(x, self.grid_size, padding=0)
        self.returnFunc = make_grid

    def get_tile(self, img, n):
        w = float(img.size[0]) / self.grid_size
        y = int(n / self.grid_size)
        x = n % self.grid_size
        tile = img.crop([x * w, y * w, (x + 1) * w, (y + 1) * w])
        tile = self._augment_tile(tile)
        return tile

    def get_image(self, item):

        sample_domain = self.domains[item]
        if sample_domain=='sketch' or sample_domain=='quickdraw':
            sample = ImageOps.invert(Image.open(self.fls[item])).convert(mode='RGB')
        else:
            sample = Image.open(self.fls[item]).convert(mode='RGB')
        return self._image_transformer(sample)

    def __getitem__(self, item):

        img = self.get_image(item)
        n_grids = self.grid_size ** 2
        tiles = [None] * n_grids
        for n in range(n_grids):
            tiles[n] = self.get_tile(img, n)

        order = np.random.randint(len(self.permutations) + 1) # added 1 for class 0: unsorted
        if self.bias_whole_image:
            if self.bias_whole_image > random():
                order = 0

        if order == 0:
            data = tiles
        else:
            data = [tiles[self.permutations[order - 1][t]] for t in range(n_grids)]

        data = torch.stack(data, 0)
        data = self.returnFunc(data)

        return torch.cat([self._augment_tile(img), data], 0), order, self.clss[item]

    def __len__(self):
        return len(self.fls)

    def __retrieve_permutations(self, classes):
        all_perm = np.load(os.path.join(_BASE_PATH, 'permutations_%d.npy' % (classes)))
        # from range [1,9] to [0,8]
        if all_perm.min() == 1:
            all_perm = all_perm - 1

        return all_perm


import os, math, glob, json, collections
from os.path import join

import nesoni
from nesoni import config, workspace, working_directory, reference_directory, io, reporting, grace, annotation, selection, span_index

import tail_tools
from . import clip_runs, extend_sam, proportions, tail_lengths, web, alternative_tails, bigwig, web, peaks

def _do_nothing():
    pass

def _make_each(actions):
    for item in actions:
        item.make()

def _make(action):
    action.make()

class _call(object):
    def __init__(self, func, *args, **kwargs):
        self.func = func
        self.args = args
        self.kwargs = kwargs

    def __call__(self):
        self.func(*self.args, **self.kwargs)

def _serial(*items):
    for item in items:
        item()

def _parallel(*items):
    with nesoni.Stage() as stage:
        for item in items:
            stage.process(item)

@config.help(
'Create nesoni working directories for a single sample.',
"""\
Reads are clipped using "clip-runs:", aligned using SHRiMP, extended using "extend-sam:" and \
imported into a nesoni working directory.

Directories are created both using all reads and for reads with a poly-A tail.
""")
@config.Bool_flag('consensus', 'Look for SNPs and indels.')
@config.Bool_flag('discard_multimappers', 'If "yes", multimapping reads are discarded. If "no", multimapping reads are assigned at random.')
@config.Bool_flag('delete_files', 'Delete files not needed by downstream processing.')
@config.Section('tags', 'Tags for this sample. (See "nesoni tag:".)')
@config.Positional('reference', 'Reference directory created by "nesoni make-reference:"')
@config.Section('reads', 'Fastq files containing SOLiD or Illumina reads.')
@config.Configurable_section('clip_runs_colorspace', 'Configuration options for clip-runs-colorspace:. Will be used with colorspace reads.')
@config.Configurable_section('clip_runs_basespace', 'Configuration options for clip-runs-basespace:. Will be used with basespace reads.')
@config.String_flag("aligner", "Aligner to use, basespace only. Options are 'bowtie2' or 'STAR'.")
@config.Int_flag("min_match", "STAR only: minimum number of matches required for alignment.")
@config.Float_flag('extension_prop_a', 'Extending alignments over genomic "A"s, what is the lowest proportion of "A"s allowed? (Basespace only.)')
@config.Int_flag('min_tail',
    'Minimum number of As to be called a poly(A) tail. Note if using Analyse_polya_batch (as usual), the setting in Analyse_poly_batch will override this value.'
    )
class Analyse_polya(config.Action_with_output_dir):
    reference = None
    tags = [ ]
    reads = [ ]
    consensus = False
    discard_multimappers = False
    delete_files = True

    aligner = "star"
    min_match = 0

    min_tail = 4

    clip_runs_colorspace = clip_runs.Clip_runs_colorspace()
    clip_runs_basespace = clip_runs.Clip_runs_basespace()
    extension_prop_a = 0.6

    _workspace_class = working_directory.Working

    #def get_polya_dir(self):
    #    return os.path.normpath(self.output_dir) + '-polyA'

    def get_filter_tool(self):
        if self.consensus:
            filter_tool = nesoni.Consensus
        else:
            filter_tool = nesoni.Filter
        return filter_tool(
            monogamous=self.discard_multimappers,
            random=True,
            infidelity=0,
            userplots=False,
        )

    def get_filter_action(self):
        return self.get_filter_tool()(working_dir = self.output_dir)

    #def get_polya_filter_action(self):
    #    return self.get_filter_tool()(working_dir = self.get_polya_dir())

    def run(self):
        assert self.reads, 'No read files given.'
        colorspace = [ io.is_colorspace(item) for item in self.reads ]
        assert len(set(colorspace)) == 1, 'Mixture of colorspace and basespace reads is not currently supported.'
        colorspace = colorspace[0]

        #polya_dir = self.get_polya_dir()

        working = working_directory.Working(self.output_dir, must_exist=False)
        working.set_reference(self.reference)
        reference = working.get_reference()

        #polya_working = working_directory.Working(polya_dir, must_exist=False)
        #polya_working.set_reference(self.reference)

        clipped_prefix = working/'clipped_reads'
        clipped_filename = clipped_prefix+('.csfastq.gz' if colorspace else '.fastq.gz')

        raw_filename = working/'alignments_raw.sam.gz'
        extended_filename = working/'alignments_extended.sam.gz'

        aligner = self.aligner.lower()
        assert aligner in ("bowtie2", "star")

        #polya_filename = working/'alignments_filtered_polyA.sam.gz'

        if colorspace:
            self.clip_runs_colorspace(
                filenames=self.reads,
                prefix=clipped_prefix,
                sample=working.name,
            ).make()
        else:
            self.clip_runs_basespace(
                filenames=self.reads,
                prefix=clipped_prefix,
                sample=working.name,
            ).make()

        cores = nesoni.coordinator().get_cores()

        if colorspace:
            nesoni.Execute(
                command = reference.shrimp_command(cs=colorspace, parameters=[ clipped_filename ]) + [ '--qv-offset', '33' ],
                execution_options = [ '-N', str(cores) ],
                output=raw_filename,
                cores=cores,
                prefix=working/'run_alignment'
                ).make()

        elif aligner == "bowtie2":
            nesoni.Execute(
                command = [
                    'bowtie2',
                    '--rg-id', '1',
                    '--rg', 'SM:'+working.name,
                    '--sensitive-local',
                    '-k', '10', #Up to 10 alignments per read
                    '-x', reference.get_bowtie_index_prefix(),
                    '-U', clipped_filename,
                    ],
                execution_options = [ '--threads', str(cores) ],
                output=raw_filename,
                cores=cores,
                prefix=working/'run_alignment'
                ).make()
        else:
            nesoni.Execute(
                command = [
                    'STAR',
                    '--genomeDir', reference/'star',
                    '--outFileNamePrefix', working/'star',
                    '--outSAMtype', 'SAM', #'Unsorted',
                    '--outStd', 'SAM',
                    '--readFilesIn', clipped_filename,
                    '--readFilesCommand', 'zcat',
                    # If we wanted no minimum proportion alignment could use this:
                    # (reads are clipped, so safe to leave default of 2/3 alignment)
                    #'--outFilterScoreMinOverLread', '0',
                    #'--outFilterMatchNminOverLread', '0',
                    '--outMultimapperOrder', 'Random',
                    # Only output 1 alignment for multmappers (NH still set)
                    #'--outSAMmultNmax', '1',
                    # Require alignment from start (not needed for clipped reads)
                    #'--alignEndsType', 'Extend5pOfRead1',
                    # No de novo introns, annotated introns will still be used
                    '--alignIntronMax', '20',
                    ] +
                    ([ '--outFilterMatchNmin', str(self.min_match) ] if self.min_match else [ ]),
                execution_options = [ '--runThreadN', str(cores) ],
                output=raw_filename,
                cores=cores,
                prefix=working/'run_alignment'
                ).make()

        if colorspace:
            extend_sam.Extend_sam_colorspace(
                input=raw_filename,
                output=extended_filename,
                reads=self.reads,
                reference_filenames=[ reference.reference_fasta_filename() ],
                tail=self.min_tail,
            ).make()
        else:
            extend_sam.Extend_sam_basespace(
                input=raw_filename,
                output=extended_filename,
                clips=[ clipped_prefix+'.clips.gz' ],
                reference_filenames=[ reference.reference_fasta_filename() ],
                prop_a = self.extension_prop_a,
                tail=self.min_tail,
            ).make()

        nesoni.Import(
            input=extended_filename,
            output_dir=self.output_dir,
            reference=[ self.reference ],
        ).make()

        self.get_filter_action().make()

        #Tail_only(
        #    input=working/'alignments_filtered.bam',
        #    output=polya_filename,
        #).make()

        #nesoni.Import(
        #    input=polya_filename,
        #    output_dir=polya_dir,
        #    reference=[ self.reference ],
        #).make()

        # This shouldn't actually filter out any alignments.
        # We do it to produce depth of coverage plots
        # and position-sorted BAM files.
        #self.get_polya_filter_action().make()

        nesoni.Tag(self.output_dir, tags=self.tags).make()
        #nesoni.Tag(polya_dir, tags=self.tags).make()

        if self.delete_files:
            # Delete unneeded files
            os.unlink(clipped_prefix+'.state')
            os.unlink(clipped_filename)
            os.unlink(working/'alignments.bam')
            os.unlink(working/'alignments_filtered.bam')
            os.unlink(working/'run_alignment.state')
            os.unlink(raw_filename)
            os.unlink(extended_filename)
            #os.unlink(polya_filename)

@config.help(
    "Use a sample from a previous pipeline run.",
    "")
@config.String_flag("abductee", "Path to sample.")
@config.Positional('reference', 'Reference directory created by "nesoni make-reference:". Filled in by Analyse_polya_batch.')
class Abduct_polya(config.Action_with_output_dir):
    abductee = None
    reference = None

    # No reads (don't participate in duplicate filename checking)
    reads = [ ]

    def __init__(self, *args, **argd):
        super(Abduct_polya, self).__init__(*args, **argd)
        if self.abductee:
            abductee = working_directory.Working(self.abductee, must_exist=True)
            self.tags = abductee.param["tags"]

    def run(self):
        working = working_directory.Working(self.output_dir, must_exist=False)
        abductee = working_directory.Working(self.abductee, must_exist=True)

        for filename in [
                "parameters",
                "alignments_filtered_sorted.bam",
                "alignments_filtered_sorted.bam.bai",
                "depths.pickle.gz",
                "clipped_reads_log.txt",
                "filter_log.txt",
                ]:
            io.symbolic_link(abductee/filename, working/filename)

@config.help(
'Analyse a set of samples and produce an HTML report, including depth of coverage plots, heatmaps, etc.',
"""\

""")
@config.String_flag('title', 'Analysis report title.')
@config.String_flag('file_prefix', 'Prefix for report filenames.')
@config.Bool_flag('do_fragile', 'Do steps that might fail? Some steps fail if too few peaks are called.')
@config.Bool_flag('do_bigwigs', 'Do bigwig file creation?')
@config.Int_flag('adaptor',
    'Minimum number of adaptor bases required, 0 for no filtering.'
    )
@config.Int_flag('min_tail',
    'Minimum number of non-templated As for a read to be called as having a poly(A) tail. Affects both peak calling and estimation of average tail lengths.'
    )
@config.Int_flag('clip_tail',
    'Tails longer than this will be reduced to this length. 0 for no clipping.'
    )
@config.Int_flag('peak_min_depth',
    'Number of poly(A) reads ending at nearly the same position required in order to call a peak.'
    )
@config.Bool_flag('peak_polya',
    'Call peaks using poly(A) reads only.'
    )
@config.Int_flag('peak_length', 'Length of peak features, ie how far back a read may be from a peak and still be counted. Should be around read length or a little shorter.')
@config.Float_flag('peak_min_tail', 'Minimum average tail length required for a peak.')
@config.Bool_flag('peak_pair_utr', 'For the older peak-pair tests, consider only peaks in the 3\'UTR?')
@config.Int_flag('extension', 'How far downstrand of the given annotations a read or peak belonging to a gene might be. However tail-tools (with a recently created reference directory) will not extend over coding sequence.')
@config.String_flag('types', 'Comma separated list of feature types to use as genes. Default is "gene".')
@config.String_flag('parts', 'Comma separated list of feature types that make up features. Default is "exon". Alternatively you might use "three_prime_utr" for a stricter definition of where we expect reads or "gene" for a broad definition including introns.')
@config.Bool_flag('umis',
    'Should counting be based on UMIs (contained in read names)?')
@config.String_flag("species", 'Species for GO analysis. Currently supports Human ("Hs"), Saccharomyces cerevisiae ("Sc"), Caenorhabditis elegans ("Ce"), Mus musculus ("Mm")')
@config.String_flag("shiny_report_url", 'The main report contains a link to a shiny-app section. What URL will the shiny-app be served from?')
#@config.String_flag('blurb', 'Introductory HTML text for report')
#@config.String_flag('genome', 'IGV .genome file, to produce IGV plots')
#@config.String_flag('genome_dir', 'IGV directory of reference sequences to go with .genome file')
#@config.Bool_flag('include_plots', 'Include plots in report?')
#@config.Bool_flag('include_genome', 'Include genome in IGV plots tarball?')
#@config.Bool_flag('include_bams', 'Include BAM files in report?')
@config.Positional('reference', 'Reference directory created by "nesoni make-reference:"')
@config.Configurable_section('template',
    'Common options for each "sample:". '
    'Put this section before the actual "sample:" sections.',
    presets = [ ('default', lambda obj: Analyse_polya(), 'default "analyse-polya:" options') ],
    )
@config.Configurable_section_list('samples',
    'Samples for analysis. Give one "sample:" section for each sample.',
    templates = [ ],
    sections = [ ('sample', lambda obj: obj.template, 'A sample, parameters as per "analyse-polya:".') ],
    )
#@config.Configurable_section('analyse_tail_lengths', 'Parameters for "analyse-tail-lengths:"')
#@config.Section('extra_files', 'Extra files to include in report')
@config.Section('groups', 'Sample groups, given as a list of <selection>=<name>.')
@config.Configurable_section_list('tests',
    'Differential tests to perform.',
    templates = [ ],
    sections = [ ('test', lambda obj: tail_tools.Test(), 'A differential test, parameters as per "test:".') ],
    )
class Analyse_polya_batch(config.Action_with_output_dir):
    file_prefix = ''
    title = 'PAT-Seq analysis'

    extension = 1000

    do_fragile = True
    do_bigwigs = True

    adaptor = 0
    min_tail = 4
    clip_tail = 0

    peak_min_depth = 50
    peak_polya = True
    peak_length = 300
    peak_min_tail = 15.0
    peak_pair_utr = True

    types = "gene"
    parts = "exon"

    umis = False

    species = ""
    shiny_report_url = "shiny/"

    reference = None
    samples = [ ]

    groups = [ ]

    tests = [ ]

    def run(self):
        #===============================================
        #                Sanity checks
        #===============================================

        assert len(set([ item.output_dir for item in self.samples ])) == len(self.samples), "Duplicate sample name."

        all_inputs = [ ]
        for sample in self.samples:
            all_inputs.extend(sample.reads)
        assert len(set(all_inputs)) == len(all_inputs), "Duplicate read filename."

        assert len(set([ item.output_dir for item in self.tests ])) == len(self.tests), "Duplicate test name."

        for test in self.tests:
            assert not test.analysis, "analysis parameter for tests should not be set, will be filled in automatically"

        #===============================================
        #                Run pipeline
        #===============================================

        names = [ sample.output_dir for sample in self.samples ]

        reference = reference_directory.Reference(self.reference, must_exist=True)

        workspace = io.Workspace(self.output_dir, must_exist=False)
        samplespace = io.Workspace(workspace/'samples', must_exist=False)
        expressionspace = io.Workspace(workspace/'expression', must_exist=False)
        testspace = io.Workspace(workspace/'test', must_exist=False)

        self._create_json()

        file_prefix = self.file_prefix
        if file_prefix and not file_prefix.endswith('-'):
            file_prefix += '-'

        samples = [ ]
        for sample in self.samples:
            samples.append(sample(
                samplespace / sample.output_dir,
                reference = self.reference,
                min_tail = self.min_tail,
                ))

        dirs = [ item.output_dir for item in samples ]

        clipper_logs = [ join(item.output_dir, 'clipped_reads_log.txt') for item in samples ]
        filter_logs = [ join(item.output_dir, 'filter_log.txt') for item in samples ]

        analyse_template = tail_lengths.Analyse_tail_counts(
            working_dirs = dirs,
            extension = self.extension,
            annotations = reference/'reference.gff',
            types = self.types,
            parts = self.parts,
            adaptor = self.adaptor,
            tail = self.min_tail,
            clip_tail = self.clip_tail,
            umis = self.umis,
            )

        with nesoni.Stage() as stage:
            for item in samples:
                item.process_make(stage)

        job_gene_counts = analyse_template(
            output_dir = expressionspace/'genewise',
            extension = self.extension,
            title = 'Genewise expression - ' + self.title,
            file_prefix = file_prefix+'genewise-',
            ).make

        job_peaks = _call(self._run_peaks,
            workspace=workspace,
            expressionspace=expressionspace,
            reference=reference,
            dirs = dirs,
            analyse_template = analyse_template,
            file_prefix=file_prefix,
            )

        job_norm = nesoni.Norm_from_samples(
            workspace/'norm',
            working_dirs = dirs
            ).make

        job_bigwig = bigwig.Polya_bigwigs(
            workspace/'bigwigs',
            working_dirs = dirs,
            norm_file = workspace/"norm.csv",
            peaks_file = workspace/("peaks", "relation-child.gff"),
            title = "IGV tracks - "+self.title
            ).make if self.do_bigwigs else _do_nothing

        job_norm_bigwig = _call(_serial, job_norm, job_bigwig)

        job_utrs = tail_tools.Call_utrs(
            workspace/('peaks','primary-peak'),
            self.reference,
            self.output_dir,
            extension=self.extension
            ).make if self.do_fragile else _do_nothing

        job_primpeak_counts = analyse_template(
            expressionspace/'primarypeakwise',
            annotations=workspace/('peaks','primary-peak-peaks.gff'),
            extension=0,
            types='peak',
            parts='peak',
            title='Primary-peakwise expression - ' + self.title,
            file_prefix=file_prefix+'primarypeakwise-',
            ).make if self.do_fragile else _do_nothing

        job_primpeak = _call(_serial, job_utrs, job_primpeak_counts)

        job_peak_primpeak_bigwig = _call(_serial,
            job_peaks,
            _call(_parallel, job_norm_bigwig, job_primpeak))

        job_count = _call(_parallel, job_gene_counts, job_peak_primpeak_bigwig)

        test_jobs = [ ]
        for test in self.tests:
            test_jobs.append(test(
                output_dir = testspace/test.output_dir,
                analysis = self.output_dir,
                ).make)

        job_test = _call(_parallel, *test_jobs)

        job_raw = self._extract_raw

        job_all = _call(_serial, job_count, _call(_parallel, job_raw, job_test))

        job_all()

        #===============================================
        #                   Report
        #===============================================

        r = reporting.Reporter(workspace/'report', self.title, self.file_prefix, style=web.style())

        if self.do_bigwigs:
            io.symbolic_link(source=workspace/'bigwigs', link_name=r.workspace/'bigwigs')
            r.write('<div style="font-size: 150%; margin-top: 1em; margin-bottom: 1em;"><a href="bigwigs/index.html">→ Load tracks into IGV</a></div>')

        tail_tools.Shiny(workspace/('report','shiny'), self.output_dir, title=self.title, species=self.species).run()
        r.write('<div style="font-size: 150%; margin-top: 1em; margin-bottom: 1em;"><a href="'+self.shiny_report_url+'" target="_blank">→ Interactive report (shiny)</a></div>')

        r.heading('Alignment to reference')

        r.report_logs('alignment-statistics',
            #[ workspace/'stats.txt' ] +
            clipper_logs + filter_logs + #filter_polya_logs +
            [ expressionspace/('genewise','aggregate-tail-counts'+('-umi' if self.umis else '')+'_log.txt') ],
            filter=lambda sample, field: (
                field not in [
                    'fragments','fragments aligned to the reference','reads kept',
                    'average depth of coverage, ambiguous',
                    'average depth of coverage, unambiguous',
                    ]
            ),
        )

        r.heading('Genewise expression')

        r.p("This is based on all reads within each gene (possibly from multiple peaks, or decay products).")

        io.symbolic_link(source=expressionspace/('genewise','report'),link_name=r.workspace/'genewise')
        r.p('<a href="genewise/index.html">→ Genewise expression</a>')

        r.heading('Peakwise expression')

        r.p("This shows results from all called peaks.")

        peak_filename = expressionspace/('peakwise','features-with-data.gff')
        r.p(r.get(peak_filename, name='peaks.gff') + ' - peaks called')

        self._describe_peaks(r)

        io.symbolic_link(source=expressionspace/('peakwise','report'),link_name=r.workspace/'peakwise')
        r.p('<a href="peakwise/index.html">→ Peakwise expression</a>')

        if self.do_fragile:
            r.subheading('Primary-peakwise expression')

            r.p("This is based on the most prominent peak in the 3'UTR for each gene. (Peak can be up to %d bases downstrand of the annotated 3'UTR end, but not inside another gene on the same strand.)" % self.extension)

            io.symbolic_link(source=expressionspace/('primarypeakwise','report'),link_name=r.workspace/'primarypeakwise')
            r.p('<a href="primarypeakwise/index.html">→ Primary-peakwise expression</a>')

            r.p(r.get(workspace/('peaks','primary-peak-peaks.gff')) + ' - primary peaks for each gene.')
            r.p(r.get(workspace/('peaks','primary-peak-utrs.gff')) + ' - 3\' UTR regions, based on primary peak call.')
            r.p(r.get(workspace/('peaks','primary-peak-genes.gff')) + ' - full extent of gene, based on primary peak call.')

        if self.tests:
            r.heading('Differential tests')
            for test in self.tests:
                io.symbolic_link(source=testspace/test.output_dir,link_name=r.workspace/('test-'+test.output_dir))
                r.p('<a href="test-%s">→ %s</a> '
                    % (test.output_dir, test.get_title()))

        web.Geneview_webapp(r.workspace/'view').run()

        r.heading('Gene viewers')
        r.p('Having identified interesting genes from heatmaps and differential tests above, '
            'these viewers allow specific genes to be examined in detail.')

        if self.groups and os.path.exists(workspace/('peak-shift','grouped.json')):
            r.get(workspace/('peak-shift','grouped.json'))
            r.p('<a href="view.html?json=%sgrouped.json">→ Gene viewer, grouped samples</a>' % r.file_prefix)

        if os.path.exists(workspace/('peak-shift','individual.json')):
            r.get(workspace/('peak-shift','individual.json'))
            r.p('<a href="view.html?json=%sindividual.json">→ Gene viewer, individual samples</a>' % r.file_prefix)

        r.heading('Raw data')

        r.p(r.tar('csv-files',glob.glob(workspace/('raw','*.csv'))))

        r.write('<ul>\n')
        r.write('<li> -info.csv = gene name and product, etc\n')
        r.write('<li> -count.csv = ' + ('UMI' if self.umis else 'read') + ' count\n')
        if self.umis:
            r.write('<li> -alignments.csv = actual aligning read count, before UMI based deduplication\n')
        r.write('<li> -mlog2-RPM.csv = moderated log2 Reads Per Million\n')
        r.write('<li> -tail.csv = average poly(A) tail length\n')
        r.write('<li> -tail-count.csv = poly(A) read count\n')
        r.write('<li> -proportion.csv = proportion of reads with poly(A)\n')
        r.write('<li> -norm.csv = read count normalization used for log2 transformation, heatmaps, differential tests, etc etc\n')
        r.write('</ul>\n')

        r.p('This set of genes was used in the analysis:')

        r.p(r.get(reference/'reference.gff') + ' - Reference annotations in GFF3 format')
        r.p(r.get(reference/'utr.gff') + ' - 3\' UTR regions')

        r.p('<b>%d further bases 3\' extension was allowed</b> beyond the GFF files above (but not extending into the next gene on the same strand).' % self.extension)

        r.write('<p/><hr>\n')
        r.subheading('About normalization and log transformation')

        r.p('Counts are converted to '
            'log2 Reads Per Million using Anscombe\'s variance stabilizing transformation '
            'for the negative binomial distribution, implemented in '
            'R package "varistran".')

        r.write('<p/><hr>\n')

        r.p('Reference directory '+self.reference)
        r.p('Tail Tools version '+tail_tools.VERSION)
        r.p('Nesoni version '+nesoni.VERSION)

        r.close()

    def _run_peaks(self, workspace, expressionspace, reference, dirs, analyse_template, file_prefix):
        shiftspace = io.Workspace(workspace/'peak-shift')

        peaks.Call_peaks(
            workspace/'peaks',
            annotations = reference/'reference.gff',
            extension = self.extension,
            min_depth = self.peak_min_depth,
            polya = self.peak_polya,
            min_tail = self.peak_min_tail,
            peak_length = self.peak_length,
            samples = dirs,
            ).make()

        analyse_template(
            expressionspace/'peakwise',
            annotations=workspace/('peaks','relation-child.gff'),
            extension=0,
            types='peak',
            parts='peak',
            title='Peakwise expression - ' + self.title,
            file_prefix=file_prefix+'peakwise-',
            ).make()

        if self.do_fragile:
            alternative_tails.Compare_peaks(
                shiftspace/'individual',
                norm_file=expressionspace/('peakwise','norm.csv'),
                utrs=reference/'utr.gff',
                utr_only=self.peak_pair_utr,
                top=2,
                reference=reference/'reference.fa',
                parents=workspace/('peaks','relation-parent.gff'),
                children=workspace/('peaks','relation-child.gff'),
                counts=expressionspace/('peakwise','counts.csv'),
                ).make()

        if self.groups:
            tail_lengths.Collapse_counts(
                shiftspace/'grouped-counts',
                counts=expressionspace/('peakwise','counts.csv'),
                groups=self.groups
                ).make()

            alternative_tails.Compare_peaks(
                shiftspace/'grouped',
                utrs=reference/'utr.gff',
                utr_only=True,
                top=2,
                reference=reference/'reference.fa',
                parents=workspace/('peaks','relation-parent.gff'),
                children=workspace/('peaks','relation-child.gff'),
                counts=shiftspace/'grouped-counts.csv',
                ).make()

    def _extract_raw(self):
        work = io.Workspace(self.output_dir, must_exist=False)
        raw = io.Workspace(work/'raw', must_exist=False)

        for name, counts, norms in [
                ('genewise',
                    work/('expression','genewise','counts.csv'),
                    work/('expression','genewise','norm.csv'),
                    ),
                ('primarypeakwise',
                    work/('expression','primarypeakwise','counts.csv'),
                    work/('expression','primarypeakwise','norm.csv'),
                    ),
                ('peakwise',
                    work/('expression','peakwise','counts.csv'),
                    work/('expression','peakwise','norm.csv'),
                    ),
                ('pairwise',
                    work/('peak-shift','individual-pairs.csv'),
                    work/('peak-shift','individual-pairs-norm.csv'),
                    ),
                ]:

            if not self.do_fragile and name in ('primarypeakwise','pairwise'):
                continue

            nesoni.Vst(
                raw/(name+'-mlog2-RPM'),
                counts,
                norm_file = norms
                ).make()

            counts_table = io.read_grouped_table(counts)
            io.write_csv_2(raw/(name+'-info.csv'), counts_table['Annotation'])
            io.write_csv_2(raw/(name+'-count.csv'), counts_table['Count'])
            io.write_csv_2(raw/(name+'-tail.csv'), counts_table['Tail'])
            io.write_csv_2(raw/(name+'-tail-count.csv'), counts_table['Tail_count'])
            io.write_csv_2(raw/(name+'-proportion.csv'), counts_table['Proportion'])

            if 'Alignments' in counts_table:
                io.write_csv_2(raw/(name+'-alignments.csv'), counts_table['Alignments'])

            norm_table = io.read_grouped_table(norms)
            io.write_csv_2(raw/(name+'-norm.csv'), norm_table['All'])

    def _create_json(self):
        workspace = io.Workspace(self.output_dir, must_exist=False)

        samples = [ ]
        groups = [ ]
        for sample in self.samples:
            this_groups = [ ]
            for item in self.groups:
                if selection.matches(
                        selection.term_specification(item),
                        sample.tags + [ sample.output_dir ]
                        ):
                    this_groups.append(selection.term_name(item))
            group = ','.join(this_groups) if this_groups else 'ungrouped'
            if group not in groups: groups.append(group)

            item = {
                'name' : sample.output_dir,
                'bam' : os.path.abspath(
                    workspace/('samples',sample.output_dir,'alignments_filtered_sorted.bam')
                    ),
                'group' : group,
                'tags' : sample.tags,
                }
            samples.append(item)

        obj = collections.OrderedDict()
        obj['reference'] = os.path.abspath( self.reference )
        obj['extension'] = self.extension
        obj['genes'] = os.path.abspath( workspace/('peaks','relation-parent.gff') )
        obj['peaks'] = os.path.abspath( workspace/('peaks','relation-child.gff') )
        obj['groups'] = groups
        obj['samples'] = samples

        with open(workspace/"plotter-config.json","wb") as f:
            json.dump(obj, f, indent=4)

    def _describe_peaks(self, r):
        workspace = io.Workspace(self.output_dir, must_exist=False)

        counts = io.read_grouped_table(workspace/("expression","peakwise","counts.csv"))["Count"]

        peak_counts = collections.defaultdict(int)
        read_counts = collections.defaultdict(int)
        total = 0
        for item in annotation.read_annotations(workspace/("peaks","relation-child.gff")):
            peak_counts[item.attr.get("Relation","None")] += 1
            read_counts[item.attr.get("Relation","None")] += sum(int(c) for c in counts[item.get_id()].values())
            total += 1

        total_reads = sum(read_counts.values())

        r.write("<p>\n")
        r.write("%d peaks\n" % total)

        for name, desc in [
                ("3'UTR", "in a 3' UTR"),
                ("Exon", "otherwise in an exon"),
                ("Downstrand", "otherwise downstrand of a non-coding RNA"),
                ("Intron", "otherwise in an intron"),
                ("Antisense", "otherwise antisense to a gene"),
                ("None", "couldn't be related to annotated genes"),
                ]:
            r.write("<br/>%d peaks and %.1f%% of reads %s\n" % (peak_counts[name], read_counts[name]*100.0/total_reads, desc))
        r.write("</p>\n")


import requests, urllib3
from config import Config

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

class UnauthorizedApiCall(Exception):
    pass

class ClientTmdbApi:
    url = "https://api.themoviedb.org/3/"
    cookie_tag = "Set-Cookie"
    session = requests.Session()
    session.headers["Accept"] = "application/json"
    session.headers["Authorization"] = Config.TMDB_API_CREDENTIALS

    def _request(self, api_method, json=None):
        if json is None:
            response = self.session.get(self.url + api_method, verify=False)
        else:
            response = self.session.post(self.url + api_method, json=json, verify=False)
            if response.status_code == 401:
                raise UnauthorizedApiCall()
        # required to avoid new session creation on every request
        if not self.session.headers.get(self.cookie_tag):
            self.session.headers[self.cookie_tag] = response.headers.get(
                self.cookie_tag
            )
        return response.json()

    def get_person_object(
        self, id_person=40239, object_name="movie_credits", language="cz-CZ"
    ):
        return self._request(
            "{}/{}/{}?language={}".format(
                "person", str(id_person), object_name, language
            )
        )

    def get_object_details(self, id_movie, object_name="movie"):
        return self._request("{}/{}".format(object_name, str(id_movie)))


import numpy as np
import graphviz
import random
from copy import deepcopy
from typing import Callable, List, Tuple, Optional
import matplotlib.pyplot as plt
import ast
import json as std_json

'''
==== Formal definition ====
P.S. n is number of total vertices.
     v is vertex.

------------------------------------------------------
  Name           Type              Shape      Notes
-----------------------------------------------
  strategies     List[float]       n(1D)      each v's strategy
  edges          ndarray(float)    nxn(2D)    1=connect, 0=disconnect; MUST be bi-directed if it's undirected.
  weights        List[float]       n(1D)      each v's special value. You can ignore this. You can use it as weight,
                                              or even use it to represent other meaning.
'''

class Graph:
    def __init__(self, strategies: Optional[List[float]], edges: Optional[np.ndarray], weights: Optional[List[float]]):
        """
        :param strategies: List with length n. (n is the number of vertices)
        :param edges: n by n ndarray.
        """
        self.strategies: Optional[List[float]] = strategies
        self.edges: Optional[np.ndarray] = edges
        self.weights: Optional[List[float]] = weights

    def __repr__(self):
        """Show the graph as a string."""
        representation = "\nVertexNo\tEdges\tStrategies\tWeights\n"
        for i in range(len(self.edges) + 1):
            if i > 0:
                for j in range(len(self.edges) + 1):
                    if j == 0:
                        representation += f"{i:03} -> [ "
                    else:
                        representation += str(self.edges[i - 1][j - 1]) + " "
                representation += f"]\ts{self.strategies[i - 1]}\tw{self.weights[i - 1]}\n"
        return representation

    def n_vertices(self) -> int:
        """Total number of vertices."""
        return int(len(self.edges))

    def plot_graph(self, filename='plotted_graph',
                   show=False,
                   picked_vertex: Optional[int] = None,
                   have_better_choice: Optional[List[int]] = None):
        """
        :param filename:
        :param show: whether plt show the graph
        :param have_better_choice: highlight those vertices which has other strategy to improve its payoff in this round
                                   (shown as a octagon)
        :param picked_vertex: highlight  the vertex picked
                              (may indicate its the one change its strategy to improve its payoff)
                              (shown as a tripleoctagon)
        """
        d = graphviz.Digraph(filename=filename)
        d.format = 'png'
        d.attr(rankdir='LR', concentrate='True')
        # Vertex index from 1        # Vertices
        d.attr('node', shape='circle')
        for vertex in range(self.n_vertices()):
            color = 'lightblue2' if self.strategies[vertex] != 0. else 'black'
            style = 'filled' if self.strategies[vertex] != 0. else ''
            shape = 'octagon' if (have_better_choice is not None and vertex in have_better_choice) else 'circle'
            shape = 'tripleoctagon' if (picked_vertex is not None and vertex == picked_vertex) else shape
            v_name = f'{vertex}\n(w{self.weights[vertex]})'
            d.node(v_name, color=color, style=style, shape=shape)
        for i in range(self.n_vertices()):
            for j in range(self.n_vertices()):
                if self.edges[i][j] != 0.:
                    d.edge(f'{i}\n(w{self.weights[i]})', f'{j}\n(w{self.weights[j]})')
        d.render(view=False)
        if show is True:
            plt.imshow(plt.imread(f'{filename}.png'))
            plt.axis('off')
            plt.show()

    def strategy_1_weight_sum(self) -> float:
        """Sum of the weight of all vertices which pick strategy 1"""
        s_w_sum = 0.0
        for v, s in enumerate(self.strategies):
            if s != 0.0:
                s_w_sum += self.weights[v]
        return s_w_sum

    def deserialize(self, codes: str) -> None:
        """Fetch strategies, edges, and weights from a string."""
        contents = ast.literal_eval(codes)
        self.strategies = contents['strategies']
        self.edges = np.array(contents['edges'])
        self.weights = contents['weights']

    @staticmethod
    def deserialize_edges(s: str) -> np.ndarray:
        return np.array([line.split(' ') for line in s.split('\n')])

    @staticmethod
    def serialize_edges(edges: np.ndarray) -> str:
        s = ''
        for row in edges:
            for element in row:
                s += element + ' '
        return s.strip()

    @staticmethod
    def deserialize_strategies(s: str) -> List[float]:
        return list(map(float, s.split(' ')))

    @staticmethod
    def serialize_strategies(strategies: List[float]) -> str:
        return ' '.join(map(str, strategies))

    @staticmethod
    def deserialize_weights(s: str) -> List[float]:
        return list(map(float, s.split(' ')))

    @staticmethod
    def serialize_weights(weights: List[float]) -> str:
        return ' '.join(map(str, weights))

    def serialize(self) -> str:
        return std_json.dumps(
            {'edges': self.edges.tolist(), 'strategies': list(self.strategies),
             'weights': list(map(float, self.weights))})

    @staticmethod
    def create_graph_with_edges(edges: np.ndarray) -> "Graph":
        """Create a default graph only given edges. Strategies -> all zero. Weights -> all 1."""
        return Graph(strategies=Strategy.zeros(len(edges)), edges=edges, weights=VertexWeight.same(len(edges)))

class Strategy:
    @staticmethod
    def zeros(n_vertices) -> List[float]:
        return np.zeros((n_vertices,)).tolist()

    @staticmethod
    def ones(n_vertices) -> List[float]:
        return np.ones((n_vertices,)).tolist()

    @staticmethod
    def partial_ones(n_vertices, prob_one: float) -> List[float]:  # prob_one: probability of one to appear
        """Each vertex has a chance to select strategy one, otherwise zero."""
        strategies = np.zeros((n_vertices,))  # 0 is default action.
        options = [0.0, 1.0]
        distribution = [1 - prob_one, prob_one]
        for i in range(n_vertices):
            strategies[i] = random.choices(options, distribution)[0]
        return strategies.tolist()

class VertexWeight:

    @staticmethod
    def same(n_vertices) -> List[float]:
        return np.ones((n_vertices,)).tolist()

    @staticmethod
    def ones(n_vertices) -> List[float]:
        return VertexWeight.same(n_vertices)

    @staticmethod
    def random(n_vertices) -> List[float]:
        """Permutation all weights and distribute them."""
        # Other type you may try:
        # return (np.floor(np.random.random((n_vertices,)) * 5) % 5 + 1) / 50 + 0.97
        # return (np.floor(np.random.random((n_vertices,)) * 5) % 5 + 1)
        # return np.random.normal(10, 3, (n_vertices,))
        # return np.random.randint(1, n_vertices + 1, size=(n_vertices,))
        # return np.random.randint(0, n_vertices, size=(n_vertices,))
        pool = np.arange(n_vertices)
        np.random.shuffle(pool)
        return pool.tolist()

class Edge:
    @staticmethod
    def vertex_index(n_vertices: int, idx: int) -> int:
        """Adjust idx if it is not in the bound."""
        if idx < 0:
            return n_vertices + idx
        if idx >= n_vertices:
            return idx - n_vertices
        return idx

    @staticmethod
    def ws_model(n_vertices, k_nearest, rewire_prob) -> np.ndarray:
        """
        Each vertex connects to its k_nearest neighbors.
        K here only support even numbers.
        Half of k will try rewiring to other vertices.
        Vertex numbers are from 0 to (n_vertices - 1)
        Edge Matrix:
            - Indices are vertex no.
            - 0 means unconnected; 1 means connected.
            - Init state is bi-directional.
        Wiki: https://en.wikipedia.org/wiki/Watts%E2%80%93Strogatz_model
        """
        edges = np.zeros((n_vertices, n_vertices))

        # Connect k nearest neighbors
        if k_nearest % 2 != 0:
            raise NotImplementedError('K-nearest must be even.')
        for vertex in range(n_vertices):
            for i in range(k_nearest):
                if i < k_nearest // 2:
                    target = vertex + (i - k_nearest // 2)
                else:
                    target = vertex + (i - k_nearest // 2 + 1)
                # Handle targets not in 0~n_vertices
                target = Edge.vertex_index(n_vertices, target)
                #
                edges[vertex][target] = 1.0
        # Rewire with a probability
        if k_nearest % 2 != 0:
            raise NotImplementedError('K-nearest must be even.')
        random_options = [True, False]
        random_distribution = [rewire_prob, 1. - rewire_prob]
        for vertex in range(n_vertices):
            for i in range(k_nearest // 2):
                do_rewire = random.choices(random_options, random_distribution)
                if do_rewire[0]:
                    old_target = Edge.vertex_index(n_vertices, vertex + i + 1)
                    # Erase original edge
                    edges[vertex][old_target] = 0.
                    edges[old_target][vertex] = 0.
                    # Rewire
                    available_vertices = []
                    for possible_target in range(n_vertices):
                        if possible_target != vertex and possible_target != old_target and \
                                edges[vertex][possible_target] == 0.:
                            available_vertices.append(possible_target)
                    new_target = available_vertices[np.random.randint(len(available_vertices))]
                    edges[vertex][new_target] = 1.
                    edges[new_target][vertex] = 1.
        return edges

class Utility:
    """Each utility function here must have at least 2 arguments: graph & vertex."""

    @staticmethod
    def maximal_independent_set(graph: Graph, vertex: int, alpha=2.0) -> float:
        """Unweighted MIS game.
        :param graph:
        :param vertex: the vertex to calculate utility
        :param alpha:
        """
        if graph.strategies[vertex] != 0.0:
            utility = 1.0
            for neighbor in range(graph.n_vertices()):
                if graph.edges[vertex][neighbor] != 0. and vertex != neighbor:
                    utility += -1 * alpha * graph.strategies[vertex] * graph.strategies[neighbor]
        else:
            utility = 0.
        return utility

    @staticmethod
    def weighted_maximal_independent_set_1(graph: Graph, vertex: int, alpha=2.0) -> float:
        """WMIS with priority function: w(v)/((degree(v)+1)"""
        priority = []
        for v in range(len(graph.edges)):
            degree = sum(graph.edges[v])
            priority.append(graph.weights[v] / (degree + 1))
        #
        if graph.strategies[vertex] != 0.0:
            utility = 1.0
            for neighbor in range(graph.n_vertices()):
                if graph.edges[vertex][neighbor] != 0. and vertex != neighbor and \
                        priority[vertex] <= priority[neighbor]:  # using priority
                    utility += -1 * alpha * graph.strategies[vertex] * graph.strategies[neighbor]
        else:
            utility = 0.
        return utility

    @staticmethod
    def weighted_maximal_independent_set_2(graph: Graph, vertex: int, alpha=2.0) -> float:
        """WMIS with priority function: w(v)/sum of all v's closed neighbors' w"""
        priority = []
        for v in range(len(graph.edges)):
            tmp_p = graph.weights[v]
            for other_v in range(len(graph.edges)):
                if graph.edges[v][other_v] != 0.0:
                    tmp_p += graph.weights[other_v]
            priority.append(graph.weights[v] / tmp_p)
        #
        if graph.strategies[vertex] != 0.0:
            utility = 1.0
            for neighbor in range(graph.n_vertices()):
                if graph.edges[vertex][neighbor] != 0. and vertex != neighbor and \
                        priority[vertex] <= priority[neighbor]:  # using priority
                    utility += -1 * alpha * graph.strategies[vertex] * graph.strategies[neighbor]
        else:
            utility = 0.
        return utility

class Game:
    """
    In this game, players only have 2 actions:
            1. Turn off (0)
            2. Turn on (1)
    """

    def __init__(self, graph: Graph, utility_func: Callable):
        self.graph = graph
        self.utility_func = utility_func

    @staticmethod
    def check_nash_equilibrium(graph: Graph, utility_func: Callable, alpha=None) -> List[int]:
        """Check what vertices can improve its utility function by changing its original strategy.
        :param graph:
        :param utility_func:
        :param alpha:
        return the idx of those vertices which can improve its utitlity.
        """
        better_choice_vertices = []
        for vertex in range(graph.n_vertices()):
            temp = graph.strategies[vertex]
            if alpha is None:
                cur_utility = utility_func(graph=graph, vertex=vertex)
                graph.strategies[vertex] = 1. if temp == 0. else 0.
                new_utility = utility_func(graph=graph, vertex=vertex)
            else:
                cur_utility = utility_func(graph=graph, vertex=vertex, alpha=alpha)
                graph.strategies[vertex] = 1. if temp == 0. else 0.
                new_utility = utility_func(graph=graph, vertex=vertex, alpha=alpha)
            #
            graph.strategies[vertex] = temp
            #
            if new_utility > cur_utility:
                better_choice_vertices.append(vertex)
        return better_choice_vertices

    @staticmethod
    def run_a_time(graph, utility_func, alpha=None) -> Tuple[bool, List, Graph, Optional[int]]:  # True means terminated
        better_choice_vertices = Game.check_nash_equilibrium(graph=graph, utility_func=utility_func, alpha=alpha)
        if len(better_choice_vertices) == 0:
            return True, better_choice_vertices, graph, None  # Reach NE
        # Randomly pick one to change its action to improve its utility
        selected_index = np.random.randint(len(better_choice_vertices))
        selected_vertex = better_choice_vertices[selected_index]
        # Switch from 0/1 to 1/0
        graph.strategies[selected_vertex] = 0. if graph.strategies[selected_vertex] == 1. else 1.
        return False, better_choice_vertices, graph, selected_vertex

    def run(self, show_each_move=False, alpha=None) -> Tuple[Graph, int]:
        graph = deepcopy(self.graph)
        move_cnt = 0
        while True:
            if show_each_move:
                graph.plot_graph(show=True)
            if self.run_a_time(graph=graph, utility_func=self.utility_func, alpha=alpha)[0]:
                break
            move_cnt += 1
        return graph, move_cnt

    @staticmethod
    def check_real_independent_set(graph: Graph):
        # Method: test each vertex
        #         Turn a vertex with strategy 0 to 1, and see if it doesn't connect any vertices with strategy 1.
        #         If so, the new graph is an independent set so the original graph is not a "maximal" independent set.
        n_vertices = len(graph.edges)
        for v in range(n_vertices):
            if graph.strategies[v] != 0.:
                continue
            no_neighbor_with_s1 = True
            # See if a strategy-0 vertex's all neighbors are with strategy-0.
            for other_v in range(n_vertices):
                if other_v == v:
                    continue
                if graph.edges[v][other_v] == 1. and graph.strategies[other_v] == 1.:
                    no_neighbor_with_s1 = False
                    break
            if no_neighbor_with_s1 is True:
                return False  # TODO: check again
        return True

# times = 100
# avg_moves = []
# avg_cardinality = []
# all_p = np.arange(10.) / 10.
# for rewire_p in all_p:
#     total_moves = 0
#     total_cardinality = 0
#     for _ in range(times):
#         init_edges = Edge.ws_model(n_vertices=30, k_nearest=4, rewire_prob=rewire_p)
#         init_strategies = Strategy.partial_ones(n_vertices=len(init_edges), prob_one=0.5)
#         init_weights = VertexWeight.random(n_vertices=len(init_edges))
#         g = Graph(strategies=init_strategies, edges=init_edges, weights=init_weights)
#         m = Game(graph=g, utility_func=Utility.maximal_independent_set)
#         result_graph, moves = m.run(show_each_move=False)
#         total_moves += moves
#         total_cardinality += sum(result_graph.strategies)
#         g.plot_graph(show=True)
#         exit(1)
#     avg_moves.append(total_moves / times)
#     avg_cardinality.append(total_cardinality / times)
# plt.plot(all_p, avg_moves, label='avg_moves')
# plt.plot(all_p, avg_cardinality, label='avg_cardinality')
# plt.legend()
# plt.show()

# TODO: verify that the game state is a valid solution


class Node:
    def __init__(self, value=None):
        self.val = value
        self.next = None

def print_node(first):
    current = first.next
    while current is not None:
        print(current.val, "-->", end="")
        current = current.next

def make_linked_list(tab):
    first = Node()
    current = first
    for i in tab:
        temp = Node(i)
        current.next = temp
        current = temp
    return first

def dod(first1, first2):
    current1 = first1.next
    current2 = first2.next
    new_list = Node()
    new_current = new_list
    przen = 0
    while current1.next is not None and current2.next is not None:
        new_digit = current1.val + current2.val + przen
        przen = 0
        if new_digit > 9:
            przen = 1
            new_digit -= 10
        temp = Node(new_digit)
        new_current.next = temp
        new_current = temp
        current1 = current1.next
        current2 = current2.next
    # if current1.next is None and current2.next is None:
    new_digit = current1.val + current2.val + przen
    przen = 0
    if new_digit > 9:
        przen = 1
        new_digit -= 10
    temp = Node(new_digit)
    new_current.next = temp
    new_current = temp
    while current1.next is not None:
        new_digit = current1.val + przen
        print(przen,new_digit)
        przen = 0
        if new_digit > 9:
            przen = 1
            new_digit -= 10
        temp = Node(new_digit)
        new_current.next = temp
        new_current = temp
        current1=current1.next
    while current2.next is not None:
        new_digit =  current2.val + przen
        przen = 0
        if new_digit > 9:
            przen = 1
            new_digit -= 10
        temp = Node(new_digit)
        new_current.next = temp
        new_current = temp
        current2=current2.next
    if przen == 1:
        temp = Node(1)
        new_current.next = temp
        # new_current = temp
    return new_list

tab = [1, 2, 3, 4,5]
tab2 = [5, 6, 7, 8]
lista = make_linked_list(tab)
print_node(lista)
print()
lista2 = make_linked_list(tab2)
print_node(lista2)
print()
new = dod(lista, lista2)
print()
print_node(new)


import json
import logging
import os

import pytest
from librelingo_fakes import fakes
from librelingo_json_export.export import _export_skill

@pytest.fixture
def export_path():
    return fakes.path()

def test_creates_the_challenges_file(fs, export_path):
    random_name, fake_skill = fakes.get_fake_skill()
    _export_skill(export_path, fake_skill, fakes.course1)
    assert os.path.exists(export_path / "challenges" / f"animals-{random_name}.json")

def test_creates_the_introduction_file(fs, export_path):
    fake_name = fakes.fake_string()
    introduction = f"# *Hello* (https://example.com)[_{fake_name}_]!"
    random_name, fake_skill = fakes.get_fake_skill(introduction=introduction)
    _export_skill(export_path, fake_skill, fakes.course1)
    with open(
        export_path / "introduction" / f"animals-{random_name}.md"
    ) as animals_file:
        introduction_file_content = animals_file.read()
        assert introduction_file_content == introduction

def test_does_not_create_an_introduction_file_if_theres_no_introduction(
    fs, export_path
):
    random_name, fake_skill = fakes.get_fake_skill()
    _export_skill(export_path, fake_skill, fakes.course1)
    assert not os.path.exists(
        export_path / "introduction" / f"animals-{random_name}.md"
    )

@pytest.fixture
def mock_get_skill_data(mocker):
    return mocker.patch("librelingo_json_export.export._get_skill_data")

def test_calls__get_skill_data_with_correct_value(fs, export_path, mock_get_skill_data):
    mock_get_skill_data.return_value = []
    _export_skill(export_path, fakes.skillWithPhraseAndWord, fakes.course1)
    mock_get_skill_data.assert_called_with(fakes.skillWithPhraseAndWord, fakes.course1)

def test_writes_correct_value_into_json_file(fs, export_path, mock_get_skill_data):
    fake_skill_data = {"fake_skill_data": 1000}
    mock_get_skill_data.return_value = fake_skill_data
    _export_skill(export_path, fakes.skillWithPhraseAndWord, fakes.course1)
    with open(export_path / "challenges" / "masculine.json") as masculine_file:
        assert json.loads(masculine_file.read()) == fake_skill_data

def test_assert_logs_correctly(fs, caplog, export_path):
    with caplog.at_level(logging.INFO, logger="librelingo_json_export"):
        _, fake_skill = fakes.get_fake_skill()
        _export_skill(export_path, fake_skill, fakes.course1)
        assert caplog.record_tuples[0] == (
            "librelingo_json_export",
            logging.INFO,
            f"Writing skill '{fake_skill.name}'",
        )


def __intermediate_dns_servers(self, uci, address):
        """
        determines UCI interface "dns" option
        """
        # allow override
        if 'dns' in uci:
            return uci['dns']
        # ignore if using DHCP or if "proto" is none
        if address['proto'] in ['dhcp', 'dhcpv6', 'none']:
            return None
        dns = self.netjson.get('dns_servers', None)
        if dns:
            return ' '.join(dns)


from django.forms import ModelForm
from django.utils.translation import gettext_lazy as _
from .models import Profile, Tool
from django import forms
from django.contrib.auth.models import User

class ProfileForm(ModelForm):
    class Meta:
        model = Profile
        fields = ('cc_number', 'cc_exp', 'cc_fullname','zip_code', 'phone', 'profile_pic', 'terms_and_cond',)
        labels = {
            'cc_number': _('* Credit Card #:'),
            'cc_exp': _('* Credit Card Exp:'),
            'cc_fullname': _('* Full name on Credit Card:'),
            'zip_code': _('Zip Code:'),
            'profile_pic': _('Profile Image:'),
            'terms_and_cond': _('I agree to the terms and conditions:'),
        }
    def clean(self):
        super(ProfileForm, self).clean()
        tac = self.cleaned_data.get('terms_and_cond')
        if tac != True:
            self._errors['terms_and_cond'] = self.error_class(['You must agree to the terms and conditions to use this site!'])
        return self.cleaned_data

class ToolForm(ModelForm):
    class Meta:
        model = Tool
        fields = ('name', 'description', 'tool_value', 'for_sale', 'visible', 'tool_pic')
        labels = {
            'tool_value': _('Tool Value ($ USD)')
        }

class UserUpdateForm(ModelForm):
    email = forms.EmailField(
    max_length=100,
    required = True,
    widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Email'}),
    )
    username = forms.CharField(
    max_length=100,
    required = True,
    widget=forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Username'}),
    )
    class Meta:
        model = User
        fields = ['username', 'email',]

class ProfileUpdateForm(ModelForm):
    class Meta:
        model = Profile
        fields = ('cc_number', 'cc_exp', 'cc_fullname','zip_code', 'phone', 'profile_pic',)
        labels = {
            'cc_number': _('* Credit Card #:'),
            'cc_exp': _('* Credit Card Exp:'),
            'cc_fullname': _('* Full name on Credit Card:'),
            'zip_code': _('Zip Code:'),
            'profile_pic': _('Profile Image:'),
        }


#!/usr/bin/env python3

import hashlib, binascii

class Transaction(object):

    def __init__(self, sender, recipient, amount):
        self.sender = sender
        self.recipient = recipient
        self.amount = amount

    def transactionhash(self):
        #concatenation of the sender, recipient, amount in appropriate order
        concat = self.sender + self.recipient + self.amount
        hashed = hashlib.sha256(bytes(concat, 'utf-8')).hexdigest()
        return hashed

class CoinbaseTransaction(Transaction):

    def __init__(self, amount = '50'):
        self.sender = bytes(b'0' * 34).decode()
        self.amount = amount
        self.recipient = None


def modelsInfo(self, modelIDs):
    """ Get ALL info for a set of models

    WARNING!!!: The order of the results are NOT necessarily in the same order as
    the order of the model IDs passed in!!!

    Parameters:
    ----------------------------------------------------------------
    modelIDs:    list of model IDs
    retval:      list of nametuples containing all the fields stored for each
                    model.
    """
    assert isinstance(modelIDs, self._SEQUENCE_TYPES), (
      "wrong modelIDs type: %s") % (type(modelIDs),)
    assert modelIDs, "modelIDs is empty"

    rows = self._getMatchingRowsWithRetries(
      self._models, dict(model_id=modelIDs),
      [self._models.pubToDBNameDict[f]
       for f in self._models.modelInfoNamedTuple._fields])

    results = [self._models.modelInfoNamedTuple._make(r) for r in rows]

    # NOTE: assetion will also fail if modelIDs contains duplicates
    assert len(results) == len(modelIDs), "modelIDs not found: %s" % (
      set(modelIDs) - set(r.modelId for r in results))

    return results


import os
from pathlib import Path
import logging

logging.basicConfig(level=logging.INFO)

project_name = "Currency_Converter_Chatbot"

list_of_files = [
    # ".github/workflows/.gitkeep",
    f"src/{project_name}/components.py",
    f"src/{project_name}/entity/config_entity.py",
    f"src/{project_name}/exception.py ",
    f"src/{project_name}/logger.py ",
    f"src/{project_name}/utils.py ",
    "app.py",
    "requirements.txt",
    "setup.py",
]

for filepath in list_of_files:
    filepath = Path(filepath)
    filedir, filename = os.path.split(filepath)

    if filedir != "":
        os.makedirs(filedir, exist_ok=True)
        logging.info(f"Creating directory:{filedir} for the file {filename}")

    if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0):
        with open(filepath,'w') as f:
            pass
            logging.info(f"Creating empty file: {filepath}")

    else:
        logging.info(f"{filename} is already exists")


import unittest
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains

from helpers.base_test import BaseTest
from helpers.decorators import screenshot_decorator

class HoversPageTests(BaseTest):
    @screenshot_decorator
    def testHovers_whenAvatarIsHoveredOver_shouldCaptionAppearUnderTheImage(self):
        page_url = self.base_url + "hovers"
        self.driver.get(page_url)

        actions = ActionChains(self.driver)

        avatar = self.driver.find_elements(By.XPATH, '//*[@class="figure"]/img')[0]
        actions.move_to_element(avatar)
        actions.perform()

        caption = self.driver.find_elements(By.XPATH, '//*[@class="figcaption"]/h5')[0]
        caption_text = caption.text

        expected_caption_text = "name: user1"

        self.assertEqual(expected_caption_text, caption_text,
                         f"Expected paragraph text differ from actual paragraph text for page url:"
                         f" {self.driver.current_url}.")

if __name__ == '__main__':
    unittest.main()


import streamlit as st
import string
from nltk.tokenize import word_tokenize
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from datetime import datetime
import Stemmer                              # Descargar PyStemmer, pip install PyStemmer
import pandas as pd
import plotly.express as px
import plotly.figure_factory as plt
import joblib

# Funcion que cuenta el numero de fichero que incluye una lista
def contar_ficheros(lista_ficheros: list):
    return len(lista_ficheros)

# Funcion que convierte a minusculas el texto pasado como argumento y lo tokeniza/separa las palabras del texto
def tokenizar_texto(texto):
    lista_tokens = word_tokenize(str(texto.read(), encoding="utf8").lower(), language="spanish")
    return lista_tokens

# Funcion en la que aplicamos un lista de parada, es decir, le pasamos un fichero con las palabras o tokens que nos
# interesa eliminar del texto procesado y las quitamos
def limpiar_texto(lista_tokens: list):
    palabras = []
    # Abrimos nuestra lista de parada
    fichero_parada = open("Lista_Stop_Words.txt", "r", encoding="utf8")
    # Hemos leido el fichero como un string y necesitamos una lista de strings con las palabras a eliminar,
    # para ello aplicamos la funcion split para separar el texto por los espacios
    lista_parada = fichero_parada.read().split("\n")
    # Creamos una lista con los diferentes caracteres(!"#$%&'()*+, -./:;<=>?@[\]^_`{|}~) y los añadimos a nuestra
    # lista de parada, pues no nos son relevantes tampoco a la hora de aplicar nuestro algoritmo de aprendizaje
    puntuacion = list(string.punctuation)
    lista_parada += puntuacion
    # Aplicamos la lista de parada y nos quedamos solo con las palabras/tokens que nos interesan
    for palabra in lista_tokens:
        if palabra not in lista_parada:
            palabras.append(palabra)
    return palabras

# Funcion con la que aplicaremos el stemming (esta técnica nos ayudará a simplificar o reducir los tokens o palabras al stem)
def stemming(lista_palabras: list):
    texto = ""
    # Cogemos el stemmer de snowball en español
    stemmer = Stemmer.Stemmer('spanish')
    for palabra in lista_palabras:
        # Cogemos el stem de cada palabra
        s = stemmer.stemWord(palabra)
        texto = texto + " " + s
    return texto

# Funcion que genera una coleccion con los textos procesados, es decir, aplicando la tokenizacion, la transformacion a
# minusculas, eliminar las palabras que no se quieren a partir de una lista de parada y aplicacion de un stemmer(Snowball)
@st.cache
def generar_coleccion(lista_textos: list):
    coleccion = []
    for texto in lista_textos:
        # Aplicamos tokenizacion del texto
        tokens = tokenizar_texto(texto)
        # Aplicamos lista de parada
        lista_limpia = limpiar_texto(tokens)
        # Aplicamos stemmer
        texto = stemming(lista_limpia)
        coleccion.append(texto)
    return coleccion

def seleccionar_algoritmo(algoritmo: str):
    model = ""
    if algoritmo == "Gradient Boosted Tree":
        model = GradientBoostingClassifier()
    elif algoritmo == "Support Vector Machine":
        model = SVC()
    elif algoritmo == "Arbol Decision":
        model = DecisionTreeClassifier()
    return model

# Funcion que asigna el tipo de noticia (odio o no odio) a la que pertenece el texto
@st.cache
def asociar_clase(odio: list, no_odio: list):
    clase_odio = []
    clase_no_odio = []
    for texto_odio in odio:
        clase_odio.append('Odio')
    for texto_no in no_odio:
        clase_no_odio.append('No Odio')
    return clase_odio + clase_no_odio

def precision(tp: int, fp: int):
    return tp/(tp + fp)

def recall(tp: int, fn: int):
    return tp/(tp + fn)

def entrenar_modelo(algoritmo: str, coleccion_documentos: list, clases: list):
    tf = TfidfVectorizer()
    matriz_idf = tf.fit_transform(coleccion_documentos).toarray()
    df = pd.DataFrame(matriz_idf , columns=tf.get_feature_names_out())
    clf = seleccionar_algoritmo(algoritmo)
    X_train, X_test, Y_train, Y_test = train_test_split(df, clases, test_size=0.3)

    X_train = pd.get_dummies(X_train)
    X_test = pd.get_dummies(X_test)
    modelo = clf.fit(X_train, Y_train)
    y_pred = modelo.predict(X_test)
    tn, fp, fn, tp = confusion_matrix(Y_test, y_pred).ravel()
    cm = [[tn, fp], [fn, tp]]
    col1, col2, col3 = st.columns(3)
    with col1:
        st.metric("Precision: ", "{:.2f}".format((precision(tp, fp) * 100)) + "%", delta="{:.2f}".format((precision(tp, fp) * 100) - 95) + "%")
    with col2:
        st.metric("Accuracy: ", "{:.2f}".format(modelo.score(X_test, Y_test) * 100) + "%", delta="{:.2f}".format((modelo.score(X_test, Y_test) * 100) - 95) + "%")
    with col3:
        st.metric("Recall: ", "{:.2f}".format((recall(tp, fn) * 100)) + "%", delta="{:.2f}".format((recall(tp, fn) * 100) - 95) + "%")
    st.write("Resultados Entrenamiento: ")
    figura = plt.create_annotated_heatmap(cm, x=['Verdadero No Odio', 'Verdadero Odio'], y=['Predicho No Odio', 'Predicho Odio'], colorscale='Viridis')
    st.plotly_chart(figura)
    joblib.dump(tf.vocabulary_, 'vocabulario.bin')
    return modelo

def visualizacion_previa(odio, no_odio, algoritmo):
    num_odio = contar_ficheros(odio)
    num_no_odio = contar_ficheros(no_odio)

    ejemplares = {'Odio': num_odio, 'No Odio': num_no_odio}
    st.text_area("Vista Previa", "Ejemplares 'Odio': " + "\t" + str(num_odio) + "\nEjemplares 'No Odio': " + "\t" + str(num_no_odio) + "\nTotal: " + "\t" + str(num_odio + num_no_odio) + "\nAlgoritmo Seleccionado: " + "\t" + algoritmo, height=130)

    df = pd.DataFrame.from_dict(ejemplares, orient='index', columns=['Ejemplares'])
    df = df.rename_axis('Clase')
    figura = px.bar(df, y="Ejemplares", color="Ejemplares")
    st.plotly_chart(figura)


import logging

from django.core.mail import send_mail

# Create your views here.
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.generics import CreateAPIView

from .models import LandingModel, ManagersModel, FormBlockAdminModel
from .serializers import LandingModelAdminSerializer, FormBlockSerializer
from .tasks import send_email

logger = logging.getLogger('main')

class LandingPageFormView(CreateAPIView):
    """View to send a form"""
    queryset = FormBlockAdminModel.objects.all()

    def perform_create(self, serializer):
        try:
            mail_to = ManagersModel.objects.all().values()
            data = self.request.data
            name = data["form_customer_name"]
            phone = data["form_customer_phone"]
            message = data["form_customer_phone"]
            for recipient in mail_to:
                send_to = recipient['managers_email']
                send_email.delay(send_to, name, phone, message)
            serializer.save()
        except Exception as e:
            logger.warning(e)

class LandingPageView(APIView):
    """View to get landing context"""

    def get(self, request, *args, **kwargs):
        try:
            queryset = LandingModel.objects.latest('id')
            serializer = LandingModelAdminSerializer(queryset, context={'request': request})
            return Response(serializer.data)
        except Exception as e:
            logger.warning(e)