import argparse
import itertools
import json
import multiprocessing as mp
import os
import pickle
import random
import re
import string
import sys
import time
import json
import math
import copy
from collections import Counter, OrderedDict

import numpy as np
import pandas as pd
import torch
from torch.utils.data import DataLoader, RandomSampler, TensorDataset
import torch.nn as nn
import torch.nn.functional as F
from tqdm import tqdm
from transformers import AdamW, BertModel, BertTokenizer, get_linear_schedule_with_warmup

from util import load_pickle, save_pickle, count_parameters, compute_metrics, compute_metrics_from_logits
import logging

logging.basicConfig(level = logging.INFO, \
                    format = '%(asctime)s  %(levelname)-5s %(message)s', \
                    datefmt =  "%Y-%m-%d-%H-%M-%S")

def cprint(*args):
    text = ""
    for arg in args:
        text += "{0} ".format(arg)
    logging.info(text)

def tokenize_conversations(data, tokenizer, max_sent_len):
    new_data = []
    for conv in tqdm(data):
        new_conv = []
        for i, (speaker, sents) in enumerate(conv):
            # each utterance has been segmented into multiple sentences
            if i==0:
                word_limit = 90
            else:
                word_limit = max_sent_len

            tokenized_sent = []
            for sent in sents:
                tokenized = tokenizer.tokenize(sent)
                if len(tokenized_sent) + len(tokenized) <= word_limit:
                    tokenized_sent.extend(tokenized)
                else:
                    break
            if len(tokenized_sent) == 0:
                tokenized_sent = tokenized[:word_limit]
            new_conv.append((speaker, tokenized_sent))
        new_data.append(new_conv)
    return new_data

def tokenize_personas(data, tokenizer, all_speakers, num_personas):
    # average: each speaker corresponds to a list of tokens, separated by [SEP] between sents
    # memnet: each speaker corresponds to a 2D list of tokens
    new_data = {}
    for k, sents in tqdm(data):
        if k in all_speakers:
            tokenized_words = []
            for sent in sents[:num_personas]:
                tokenized_words.extend(tokenizer.tokenize(sent) + ["[SEP]"])
            if len(tokenized_words) > 1:
                tokenized_words.pop() # remove the last [SEP]
                new_data[k] = tokenized_words
            else:
                new_data[k] = ["."]
    return new_data

def create_context_and_response(data):
    new_data = []
    for conv in tqdm(data):
        context = []
        for s, ts in conv[:-1]:
            context.extend(ts + ["[SEP]"])
        context.pop() # pop the last [SEP]
        response = conv[-1][1]
        # if len(context) > 0 and len(response) > 0:
        #     new_data.append((context, response, conv[-1][0]))
        if len(response) == 0:
            response.extend(" ")
        new_data.append((context, response, conv[-1][0]))
    return new_data

def convert_conversations_to_ids(data, persona, tokenizer, max_seq_len, max_sent_len, num_personas):
    def pad_tokens(tokens, max_len, sentence_type, num_personas=0, response_ids=None):
        # note token_type_ids to differentiate context utterances
        # speaker A has 0, speaker B has 1, response is speaker B and has 1, persona has 1
        # persona does not have positional embedding
        if sentence_type == "persona" and num_personas > 0:
            # filter persona sentences that appeared in response_ids
            if response_ids is not None:
                response_sent = " ".join(tokenizer.convert_ids_to_tokens(response_ids, skip_special_tokens=True))

                all_persona_sent_ids = []
                for t_id in tokens:
                    if t_id in [101]:
                        sent_ids = []
                    if t_id in [102]:
                        all_persona_sent_ids.append(sent_ids)
                        sent_ids = []
                    if t_id not in tokenizer.all_special_ids:
                        sent_ids.append(t_id)

                # convert ids to tokens
                filtered_tokens = []
                for sent_ids in all_persona_sent_ids:
                    sent = " ".join(tokenizer.convert_ids_to_tokens(sent_ids))
                    if sent not in response_sent:
                        filtered_tokens.extend(sent_ids + [tokenizer.convert_tokens_to_ids("[SEP]")])
                filtered_tokens.insert(0, tokenizer.convert_tokens_to_ids("[CLS]"))

                tokens = filtered_tokens

            # remove additional persona sentences
            persona_sent_count = 0
            truncated_tokens = []
            for token_id in tokens:
                if token_id == tokenizer.convert_tokens_to_ids("[SEP]"):
                    persona_sent_count += 1
                    if persona_sent_count == num_personas:
                        break
                truncated_tokens.append(token_id)
            tokens = truncated_tokens

        # clip too much tokens
        while len(tokens) >= max_len and sentence_type == "context":
            pos = tokens.index(tokenizer.convert_tokens_to_ids('[SEP]')) + 1
            tokens = tokens[pos:]
            pos = tokens.index(tokenizer.convert_tokens_to_ids('[SEP]')) + 1
            tokens = tokens[pos:]
            tokens.insert(0, tokenizer.convert_tokens_to_ids("[CLS]"))
        assert max_len >= len(tokens)
        attention_mask = [1]*len(tokens)
        padding_length = max_len - len(tokens)
        attention_mask = attention_mask + ([0] * padding_length)

        if sentence_type == "context":
            token_type_ids = []
            token_type = 0
            for token_id in tokens:
                token_type_ids.append(token_type)
                if token_id == tokenizer.convert_tokens_to_ids("[SEP]"):
                    token_type = int(1-token_type)
            token_type_ids = token_type_ids + [0] * padding_length
        else:
            token_type_ids = [0] * max_len

        tokens = tokens + [0] * padding_length
        return tokens, attention_mask, token_type_ids

    all_context_ids = []
    all_context_attention_mask = []
    all_context_token_type_ids = []
    all_response_ids = []
    all_response_attention_mask = []
    all_response_token_type_ids = []
    all_persona_ids = []
    all_persona_attention_mask = []
    all_persona_token_type_ids = []
    max_persona_len = 23*num_personas+1
    context_lens = []
    for context, response, speaker in tqdm(data):
        context_ids = tokenizer.encode(context, add_special_tokens=True) # convert to token ids, add [cls] and [sep] at beginning and end
        response_ids = tokenizer.encode(response, add_special_tokens=True)
        context_lens.append(len(context_ids))

        context_ids, context_attention_mask, context_token_type_ids = pad_tokens(context_ids, max_seq_len, "context")
        response_ids, response_attention_mask, response_token_type_ids = pad_tokens(response_ids, max_sent_len+2, "response")

        all_context_ids.append(context_ids)
        all_context_attention_mask.append(context_attention_mask)
        all_context_token_type_ids.append(context_token_type_ids)
        all_response_ids.append(response_ids)
        all_response_attention_mask.append(response_attention_mask)
        all_response_token_type_ids.append(response_token_type_ids)

        if persona is not None:
            persona_ids = tokenizer.encode(persona[speaker], add_special_tokens=True)
            persona_ids, persona_attention_mask, persona_token_type_ids = pad_tokens(persona_ids, max_persona_len, "persona", num_personas, response_ids)
            # persona_ids, persona_attention_mask, persona_token_type_ids = pad_tokens(persona_ids, max_persona_len, "persona", num_personas)
            all_persona_ids.append(persona_ids)
            all_persona_attention_mask.append(persona_attention_mask)
            all_persona_token_type_ids.append(persona_token_type_ids)

    # (num_examples, max_seq_len)
    all_context_ids = torch.tensor(all_context_ids, dtype=torch.long)
    all_context_attention_mask = torch.tensor(all_context_attention_mask, dtype=torch.long)
    all_context_token_type_ids = torch.tensor(all_context_token_type_ids, dtype=torch.long)

    # (num_examples, max_sent_len)
    all_response_ids = torch.tensor(all_response_ids, dtype=torch.long)
    all_response_attention_mask = torch.tensor(all_response_attention_mask, dtype=torch.long)
    all_response_token_type_ids = torch.tensor(all_response_token_type_ids, dtype=torch.long)

    if persona is not None:
        # (num_examples, max_persona_len)
        all_persona_ids = torch.tensor(all_persona_ids, dtype=torch.long)
        all_persona_attention_mask = torch.tensor(all_persona_attention_mask, dtype=torch.long)
        all_persona_token_type_ids = torch.tensor(all_persona_token_type_ids, dtype=torch.long)

    cprint(all_context_ids.shape, all_context_attention_mask.shape, all_context_token_type_ids.shape)
    cprint(all_response_ids.shape, all_response_attention_mask.shape, all_response_token_type_ids.shape)

    if persona is not None:
        cprint(all_persona_ids.shape, all_persona_attention_mask.shape, all_persona_token_type_ids.shape)
        dataset = TensorDataset(all_context_ids, all_context_attention_mask, all_context_token_type_ids, \
            all_response_ids, all_response_attention_mask, all_response_token_type_ids, \
                all_persona_ids, all_persona_attention_mask, all_persona_token_type_ids)
    else:
        dataset = TensorDataset(all_context_ids, all_context_attention_mask, all_context_token_type_ids, \
            all_response_ids, all_response_attention_mask, all_response_token_type_ids)

    cprint("context lens stats: ", min(context_lens), max(context_lens), \
        np.mean(context_lens), np.std(context_lens))
    return dataset

def match(model, matching_method, x, y, x_mask, y_mask):
    # Multi-hop Co-Attention
    # x: (batch_size, m, hidden_size)
    # y: (batch_size, n, hidden_size)
    # x_mask: (batch_size, m)
    # y_mask: (batch_size, n)
    assert x.dim() == 3 and y.dim() == 3
    assert x_mask.dim() == 2 and y_mask.dim() == 2
    assert x_mask.shape == x.shape[:2] and y_mask.shape == y.shape[:2]
    m = x.shape[1]
    n = y.shape[1]

    attn_mask = torch.bmm(x_mask.unsqueeze(-1), y_mask.unsqueeze(1)) # (batch_size, m, n)
    attn = torch.bmm(x, y.transpose(1,2)) # (batch_size, m, n)
    model.attn = attn
    model.attn_mask = attn_mask

    x_to_y = torch.softmax(attn * attn_mask + (-5e4) * (1-attn_mask), dim=2) # (batch_size, m, n)
    y_to_x = torch.softmax(attn * attn_mask + (-5e4) * (1-attn_mask), dim=1).transpose(1,2) # # (batch_size, n, m)

    # x_attended, y_attended = None, None # no hop-1
    x_attended = torch.bmm(x_to_y, y) # (batch_size, m, hidden_size)
    y_attended = torch.bmm(y_to_x, x) # (batch_size, n, hidden_size)

    # x_attended_2hop, y_attended_2hop = None, None # no hop-2
    y_attn = torch.bmm(y_to_x.mean(dim=1, keepdim=True), x_to_y) # (batch_size, 1, n) # true important attention over y
    x_attn = torch.bmm(x_to_y.mean(dim=1, keepdim=True), y_to_x) # (batch_size, 1, m) # true important attention over x

    # truly attended representation
    x_attended_2hop = torch.bmm(x_attn, x).squeeze(1) # (batch_size, hidden_size)
    y_attended_2hop = torch.bmm(y_attn, y).squeeze(1) # (batch_size, hidden_size)

    # # hop-3
    # y_attn, x_attn = torch.bmm(x_attn, x_to_y), torch.bmm(y_attn, y_to_x) # (batch_size, 1, n) # true important attention over y
    # x_attended_3hop = torch.bmm(x_attn, x).squeeze(1) # (batch_size, hidden_size)
    # y_attended_3hop = torch.bmm(y_attn, y).squeeze(1) # (batch_size, hidden_size)
    # x_attended_2hop = torch.cat([x_attended_2hop, x_attended_3hop], dim=-1)
    # y_attended_2hop = torch.cat([y_attended_2hop, y_attended_3hop], dim=-1)

    x_attended = x_attended, x_attended_2hop
    y_attended = y_attended, y_attended_2hop

    return x_attended, y_attended

def aggregate(model, aggregation_method, x, x_mask):
    # x: (batch_size, seq_len, emb_size)
    # x_mask: (batch_size, seq_len)
    assert x.dim() == 3 and x_mask.dim() == 2
    assert x.shape[:2] == x_mask.shape
    # batch_size, seq_len, emb_size = x.shape

    if aggregation_method == "mean":
        return (x * x_mask.unsqueeze(-1)).sum(dim=1)/x_mask.sum(dim=-1, keepdim=True).clamp(min=1) # (batch_size, emb_size)

    if aggregation_method == "max":
        return x.masked_fill(x_mask.unsqueeze(-1)==0, -5e4).max(dim=1)[0] # (batch_size, emb_size)

    if aggregation_method == "mean_max":
        return torch.cat([(x * x_mask.unsqueeze(-1)).sum(dim=1)/x_mask.sum(dim=-1, keepdim=True).clamp(min=1), \
            x.masked_fill(x_mask.unsqueeze(-1)==0, -5e4).max(dim=1)[0]], dim=-1) # (batch_size, 2*emb_size)

def fuse(model, matching_method, aggregation_method, batch_x_emb, batch_y_emb, batch_persona_emb, \
    batch_x_mask, batch_y_mask, batch_persona_mask, batch_size, num_candidates):

    batch_x_emb, batch_y_emb_context = match(model, matching_method, batch_x_emb, batch_y_emb, batch_x_mask, batch_y_mask)
    # batch_x_emb: ((batch_size*num_candidates, m, emb_size), (batch_size*num_candidates, emb_size))
    # batch_y_emb_context: (batch_size*num_candidates, n, emb_size), (batch_size*num_candidates, emb_size)

    # hop 2 results
    batch_x_emb_2hop = batch_x_emb[1]
    batch_y_emb_context_2hop = batch_y_emb_context[1]

    # mean_max aggregation for the 1st hop result
    batch_x_emb = aggregate(model, aggregation_method, batch_x_emb[0], batch_x_mask) # batch_x_emb: (batch_size*num_candidates, 2*emb_size)
    batch_y_emb_context = aggregate(model, aggregation_method, batch_y_emb_context[0], batch_y_mask) # batch_y_emb_context: (batch_size*num_candidates, 2*emb_size)

    if batch_persona_emb is not None:
        batch_persona_emb, batch_y_emb_persona = match(model, matching_method, batch_persona_emb, batch_y_emb, batch_persona_mask, batch_y_mask)
        # batch_persona_emb: (batch_size*num_candidates, m, emb_size), (batch_size*num_candidates, emb_size)
        # batch_y_emb_persona: (batch_size*num_candidates, n, emb_size), (batch_size*num_candidates, emb_size)

        batch_persona_emb_2hop = batch_persona_emb[1]
        batch_y_emb_persona_2hop = batch_y_emb_persona[1]

        # # no hop-1
        # return torch.bmm(torch.cat([batch_x_emb_2hop, batch_persona_emb_2hop], dim=-1).unsqueeze(1), \
        #             torch.cat([batch_y_emb_context_2hop, batch_y_emb_persona_2hop], dim=-1)\
        #                 .unsqueeze(-1)).reshape(batch_size, num_candidates)

        batch_persona_emb = aggregate(model, aggregation_method, batch_persona_emb[0], batch_persona_mask) # batch_persona_emb: (batch_size*num_candidates, 2*emb_size)
        batch_y_emb_persona = aggregate(model, aggregation_method, batch_y_emb_persona[0], batch_y_mask) # batch_y_emb_persona: (batch_size*num_candidates, 2*emb_size)

        # # no hop-2
        # return torch.bmm(torch.cat([batch_x_emb, batch_persona_emb], dim=-1).unsqueeze(1), \
        #             torch.cat([batch_y_emb_context, batch_y_emb_persona], dim=-1)\
        #                 .unsqueeze(-1)).reshape(batch_size, num_candidates)
        return torch.bmm(torch.cat([batch_x_emb, batch_x_emb_2hop, batch_persona_emb, batch_persona_emb_2hop], dim=-1).unsqueeze(1), \
                    torch.cat([batch_y_emb_context, batch_y_emb_context_2hop, batch_y_emb_persona, batch_y_emb_persona_2hop], dim=-1)\
                        .unsqueeze(-1)).reshape(batch_size, num_candidates)
    else:
        return torch.bmm(torch.cat([batch_x_emb, batch_x_emb_2hop], dim=-1).unsqueeze(1), \
                    torch.cat([batch_y_emb_context, batch_y_emb_context_2hop], dim=-1)\
                        .unsqueeze(-1)).reshape(batch_size, num_candidates)

def dot_product_loss(batch_x_emb, batch_y_emb):
    """
        if batch_x_emb.dim() == 2:
            # batch_x_emb: (batch_size, emb_size)
            # batch_y_emb: (batch_size, emb_size)

        if batch_x_emb.dim() == 3:
            # batch_x_emb: (batch_size, batch_size, emb_size), the 1st dim is along examples and the 2nd dim is along candidates
            # batch_y_emb: (batch_size, emb_size)
    """
    batch_size = batch_x_emb.size(0)
    targets = torch.arange(batch_size, device=batch_x_emb.device)

    if batch_x_emb.dim() == 2:
        dot_products = batch_x_emb.mm(batch_y_emb.t())
    elif batch_x_emb.dim() == 3:
        dot_products = torch.bmm(batch_x_emb, batch_y_emb.unsqueeze(0).repeat(batch_size, 1, 1).transpose(1,2))[:, targets, targets] # (batch_size, batch_size)

    # dot_products: [batch, batch]
    log_prob = F.log_softmax(dot_products, dim=1)
    loss = F.nll_loss(log_prob, targets)
    nb_ok = (log_prob.max(dim=1)[1] == targets).float().sum()
    return loss, nb_ok

def evaluate_epoch(data_iter, models, num_personas, gradient_accumulation_steps, device, dataset, epoch, \
    apply_interaction, matching_method, aggregation_method, tokenizer, info_, num_cand, rank1_k, rank2_k):
    epoch_loss = []
    ok = 0
    total = 0
    recall = []
    MRR = []

    rank_KoK_context = []
    rank_KoK_response = []
    rank_KoK_logits = []
    rank_KoK_persona = []
    rank_KoK_info = []

    if len(models) == 1:
        if num_personas == 0:
            context_model, response_model = models[0], models[0]
        else:
            context_model, response_model, persona_model = models[0], models[0], models[0]
    if len(models) == 2:
        context_model, response_model = models
    if len(models) == 3:
        context_model, response_model, persona_model = models
    ##################################################################
    print("rank 1 processing...")
    count = 0
    ##################################################################
    for batch_idx, batch, in enumerate(data_iter):
        batch = tuple(t.to(device) for t in batch)
        batch_y = {"input_ids": batch[3], "attention_mask": batch[4], "token_type_ids": batch[5]}
        ##################################################################
        count += 1
        input_batch_x = []
        input_batch_y = []
        input_batch_persona = []
        for k in batch[0]:
            tokens = tokenizer.convert_ids_to_tokens(k.cpu())
            tokens.remove('[CLS]')
            while '[PAD]' in tokens: tokens.remove('[PAD]')
            tokens = [item.replace("[SEP]", "    ") for item in tokens]
            strings = tokenizer.convert_tokens_to_string(tokens)
            # input_batch_x.append(' '.join(strings))
            input_batch_x.append(strings)
        for k in batch[3]:
            # tokens = tokenizer.convert_ids_to_tokens(k.cpu())
            # strings = tokenizer.convert_tokens_to_string(tokens)
            # t = ' '.join(tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(k.cpu())))
            tokens = tokenizer.convert_ids_to_tokens(k.cpu())
            tokens.remove('[CLS]')
            try:
                while '[PAD]' in tokens: tokens.remove('[PAD]')
            except ValueError:
                print(tokens)
            try:
                while '[SEP]' in tokens: tokens.remove('[SEP]')
            except ValueError:
                print(tokens)
            strings = tokenizer.convert_tokens_to_string(tokens)
            # input_batch_y.append(t)
            input_batch_y.append(strings)
        for k in batch[6]:
            # input_batch_persona.append(' '.join(tokenizer.convert_ids_to_tokens(k.cpu())))
            tokens = tokenizer.convert_ids_to_tokens(k.cpu())
            tokens.remove('[CLS]')
            # tokens.remove('[PAD]')
            # tokens.remove('[SEP]')
            while '[PAD]' in tokens: tokens.remove('[PAD]')
            while '[SEP]' in tokens: tokens.remove('[SEP]')
            strings = tokenizer.convert_tokens_to_string(tokens)
            input_batch_persona.append(strings)
        ##################################################################
        has_persona = len(batch) > 6

        # get context embeddings in chunks due to memory constraint
        batch_size = batch[0].shape[0]
        chunk_size = 20
        num_chunks = math.ceil(batch_size/chunk_size)

        if apply_interaction:
            # batch_x_mask = batch[0].ne(0).float()
            # batch_y_mask = batch[3].ne(0).float()
            batch_x_mask = batch[1].float()
            batch_y_mask = batch[4].float()

            batch_x_emb = []
            batch_x_pooled_emb = []
            with torch.no_grad():
                for i in range(num_chunks):
                    mini_batch_x = {
                        "input_ids": batch[0][i*chunk_size: (i+1)*chunk_size],
                        "attention_mask": batch[1][i*chunk_size: (i+1)*chunk_size],
                        "token_type_ids": batch[2][i*chunk_size: (i+1)*chunk_size]
                        }
                    mini_output_x = context_model(**mini_batch_x)
                    batch_x_emb.append(mini_output_x[0]) # [(chunk_size, seq_len, emb_size), ...]
                    batch_x_pooled_emb.append(mini_output_x[1])
                batch_x_emb = torch.cat(batch_x_emb, dim=0) # (batch_size, seq_len, emb_size)
                batch_x_pooled_emb = torch.cat(batch_x_pooled_emb, dim=0)
                emb_size = batch_x_emb.shape[-1]
            if has_persona:
                # batch_persona_mask = batch[6].ne(0).float()
                batch_persona_mask = batch[7].float()
                batch_persona_emb = []
                batch_persona_pooled_emb = []
                with torch.no_grad():
                    for i in range(num_chunks):
                        mini_batch_persona = {
                            "input_ids": batch[6][i*chunk_size: (i+1)*chunk_size],
                            "attention_mask": batch[7][i*chunk_size: (i+1)*chunk_size],
                            "token_type_ids": batch[8][i*chunk_size: (i+1)*chunk_size]
                            }
                        mini_output_persona = persona_model(**mini_batch_persona)
                        # [(chunk_size, emb_size), ...]
                        batch_persona_emb.append(mini_output_persona[0])
                        batch_persona_pooled_emb.append(mini_output_persona[1])
                    batch_persona_emb = torch.cat(batch_persona_emb, dim=0)
                    batch_persona_pooled_emb = torch.cat(batch_persona_pooled_emb, dim=0)

            with torch.no_grad():
                output_y = response_model(**batch_y)
                batch_y_emb = output_y[0]

            batch_size, sent_len, emb_size = batch_y_emb.shape

            # interaction
            # context-response attention
            num_candidates = batch_size

            with torch.no_grad():
                # evaluate per example
                logits = []
                for i in range(batch_size):
                    x_emb = batch_x_emb[i:i+1].repeat_interleave(num_candidates, dim=0) # (num_candidates, context_len, emb_size)
                    x_mask = batch_x_mask[i:i+1].repeat_interleave(num_candidates, dim=0) # (batch_size*num_candidates, context_len)
                    persona_emb, persona_mask = None, None
                    if has_persona:
                        persona_emb = batch_persona_emb[i:i+1].repeat_interleave(num_candidates, dim=0)
                        persona_mask = batch_persona_mask[i:i+1].repeat_interleave(num_candidates, dim=0)
                    logits_single = fuse(context_model, matching_method, aggregation_method, \
                        x_emb, batch_y_emb, persona_emb, x_mask, batch_y_mask, persona_mask, 1, num_candidates).reshape(-1)

                    logits.append(logits_single)
                logits = torch.stack(logits, dim=0)

                # compute loss
                targets = torch.arange(batch_size, dtype=torch.long, device=batch[0].device)
                loss = F.cross_entropy(logits, targets)
            ##################################################################
            input_logits = []
            for l in logits.cpu():
                row = []
                for r in l:
                    row.append(r.item())
                input_logits.append(row)
            rank_arr = input_logits[0]

            dict = {"context": input_batch_x, "response": input_batch_y, "logits": rank_arr, "persona": input_batch_persona}
            df = pd.DataFrame(dict)
            df = df.sort_values(by=['logits'], axis=0, ascending=False)

            # rank_KoK.append(df[0:rank1_k])
            for i in range(rank1_k):
                rank_KoK_context.append(df.iloc[i, 0])
                rank_KoK_response.append(df.iloc[i, 1])
                rank_KoK_logits.append(df.iloc[i, 2])
                rank_KoK_persona.append(df.iloc[i, 3])
            input_logits_argmax = []
            for l in logits:
                input_logits_argmax.append(l.cpu().float().argmax().item())
            ####################################################################
            num_ok = (targets.long() == logits.float().argmax(dim=1)).sum()
            valid_recall, valid_MRR = compute_metrics_from_logits(logits, targets)
        else:
            batch_x_emb = []
            with torch.no_grad():
                for i in range(num_chunks):
                    mini_batch_x = {
                        "input_ids": batch[0][i*chunk_size: (i+1)*chunk_size],
                        "attention_mask": batch[1][i*chunk_size: (i+1)*chunk_size],
                        "token_type_ids": batch[2][i*chunk_size: (i+1)*chunk_size]
                        }
                    mini_output_x = context_model(**mini_batch_x)
                    batch_x_emb.append(mini_output_x[0].mean(dim=1)) # [(chunk_size, emb_size), ...]
                batch_x_emb = torch.cat(batch_x_emb, dim=0) # (batch_size, emb_size)
                emb_size = batch_x_emb.shape[-1]

            if has_persona:
                batch_persona_emb = []
                with torch.no_grad():
                    for i in range(num_chunks):
                        mini_batch_persona = {
                            "input_ids": batch[6][i*chunk_size: (i+1)*chunk_size],
                            "attention_mask": batch[7][i*chunk_size: (i+1)*chunk_size],
                            "token_type_ids": batch[8][i*chunk_size: (i+1)*chunk_size]
                            }
                        mini_output_persona = persona_model(**mini_batch_persona)

                        # [(chunk_size, emb_size), ...]
                        batch_persona_emb.append(mini_output_persona[0].mean(dim=1))

            with torch.no_grad():
                batch_persona_emb = torch.cat(batch_persona_emb, dim=0)
                batch_x_emb = (batch_x_emb + batch_persona_emb)/2
                output_y = response_model(**batch_y)
                batch_y_emb = output_y[0].mean(dim=1)

            # compute loss
            loss, num_ok = dot_product_loss(batch_x_emb, batch_y_emb)
            valid_recall, valid_MRR = compute_metrics(batch_x_emb, batch_y_emb)

        ok += num_ok.item()
        total += batch[0].shape[0]

        # compute valid recall
        recall.append(valid_recall)
        MRR.append(valid_MRR)

        if gradient_accumulation_steps > 1:
            loss = loss / gradient_accumulation_steps
        epoch_loss.append(loss.item())

        # if batch_idx%print_every == 0:
        #     cprint("loss: ", np.mean(epoch_loss[-print_every:]))
        #     cprint("valid recall: ", np.mean(recall[-print_every:], axis=0))
        #     cprint("valid MRR: ", np.mean(MRR[-print_every:], axis=0))
    print("info_ len: ", len(info_))
    # info_idx = 0
    # for i in range(0, len(info_)):
    #     if info_idx >= len(info_):
    #         break
    #     num_exp_persona = info_[info_idx][2]
    #     for j in range(num_exp_persona):
    #         if info_idx + j >= len(info_):
    #             break
    #         rank_KoK_info.append(info_[info_idx])
    #     info_idx += num_exp_persona
    for i in range(0, len(info_), num_cand):
        rank_KoK_info.append(info_[i])
    acc = ok/total
    # compute recall for validation dataset
    recall = np.mean(recall, axis=0)
    MRR = np.mean(MRR)
    rank1_dict = {'context': rank_KoK_context, 'response': rank_KoK_response,
            'logits': rank_KoK_logits, 'persona': rank_KoK_persona, 'info': rank_KoK_info}

    print("rank 2 processing...")
    c = 0
    rank2_context = []
    rank2_response = []
    rank2_logits = []
    rank2_info = []

    print(len(rank1_dict['context']))
    print(len(rank1_dict['response']))
    print(len(rank1_dict['logits']))
    print(len(rank1_dict['persona']))
    print(len(rank1_dict['info']))

    while c < len(rank1_dict['response']):
        num_exp_persona = rank1_dict['info'][c][2]

        context = []
        response = []
        logits = []
        info = []
        for d in range(num_exp_persona):
            if c + d >= len(rank1_dict['response']):
                break
            context.append(rank1_dict['context'][c + d])
            response.append(rank1_dict['response'][c + d])
            logits.append(rank1_dict['logits'][c + d])
            info.append(rank1_dict['info'][c + d])
        c += num_exp_persona
        rank2_dict = {'context': context, 'response': response, 'logits': logits, 'info': info}
        df = pd.DataFrame(rank2_dict)
        df = df.sort_values(by=['logits'], axis=0, ascending=False)
        for e in range(rank2_k):
            if e >= len(response):
                break
            rank2_context.append(df.iloc[e, 0])
            rank2_response.append(df.iloc[e, 1])
            rank2_logits.append(df.iloc[e, 2])
            rank2_info.append(df.iloc[e, 3])
    # while c < len(rank1_dict['response']):
    #     num_exp_persona = rank1_dict['info'][c][2]
    #     response = []
    #     logits = []
    #     info = []
    #     rank2_dict = {'response': response[c:(c + num_exp_persona)], 'logits': logits[c:(c + num_exp_persona)],
    #                   'info': info[c:(c + num_exp_persona)]}
    #     c += num_exp_persona
    #     df = pd.DataFrame(rank2_dict)
    #     df = df.sort_values(by=['logits'], axis=0, ascending=False)
    #     end = min(num_exp_persona, rank2_k)
    #     for e in range(end):
    #         if e >= len(response):
    #             break
    #         rank2_response.append(df.iloc[e, 0])
    #         rank2_logits.append(df.iloc[e, 1])
    #         rank2_info.append(df.iloc[e, 2])
    rank2_dict = {'context': rank2_context, 'response': rank2_response, 'logits': rank2_logits, 'info': rank2_info}

    return np.mean(epoch_loss), (acc, recall, MRR), rank1_dict, rank2_dict

def main(config, progress):
    # save config
    with open("./log/configs.json", "a") as t:
        json.dump(config, t)
        t.write("\n")
    cprint("*"*80)
    cprint("Experiment progress: {0:.2f}%".format(progress*100))
    cprint("*"*80)
    metrics = {}

    # data hyper-params
    train_path = config["train_path"]
    valid_path = config["valid_path"]
    test_path = config["test_path"]
    dataset = train_path.split("/")[1]
    test_mode = bool(config["test_mode"])
    load_model_path = config["load_model_path"]
    save_model_path = config["save_model_path"]
    personaChat_file_path = config["personaChat_file_path"]
    num_candidates = config["num_candidates"]
    num_personas = config["num_personas"]
    persona_path = config["persona_path"]
    max_sent_len = config["max_sent_len"]
    max_seq_len = config["max_seq_len"]
    PEC_ratio = config["PEC_ratio"]
    train_ratio = config["train_ratio"]
    rank1_k = config["rank1_k"]
    rank2_k = config["rank2_k"]

    if PEC_ratio != 0 and train_ratio != 1:
        raise ValueError("PEC_ratio or train_ratio not qualified!")

    # model hyper-params
    config_id = config["config_id"]
    model = config["model"]
    shared = bool(config["shared"])
    apply_interaction = bool(config["apply_interaction"])
    matching_method = config["matching_method"]
    aggregation_method = config["aggregation_method"]
    output_hidden_states = False

    # training hyper-params
    batch_size = config["batch_size"]
    epochs = config["epochs"]
    warmup_steps = config["warmup_steps"]
    gradient_accumulation_steps = config["gradient_accumulation_steps"]
    lr = config["lr"]
    weight_decay = 0
    seed = config["seed"]
    device = torch.device(config["device"])
    fp16 = bool(config["fp16"])
    fp16_opt_level = config["fp16_opt_level"]

    # set seed
    random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)

    if test_mode and load_model_path == "":
        raise ValueError("Must specify test model path when in test mode!")

    # load data
    cprint("Loading conversation data...")
    train = load_pickle(train_path)
    train = train[0]
    valid = load_pickle(valid_path)
    test = load_pickle(test_path)
    valid_path = test_path
    valid = test[0]
    valid_info = test[1]
    print("valid len: ", len(valid))
    cprint("sample train data: ", train[0])
    cprint("sample valid data: ", valid[0])

    # tokenization
    cprint("Tokenizing ...")
    tokenizer = BertTokenizer.from_pretrained(model)
    cached_tokenized_train_path = train_path.replace(".pkl", "_tokenized.pkl")
    cached_tokenized_valid_path = valid_path.replace(".pkl", "_tokenized.pkl")
    if os.path.exists(cached_tokenized_train_path):
        cprint("Loading tokenized dataset from ", cached_tokenized_train_path)
        train = load_pickle(cached_tokenized_train_path)
    else:
        train = tokenize_conversations(train, tokenizer, max_sent_len)
        cprint("Saving tokenized dataset to ", cached_tokenized_train_path)
        save_pickle(train, cached_tokenized_train_path)

    if os.path.exists(cached_tokenized_valid_path):
        cprint("Loading tokenized dataset from ", cached_tokenized_valid_path)
        valid = load_pickle(cached_tokenized_valid_path)

    else:
        valid = tokenize_conversations(valid, tokenizer, max_sent_len)
        cprint("Saving tokenized dataset to ", cached_tokenized_valid_path)
        save_pickle(valid, cached_tokenized_valid_path)

    persona = None
    if num_personas > 0:
        cprint("Tokenizing persona sentences...")
        cached_tokenized_persona_path = persona_path.replace(".pkl", "_tokenized.pkl")
        if os.path.exists(cached_tokenized_persona_path):
            cprint("Loading tokenized persona from file...")
            persona = load_pickle(cached_tokenized_persona_path)
        else:
            cprint("Loading persona data...")
            persona = load_pickle(persona_path)
            all_speakers = set([s for conv in load_pickle(config["train_path"])[0] + \
                load_pickle(config["valid_path"])[0] + load_pickle(config["test_path"])[0] for s, sent in conv])
            cprint("Tokenizing persona data...")
            persona = tokenize_personas(persona, tokenizer, all_speakers, num_personas)
            cprint("Saving tokenized persona to file...")
            save_pickle(persona, cached_tokenized_persona_path)
        cprint("Persona dataset statistics (after tokenization):", len(persona))
        cprint("Sample tokenized persona:", list(persona.values())[0])

    cprint("Sample tokenized data: ")
    cprint(train[0])
    cprint(valid[0])

    # select subsets of training and validation data for casualconversation
    cprint(dataset)

    # create context and response
    train = create_context_and_response(train)
    valid = create_context_and_response(valid)
    cprint("Sample context and response: ")
    cprint(train[0])
    cprint(valid[0])

    # convert to token ids
    cprint("Converting conversations to ids: ")
    valid_dataset = convert_conversations_to_ids(valid, persona, tokenizer, \
        max_seq_len, max_sent_len, num_personas)
    ####################################################
    valid_dataloader = DataLoader(valid_dataset, batch_size=num_candidates)

    # create model
    cprint("Building model...")
    model = BertModel.from_pretrained(model, output_hidden_states=output_hidden_states)
    cprint(model)
    cprint("number of parameters: ", count_parameters(model))

    if shared:
        cprint("number of encoders: 1")
        models = [model]
    else:
        if num_personas == 0:
            cprint("number of encoders: 2")
            # models = [model, copy.deepcopy(model)]
            models = [model, pickle.loads(pickle.dumps(model))]
        else:
            cprint("number of encoders: 3")
            # models = [model, copy.deepcopy(model), copy.deepcopy(model)]
            models = [model, pickle.loads(pickle.dumps(model)), pickle.loads(pickle.dumps(model))]

    cprint("Loading weights from ", load_model_path)
    model.load_state_dict(torch.load(load_model_path))
    models = [model]

    for i, model in enumerate(models):
        cprint("model {0} number of parameters: ".format(i), count_parameters(model))
        model.to(device)

    # optimization
    amp = None
    if fp16:
        from apex import amp

    no_decay = ["bias", "LayerNorm.weight"]
    optimizers = []
    schedulers = []
    for i, model in enumerate(models):
        optimizer_grouped_parameters = [
            {
                "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
                "weight_decay": weight_decay,
            },
            {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0},
        ]
        optimizer = AdamW(optimizer_grouped_parameters, lr=lr, eps=1e-8)

        if fp16:
            model, optimizer = amp.initialize(model, optimizer, opt_level=fp16_opt_level)
            models[i] = model
        optimizers.append(optimizer)

    # evaluation
    for model in models:
        model.eval()
    valid_iterator = tqdm(valid_dataloader, desc="Iteration")
    valid_loss, (valid_acc, valid_recall, valid_MRR), rank1_dict, rank2_dict = \
        evaluate_epoch(valid_iterator, models, num_personas, gradient_accumulation_steps, device, dataset, 0,
                       apply_interaction, matching_method, aggregation_method, tokenizer, valid_info, num_candidates, rank1_k, rank2_k)
    cprint("test loss: {0:.4f}, test acc: {1:.4f}, test recall: {2}, test MRR: {3:.4f}"
        .format(valid_loss, valid_acc, valid_recall, valid_MRR))

    directory = "./ranking_dataset"
    try:
        if not os.path.exists(directory):
            os.makedirs(directory)
    except OSError:
        print('Error: Creating directory. ' + directory)

    print(len(rank1_dict['context']))
    print(len(rank1_dict['response']))
    print(len(rank1_dict['logits']))
    print(len(rank1_dict['persona']))
    print(len(rank1_dict['info']))

    df = pd.DataFrame(rank1_dict)
    df.to_csv("./ranking_dataset/rank_1.tsv", sep='\t', na_rep='NaN')
    with open("./ranking_dataset/rank_1.json", "w") as p:
        json.dump(rank1_dict, p)

    df = pd.DataFrame(rank2_dict)
    df.to_csv("./ranking_dataset/rank_2.tsv", sep='\t', na_rep='NaN')
    with open("./ranking_dataset/rank_2.json", "w") as p:
        json.dump(rank2_dict, p)

    cached_personaChat_file_path = personaChat_file_path.replace(".json", "_ranked_" + str(rank2_k) + ".json")
    with open(cached_personaChat_file_path, 'w') as p:
        data = afterProcess(personaChat_file_path, rank2_dict, rank2_k)
        json.dump(data, p)
    print("ranking finished")
    sys.exit()

def afterProcess(read_path, dict, rank_k):
    with open(read_path, 'r') as p:
        data = json.load(p)
        idx = 0
        info = dict['info']
        print(len(info))
        for k in range(len(info)):
            i, j, num_exp_persona, pair, tv = info[idx][0], info[idx][1], info[idx][2], info[idx][3], info[idx][4]
            end = min(rank_k, num_exp_persona)
            if idx + end >= len(info):
                break
            if tv == 't':
                aug = []
                for l in range(end):
                    aug.append({'exp_persona': data['train'][i]['utterances'][j]['exp_persona'][info[idx + l][3]],
                                'new_responses': dict['response'][idx + l]})
                del data['train'][i]['utterances'][j]['new_responses']
                del data['train'][i]['utterances'][j]['exp_persona']
                data['train'][i]['utterances'][j]['aug'] = aug
                idx += end
            elif tv == 'v':
                aug = []
                for l in range(end):
                    aug.append({'exp_persona': data['valid'][i]['utterances'][j]['exp_persona'][info[idx + l][3]],
                                'new_responses': dict['response'][idx + l]})
                del data['valid'][i]['utterances'][j]['new_responses']
                del data['valid'][i]['utterances'][j]['exp_persona']
                data['valid'][i]['utterances'][j]['aug'] = aug
                idx += end
        for dialogue in data['train']:
            for context in dialogue['utterances']:
                if "exp_persona" in context or "new_responses" in context and "aug" not in context:
                    try:
                        del context["exp_persona"]
                    except:
                        pass
                    try:
                        del context["new_responses"]
                    except:
                        pass
                    context['aug'] = [{"exp_persona": "", "new_responses": ""}]
        for dialogue in data['valid']:
            for context in dialogue['utterances']:
                if "exp_persona" in context or "new_responses" in context and "aug" not in context:
                    try:
                        del context["exp_persona"]
                    except:
                        pass
                    try:
                        del context["new_responses"]
                    except:
                        pass
                    context['aug'] = [{"exp_persona": "", "new_responses": ""}]
    return data

def clean_config(configs):
    cleaned_configs = []
    for config in configs:
        if config not in cleaned_configs:
            cleaned_configs.append(config)
    return cleaned_configs

def merge_metrics(metrics):
    avg_metrics = {"score" : 0}
    num_metrics = len(metrics)
    for metric in metrics:
        for k in metric:
            if k != "config":
                avg_metrics[k] += np.array(metric[k])

    for k, v in avg_metrics.items():
        avg_metrics[k] = (v/num_metrics).tolist()

    return avg_metrics

if __name__ == "__main__":
    mp.set_start_method('spawn', force=True)
    parser = argparse.ArgumentParser(description="Model for Transformer-based Dialogue Generation with Controlled Emotion")
    parser.add_argument('--config', help='Config to read details', required=True)
    parser.add_argument('--note', help='Experiment note', default="")
    args = parser.parse_args()
    cprint("Experiment note: ", args.note)
    with open(args.config) as configfile:
        config = json.load(configfile) # config is now a python dict

    # pass experiment config to main
    parameters_to_search = OrderedDict() # keep keys in order
    other_parameters = {}
    keys_to_omit = ["kernel_sizes"] # keys that allow a list of values
    for k, v in config.items():
        # if value is a list provided that key is not device, or kernel_sizes is a nested list
        if isinstance(v, list) and k not in keys_to_omit:
            parameters_to_search[k] = v
        elif k in keys_to_omit and isinstance(config[k], list) and isinstance(config[k][0], list):
            parameters_to_search[k] = v
        else:
            other_parameters[k] = v

    if len(parameters_to_search) == 0:
        config_id = time.perf_counter()
        config["config_id"] = config_id
        cprint(config)
        output = main(config, progress=1)
        cprint("-"*80)
        cprint(output["config"])
        cprint(output["epoch"])
        cprint(output["score"])
        cprint(output["recall"])
        cprint(output["MRR"])
    else:
        all_configs = []
        for i, r in enumerate(itertools.product(*parameters_to_search.values())):
            specific_config = {}
            for idx, k in enumerate(parameters_to_search.keys()):
                specific_config[k] = r[idx]

            # merge with other parameters
            merged_config = {**other_parameters, **specific_config}
            all_configs.append(merged_config)

        #   cprint all configs
        for config in all_configs:
            config_id = time.perf_counter()
            config["config_id"] = config_id
            logging.critical("config id: {0}".format(config_id))
            cprint(config)
            cprint("\n")

        # multiprocessing
        num_configs = len(all_configs)
        # mp.set_start_method('spawn')
        pool = mp.Pool(processes=config["processes"])
        results = [pool.apply_async(main, args=(x,i/num_configs)) for i,x in enumerate(all_configs)]
        outputs = [p.get() for p in results]

        # if run multiple models using different seed and get the averaged result
        if "seed" in parameters_to_search:
            all_metrics = []
            all_cleaned_configs = clean_config([output["config"] for output in outputs])
            for config in all_cleaned_configs:
                metrics_per_config = []
                for output in outputs:
                    if output["config"] == config:
                        metrics_per_config.append(output)
                avg_metrics = merge_metrics(metrics_per_config)
                all_metrics.append((config, avg_metrics))
            # log metrics
            cprint("Average evaluation result across different seeds: ")
            for config, metric in all_metrics:
                cprint("-"*80)
                cprint(config)
                cprint(metric)

            # save to log
            with open("./log/{0}.txt".format(time.perf_counter()), "a+") as f:
                for config, metric in all_metrics:
                    f.write(json.dumps("-"*80) + "\n")
                    f.write(json.dumps(config) + "\n")
                    f.write(json.dumps(metric) + "\n")

        else:
            for output in outputs:
                cprint("-"*80)
                cprint(output["config"])
                cprint(output["score"])
                cprint(output["recall"])
                cprint(output["MRR"])
                cprint("Best result at epoch {0}: ".format(output["epoch"]))
                cprint(output["recall"][output["epoch"]], output["MRR"][output["epoch"]])

            # save to log
            with open("./log/{0}.txt".format(time.perf_counter()), "a+") as f:
                for output in outputs:
                    f.write(json.dumps("-"*80) + "\n")
                    f.write(json.dumps(output) + "\n")


import re
import json
import math
import time
import datetime

from explorer.views import homepage
from explorer.settings import LOGGING
import logging

#from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, FIRST_COMPLETED

logging.config.dictConfig(LOGGING)

logger = logging.getLogger('django.request')

from django.shortcuts import render
from django.db.models import Q
from django.http import HttpResponse
from .models import *
from .chain import Chain

##############################################
from django.template import loader
from pyecharts.charts  import Line3D
import pyecharts.options as opts

def block_update_finished(request):
    blocks_list = Block.objects.all()
    context = {
        'blocks': blocks_list,
    }
    context = { 'blocks': blocks_list }
    return render(request, 'block/block_list.html', context)

def block_detail(request):
    hash  = request.GET.get('hash')
    logger.info("hash: %s" %(hash))
    ablock = Block.objects.get(hash=hash)

    blmessages = BlsMessage.objects.filter(block=ablock)

    context = {
        'ablock': ablock,
        'blmessages': blmessages
    }
    return render(request, 'block/block_detail.html', context)

def block_count(request):
    to     = request.GET.get('to')
    method = request.GET.get('method')
    logger.info("to: %s, method：%s" %(to, method))
    # 查询集
    blsmessages = BlsMessage.objects.filter(to=to,method=method)
    blsmessages = blsmessages.order_by('block')

    context = {
        'to': to,
        'method': method,
        'blsmessages': blsmessages
    }

    return render(request, 'block/block_count.html', context)

def block_search(request):
    search = request.GET.get('search')
    blocks_list = Block.objects.all()
    # 搜索查询集
    logger.info("search: %s" %(search))
    if search:
        blocks_list = blocks_list.filter(
            Q(miner__icontains=search) |
            Q(height__icontains=search) |
            Q(hash__icontains=search)
        )
    else:
        # 将 search 参数重置为空
        search = ''

    blocks_list.order_by('height')
    context = {
        'blocks': blocks_list,
        'search': search,
    }

    return render(request, 'block/block_search.html', context)

# get blocklist with date
def block_list(request):
    date = request.GET.get('date')
    if (None == date):
        date = datetime.date.today()

    startTimestr = ("%s %s" %(date, " 00:00:00"))
    startTimestamp = time.mktime(time.strptime(startTimestr, "%Y-%m-%d %H:%M:%S"))

    endTimeStr = ("%s %s" %(date, " 23:59:59"))
    endTimestamp = time.mktime(time.strptime(endTimeStr, "%Y-%m-%d %H:%M:%S"))

    blocks_list = Block.objects.all().filter(timestamp__gte=startTimestamp, timestamp__lte=endTimestamp)
    blocks_list.order_by('height')

    # 需要传递给模板（templates）的对象
    context = {
        'blocks': blocks_list,
        'date': date,
        'startTimestamp' : startTimestamp,
        'endTimestamp' : endTimestamp,
        'currentTimestamp' : startTimestamp,
    }

    # render函数：载入模板，并返回context对象
    return render(request, 'block/block_list.html', context)


from importlib import import_module

import os
import sys
import argparse
import unittest
import pytest
import shutil

TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(os.path.dirname(TOOLS_DIR))

def handle_input_and_run_tests_for_package(package_name, path_list):
    parser = argparse.ArgumentParser(description="Run tests for {}.".format(package_name))
    parser.add_argument('--coverage',
                        nargs='?',
                        const=True,
                        default=False,
                        help='compute test coverage (--coverage combine to combine with previous reports)')
    parser.add_argument('--pytest', '-p',
                        default=False,
                        action='store_true',
                        help="run tests with pytest")
    parser.add_argument('--verbose', '-v',
                        default=False,
                        action='store_true',
                        help='show verbose output (with pytest)')
    options = parser.parse_args()

    test_session_title = ' Running tests for {} '.format(package_name)
    header = test_session_title.center(shutil.get_terminal_size().columns, '#')
    print(header)

    if options.coverage:
        import coverage
        cov = coverage.Coverage(config_file="tools/.coveragerc")
        if options.coverage == 'combine':
            cov.load()
        cov.start()

    if options.pytest:
        location_to_run_in = os.path.join(TOOLS_DIR, '..', *path_list)
        paths_to_test = ['.']
        pytest_options = [
            '-s',    # show output from tests; this hides the progress bar though
            '-x',    # stop on first test failure
            '--ff',  # runs last failure first
        ]
        pytest_options += (['-v'] if options.verbose else [])
        os.chdir(location_to_run_in)
        result = pytest.main(paths_to_test + pytest_options)
        if result != 0:
            sys.exit(1)
        failures = False
    else:
        # Codecov seems to work only when using loader.discover. It failed to capture line executions
        # for functions like loader.loadTestFromModule or loader.loadTestFromNames.
        test_suites = unittest.defaultTestLoader.discover(os.path.join(*path_list))
        suite = unittest.TestSuite(test_suites)
        runner = unittest.TextTestRunner(verbosity=2)
        result = runner.run(suite)
        failures = result.failures
        if result.failures or result.errors:
            sys.exit(1)

    if not failures and options.coverage:
        cov.stop()
        cov.data_suffix = False  # Disable suffix so that filename is .coverage
        cov.save()
        cov.html_report()
        print("HTML report saved in directory 'htmlcov'.")


import pandas as pd
import json
from elasticsearch import Elasticsearch
import elasticsearch.helpers
from sklearn import preprocessing

def search():
    es = Elasticsearch([{'host': '10.134.175.251', 'port': 9200}])

    body={"query": {"match": {"state": "IL" } } }
    results = elasticsearch.helpers.scan(es, query=body, index="business")
    df_business = pd.DataFrame.from_dict([document['_source'] for document in results])
    print(df_business)
    df_num=df_business[["stars","review_count"]]
    names = df_num.columns
# Create the Scaler object
    scaler = preprocessing.StandardScaler()
# Fit your data on the scaler object
    scaled_df = scaler.fit_transform(df_num)
    scaled_df = pd.DataFrame(scaled_df, columns=names)
    corr_data = scaled_df.corr()
    print (corr_data)
    grouped_data=df_business.groupby('postal_code').mean()
    print (grouped_data['stars'].plot(kind='bar'))
    print (grouped_data['review_count'].plot(kind='bar'))

if __name__ == "__main__":

    search()


# -*- coding: utf-8 -*-

import sys
#import pokemons
import getFiles
#from Combats import JCJ
from dresseurs import Dresseur

class Game:
    def __init__(self):
        self.pokemonDictionary = getFiles.creer_mesPokemons()
        self.attaqueDictionary = getFiles.creer_mesAttaques()
        self.defenseDictionary = getFiles.creer_mesDefenses()
        self.listDresseurs = []
        self.utilisateur = None

    def afficherBienvenue(self):
        print("#-------------------------------------------#")
        print("#         Bienvenue dans POOkemon!          #")
        print("#   Le jeu de Pokemon Oriente Objet  :D     #")
        print("#-------------------------------------------#")

    def getNomUtilisateur(self):
        self.utilisateur = Dresseur(input("Quel est votre nom? "),self.pokemonDictionary, self.attaqueDictionary, self.defenseDictionary)
        self.listDresseurs.append(self.utilisateur)
        print ("Voila votre dresseur : {}: {} Pokemon.".format(self.utilisateur.nom, len(self.utilisateur.deck) + len(self.utilisateur.pokemonListe) ))

    def afficherMenuPrincipal(self):
        print("\n----  Menu Principal  ----")
        print("0/ Voir vos pokemon")
        print("1/ Changer le deck")
        print("2/ Combattre / Capturer un pokemon")
        print("3/ Combattre un dresseur")
        print("4/ Creer un nouveau dresseur")
        print("5/ Quitter le jeu ")

    def initialiserGame(self):
        self.afficherBienvenue()
        self.getNomUtilisateur()
        self.utilisateur.afficherDeck()

    def main(self):

        self.afficherMenuPrincipal()
        # Verifier si la saisie est valide
        while True:
            try:
                choix = int (input("Que voulez vous faire? (0-5) : "))
            except ValueError:
                print("Saisie invalide, veuillez reesayer.")
                #better try again... Return to the start of the loop
                continue
            if choix < 0 or choix > 5:
                print("Saisie invalide, veuillez reesayer.")
                continue
            else:
                break

        if choix == 0 :
            self.utilisateur.afficherTousLesPokemons()
        elif choix == 1:
            self.utilisateur.changerDeck()
        elif choix == 2:
            self.utilisateur.combatrePokemonLibre(self)
        elif choix == 3:
            self.utilisateur.combatreDresseur(self)
        elif choix==4 :
            self.creerNouveauDresseur()

        elif choix== 5:
            sys.exit()

    # Lorsque le joueur principal souhaite creer un nouveu dresseur
    def creerNouveauDresseur(self):
        print("\n-- Creation d'un nouveau dresseur --")
        nom_nouveau_dresseur = input("Veuillez saisir le nom du nouveau dresseur : ")

        for dresseur in self.listDresseurs:
            if dresseur.nom == nom_nouveau_dresseur :
                print("Ce nom existe deja, veuillez choisir un autre.")
                self.creerNouveauDresseur()

        nouveau_dresseur = Dresseur(nom_nouveau_dresseur, self.pokemonDictionary, self.attaqueDictionary, self.defenseDictionary)
        # Append nouveau dresseur à la liste des dresseurs
        self.listDresseurs.append(nouveau_dresseur)

        print("\nVoici le deck de {} : ".format(nouveau_dresseur.nom))
        nouveau_dresseur.afficherDeck()
        print("\nLe rest des pokemons : ")
        nouveau_dresseur.afficherPokemonListe()
        print()

#%%mh
myGame = Game()

myGame.initialiserGame()

while(True):
    myGame.main()


import pandas as pd
import pytest

from openclean.data.util import repair_mapping, to_set

@pytest.fixture
def dataset():
    """Get simple dataset with the name and age of two people."""
    return pd.DataFrame(
        data=[['Alice', 29], ['Bob', 32]],
        columns=['Name', 'Age']
    )

# -- repair_lookup ------------------------------------------------------------

def test_repair_mapping():
    """Test the repair lookup generator function."""
    df = pd.DataFrame(data=[[1, 2, 3], [1, 1, 4]], columns=['A', 'B', 'C'])
    assert repair_mapping(df, key='A', value='B') == {1: 2}
    assert repair_mapping(df, key=['A', 'B'], value='C') == {(1, 2): 3, (1, 1,): 4}
    # Error if no valid mapping exists.
    with pytest.raises(ValueError):
        repair_mapping(df, key='A', value='C')

# -- to_set -------------------------------------------------------------------

def test_df_to_set(dataset):
    """Test converting a data frame with multiple columns into a set of
    tuples.
    """
    names = to_set(dataset[['Name']])
    assert len(names) == 2
    assert 'Alice' in names
    assert 'Bob' in names
    tuples = to_set(dataset[['Name', 'Age']])
    assert ('Alice', 29) in tuples
    assert ('Bob', 32) in tuples
    assert ('Alice', 32) not in tuples

def test_invalid_to_set():
    """Test error when invalid object is passed to the to_set function."""
    with pytest.raises(ValueError):
        to_set(['A', 'B'])

def test_series_to_set(dataset):
    """Test converting a data series into a set of values."""
    names = to_set(dataset['Name'])
    assert 'Alice' in names
    assert 'Bob' in names
    assert 'Claire' not in names


from get_team_stats import get_team_stats
from correct_name import correct_name
import json

def build_dict():
	with open('data/teams.json', 'r') as file:
		teams = json.load(file)
	for conf in teams:
		for t in teams[conf]:
			n = correct_name(teams[conf][t]['team'])
			if n != '':
				j = get_team_stats(n)
				j[n]['seed'] = teams[conf][t]['seed']
				with open(f'teams/{n}.json', 'w+', encoding='utf8') as file:
					file.write(json.dumps(j, ensure_ascii=False, indent =4))


minutes_for_walking_day = int(input())
number_of_walking_day = int(input())
calories_for_day = int(input())

total_walking = minutes_for_walking_day * number_of_walking_day
burned_for_day = total_walking * 5
percent_of_daily = calories_for_day / 2

if burned_for_day >= percent_of_daily:
    print(f"Yes, the walk for your cat is enough. Burned calories per day: {burned_for_day}.")
else:
    print(f"No, the walk for your cat is not enough. Burned calories per day: {burned_for_day}.")


def unsorted_subarray(arr, n):
    i = 0
    j = n - 1

    while arr[i + 1] >= arr[i]:
        i += 1

    while arr[j-1] <= arr[j]:
        j -= 1

    min_value = min(arr[i+1:j-1])
    max_value = max(arr[i+1:j-1])

    i = 0
    j = n - 1
    while min_value >= arr[i]:
        i += 1

    while max_value <= arr[j]:
        j -= 1

    print(i, j)

arr = [10, 12, 20, 30, 25, 40, 32, 31, 35, 50, 60]
unsorted_subarray(arr, len(arr))


from collections import OrderedDict
from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple

import dimod
import networkx as nx
import numpy as np
import scipy.optimize

from dimod.typing import GraphLike, Variable

from penaltymodel.exceptions import ImpossiblePenaltyModel
from penaltymodel.utils import as_graph

__all__ = []

class Index:
    """Tracks the column indices of the various values of interest"""

    def __init__(self,
                 decision: Iterable[Variable],
                 auxiliary: Iterable[Variable],
                 interactions: Iterable[Tuple[Variable, Variable]]
                 ):

        start = 2  # after gap and offset
        self._decision = dict((v, i) for i, v in enumerate(decision, start))
        start += len(self._decision)
        self._auxiliary = dict((v, i) for i, v in enumerate(auxiliary, start))
        start += len(self._auxiliary)
        self._interactions = dict(
            (frozenset(e), i) for i, e in enumerate(interactions, start))
        self._stop = start + len(self._interactions)

    def __len__(self) -> int:
        return self._stop

    def auxiliaries(self) -> Sequence[int]:
        return range(2 + len(self._decision), 2 + self.num_variables())

    def decisions(self) -> Sequence[int]:
        return range(2, 2 + len(self._decision))

    @staticmethod
    def gap() -> int:
        return 0

    def interaction(self, u: Variable, v: Variable) -> int:
        return self._interactions[frozenset((u, v))]

    def num_variables(self) -> int:
        return len(self._decision) + len((self._auxiliary))

    def make_bounds(self,
                    min_classical_gap: float,
                    linear_bound: Tuple[float, float],
                    quadratic_bound: Tuple[float, float],
                    ) -> Sequence[Tuple[Optional[float], Optional[float]]]:
        bounds = [(min_classical_gap, None), (None, None)]
        bounds.extend(linear_bound for _ in range(self.num_variables()))
        bounds.extend(quadratic_bound for _ in range(len(bounds), len(self)))
        return bounds

    @staticmethod
    def offset() -> int:
        return 1

    def variable(self, v: Variable) -> int:
        try:
            return self._decision[v]
        except KeyError:
            try:
                return self._auxiliary[v]
            except KeyError:
                raise ValueError(f"no variable {v!r}") from None

    def variables(self) -> Sequence[int]:
        return range(2, 2 + self.num_variables())

def all_possible(num_variables: int) -> np.ndarray:
    """Create an array of all possible spin configurations."""
    assert 0 <= num_variables < 1 << 8
    a = np.unpackbits(np.arange(1 << num_variables, dtype=np.uint8).reshape(1 << num_variables, 1),
                      axis=1, count=num_variables, bitorder='little').astype(np.int8)
    a *= 2
    a -= 1
    return a

def next_auxiliary(state: Tuple[int, ...]) -> Tuple[int, ...]:
    s = list(state)
    for i in range(len(s)-1, -1, -1):
        if s[i] <= 0:
            s[i] = 1
            break
        else:
            s[i] = -1
    return tuple(s)

def generate(graph_like: GraphLike,
             samples_like,
             *,
             linear_bound: Tuple[float, float] = (-2, 2),
             quadratic_bound: Tuple[float, float] = (-1, 1),
             min_classical_gap: float = 2,
             ) -> Tuple[dimod.BinaryQuadraticModel, float, Dict[Tuple[int, ...], Tuple[int, ...]]]:
    """Generate a penalty model.

    This function is considered internal, it is recommended to use
    :func:`~penaltymodel.get_penalty_model` with ``use_cache=False`` instead.
    """
    graph = as_graph(graph_like)
    samples, decision = dimod.as_samples(samples_like)

    if any(v not in graph.nodes for v in decision):
        raise ValueError("the decision variables must be a subset of the graph nodes")

    # let's make things easier for ourselves by casting the samples into -1, +1
    if (samples == 0).any():
        samples = 2*samples - 1
    if not ((samples == +1) ^ (samples == -1)).all():
        raise ValueError("given samples should be 0/1 or -1/+1")

    auxiliaries = list(graph.nodes - decision)
    num_samples = samples.shape[0]
    num_variables = len(graph.nodes)
    num_auxiliary = num_variables - len(decision)

    if isinstance(samples_like, dimod.SampleSet):
        energies = samples_like.record.energy
    else:
        energies = np.zeros(num_samples)

    # construct the table
    if len(energies):
        table = {tuple(map(int, state)): energy for state, energy in zip(samples, energies)}
    else:
        table = {}

    # todo: more correctness checks, max variables==8

    # some edge cases we can easily eliminate
    if not table or not decision:
        bqm = dimod.BinaryQuadraticModel('SPIN')
        bqm.add_linear_from((v, 0) for v in graph.nodes)
        bqm.add_quadratic_from((u, v, 0) for u, v in graph.edges)
        return bqm, float('inf'), {}

    # create an object to track the columns in the LP matrix

    indexer = Index(decision, auxiliaries, graph.edges)

    # we'll use this to track where the ground states are. Note that we could
    # avoiding needing to store this in memory with some clever indexing, but
    # let's not worry too much about that unless it becomes a problem later
    ground: Dict[Tuple[int, ...], Dict[Tuple[int, ...], int]] = dict()

    # ok, let's build our matrices for the LP

    b = np.full(1 << num_variables, max(table.values(), default=0), dtype=float)

    A = np.empty((1 << len(graph), len(indexer)), dtype=np.int8)
    A[:, indexer.variables()] = all_possible(num_variables)
    for u, v in graph.edges:
        A[:, indexer.interaction(u, v)] = A[:, indexer.variable(u)] * A[:, indexer.variable(v)]
    A[:, indexer.offset()] = 1

    # the gap and b are how we distinguish between values in the table and
    # not
    for i in range(1 << num_variables):
        decision_state = tuple(A[i, indexer.decisions()])

        if decision_state in table:
            A[i, indexer.gap()] = 0
            b[i] = table[decision_state]
            ground.setdefault(decision_state, {})[tuple(A[i, indexer.auxiliaries()])] = i
        else:
            A[i, indexer.gap()] = -1

    # for now we're just trying to find feasibility, we'll optimize at the end
    c = np.zeros(len(indexer))

    # bounds are fixed
    bounds = indexer.make_bounds(min_classical_gap, linear_bound, quadratic_bound)

    # ok, we have everything in hand to start solving!
    upper_bound = list(range(A.shape[0]))
    equality: List[int] = []
    auxiliary_configurations: Dict[Tuple[int, ...], Tuple[int, ...]] = OrderedDict()

    # WLOG, we can fix one right away
    decision_state = next(state for state in ground if state not in auxiliary_configurations)
    auxiliary_configurations[decision_state] = auxiliary_state = (-1,)*num_auxiliary
    i = ground[decision_state][auxiliary_state]
    upper_bound.remove(i)
    equality.append(i)

    while True:
        A_eq = A[equality, :]
        b_eq = b[equality]
        A_ub = -A[upper_bound, :]  # negate because we want A_ub <= b_ub
        b_ub = -b[upper_bound]

        res = scipy.optimize.linprog(c, A_ub, b_ub, A_eq, b_eq, bounds=bounds, method='highs')

        if res.success:
            if len(auxiliary_configurations) == len(table):
                break

            # fix a new state
            decision_state = next(state for state in ground if state not in auxiliary_configurations)
            auxiliary_configurations[decision_state] = auxiliary_state = (-1,)*num_auxiliary
            i = ground[decision_state][auxiliary_state]
            upper_bound.remove(i)
            equality.append(i)
        else:
            # ok, we didn't succeed. So first try changing the aux state of the
            # last set
            try:
                decision_state, auxiliary_state = auxiliary_configurations.popitem()
                upper_bound.append(equality.pop())  # put it back into inequality
                while all(s == 1 for s in auxiliary_state):
                    decision_state, auxiliary_state = auxiliary_configurations.popitem()
                    upper_bound.append(equality.pop())  # put it back into inequality
            except KeyError:
                raise ImpossiblePenaltyModel("There is no BQM that can encode the given constraint") from None

            # iterate the auxiliary state
            auxiliary_configurations[decision_state] = auxiliary_state = next_auxiliary(auxiliary_state)
            i = ground[decision_state][auxiliary_state]
            upper_bound.remove(i)
            equality.append(i)

    assert res.success

    # having found something feasible, let's do one last run, this time optimizing the gap
    c[indexer.gap()] = -1
    res_opt = scipy.optimize.linprog(c, A_ub, b_ub, A_eq, b_eq, bounds=bounds, method='highs')
    if res_opt.success:
        res = res_opt
        gap = res.x[indexer.gap()]
    elif res_opt.status == 3:
        # error code 3 is unbounded objective, which can happen for a fully
        # specified problem
        gap = float('inf')
    else:
        raise RuntimeError("something went wrong")

    # let's make the BQM!
    bqm = dimod.BinaryQuadraticModel('SPIN')
    bqm.add_linear_from((v, res.x[indexer.variable(v)]) for v in graph.nodes)
    bqm.add_quadratic_from((u, v, res.x[indexer.interaction(u, v)]) for u, v in graph.edges)
    bqm.offset = res.x[indexer.offset()]

    # return which auxiliary variables are which
    aux = dict((state, dict(zip(auxiliaries, aux))) for state, aux in auxiliary_configurations.items())

    return bqm, gap, aux


tasks = [
    {
        "length": 8,
        "deadline": 4,
        "scheduled": False,
        "scheduled_at_vm": -1,
        "scheduled_step_at": -1,
        "execution_time": -1,

    },
    {
        "length": 13,
        "deadline": 3,
        "scheduled": False,
        "scheduled_at_vm": -1,
        "scheduled_step_at": -1,
        "execution_time": -1,
    },
    {
        "length": 3,
        "deadline": 2,
        "scheduled": False,
        "scheduled_at_vm": -1,
        "scheduled_step_at": -1,
        "execution_time": -1,
    },
    {
        "length": 5,
        "deadline": 4,
        "scheduled": False,
        "scheduled_at_vm": -1,
        "scheduled_step_at": -1,
        "execution_time": -1,
    },
    {
        "length": 10,
        "deadline": 3,
        "scheduled": False,
        "scheduled_at_vm": -1,
        "scheduled_step_at": -1,
        "execution_time": -1,
    },
]

vms = [
    2, 3, 4
]

vms_work_time = [0, 0, 0]

computed_map = []

for i, t in enumerate(tasks):
    for j, v in enumerate(vms):
        time_needed = t['length']/v
        computed_map.append({
            "time_needed": time_needed,
            "task_index": i,
            "vm_index": j,
        })

computed_map_sorted = sorted(computed_map, key=lambda x: x['time_needed'])

def findMinimumIndex(r):
    min_value = 10000000000
    min_index = -1
    for i, j in enumerate(r):
        if min_value > j:
            min_value = j
            min_index = i

    return min_index
any_task_scheduled = False
scheduled_step_at = 1

def allTaskScheduled(t):
    for v in t:
        if v['scheduled'] == False:
            return False
    return True

while allTaskScheduled(tasks) == False:
    for i, c in enumerate(computed_map_sorted):
        task_index = c['task_index']
        vm_index = c['vm_index']
        time_needed = c['time_needed']

        # continue if task already assigned
        if tasks[c['task_index']]['scheduled']:
            continue

        # p = input()

        if any_task_scheduled == False:
            vms_work_time[vm_index] += c['time_needed']
            tasks[task_index]['scheduled'] = True
            tasks[task_index]['scheduled_at_vm'] = vm_index+1
            tasks[task_index]['execution_time'] = time_needed
            tasks[task_index]['scheduled_step_at'] = scheduled_step_at
            scheduled_step_at += 1

            any_task_scheduled = True
        else:
            priority_vm_index = findMinimumIndex(vms_work_time)
            if priority_vm_index == vm_index or vms_work_time[priority_vm_index] == vms_work_time[vm_index]:
                vms_work_time[vm_index] += c['time_needed']
                tasks[task_index]['scheduled'] = True
                tasks[task_index]['scheduled_at_vm'] = vm_index+1
                tasks[task_index]['execution_time'] = time_needed
                tasks[task_index]['scheduled_step_at'] = scheduled_step_at
                print("Task", task_index+1, "scheduled")
                scheduled_step_at += 1

for t in tasks:
    print(t)

print(vms_work_time)


import re
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from typing import List, Dict, Tuple, Union, TYPE_CHECKING, Optional
from PyAltium365.Connections import JsonConSearchAsync
from PyAltium365.Data.DataConVault import AluItem

if TYPE_CHECKING:
    from PyAltium365.AltiumApi import AltiumApi
else:
    from PyAltium365.Fix import AltiumApi

class SearchType(Enum):
    CONDITIONAL_QUERY = "DtoSearchConditionStrictQuery"
    CONDITIONAL_RANGE_QUERY = "DtoSearchConditionRangeQuery"
    CONDITIONAL_WILDCARD = "DtoSearchConditionWildcardQuery"

class DataType(Enum):
    NO_TYPE = ""
    CAPACITANCE = "0629FA77_2DBB3E_2D4045_2D91F0_2DCC1164D3D0AADD420E8DDD8B445E911A0601BB2B6D53"
    RESISTANCE = "B90F0DAE_2DB695_2D41F5_2DBCB0_2D0DE5F75C9E50DD420E8DDD8B445E911A0601BB2B6D53"
    VOLTAGE = "28379228_2DD94F_2D4F3B_2D8038_2D3FE85C36E292DD420E8DDD8B445E911A0601BB2B6D53"
    TOLERANCE = "935791AE_2D95D8_2D4D5E_2DB810_2DE9B66A56E5A5DD420E8DDD8B445E911A0601BB2B6D53"
    FREQUENCY = "F4CAEC49_2DEE33_2D46C9_2DAF4A_2D260721F7AA26DD420E8DDD8B445E911A0601BB2B6D53"
    DISTANCE = "B0C104C5_2D1A99_2D4DE5_2DA539_2DD5FC817D4B3EDD420E8DDD8B445E911A0601BB2B6D53"
    POWER = "A40E567D_2D1324_2D4823_2D8379_2DFB7E896E18B9DD420E8DDD8B445E911A0601BB2B6D53"

    @classmethod
    def _missing_(cls, value):  # type: ignore
        return DataType.NO_TYPE

@dataclass
class Counter:
    Name: str = ""
    Type: DataType = DataType.NO_TYPE
    Count: int = 0
    Values: Dict[str, int] = field(default_factory=lambda: {})

class CounterRange(Counter):
    Values: Dict[float, int] = field(default_factory=lambda: {})  # type: ignore
    MinValue: float = 0
    MaxValue: float = 0

@dataclass
class Counters:
    TotalCount: int = 0
    Counters: List[Counter] = field(default_factory=lambda: [])

@dataclass
class SearchDataBase:
    altium_api: AltiumApi
    Id: str = ""
    Parameters: Dict[str, Union[str, float]] = field(default_factory=lambda: {})
    HRID: str = ""
    CreatedAt: datetime = datetime(1, 1, 1)
    CreatedBy: str = ""
    ModifiedBy: str = ""
    LatestRevision: bool = False
    Updated: datetime = datetime(1, 1, 1)
    CreatedByGUID: str = ""
    UpdatedByGUID: str = ""
    AppType: str = ""
    Url: str = ""
    SubmitDate: datetime = datetime(1, 1, 1)
    Language: str = ""
    FolderGUID: str = ""
    FolderFullPath: str = ""
    Description: str = ""
    NamingSchemeGuid: str = ""
    Cat: str = ""
    ContentTypeGUID: str = ""
    Text: str = ""
    LifeCycle: str = ""
    ACL: str = ""
    ItemHRID: str = ""
    Comment: str = ""
    LifeCycleStateGUID: str = ""
    ItemGUID: str = ""
    RevisionId: int = -1
    AncestorRevisionGUID: str = ""
    ItemDescription: str = ""
    ReleaseNote: str = ""
    ReleaseDate: datetime = datetime(1, 1, 1)
    ReleaseDateNum: datetime = datetime(1, 1, 1)
    LifeCycleDefinitionGUID: str = ""

    def get_folder(self) -> None:
        raise NotImplementedError()

    def get_created_by(self) -> None:
        raise NotImplementedError()

    def get_updated_by(self) -> None:
        raise NotImplementedError()

    def get_content_type(self) -> None:
        raise NotImplementedError()

    def get_item(self) -> Optional[AluItem]:
        return self.altium_api.get_item_from_guid(self.ItemGUID)

    def get_life_cycle_state(self) -> None:
        raise NotImplementedError()

    def get_life_cycle_definition(self) -> None:
        raise NotImplementedError()

    def get_ancestor_revision(self) -> None:
        raise NotImplementedError()

@dataclass
class SearchDataComponent(SearchDataBase):
    FootprintName: List[str] = field(default_factory=lambda: [])
    FootprintDescription: List[str] = field(default_factory=lambda: [])

@dataclass
class SearchDataFootprint(SearchDataBase):
    pass

@dataclass
class SearchDataSymbol(SearchDataBase):
    pass

@dataclass
class SearchDataDatasheet(SearchDataBase):
    FileName: str = ""
    Caption: str = ""
    FileHash: str = ""
    FileSize: str = ""
    DynamicData: str = ""

@dataclass
class SearchDataProjectTemplate(SearchDataBase):
    MainFilename: str = ""
    Kind: str = ""
    DynamicData: str = ""

@dataclass
class SearchDataSchematicTemplate(SearchDataBase):
    pass

@dataclass
class SearchDataComponentTemplate(SearchDataBase):
    DynamicData: str = ""

@dataclass
class SearchDataScript(SearchDataBase):
    DynamicData: str = ""
    MainFilename: str = ""
    Kind: str = ""

@dataclass
class SearchDataLayerstack(SearchDataBase):
    pass

class SearchDataType(Enum):
    NO_TYPE = ("", SearchDataBase)
    COMPONENT = ("Component", SearchDataComponent)
    FOOTPRINT = ("Footprint", SearchDataFootprint)
    SYMBOL = ("Symbol", SearchDataSymbol)
    DATASHEET = ("Datasheet", SearchDataDatasheet)
    PROJECT_TEMPLATE = ("Project Template", SearchDataProjectTemplate)
    SCHEMATIC_TEMPLATE = ("Schematic Template", SearchDataSchematicTemplate)
    COMPONENT_TEMPLATE = ("Component Template", SearchDataComponentTemplate)
    SCRIPT = ("Script", SearchDataScript)
    LAYERSTACK = ("Layerstack", SearchDataLayerstack)

    @classmethod
    def _missing_(cls, value):  # type: ignore
        return SearchDataType.NO_TYPE

    def get_from_str(self, value: str):  # type: ignore
        for orig in SearchDataType:
            if orig.value[0].lower() == value.lower():
                return orig
        return SearchDataType._missing_(None)

encoding_decoding_naming = {
    "_5F": "_",
    "_20": " ",
    "_28": "(",
    "_29": ")",
    "_2C": ",",
    "_2D": "-",
    "_2E": ".",
    "_2F": "/",
    "_40": "@",
    "_5B": "[",
    "_5D": "]",
    "_5E": "^",
}

not_counted_search_parameters = [
    Counter(Name="LatestRevision"),
    Counter(Name="FolderFullPath")
]

class AsyncSearchObject:
    def __init__(self, searcher: JsonConSearchAsync, altium_api: AltiumApi):  # type: ignore
        self._searcher: JsonConSearchAsync = searcher  # type: ignore
        self._altium_api = altium_api
        self._search_parameters: List[Dict] = []
        self._counters: Counters = Counters()
        self._counters_up_to_date = False
        self._update_search_names_and_counters()

    def add_search_parameter(self, name: str, value: Union[str, List[str]], dtype: DataType = DataType.NO_TYPE, remove_old: bool = False) -> bool:
        self._update_search_names_and_counters()
        if type(value) == str:
            value = [value]
        counter = None
        for c in self._counters.Counters:
            if c.Name == name and c.Type == dtype:
                counter = c
                break
        if counter is None:
            return False
        full_name = self._get_index_name_from_name_and_type(counter.Name, counter.Type)
        for parameter in self._search_parameters:
            if parameter['Field'] == full_name:
                if remove_old:
                    parameter['Field'] = []
                for val in value:
                    if val not in parameter['Value']:
                        parameter['Value'].append(val)
                return True
        self._search_parameters.append({
            'Type': SearchType.CONDITIONAL_QUERY,
            'Field': full_name,
            'Value': value
        })
        self._counters_up_to_date = False
        return True

    def add_search_content_type(self, data_type: Union[SearchDataType, List[SearchDataType]], remove_old: bool = False) -> bool:
        if type(data_type) is not List:
            data_type = [data_type]  # type: ignore
        search_l = []
        for d_type in data_type:
            search_l.append(d_type.value[0])
        return self.add_search_parameter('ContentType', search_l, remove_old=remove_old)

    def add_search_parameter_range(self, name: str, minv: float, maxv: float, dtype: DataType = DataType.NO_TYPE, max_included: bool = True,
                                   min_included: bool = True, remove_old: bool = False) -> bool:
        self._update_search_names_and_counters()
        counter = None
        for c in self._counters.Counters:
            if c.Name == name and c.Type == dtype:
                counter = c
                break
        if counter is None:
            return False
        if remove_old:
            self.remove_search_parameter(name, dtype)
        full_name = self._get_index_name_from_name_and_type(counter.Name, counter.Type)
        for parameter_i in range(len(self._search_parameters)):
            parameter = self._search_parameters[parameter_i]
            if parameter['Field'] == full_name:
                self._search_parameters.pop(parameter_i)
                break

        self._search_parameters.append({
            'Type': SearchType.CONDITIONAL_RANGE_QUERY,
            'Field': full_name,
            'Min': minv,
            'Max': maxv,
            'MinInc': min_included,
            'MaxInc': max_included
        })
        self._counters_up_to_date = False
        return True

    def set_search_wildcard(self, wildcard: Optional[str]) -> bool:
        param = None
        for parameter_i in range(len(self._search_parameters)):
            parameter = self._search_parameters[parameter_i]
            if parameter['Type'] == SearchType.CONDITIONAL_WILDCARD:
                if wildcard is None:
                    self._search_parameters.pop(parameter_i)
                    return True
                param = parameter
        if wildcard is None:
            return True
        if param is None:
            param = {
                'Type': SearchType.CONDITIONAL_WILDCARD,
                'Field': []
            }
            self._search_parameters.append(param)
        param['Value'] = wildcard
        return True

    def remove_search_parameter(self, name: str, dtype: DataType = DataType.NO_TYPE) -> bool:
        counter = None
        for c in self._counters.Counters:
            if c.Name == name and c.Type == dtype:
                counter = c
                break
        if counter is None:
            return False
        full_name = self._get_index_name_from_name_and_type(counter.Name, counter.Type)
        for i in range(len(self._search_parameters)):
            if self._search_parameters[i]['Field'] == full_name:
                self._search_parameters.pop(i)
                return True
        return False

    def get_current_count(self) -> int:
        self._update_search_names_and_counters()
        return self._counters.TotalCount

    def get_all_search_names_and_type(self) -> List[Tuple[str, DataType]]:
        l: List[Tuple[str, DataType]] = []
        for c in self._counters.Counters:
            l.append((c.Name, c.Type))
        return l

    def get_all_search_names(self) -> List[str]:
        l: List[str] = []
        for c in self._counters.Counters:
            l.append(c.Name)
        return l

    def get_all_search_names_and_type_without_unusefull(self) -> List[Tuple[str, DataType]]:
        l: List[Tuple[str, DataType]] = []
        for c in self._counters.Counters:
            if len(c.Values) == 1:
                if c.Values.values()[0] == c.Count:  # type: ignore
                    l.append((c.Name, c.Type))
        return l

    def get_all_search_names_and_type_without_usefull(self) -> List[Tuple[str, DataType]]:
        l: List[Tuple[str, DataType]] = []
        for c in self._counters.Counters:
            if len(c.Values) > 1:
                l.append((c.Name, c.Type))
        return l

    def get_results(self, max_amount: int = 500) -> List[SearchDataBase]:
        if max_amount <= 0 or max_amount > 10000:
            max_amount = 10000
        converting = True
        results: List[SearchDataBase] = []
        while converting:
            raw_results = self._searcher.get_search_interface(self, len(results), min(10000, max_amount - len(results)))  # type: ignore
            if 'Documents' not in raw_results:
                return []
            for item in raw_results['Documents']:
                fields = item['Fields']

                part: SearchDataBase = SearchDataBase(self._altium_api)
                for field1 in fields:
                    raw_name = field1['Name']
                    value = field1['Value']
                    name, _ = self._get_facet_name_and_type(raw_name)
                    if name == 'ContentType':
                        part = SearchDataType.get_from_str(SearchDataType.NO_TYPE, value).value[1](self._altium_api)
                        break

                for field1 in fields:
                    raw_name = field1['Name']
                    value = field1['Value']
                    name, _ = self._get_facet_name_and_type(raw_name)

                    if name == 'ContentType':
                        continue

                    m = re.search(r'\d+$', name)
                    if m is not None:
                        sub_name = name[:-len(m.group())]
                        if hasattr(part, sub_name):
                            if type(getattr(part, sub_name)) == list:
                                li = getattr(part, sub_name)
                                index = int(m.group())
                                while len(li) < index:
                                    li.append("")
                                li[index - 1] = value
                                continue
                    if hasattr(part, name):
                        if type(getattr(part, name)) == datetime:
                            if value.replace('.', '', 1).isdecimal():
                                d = datetime(1899, 12, 31)
                                setattr(part, name, d + timedelta(days=float(value)))
                            else:
                                setattr(part, name, datetime.strptime(value, "%m/%d/%Y %H:%M:%S"))
                        elif type(getattr(part, name)) == int:
                            setattr(part, name, int(value))
                        elif type(getattr(part, name)) == bool:
                            setattr(part, name, int(value) > 0)
                        else:
                            setattr(part, name, value)
                        continue
                    part.Parameters[name] = value

                results.append(part)

            if len(results) == min(max_amount, raw_results['Total']):
                converting = False
        return results

    def _update_search_names_and_counters(self) -> None:
        if not self._counters_up_to_date:
            temp_counters = self._searcher.get_part_counters(self)  # type: ignore
            new_counters = Counters()
            new_counters.Counters.extend(not_counted_search_parameters)
            try:
                new_counters.TotalCount = temp_counters['Total']
                temp_counter = temp_counters['FacetedCounters']
                if type(temp_counter) is not list:
                    temp_counter = [temp_counter]
                for temp_c in temp_counter:
                    if 'SupportRange' in temp_c:
                        n_counter = CounterRange()
                        n_counter.MaxValue = float(temp_c['MaxValue'])
                        n_counter.MinValue = float(temp_c['MinValue'])
                    else:
                        n_counter = Counter()  # type: ignore
                    n_counter.Name, n_counter.Type = self._get_facet_name_and_type(temp_c['FacetName'])
                    if n_counter.Name.endswith('_T@x^'):
                        continue
                    n_counter.Count = temp_c['TotalHitCount']
                    for temp_cv in temp_c['Counters']:
                        if 'SupportRange' in temp_c:
                            val = float(temp_cv['Value'])
                        else:
                            val = temp_cv['Value']
                        n_counter.Values[val] = temp_cv['Count']
                    new_counters.Counters.append(n_counter)
                self._counters = new_counters
                self._counters_up_to_date = True
            except Exception:
                print("Data error")

    def _get_facet_name_and_type(self, indexed_name: str) -> Tuple[str, DataType]:
        m1 = re.match("^(.*)_5F([0-9A-F]{8}_[0-9A-F]{6}_[0-9A-F]{6}_[0-9A-F]{6}_[0-9A-F]{46})$", indexed_name)
        m2 = re.match("^(.*)([0-9A-F]{32})$", indexed_name)
        if m1 is not None:
            name = m1.group(1)
            index = m1.group(2)
        elif m2 is not None:
            name = m2.group(1)
            index = m2.group(2)
        else:
            name = indexed_name
            index = ""
        dtype = DataType(index)
        name = self._decode_naming_chars(name)
        return name, dtype

    def _get_index_name_from_name_and_type(self, name: str, dtype: DataType) -> str:
        if dtype == DataType.NO_TYPE:
            full_name = self._encode_naming_chars(name)
            full_name += "DD420E8DDD8B445E911A0601BB2B6D53"
        else:
            full_name = self._encode_naming_chars(name)
            full_name += "_5F" + dtype.value
        return full_name

    def _decode_naming_chars(self, inp: str) -> str:
        for key, val in encoding_decoding_naming.items():
            inp = inp.replace(key, val)
        return inp

    def _encode_naming_chars(self, inp: str) -> str:
        for key, val in encoding_decoding_naming.items():
            inp = inp.replace(val, key)
        return inp


import sys
stdin = sys.stdin.readline

N = int(input())
arr = list()

for _ in range(N):
    name, kor, eng, math = input().split()
    arr.append((name, int(kor), int(eng), int(math)))

arr.sort(key=lambda x: (-x[1], x[2], -x[3], x[0]))

for i in arr:
    print(i[0])


import time
from typing import Union, Optional

import numpy as np
import pandas as pd

class ForestAdditions:
    def partial_fit(self, X: Union[np.array, pd.DataFrame], y: Union[np.array, pd.Series],
                    classes: Union[list, np.ndarray] = None, sample_weight: Optional[np.ndarray] = None):
        """
        Fit a single DTC using the given subset of x and y.

        This calls .fit, which is overloaded. However flags pf_call=True, so .fit() will handle calling super .fit().

        For classifiers;
          - First call needs to be supplied with the expected classes (similar to existing models with .partial_fit())
            in case not all classes are present in the first subset.

        This object sets classes_ and n_classes_ depending on the supplied classes. The Individual trees set theirs
        depending on the data available in the subset. The predict_proba method is modified to standardise shape to the
        dimensions defined in this object.

        For regressors:
          - self._check_classes is overloaded with dummy method.

        :param x:
        :param y:
        :return:
        """
        if self.verbose > 1:
            print(f"PF Call with set classes: "
                  f"{getattr(self, 'classes_', '[no classes attr]')} and input classes {classes}")

        self._check_classes(classes=classes)

        # Fit the next estimator, if not done
        if self._fit_estimators < self.max_n_estimators:
            t0 = time.time()
            self.fit(X, y,
                     pf_call=True,
                     classes_=getattr(self, 'classes_', None),
                     sample_weight=sample_weight)  # Pass classes for enforcement, if classifier.
            t1 = time.time()

            if self.verbose > 1:
                print(f"Fit estimators {self._fit_estimators} - {self._fit_estimators + self.n_estimators_per_chunk} "
                      f"/ {self.max_n_estimators}")
                print(f"Model reports {len(self.estimators_)}")
                print(f"Fit time: {round(t1 - t0, 2)}")
                print(len(self.estimators_))
            self._fit_estimators += self.n_estimators_per_chunk

            # If still not done, prep to fit next
            if self._fit_estimators < self.max_n_estimators:
                self.n_estimators += self.n_estimators_per_chunk

        else:
            if self.verbose > 0:
                print('Done')

        return self

    def _sampled_partial_fit(self, x: Union[np.array, pd.DataFrame], y: [np.ndarray, pd.Series],
                             sample_weight: Optional[np.array] = None):
        """
        This feeds partial_fit with random samples based on the spf_ parameters. Used by .fit() when not using dask.
        :param x: Data.
        :param y: Labels.
        :return:
        """

        n_samples = int(self.spf_sample_prop * x.shape[0])

        for _ in range(self.spf_n_fits):
            idx = np.random.randint(0, x.shape[0], n_samples)

            if self.verbose > 0:
                print(f"_sampled_partial_fit size: {idx.shape}")

            self.partial_fit(x[idx, :], y[idx], sample_weight=sample_weight[idx] if sample_weight is not None else None,
                             classes=np.unique(y))

        return self


# -*- coding: utf-8 -*-
"""
Created on Sun Apr  3 12:54:54 2022

@author: joach
"""

import os
import caer
import canaro
import numpy as np
import cv2 as cv
import gc
import matplotlib.pyplot as plt
from tensorflow.keras.utils import to_categorical

#%%
IMG_SIZE=(80,80)
channels=1
char_path=(r"D:/programmation/Ecole/Story_24_Computer_Vision/reconnaissance_RN/simpsons_dataset")

#%%
#Ceating a character dic, sorting it on descending order

char_dict={}
for char in os.listdir(char_path):
    char_dict[char]=len(os.listdir(os.path.join(char_path,char)))

#sort in decending order
char_dict=caer.sort_dict(char_dict,descending=True)
#%%
print(char_dict)
#%%
#getting the first 10 catégories with the most number of images

characters=[]
count=0

for i in char_dict:
    characters.append(i[0])
    count +=1
    if count>=10:
        break
#%%
print(characters)
#%%
#Create training data

train=caer.preprocess_from_dir(char_path,characters,channels=channels,IMG_SIZE=IMG_SIZE,isShuffle=True)
#%%
#Visualizing the data
plt.figure(figsize=(30,30))
plt.imshow(train[0][0],cmap="gray")
plt.show()
#%%
#Separating the array and corresponding labels
featureSet,labels=caer.sep_train(train,IMG_SIZE=IMG_SIZE)
#%%
#Normaize the featureSet =>(0,1)
featureSet=caer.normalize(featureSet)
labels=to_categorical(labels,len(characters))
#%%
x_train,x_val,y_train,y_val=caer.train_val_split(featureSet,labels,val_ratio=0.2)

#%%
del train
del featureSet
del labels
gc.collect()
#%%
BATCH_SIZE=32
EPOCHS=10

#%%
#Image data generator
datagen=canaro.generators.imageDataGenerator()
train_gen=datagen.flow(x_train,y_train,batch_size=BATCH_SIZE)

#%%
#Creating model
model=canaro.models.createSimpsonsModel(IMG_SIZE=IMG_SIZE,channels=channels,output_dim=len(characters),
                                        loss="binary_crossentropy",decay=1e-6,learning_rate=0.001,
                                       momentum=0.9,nesterov=True )
#%%
print(model.summary())
#%%
# Training the model
from tensorflow.keras.callbacks import LearningRateScheduler
#%%
callbacks_list = [LearningRateScheduler(canaro.lr_schedule)]

training = model.fit(train_gen,
                    steps_per_epoch=len(x_train)//BATCH_SIZE,
                    epochs=EPOCHS,
                    validation_data=(x_val,y_val),
                    validation_steps=len(y_val)//BATCH_SIZE,
                    callbacks = callbacks_list)
#%%
test_path=r"D:/programmation/Ecole/Story_24_Computer_Vision/reconnaissance_RN/kaggle_simpson_testset/kaggle_simpson_testset/charles_montgomery_burns_3.jpg"

img=cv.imread(test_path)
plt.imshow(img,cmap="gray")
plt.show()
#%%

def prepare(img):
    img=cv.cvtColor(img,cv.COLOR_BGR2GRAY)
    img=cv.resize(img,IMG_SIZE)
    img=caer.reshape(img,IMG_SIZE,1)
    return img
#%%

predictions=model.predict(prepare(img))

print(characters[np.argmax(predictions[0])])


import numpy as np

def picksubset(m, num, seed=None):
    """
     function [f,idx,fnot] = picksubset(m,num,seed)

     <m> is a numpy array
     <num> is
       X indicating the size of the subset to pick out
     <seed> (optional) is the rand state to use.
       default: 0.

     return:
      <f> as a vector with a random subset of <m>.
      <idx> as a vector of the indices of the elements that we picked.
      <fnot> as a vector with the remaining elements of <m>.

     note that if you try to pick out a subset bigger than <m>,
     we will just return as many elements as there are in <m>.

     example:
     from numpy.random import randn
     picksubset(randn(10,10),10)
    """
    # input
    if seed is None:
        seed = 0

    # do it
    np.random.seed(seed)

    nm = len(m.flatten())

    idx = np.random.permutation(range(nm))[range(np.min([num, nm]))]

    f = m.flatten()[idx]
    notf = np.ones(m.flatten().shape, dtype=int)
    notf[idx] = 0
    fnot = m.flatten()[notf]

    np.random.seed(None)

    return f, idx, fnot