# Code for MultiTask TArC annotation/collection

import os
import re
import sys
import csv
import glob
import torch

from fairseq.tarc_utils import *
from fairseq.data import Dictionary, TarcMultiTaskDataset
from fairseq.tasks import FairseqTask, register_task

from typing import Dict, List

LOSS_INIT_VALUE=999999.9
ER_INIT_VALUE=LOSS_INIT_VALUE

CLS_COL_IDX=1
POS_COL_IDX=4
def set_pos_col_idx(args):
    if args.sub_task == 'tarc-ext':
        return 3
    elif args.sub_task == 'tarc-full':
        return 4
    elif args.sub_task == 'madar-trs-ex':
        return 2
    elif args.sub_task == 'madar-trs-full':
        return 3
    elif args.sub_task == 'tarc-full-npos':
        return 5
    else:
        return 9999999

# Define some utility functions for reading the TArC multi-task data format (tabular format)
def process_morpho(tag):
    res_tags = []
    for c in tag:
        res_tags.append( '@' + c )
    return res_tags

def append_tags( tags, sep ):
    res_tags = []
    for i in range(len(tags)):
        if i == 0:
            res_tags.append( tags[i] )
        else:
            res_tags.append( sep + tags[i] )
    return res_tags

def process_micro_pos(tag):

    res_tags = []
    if tag != '_':
        micro_tags = tag.split('_')
        res_tags.extend( append_tags( micro_tags[:-1], '_' ) )
        if len(micro_tags) >= 2 and (micro_tags[-2] == 'ADJ' or micro_tags[-2] == 'PRON') and micro_tags[-1][0] in ['1', '2', '3']:
            morpho_tags = process_morpho( micro_tags[-1] )
            morpho_tags[0] = morpho_tags[0].replace('@', '_')
            res_tags.extend( morpho_tags )
        else:
            if micro_tags[-1] == 'SG' or micro_tags[-1] == 'PL':
                res_tags.append( '_' + micro_tags[-1] )
            else:
                if micro_tags[-1] != ':':
                    nano_tags = micro_tags[-1].split(':')
                    if len(nano_tags) > 1:
                        if len(nano_tags) != 2:
                            sys.stderr.write(' - Micro POS error: {}\n'.format(micro_tags[-1]))
                            sys.exit(1)
                        res_tags.append( '_' + nano_tags[0] )
                        morpho_tags = process_morpho(nano_tags[1])
                        morpho_tags[0] = morpho_tags[0].replace('@', ':')
                        res_tags.extend( morpho_tags )
                    else:
                        if len(micro_tags) > 1:
                            res_tags.append( '_' + micro_tags[-1] )
                        else:
                            res_tags.append( micro_tags[-1] )
                else:
                    res_tags.append( micro_tags[-1] )
    else:
        res_tags.append( tag )
    return res_tags

def process_suffix_pos(tag):

    res_tags = []
    if tag != '-':
        tag_tokens = tag.split('-')
        if len(tag_tokens) > 1:
            for i in range(len(tag_tokens)):
                t = tag_tokens[i]
                if len(t) > 2 and t[:2] == 'IV' and not (len(t) > 6 and t[:6] == 'IVSUFF'):
                    res_tags.append('IV')
                    morpho_tags = process_morpho( t[2:] )
                    res_tags.extend( morpho_tags )
                else:
                    suffix_tokens = process_micro_pos(t)
                    if i > 0:
                        suffix_tokens[0] = '-' + suffix_tokens[0]
                    res_tags.extend( suffix_tokens )
        else:
            res_tags.extend( process_micro_pos(tag) )
    else:
        res_tags.append(tag)
    return res_tags

def process_macro_pos(tag):

    res_tags = []
    if tag != '+':
        tag_tokens = tag.split('+')
        for i in range(len(tag_tokens)):
            t = tag_tokens[i]
            micro_pos = process_suffix_pos(t)
            if i > 0:
                res_tags.append('+')
            res_tags.extend(micro_pos)
    else:
        res_tags.append(tag)
    return res_tags

def process_pos_tag(tag, args, pad_flag=True, rep_flag=False):
    res_tags = []
    pos_tokens = tag.split(']')
    if len(pos_tokens) > 1:
        if len(pos_tokens) != 2:
            sys.stderr.write(' - POS format error (splitting at ]): {}\n'.format(tag))
            sys.exit(1)
        pos_pos_tokens = pos_tokens[0].split('[')
        if len(pos_pos_tokens) > 1 and len(pos_pos_tokens[0]) > 0:  # Case like 'IV2S-IV+[PREP+PRON_1S]IVSUFF_IO:1S'
            if len(pos_pos_tokens) != 2:
                sys.stderr.write(' - POS format error (splitting at [): {}\n'.format(tag))
                sys.exit(1)

            pref = pos_pos_tokens[0]
            pref_tokens = process_macro_pos( pref )
            infix = pos_pos_tokens[1]
            infix_tokens = process_macro_pos( infix )
            infix_tokens = ['+['] + infix_tokens + [']']
            post = pos_tokens[1]
            post_tokens = process_macro_pos( post )

            res_tags.extend(pref_tokens)
            res_tags.extend( infix_tokens )
            res_tags.extend( post_tokens )
        elif len(pos_pos_tokens) > 1:   # Case like '[NOUN_QUANT]ADV'
            if len(pos_pos_tokens) != 2:
                sys.stderr.write(' - POS format error (splitting at [, with first part empty): {}\n'.format(tag))
                sys.exit(1)

            pref = pos_pos_tokens[1]
            pref_tokens = process_macro_pos( pref )
            pref_tokens = ['['] + pref_tokens + [']']
            post = pos_tokens[1]
            post_tokens = process_macro_pos( post )
            res_tags.extend( pref_tokens )
            res_tags.extend( post_tokens )
        else:
            sys.stderr.write(' - POS format error, possibly unbalanced [] at pos {}\n'.format(tag))
            sys.exit(1)
    else:   # "Normal" case (normal for Arabic people...)
        pos_tokens = process_macro_pos( tag )
        res_tags.extend( pos_tokens )
    if pad_flag:
        return [start_token] + res_tags + [end_token]
    else:
        return res_tags

def create_character_list(token, args, pad_flag=True, rep_flag=False):
    if (not (token in fillers + latin_fillers)) and rep_flag and args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        token = replace_all_num(token)
        token = replace_all_pun(token)
        token = replace_all_sym(token)
    elif (not (token in fillers + latin_fillers)) and args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        token = replace_all_Lpun(token)
        token = replace_all_Lsym(token)

    tok_lst = []
    tok_idx = 0
    for t in token.split():
        if tok_idx > 0:
            tok_lst.append( space_token )
        for c in t:
            tok_lst.append( c )
        tok_idx += 1

    if args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        seq_fillers = detect_fillers(tok_lst, fillers + latin_fillers)
        if len(seq_fillers) > 0:
            new_lst = []
            prev_start = 0
            for t in seq_fillers:
                new_lst = new_lst + tok_lst[prev_start:t[0]] + [''.join(tok_lst[t[0]:t[1]])]
                prev_start = t[1]
            new_lst = new_lst + tok_lst[t[1]:]
            tok_lst = new_lst

    if pad_flag:
        return [start_token] + tok_lst + [end_token]
    else:
        return tok_lst

def process_tokenization( token, args, pad_flag=True, rep_flag=False ):

    res_tok = []
    if token != '+':
        tok_tokens = token.split('+')
        if len(tok_tokens) > 1:
            res_tok.append(start_token)
            for i in range(len(tok_tokens)):
                t = tok_tokens[i]
                if len(t) > 0:
                    if i > 0:
                        res_tok.append( '+' )
                    res_tok.extend( create_character_list( t, args, pad_flag=False, rep_flag=rep_flag ) )
                else:
                    sys.stderr.write('process_tokenization FORMAT ERROR: found empy token splitting on + >{}< (probably you must replace multiple + in your data)\n'.format(token))
                    sys.exit(1)
            res_tok.append(end_token)
        else:
            res_tok.extend( create_character_list( token, args, pad_flag=True, rep_flag=rep_flag ) )
    else:
        res_tok.append(start_token)
        res_tok.append(token)
        res_tok.append(end_token)

    return res_tok

def tiger_label_processing(label, args, pad_flag=True, rep_flag=False):

    if label != '$.':
        label_tokens = label.split('.')
    else:
        label_tokens = [label]

    if pad_flag:
        return [start_token] + label_tokens + [end_token]
    else:
        return label_tokens

def no_processing(token, args, pad_flag=True, rep_flag=False):
    return [token]

madar_translation = (create_character_list, no_processing, create_character_list)
madar_ex_translation = (create_character_list, no_processing, process_pos_tag, create_character_list)
madar_full_translation = (create_character_list, no_processing, create_character_list, process_pos_tag, create_character_list)
tarc_processing_base = (create_character_list, process_tokenization, process_pos_tag)   # Processing functions for raw tunisian, tokenized tunisian and POS
tarc_processing_ext = (create_character_list, no_processing, process_tokenization, process_pos_tag)    # Processing functions for arabish, tunisian, tokenized tunisian and POS
tarc_processing_full = (create_character_list, no_processing, create_character_list, process_tokenization, process_pos_tag)
tarc_processing_full_npos = (create_character_list, no_processing, no_processing, create_character_list, process_tokenization, process_pos_tag)
tarc_substep_1 = (create_character_list, no_processing, create_character_list)
tarc_substep_2 = (create_character_list, process_tokenization)
tarc_substep_3 = (process_tokenization, process_pos_tag)
tiger_mt_processing = (create_character_list, no_processing, no_processing)
tiger_mtext_processing = (create_character_list, no_processing, tiger_label_processing)
tiger4_mt_processing = (create_character_list, no_processing, no_processing, no_processing)

# NOTE: used to decide if tokens and chars representations must be computed and merged when their usage is specified from command line
granularity_merging_flags = {}
granularity_merging_flags['tiger-mt'] = (True, False, False)
granularity_merging_flags['tiger-mt-ext'] = (True, False, True)
granularity_merging_flags['tiger4-mt'] = (True, False, False, False)
granularity_merging_flags['madar-trs'] = (True, False, True)
granularity_merging_flags['madar-trs-ex'] = (True, False, True, True)
granularity_merging_flags['madar-trs-full'] = (True, False, True, True, True)
granularity_merging_flags['tarc-base'] = (True, False, True)
granularity_merging_flags['tarc-ext'] = (True, False, True, True)
granularity_merging_flags['tarc-full'] = (True, False, True, True, True)
granularity_merging_flags['tarc-full-npos'] = (True, False, False, True, True, True)
granularity_merging_flags['tarc-substep1'] = (True, False, True)
granularity_merging_flags['tarc-substep2'] = (True, True)
granularity_merging_flags['tarc-substep3'] = (True, True)

def choose_column_processing(num_columns, args):

    column_processing = None
    if args.sub_task == 'tiger-mt':
        print(' - TArCMultiTask, processing mode set to tiger-mt')
        sys.stdout.flush()
        column_processing = tiger_mt_processing
    elif args.sub_task == 'tiger-mt-ext':
        print(' - TArCMultiTask, processing mode set to tiger-mt-ext')
        sys.stdout.flush()
        column_processing = tiger_mtext_processing
    elif args.sub_task == 'tiger4-mt':
        print(' - TArCMultiTask, processing mode set to tiger4-mt')
        sys.stdout.flush()
        column_processing = tiger4_mt_processing
    elif args.sub_task == 'madar-trs':
        print(' - TArCMultiTask, processing mode set to madar-trs')
        sys.stdout.flush()
        column_processing = madar_translation
    elif args.sub_task == 'madar-trs-ex':
        print(' - TArCMultiTask, processing mode set to madar-trs-ex')
        sys.stdout.flush()
        column_processing = madar_ex_translation
    elif args.sub_task == 'madar-trs-full':
        print(' - TArCMultiTask, processing mode set to madar-trs-full')
        sys.stdout.flush()
        column_processing = madar_full_translation
    elif args.sub_task == 'tarc-base':
        print(' - TArCMultiTask, processing mode set to tarc-base')
        sys.stdout.flush()
        column_processing = tarc_processing_base
    elif args.sub_task == 'tarc-ext':
        print(' - TArCMultiTask, processing mode set to tarc-ext')
        sys.stdout.flush()
        column_processing = tarc_processing_ext
    elif args.sub_task == 'tarc-full':
        print(' - TArCMultiTask, processing mode set to tarc-full')
        sys.stdout.flush()
        column_processing = tarc_processing_full
    elif args.sub_task == 'tarc-full-npos':
        print(' - TArCMultiTask, processing mode set to tarc-full-npos')
        column_processing = tarc_processing_full_npos
    elif args.sub_task == 'tarc-substep1':
        print(' - TArCMultiTask, processing mode set to tarc-substep1')
        sys.stdout.flush()
        column_processing = tarc_substep_1
    elif args.sub_task == 'tarc-substep2':
        print(' - TArCMultiTask, processing mode set to tarc-substep2')
        sys.stdout.flush()
        column_processing = tarc_substep_2
    elif args.sub_task == 'tarc-substep3':
        print(' - TArCMultiTask, processing mode set to tarc-substep3')
        sys.stdout.flush()
        column_processing = tarc_substep_3
    elif args.sub_task == 'base':
        print(' - TArCMultiTask, processing mode set to base')
        sys.stdout.flush()
        column_processing = [no_processing for i in range(num_columns)]
    else:
        print(' - TArCMultiTask, setting default processing mode (no processing)')
        sys.stdout.flush()
        column_processing = [no_processing for i in range(num_columns)]

    return column_processing

def check_column_processing(num_columns, args):

    if num_columns == 3 and not args.sub_task in ['tiger-mt', 'tiger-mt-ext', 'tarc-base', 'base', 'madar-trs', 'tarc-substep1']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 2 and not args.sub_task in ['tarc-substep2', 'tarc-substep3', 'base']:
        raise ValueError(' 2 columns are expected with sub-task tarc-substep2|3')
    elif num_columns == 4 and not args.sub_task in ['tiger4-mt', 'tarc-ext', 'base','madar-trs-ex']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 5 and not args.sub_task in ['tarc-full','madar-trs-full', 'base']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 6 and not args.sub_task in ['tarc-full-npos', 'base']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif (not num_columns in [2, 3, 4, 5, 6]) and (not args.sub_task in ['base']):
        raise ValueError(' Unexpected number of columns in input file, possible values: 2, 3, 4, 5 unless base is specified. Got {}'.format(num_columns))

def apply_global_filling(args, curr_seq, cls_seq, Afillers, Lfillers, OFFcols, idx):

    (fillerFOR, fillerEMO) = Afillers
    (LfillerFOR, LfillerEMO) = Lfillers
    (CLS_COL_IDX, POS_COL_IDX) = OFFcols

    if args.sub_task in ['tarc-ext', 'tarc-full', 'madar-trs', 'madar-trs-ex', 'madar-trs-full', 'tarc-full-npos'] and idx != CLS_COL_IDX and idx != POS_COL_IDX:
        for s_idx in range(len(curr_seq)):
            if cls_seq[s_idx] == 'foreign':
                if (args.sub_task == 'tarc-full' and idx == 0) or (args.sub_task == 'tarc-full-npos' and idx == 0) or (args.sub_task == 'madar-trs' and idx == 2) or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                    curr_seq[s_idx] = LfillerFOR
                else:
                    curr_seq[s_idx] = fillerFOR
            elif cls_seq[s_idx] == 'emotag':
                if (args.sub_task == 'tarc-full' and idx == 0) or (args.sub_task == 'tarc-full-npos' and idx == 0) or (args.sub_task == 'madar-trs' and idx == 2) or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                    curr_seq[s_idx] = LfillerEMO
                else:
                    curr_seq[s_idx] = fillerEMO

    return curr_seq

def _add_t2c_entry_safe(D: Dict[str, List[str]], key: str, val: List[str]) -> Dict[str, List[str]]:

    if not (key in D):
        D[key] = val
    else:
        if len(D[key]) != len(val):
            print(' - WARNING: adding ambigous value to entry {}: 1. {} VS. 2. {}'.format(key, D[key], val))
            sys.stdout.flush()
        else:
            check_flag = True
            for i in range(len(val)):
                if val[i] != D[key][i]:
                    check_flag = False
                    break
            if not check_flag:
                print(' - WARNING: adding ambigous value to entry {}: 1. {} VS. 2. {}'.format(key, D[key], val))
                sys.stdout.flush()

    return D

def read_tarc_tabular_data(filename, args):

    with open(filename, 'rb') as f:
        lines = [l.decode('utf-8') for l in f.readlines()]
        f.close()

    POS_COL_IDX = set_pos_col_idx(args)
    test_flag = False
    if filename[-4:] == 'test':
        test_flag = True

    # NOTE: The following are lists to keep token-level and character-level information.
    #       char_seq_lengths and char_seq_lens are used to store start and end indices of tokens in the character-level information,
    #       so that when using both, the model can retrieve the list of embeddings of characters belonging to a given token, and mix them properly.
    char_sequences = []
    char_seq_lengths = []
    tok_sequences = []
    char_seqs = []
    char_seq_lens = []
    tok_seqs = []

    start_idx = 0
    num_columns = 0
    all_sequences = []
    curr_sequences = []
    for line in lines:
        tokens = line.rstrip(' \r\n').split('\t')

        #print(' - Read tokens ({}): {}'.format(len(tokens), tokens))
        #sys.stdout.flush()

        if len(tokens) > 1 and num_columns == 0:
            num_columns = len(tokens)
            all_sequences = [[] for i in range(num_columns)]
            curr_sequences = [[] for i in range(num_columns)]
        if len(tokens) > 1:

            if len(tokens) != num_columns:
                raise IOError(' Wrong data format, found different number of columns at different input lines ({} VS. {})'.format(len(tokens), num_columns))

            for i in range(num_columns):
                curr_sequences[i].append(tokens[i])
        else:
            for i in range(num_columns):
                curr_seq = apply_global_filling(args, curr_sequences[i], curr_sequences[CLS_COL_IDX], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], i)
                all_sequences[i].append( curr_seq )

            curr_sequences = [[] for i in range(num_columns)]

    print(' *** TarcMultiTask, read {} sequences'.format(len(all_sequences)))
    print(' *** TarcMultiTask, detected {} tasks in data'.format(num_columns-1))
    sys.stdout.flush()

    column_processing = choose_column_processing( num_columns, args )
    check_column_processing( num_columns, args )

    total_tokens = 0
    assert len(all_sequences) == num_columns
    num_sequences = len(all_sequences[0])
    for i in range(num_columns):
        assert len(all_sequences[i]) == num_sequences
    if args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        for i in range(num_sequences):
            seq_len = len(all_sequences[0][i])
            for j in range(num_columns):
                assert len(all_sequences[j][i]) == seq_len

    def init_llist(size):
        return [[] for i in range(size)]

    masked_tokens = 0
    token_offsets = []
    token2components = [{} for t_idx in range(num_columns)]
    for ts_idx in range(num_sequences):

        curr_seq_len = len(all_sequences[0][ts_idx])
        for s_idx in range(curr_seq_len):
            tokens = []
            for t_idx in range(num_columns):
                tokens.append( all_sequences[t_idx][ts_idx][s_idx] )

            assert len(tokens) == num_columns

            if len(char_sequences) == 0:
                char_sequences = init_llist( num_columns )
            if len(tok_sequences) == 0:
                tok_sequences = init_llist( num_columns )
            if len(char_seq_lengths) == 0:
                char_seq_lengths = init_llist( num_columns )
            if len(char_seqs) == 0:
                char_seqs = init_llist( num_columns )
            if len(char_seq_lens) == 0:
                char_seq_lens = init_llist( num_columns )
            if len(token_offsets) == 0:
                token_offsets = [0 for i in range(len(tokens))]
            if len(tok_seqs) == 0:
                tok_seqs = init_llist( num_columns )
            for idx in range(num_columns):
                if idx > 0 and args.ignore_test_output and test_flag:
                    process_res = no_processing(tokens[idx], args)
                else:
                    rep_flag = True
                    if (idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                        rep_flag = False
                    process_res = column_processing[idx]( tokens[idx], args, rep_flag=rep_flag )

                token2components[idx] = _add_t2c_entry_safe(token2components[idx], tokens[idx], process_res)

                char_seqs[idx].extend( process_res )
                char_seq_lens[idx].append( (token_offsets[idx], token_offsets[idx]+len(process_res)) )
                token_offsets[idx] += len(process_res)
                tok_seqs[idx].append( tokens[idx] )

        total_tokens += len(char_seqs[0])
        for idx in range(len(char_sequences)):
            char_sequences[idx].append( char_seqs[idx] )
            char_seq_lengths[idx].append( char_seq_lens[idx] )
            tok_sequences[idx].append( tok_seqs[idx] )
        char_seqs = []
        char_seq_lens = []
        token_offsets = []
        tok_seqs = []

    print(' - Size of char_sequences: {}'.format(len(char_sequences)))
    sys.stdout.flush()
    for ci in range(len(char_sequences)):
        print('   - Size of dim {}: {}'.format(ci, len(char_sequences[ci])))
        sys.stdout.flush()

    print(' - Loaded {} sequences, {} tokens (masked {}) from {}'.format( len(char_sequences[0]), total_tokens, masked_tokens, filename ))
    sys.stdout.flush()

    return (tok_sequences, char_sequences, char_seq_lengths, token2components)

def read_tarc_parallel_data( file_prefix, args ):

    input_file = file_prefix + '.input'
    output_files = glob.glob(file_prefix + '.output*')

    POS_COL_IDX = set_pos_col_idx(args)
    test_flag = False
    if file_prefix[-4:] == 'test':
        test_flag = True

    input_data = []
    f = open(input_file, encoding='utf-8')
    data = f.readlines()
    f.close()
    total_tokens = 0
    for s in data:
        tokens = s.rstrip(' \r\n').split()
        total_tokens += len(tokens)
        input_data.append(tokens)
    print(' - TArCMultiTask, read {} sequences, {} tokens from input data'.format(len(input_data), total_tokens))
    sys.stdout.flush()

    output_data = [[] for i in range(len(output_files))]
    for i in range( len(output_files) ):
        f = open(output_files[i], encoding='utf-8')
        data = f.readlines()
        f.close()

        total_tokens = 0
        s_idx = 0
        for s in data:
            tokens = s.rstrip(' \r\n').split()
            total_tokens += len(tokens)
            if i > 0:
                i_seq = input_data[s_idx]
                input_data[s_idx] = apply_global_filling(args, i_seq, output_data[CLS_COL_IDX-1][s_idx], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], 0)
                tokens = apply_global_filling(args, tokens, output_data[CLS_COL_IDX-1][s_idx], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], i+1)
            output_data[i].append( tokens )
            s_idx += 1
        print(' - TArCMultiTask, read {} sequences, {} tokens from output data {}'.format(len(output_data[i]), total_tokens, i))
        sys.stdout.flush()

    for i in range( len(output_data) ):
        assert len(input_data) == len(output_data[i])
        #for s_idx in range(len(input_data)):
        #    assert len( input_data[s_idx] ) == len( output_data[i][s_idx] )

    num_columns = len(output_data)+1
    column_processing = choose_column_processing(num_columns, args)
    check_column_processing(num_columns, args)

    total_tokens = 0
    # NOTE: The following are lists to keep token-level and character-level information.
    #       char_seq_lengths and char_seq_lens are used to store start and end indices of tokens in the character-level information,
    #       so that when using both, the model can retrieve the list of embeddings of characters belonging to a given token, and mix them properly.
    def init_llist(size):
        return [[] for i in range(size)]
    lsize = len(output_data)+1
    char_sequences = init_llist( lsize )
    char_seq_lengths = init_llist( lsize )
    tok_sequences = init_llist( lsize )
    char_seqs = init_llist( lsize )
    char_seq_lens = init_llist( lsize )
    tok_seqs = init_llist( lsize )
    token_offsets = [0 for i in range( lsize )]
    token2components = [{} for t_idx in range(num_columns)]
    for i in range( len(input_data) ):
        c_idx = 0
        for t in input_data[i]:
            rep_flag = True
            if (c_idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (c_idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and c_idx == 3) or (args.sub_task == 'madar-trs-full' and c_idx == 4):
                rep_flag = False
            #process_res = column_processing[idx]( tokens[idx], args, rep_flag=rep_flag )
            tok_res = column_processing[c_idx](t, args, rep_flag=rep_flag)
            total_tokens += len(tok_res)
            char_seqs[c_idx].extend( tok_res )
            char_seq_lens[c_idx].append( (token_offsets[c_idx], token_offsets[c_idx]+len(tok_res)) )
            token_offsets[c_idx] += len(tok_res)
            tok_seqs[c_idx].append( t )

            token2components[c_idx] = _add_t2c_entry_safe(token2components[c_idx], t, tok_res)

        char_sequences[c_idx].append( char_seqs[c_idx] )
        char_seq_lengths[c_idx].append( char_seq_lens[c_idx] )
        tok_sequences[c_idx].append( tok_seqs[c_idx] )
        c_idx += 1
        for j in range(len(output_data)):
            for t in output_data[j][i]:
                if args.ignore_test_output and test_flag:
                    process_res = no_processing(t)
                else:
                    rep_flag = True
                    if (c_idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (c_idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and c_idx == 3) or (args.sub_task == 'madar-trs-full' and c_idx == 4):
                        rep_flag = False
                    process_res = column_processing[c_idx](t, args, rep_flag=rep_flag)
                char_seqs[c_idx].extend( process_res )
                char_seq_lens[c_idx].append( (token_offsets[c_idx], token_offsets[c_idx]+len(process_res)) )
                token_offsets[c_idx] += len(process_res)
                tok_seqs[c_idx].append(t)

                token2components[c_idx] = _add_t2c_entry_safe(token2components[c_idx], t, process_res)

            char_sequences[c_idx].append( char_seqs[c_idx] )
            char_seq_lengths[c_idx].append( char_seq_lens[c_idx] )
            tok_sequences[c_idx].append( tok_seqs[c_idx] )
            c_idx += 1
        char_seqs = init_llist( lsize )
        char_seq_lens = init_llist( lsize )
        tok_seqs = init_llist( lsize )
        token_offsets = [0 for i in range(lsize)]

    print(' - TArCMultiTask, read {} sequences, {} tokens'.format(len(char_sequences[0]), total_tokens))
    sys.stdout.flush()

    return (tok_sequences, char_sequences, char_seq_lengths, token2components)

def map_tokens(data, dict, pad_flag):

    tensors = []
    for s in data:
        tok_lst = []
        for t in s:
            tok_lst.append( dict.add_symbol(t) )
        if pad_flag:
            tok_lst = [dict.bos()] + tok_lst + [dict.eos()]
        tensors.append( torch.LongTensor( tok_lst ) )

    return tensors

@register_task('tarc_multitask')
class TarcMultiTask(FairseqTask):

    @staticmethod
    def add_args(parser):

        # Add some command-line arguments for specifying where the data is
        # located and the maximum supported input length.
        parser.add_argument('data', metavar='FILE',
                            help='Main input file in tabular format. It must contain at least 2 columns; the first column is interpreted as the input, all the other columns are interpreted as outputs, unless --num-of-tasks and --task-indeces specify otherwise')
        parser.add_argument('--num-of-tasks', type=int, default=-1,
                            help='Number of tasks to learn simultaneously. It must correspond at most to the number of columns minus one in the main input file')
        parser.add_argument('--task-indeces', type=str, default='1:',
                            help='Indeces of columns in the main input file corresponding to the learnable tasks. It is represented as an interval string S:E')
        parser.add_argument('--additinal-data', type=str,
                            help='A semicolon separated list of files for multi-task learning')
        parser.add_argument('--data-description', type=str,
                            help='A string describing the content of each additinal input data in the format [N,S:E;]+, where N is the number of tasks, S:E the indeces of the learnable tasks')
        parser.add_argument('--pad-reference', action='store_true', default=False,
                            help='Specify to pad sequences with start and end of sequence markers')
        parser.add_argument('--max-source-positions', type=int, default=2048,
                            help='Maximum number of items in source sequences')
        parser.add_argument('--max-target-positions', type=int, default=2048,
                            help='Maximum number of items in target sequences')
        parser.add_argument('--sub-task', type=str, default='base',
                            help='Which multi-task problem to address: 1) base (default, no pre-processing) 2) tiger-mt, 2) tarc-base 3) tarc-ext')
        parser.add_argument('--data-format', type=str, default='tabular',
                            help='Format of input data: 1) tabular (default), 2) parallel')
        parser.add_argument('--sequence-separator', type=str, default='_SEQ_SEP_',
                            help='Used to separate output sequences from different tasks')
        parser.add_argument('--serialized-data', type=str,
                            help='Load data from a previously created serialization of the whole corpus. If it does not exist, read the whole corpus and serialize it.')
        parser.add_argument('--ignore-test-output', action='store_true', default=False,
                            help='Don\'t apply pre-processing to test output(s)')
        parser.add_argument('--keep-data-order', action='store_true', default=False,
                            help='Keep data in the original order, that is does not sort sequences by length.')
        parser.add_argument('--reverse-sequences', action='store_true', default=False,
                            help='Reverse sequences, that is they will be processed from right to left (e.g. for Arabic processing)')
        parser.add_argument('--reverse-tokens', action='store_true', default=False,
                            help='Reverse tokens individually, that is they will be processed from last to first char (e.g. for Arabic processing)')
        parser.add_argument('--load-madar-model', type=str, default='None',
                            help='Task specific for --sub-task=tarc-full, pre-trained model loading for parameter pre-initialization')
        parser.add_argument('--load-madar-data', type=str, default='None',
                            help='Task specific for --sub-task=tarc-full, serialized data loading for embedding pre-initialization')
        parser.add_argument('--token-sequences', action='store_true', default=False,
                            help='Use token-level information for modelling sequences')
        parser.add_argument('--char-sequences', action='store_true', default=False,
                            help='Use character-level information for modelling sequences')
        parser.add_argument('--double-learning', action='store_true', default=False,
                            help='Learn the model from both token and character representations')

    @classmethod
    def setup_task(cls, args, **kwargs):

        # Here we can perform any setup required for the task. This may include
        # loading Dictionaries, initializing shared Embedding layers, etc.
        # In this case we'll just initialize the label dictionary

        input_vocab = Dictionary(
                                 pad=pad_token,
                                 eos=eos_token,
                                 unk=unk_token,
                                 bos=bos_token,
                                 extra_special_symbols=[start_token, end_token]
        )
        output_vocab = input_vocab

        print('| [token] dictionary: {} types'.format(len(input_vocab)))
        print('| [label] dictionary: {} types'.format(len(output_vocab)))
        if (not args.token_sequences) and (not args.char_sequences):
            args.char_sequences = True

        if args.token_sequences:
            print('  - TArCMultiTask, using token-level information in sequences')
            sys.stdout.flush()
        if args.char_sequences:
            print('  - TArCMultiTask, using character-level information in sequences')
            sys.stdout.flush()

        return TarcMultiTask(args, input_vocab, output_vocab)

    def __init__(self, args, input_vocab, output_vocab):

        super().__init__(args)
        self.args = args

        self.input_vocab = input_vocab
        self.output_vocab = output_vocab

        self.sequence_separator = self.input_vocab.add_symbol(args.sequence_separator)
        self.token2components_tsr = []
        self.granularity_merging_flags = {}
        self.double_learning = args.double_learning
        self.splits = {}
        self.num_of_inputs = 1

    def set_granularity_merging_flags(self, g_flags):
        self.granularity_merging_flags = g_flags

    def get_granularity_merging_flags(self):
        return self.granularity_merging_flags

    def _t2c_to_tsr(self, t2c: Dict[str, List[str]], dict: Dictionary) -> Dict[int, torch.LongTensor]:

        res_dict = {}
        for k in t2c.keys():
            res_dict[dict.add_symbol(k)] = torch.LongTensor( [dict.add_symbol(v) for v in t2c[k]] )
        return res_dict

    def load_dataset(self, split, **kwargs):

        """Load a given dataset split (e.g., train, valid, test)."""

        if len(self.datasets) == 0:
            if self.args.serialized_data is not None and os.path.exists(self.args.serialized_data) and os.path.isfile(self.args.serialized_data):
                print(' - TArCMultiTask, reading serialized data from {}...'.format(self.args.serialized_data))
                sys.stdout.flush()
                self.splits = torch.load( self.args.serialized_data )
                self.input_vocab = self.splits['vocab']
                self.output_vocab = self.input_vocab
                self.token2components_tsr = self.splits['token2components']
            else:
                print(' - TArCMultiTask, creating dictionaries from whole corpus...')
                sys.stdout.flush()

                token2components = []
                for my_split in ['train', 'dev', 'test']:
                    data_sequences = []
                    if self.args.data_format == 'tabular':
                        data_sequences = read_tarc_tabular_data( self.args.data + '.' + my_split, self.args)
                    elif self.args.data_format == 'parallel':
                        data_sequences = read_tarc_parallel_data( self.args.data + '.' + my_split, self.args )
                    else:
                        raise NotImplementedError(' unsupported data format {}'.format(self.args.data_format))

                    if len(token2components) == 0:
                        token2components = [{bos_token : [bos_token], eos_token : [eos_token]} for t_idx in range(len(data_sequences[3]))]
                    for t_idx in range(len(data_sequences[3])):
                        token2components[t_idx].update( data_sequences[3][t_idx] )

                    if self.args.token_sequences and self.args.char_sequences:
                        assert len(data_sequences[0]) == len(data_sequences[1])
                        assert len(data_sequences[1]) == len(data_sequences[2])

                    print(' - load_dataset, read {} data split'.format(my_split))
                    print('   - got {} tasks'.format(len(data_sequences[0])-1))
                    sys.stdout.flush()

                    tensors = []
                    lengths = []
                    tok_tensors = []
                    tok_lengths = []
                    char_seq_lengths = [[] for i in range(len(data_sequences[2]))]
                    for d_idx in range( len(data_sequences[0]) ):
                        tok_tt = map_tokens(data_sequences[0][d_idx], self.input_vocab, self.args.pad_reference)
                        tok_tensors.append( tok_tt )
                        tok_ll = torch.LongTensor( [t.size(0) for t in tok_tt] )
                        tok_lengths.append( tok_ll )

                        seq_tt = []
                        for s in data_sequences[2][d_idx]:
                            if self.args.pad_reference:
                                s = [(p[0]+1, p[1]+1) for p in s]
                            char_seq_lengths[d_idx].append( torch.LongTensor( s ) )    # As many pairs as tokens...

                        tt = map_tokens(data_sequences[1][d_idx], self.input_vocab, self.args.pad_reference)
                        tensors.append( tt )
                        ll = torch.LongTensor([t.size(0) for t in tt])
                        lengths.append(ll)
                    self.splits[my_split] = ([tok_tensors, tensors], [tok_lengths, lengths, char_seq_lengths])

                for t_idx in range(len(token2components)):
                    self.token2components_tsr.append( self._t2c_to_tsr(token2components[t_idx], self.input_vocab) )

                self.splits['vocab'] = self.input_vocab
                self.splits['token2components'] = self.token2components_tsr
                if self.args.serialized_data is not None:
                    print(' - TArCMultiTask, serializing data to file {}...'.format(self.args.serialized_data))
                    sys.stdout.flush()
                    torch.save(self.splits, self.args.serialized_data)

        assert len(self.splits) == 5 and 'train' in self.splits.keys() and 'dev' in self.splits.keys() and 'test' in self.splits.keys()
        print(' - TArCMultiTask, instantiating current split {}'.format(split))
        sys.stdout.flush()
        my_split = split
        if split == 'valid':
            my_split = 'dev'

        tensors, lengths = self.splits[my_split]

        print(' - Split {} data lengths statistics:'.format(split))
        for idx, ll in enumerate(lengths[1]):
            print('\t- Max. length @{}: {}'.format(idx, torch.max(ll).item()))
            print('\t- Min. length @{}: {}'.format(idx, torch.min(ll).item()))
            print('\t-----')
        print(' - Dictionary sizes: {}, {}'.format(len(self.input_vocab), len(self.output_vocab)))
        print(' _______________')

        num_of_tasks = len(tensors[0])-1
        self.num_of_inputs = len(tensors[0]) - num_of_tasks
        self.args.num_of_tasks = num_of_tasks
        bound_idx = len(tensors[0])-num_of_tasks

        if 'base' not in granularity_merging_flags:
            granularity_merging_flags['base'] = tuple([False for i in range(num_of_tasks+1)])
        if self.args.sub_task not in granularity_merging_flags:
            granularity_merging_flags[self.args.sub_task] = tuple([False for i in range(num_of_tasks+1)])
        self.set_granularity_merging_flags(granularity_merging_flags[self.args.sub_task])

        sources = [tensors[0][0:bound_idx], tensors[1][0:bound_idx]]
        src_lengths = [lengths[0][0:bound_idx], lengths[1][0:bound_idx], lengths[2][0:bound_idx]]
        targets = [tensors[0][bound_idx:], tensors[1][bound_idx:]]
        tgt_lengths = [lengths[0][bound_idx:], lengths[1][bound_idx:], lengths[2][bound_idx:]]

        print(' - Tarc MultiTask, learning with {} input(s) (lengths: {}), {} different outputs (num. of tasks: {}, lengths: {})'.format(len(sources), len(src_lengths), len(targets), self.args.num_of_tasks, len(tgt_lengths)))
        sys.stdout.flush()

        input_feed = True
        self.datasets[split] = TarcMultiTaskDataset.TarcMultiTaskDataset(
            src=sources,
            src_sizes=src_lengths,
            src_dict=self.input_vocab,
            tgt=targets,
            tgt_sizes=tgt_lengths,
            sequence_separator=self.sequence_separator,
            tgt_dict=self.output_vocab,
            left_pad_target=False,
            max_source_positions=self.args.max_source_positions,
            max_target_positions=self.args.max_target_positions,
            input_feeding=input_feed,
            keep_data_order=self.args.keep_data_order,
            granularity_flags=(self.args.token_sequences, self.args.char_sequences),
        )

    def max_positions(self):
        """Return the max input length allowed by the task."""
        # The source should be less than *args.max_positions* and the "target"
        # has max length 1.
        return (self.args.max_source_positions, self.args.max_target_positions)

    @property
    def source_dictionary(self):
        """Return the source :class:`~fairseq.data.Dictionary`."""
        return self.input_vocab

    @property
    def target_dictionary(self):
        """Return the target :class:`~fairseq.data.Dictionary`."""
        return self.output_vocab

    # We could override this method if we wanted more control over how batches
    # are constructed, but it's not necessary for this tutorial since we can
    # reuse the batching provided by LanguagePairDataset.
    #
    # def get_batch_iterator(
    #     self, dataset, max_tokens=None, max_sentences=None, max_positions=None,
    #     ignore_invalid_inputs=False, required_batch_size_multiple=1,
    #     seed=1, num_shards=1, shard_id=0,
    # ):
    #     (...)

    def build_generator(self, args):

        from fairseq.tarc_multitask_sequence_generator import TarcSequenceGenerator

        return TarcSequenceGenerator(
            self.target_dictionary,
            max_len_a=getattr(args, "max_len_a", 0),
            max_len_b=getattr(args, "max_len_b", 200),
        )


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 23 09:13:04 2020

@author: Collabcap
"""

import os
ma_pos=os.getcwd()
output_path = os.path.join(ma_pos,"json")

import pymongo
from pymongo import MongoClient

import json
import numpy as np
import pandas as pd

import swifter

import argparse

import codecs
import re

import unicodedata

from stop_words import get_stop_words

import warnings
warnings.simplefilter('ignore', UserWarning)

import time

import logging
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO, datefmt='%m/%d/%Y %I:%M:%S %p')

#####################"

from multiprocessing import Pool, current_process

########################################"

DF = pd.ExcelFile( "list_mot_niv5.xlsx" )
tab_naf_code = pd.read_csv( "naf2008_5_niveaux.csv")

list_code_naf_niv5 = tab_naf_code.NIV5.str.replace(".","").tolist()
tab_naf_code["NIV5"] = list_code_naf_niv5
tab_naf_code["niveau5"] = list_code_naf_niv5
tab_naf_code["niveau4"] = tab_naf_code.niveau5.apply(lambda x : x[0:4])
tab_naf_code["niveau3"] = tab_naf_code.niveau5.apply(lambda x : x[0:3])
tab_naf_code["niveau2"] = tab_naf_code.niveau5.apply(lambda x : x[0:2])

###################################"

#writer = pd.ExcelWriter( "list_mot_niv4.xlsx" , engine='xlsxwriter')

writer = pd.ExcelWriter( "list_mot_niv4.xlsx" , engine='xlsxwriter')
print("niveau 4")

print(tab_naf_code.loc[0] )
list_etudie = tab_naf_code["niveau4"].unique().tolist()

for niv4 in list_etudie :
    logging.info(niv4)
    DF_temp = pd.DataFrame(None,columns = ["word","occurence"])
    list_niveau5 = tab_naf_code[ tab_naf_code.niveau4 == niv4]["niveau5"].tolist()
    for niv5 in list_niveau5 :
        temp = DF.parse( niv5 )
        DF_temp = DF_temp.append(temp )
    DF_excel = DF_temp.groupby( [ "word"] ).sum()
    DF_excel = DF_excel.sort_values(by = ['occurence'], ascending=False )
    DF_excel.to_excel(writer, sheet_name= niv4 , index = True)
    for column in DF_excel:
                column_length = max(DF_excel[column].astype(str).map(len).max(), len(column)) #ajusté a la plus grande cellue
                col_idx = DF_excel.columns.get_loc(column)
                writer.sheets[ niv4 ].set_column(col_idx, col_idx, min(column_length,30) )
writer.save()

writer = pd.ExcelWriter( "list_mot_niv3.xlsx" , engine='xlsxwriter')
print("niveau 3")

list_etudie = tab_naf_code["niveau3"].unique().tolist()

for niv3 in list_etudie :
    logging.info(niv3)
    DF_temp = pd.DataFrame(None,columns = ["word","occurence"])
    list_niveau5 = tab_naf_code[ tab_naf_code.niveau3 == niv3]["niveau5"].tolist()
    for niv5 in list_niveau5 :
        temp = DF.parse( niv5 )
        DF_temp = DF_temp.append(temp )
    DF_excel = DF_temp.groupby( [ "word"] ).sum()
    DF_excel = DF_excel.sort_values(by = ['occurence'], ascending=False )
    DF_excel.to_excel(writer, sheet_name= niv3 , index = True)
    for column in DF_excel:
                column_length = max(DF_excel[column].astype(str).map(len).max(), len(column)) #ajusté a la plus grande cellue
                col_idx = DF_excel.columns.get_loc(column)
                writer.sheets[ niv3 ].set_column(col_idx, col_idx, min(column_length,30) )
writer.save()

writer = pd.ExcelWriter( "list_mot_niv2.xlsx" , engine='xlsxwriter')

print("niveau 2")

list_etudie = tab_naf_code["niveau2"].unique().tolist()

for niv2 in list_etudie :
    logging.info(niv3)
    DF_temp = pd.DataFrame(None,columns = ["word","occurence"])
    list_niveau5 = tab_naf_code[ tab_naf_code.niveau2 == niv2]["niveau5"].tolist()
    for niv5 in list_niveau5 :
        temp = DF.parse( niv5 )
        DF_temp = DF_temp.append(temp )
    DF_excel = DF_temp.groupby( [ "word"] ).sum()
    DF_excel = DF_excel.sort_values(by = ['occurence'], ascending=False )
    DF_excel.to_excel(writer, sheet_name= niv2 , index = True)
    for column in DF_excel:
                column_length = max(DF_excel[column].astype(str).map(len).max(), len(column)) #ajusté a la plus grande cellue
                col_idx = DF_excel.columns.get_loc(column)
                writer.sheets[ niv2 ].set_column(col_idx, col_idx, min(column_length,30) )
writer.save()


# -*- coding: utf-8 -*-

from clasificador import Clasificador
from clasificador_no_encontrado import ClasificadorNoEntrenado
import utils
from copy import deepcopy

class ClasificadorDR(Clasificador):

    def __init__(self, clasificacion, clases, atributos):
        Clasificador.__init__(self, clasificacion, clases, atributos)
        self.reglas = None

    def entrena(self, entrenamiento, validacion=None):
        self.reglas = utils.obtener_total_reglas(self.atributos, entrenamiento)

    """
    Recibe como argumento un ejemplo y devuelve el valor de clasificación
    según el modelo obtenido en el entrenamiento.
    """

    def clasifica(self, ejemplo):
        if self.reglas:

            for regla, clase in self.reglas:

                # Si la regla es None, hemos llegado a la de por defecto
                if not regla:
                    return clase
                else:
                    clasificado = True
                    # Veremos si nuestro ejemplo coincide con una regla completa
                    for condicion in regla:
                        indice, valor = condicion
                        clasificado = clasificado and ejemplo[indice] == valor

                        # En el momento que una condición no cumpla evitamos comprobar el resto de ellas
                        if not clasificado:
                            break

                    if clasificado:
                        return clase
        else:
            raise ClasificadorNoEntrenado('método clasifica')

    """
    Recibe como argumento un conjunto de prueba y devuelve el rendimiento del
    modelo obtenido en el entrenamiento.
    """

    def evalua(self, prueba):
        if self.reglas:

            # Calcula ejemplos correctamente clasificados
            res = 0

            for x in prueba:
                y = deepcopy(x)
                clase = y.pop(-1)
                clase_rama = self.clasifica(y)
                if clase_rama == clase:
                    res += 1

            rendimiento = res / len(prueba)

            return rendimiento
        else:
            raise ClasificadorNoEntrenado('método evalua')

    """
    Imprime de forma legible el modelo obtenido en el entrenamiento.
    """

    def imprime(self):
        if self.reglas:
            print(self.reglas)
        else:
            raise ClasificadorNoEntrenado('método imprime')


import spacy
import tkinter as tk

class Redactor(tk.Tk):
    def __init__(self):
        super().__init__()
        self.title("Text Redactor")

        self.entry = tk.Text(self, width=50, height=10, wrap="word")
        self.entry.pack(pady=20)

        self.redact_button = tk.Button(self, text="Redact", command=self.redact_text)
        self.redact_button.pack()

        options = tk.Frame(self)
        self.variables = {}
        entities = [
            "PERSON", "NORP", "FAC", "ORG", "GPE", "LOC", "PRODUCT", "EVENT", "WORK_OF_ART",
            "LAW", "LANGUAGE", "DATE", "TIME", "PERCENT", "MONEY", "QUANTITY", "ORDINAL", "CARDINAL"]

        cols = 4

        for i, entity in enumerate(entities):
            # Create a BooleanVar for each entity
            self.variables[entity] = tk.BooleanVar()

            # Create a Checkbutton for each entity, positioned in a grid
            tk.Checkbutton(options, text=entity, variable=self.variables[entity]).grid(row=i // cols, column=i % cols)  # This will distribute the buttons in a grid layout

        self.variable_all = tk.BooleanVar()
        tk.Checkbutton(options, text="All", command=self.trigger_all, variable=self.variable_all)
        options.pack()

        self.result_label = tk.Label(self, text="Redacted Text will appear here.", wraplength=500)
        self.result_label.pack(pady=20)

    def trigger_all(self):
        value = self.variable_all.get()
        print(value)
        for entity, var in self.variables.items():
            var.set(value)

    def redact_text(self):
        text = self.entry.get("1.0", "end-1c")
        redacted_text, unredacted_items = self.redact_names(text)
        self.result_label['text'] = redacted_text
        with open('log.txt', 'w+') as f:
            f.write("Potential Classifications Errors:\n")
            for line in unredacted_items:
                f.write(line + "\n")

            f.write("In Text:\n")
            f.write(redacted_text)

    def redact_names(self, text):
        nlp = spacy.load("en_core_web_sm")
        doc = nlp(text)
        redacted_sentence = text
        unredacted_items = []
        for ent in doc.ents:
            if self.variables[ent.label_].get():
                redacted_sentence = redacted_sentence.replace(ent.text, '[REDACTED]')
            else:
                unredacted_items.append(ent.text)
        print()
        return redacted_sentence, unredacted_items

if __name__ == "__main__":
    redactor_app = Redactor()
    redactor_app.mainloop()


# -*- encoding:utf-8 -*-

"""
here we prepare the dataset in form of .pkl files
"""

from __future__ import print_function

import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import pickle
import pandas as pd
import re
np.random.seed(1337)
import sys

# transforms a csv file to pkl

def clean_text(text):
    text = text.lower()
    text = text.replace("\r","")
    text = text.replace("\n","")
    text = text.strip()
    return text

def load_data(filename):
    rows = pd.read_csv(filename, keep_default_na=False)
    sequences = []

    for i in range(rows.shape[0]):
        text = rows.loc[i].ReviewText
        if text != "":
            sequences.append(clean_text(text))
    return sequences

def main():

    embedding_file = sys.argv[1]
    param_file = sys.argv[2]
    input_file = sys.argv[3]

    # loading the data
    sequences = load_data(input_file)
    print("Data loaded")

    #tokenizer = Tokenizer()
    tokenizer = Tokenizer(5000) # we keep the 5000 most frequent words of our vocabulary

    tokenizer.fit_on_texts(sequences)

    #MAX_SEQUENCE_LENGTH = max([len(seq) for seq in sequences])
    #print(MAX_SEQUENCE_LENGTH)
    MAX_SEQUENCE_LENGTH = 100 # we keep the last 100 words of every sentence

    #MAX_NB_WORDS = len(tokenizer.word_index) + 1
    MAX_NB_WORDS = 5000

    word_index = tokenizer.word_index

    print("MAX_SEQUENCE_LENGTH: {}".format(MAX_SEQUENCE_LENGTH))
    print("MAX_NB_WORDS: {}".format(MAX_NB_WORDS))

    sequences = tokenizer.texts_to_sequences(sequences)
    sequences = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

    # loading the embedding file

    print('Now indexing word vectors...')

    embeddings_index = {}
    f = open(embedding_file, 'r')
    for line in f:
        values = line.split()
        word = values[0]
        try:
            coefs = np.asarray(values[1:], dtype='float32')
        except ValueError:
            continue
        embeddings_index[word] = coefs
    f.close()

    embedding_size = len(embeddings_index.values()[0])

    maximum = max(max(v) for v in embeddings_index.values()) # we'll use the max value of the embeddings to normalize every word embedding vector

    print("Now constructing embedding matrix...")

    num_words = min(MAX_NB_WORDS, len(word_index))
    embedding_matrix = np.zeros((num_words , embedding_size))
    for word, i in word_index.items():
        if i >= MAX_NB_WORDS:
            continue
        embedding_vector = embeddings_index.get(word)

        if embedding_vector is not None:
            # we normalize word embedding vectors
            embedding_matrix[i] = [float(x)/maximum for x in embedding_vector] #normalize the embedding_vector
        else:
            embedding_matrix[i] = np.random.normal(-0.25, 0.25, embedding_size)

    # we need to represent every input sequence using word embeddings
    temp = np.zeros((sequences.shape[0], MAX_SEQUENCE_LENGTH, embedding_size))
    for i in range(sequences.shape[0]):
        # for each sequence
        for j in range(MAX_SEQUENCE_LENGTH):
            # for each word of the sequence, get it embedding vector from the embedding matrix
            temp[i][j] = embedding_matrix[sequences[i][j]]

    sequences = temp
    print(sequences.shape)

    print("Now dumping")
    pickle.dump(sequences, open(input_file + ".pkl", "wb"))

if __name__ == '__main__':
    main()


#!/usr/bin/python3
"""Write csv file where rows from an input file matching specified criteria are split into multiple rows based on rows
and matching criteria in another input file.

Sometimes you need to clarify the details of a particular transaction using data from another source.

The script takes one required argument:
filter=Path to a csv file containing the columns: matching column a, matching regex a, comparison column a, comparison
column b, destination column(s) a, source column(s) b
split_source=Path to a csv file with data to use for performing split operation.

The script takes the following optional arguments:
input=Path to a csv file containing the designated input columns.
output=Path to write the output csv file. This file will be overwritten without warning if it exists.
"""
import csv
import argparse
import re
import sys
from copy import deepcopy

author = 'brian.k.smith@gmail.com'

parser = argparse.ArgumentParser(description='Split rows where a column matches a regex based on values in'
                                             ' second input.')
parser.add_argument('filter', help='Path to csv filter file with columns: matching column a, matching regex a,'
                                   'comparison column a, comparison column b, '
                                   'destination columns a (semicolon separated list), '
                                   'source columns b (semicolon separated list, order matching destinations),'
                                   'currency column a (column where the split values should add up to original)')
parser.add_argument('split', help='Secondary input file that contains data to use to split rows in primary.')
parser.add_argument('--add_imbalance', help='Correct for differences between actual and expected currency by adding an'
                                            'imbalance transaction.', action='store_true')
parser.add_argument('--input', help='Path to csv input file with column(s) to examine.'
                                    ' Defaults to stdin.')
parser.add_argument('--output', help='Path to csv output file. This file will be overwritten if it exists. Defaults'
                                     ' to stdout.')

args = parser.parse_args()

currency_regex = re.compile('\$?\s*(-?\d+\.\d+)')

# Take care of any default setup needed
close_input = True
if args.input is None:
    close_input = False
    args.input = 0
close_output = True
if args.output is None:
    close_output = False
    args.output = 1

# Open required files
# noinspection PyTypeChecker
with open(args.input, newline='', encoding='UTF-8', closefd=close_input) as input_file,\
        open(args.split, newline='') as split_file,\
        open(args.filter, newline='') as filter_file,\
        open(args.output, mode='w', newline='', closefd=close_output) as output_file:
    in_reader = csv.reader(input_file)
    filter_reader = csv.reader(filter_file, )
    split_reader = csv.reader(split_file)
    out_writer = csv.writer(output_file)
    # Set up filters map
    filters = []
    filter_headers = next(filter_reader)
    for filter_row in filter_reader:
        filt = {"a_match_col": int(filter_row[0]),
                "a_match_regex": re.compile(filter_row[1]),
                "a_comp_col": int(filter_row[2]),
                "b_comp_col": int(filter_row[3]),
                "a_dest_col": list(map(int, filter_row[4].split(';'))),
                "b_source_col": list(map(int, filter_row[5].split(';'))),
                "a_currency_col": int(filter_row[6])}
        filters.append(filt)
    # Set up split data map. Index by comparison column: list of lists containing the values from all of the
    # file B source columns
    split_data = {}
    split_headers = next(split_reader)
    for row in split_reader:
        for filt in filters:
            if row[filt["b_comp_col"]]:
                if row[filt["b_comp_col"]] not in split_data:
                    split_data[row[filt["b_comp_col"]]] = []
                data_list = []
                for col in filt["b_source_col"]:
                    data_list.append(row[col])
                split_data[row[filt["b_comp_col"]]].append(data_list)
    # Set up output file with input headers
    output_headers = next(in_reader)
    out_writer.writerow(output_headers)
    # Iterate through input
    row_count = 0
    for row in in_reader:
        row_count += 1
        out_row = deepcopy(row)
        row_split = False
        for filt in filters:
            match = filt["a_match_regex"].search(row[filt["a_match_col"]])
            if match:
                original_currency = float(row[filt["a_currency_col"]])
                split_currency = 0.0
                for key in split_data:
                    if row[filt["a_comp_col"]] == key:
                        for d_list in split_data[key]:
                            split_row = deepcopy(out_row)
                            col_count = 0
                            while col_count < len(filt["a_dest_col"]):
                                dest_col = filt["a_dest_col"][col_count]
                                if dest_col == filt["a_currency_col"]:
                                    match = currency_regex.match(d_list[col_count])
                                    split_string = match.group(1)
                                    split_row[dest_col] = split_string
                                    split_currency += float(split_string)
                                else:
                                    split_row[dest_col] = d_list[col_count]
                                col_count += 1
                            out_writer.writerow(split_row)
                            row_split = True
                if row_split and abs(original_currency - split_currency) > 0.001:
                    if args.add_imbalance:
                        imba_row = deepcopy(out_row)
                        col_count = 0
                        while col_count < len(filt["a_dest_col"]):
                            dest_col = filt["a_dest_col"][col_count]
                            if dest_col == filt["a_currency_col"]:
                                imba = original_currency - split_currency
                                split_row[dest_col] = "{:0.2f}".format(imba)
                            else:
                                split_row[dest_col] = "IMBALANCE"
                            col_count += 1
                        out_writer.writerow(split_row)
                    else:
                        print("ERROR: Sum of currency in split rows (${:.2f}) does not equal starting currency"
                              "(${:.2f}) at row {:d} ({})!"
                              .format(split_currency, original_currency, row_count, ",".join(row)), file=sys.stderr)
        if not row_split:
            out_writer.writerow(out_row)


import re
import pickle

from collections import defaultdict, Counter
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize, sent_tokenize
import pandas as pd
from sklearn import preprocessing
from fkscore import fkscore
import numpy as np

'''
Interaction Dynamics utils
'''
# refined_sample = pickle.load(open('samples/interaction_dynamics_sample.pkl', mode='rb'))

# returns all comment authors that received a delta
def deltas(submission):
    delta_authors = []
    id_map = {}
    for comment in submission['comments']:
        if 'author' in comment.keys():
            id_map[comment['id']] = comment['author']
            if comment['author'] == 'DeltaBot':
                author = re.search(r"(?<=\/u\/).+(?=\.)", comment['body'])
                if author is not None:
                    delta_authors.append(author.group(0))
    return delta_authors

# sorts all comments in a submission by creation time
def sort_comments(submission):
    comments = []
    for comment in submission['comments']:
        if 'created' in comment.keys() and 'author' in comment.keys():
            comments.append(comment)
    return sorted(comments, key= lambda comment: comment['created'])

# counts the number of comments made by each challenger
def num_author_comments(submission):
    id_map = {}
    op = submission['author']
    for comment in submission['comments']:
        if 'author' in comment.keys():
            id_map[comment['id']] = comment['author']
    author_comment_map = defaultdict(int)
    for comment in submission['comments']:
        if 'author' in comment.keys():
            if comment['author'] not in ['DeltaBot', op]:
                author_comment_map[comment['author']] += 1
    return author_comment_map

# counts the length of back-and-forths between challenger and op
def op_challenger_chain_counts(submission):
    id_map = {}
    op = submission['author']
    for comment in submission['comments']:
        if 'author' in comment.keys():
            id_map[comment['id']] = comment['author']
    op_chal_chains = defaultdict(int)
    for comment in submission['comments']:
        if 'author' in comment.keys() and comment['replies']:
            for reply_id in comment['replies']['data']['children']:
                if reply_id in id_map.keys():
                    if id_map[reply_id] == op:
                        op_chal_chains[comment['author']] += 1
                    elif comment['author'] == op:
                        op_chal_chains[id_map[reply_id]] += 1
    return op_chal_chains

'''
Language Indicator utils
'''
stop_words = set(stopwords.words('english'))

# simple implementation of jaccard score and other measures for two lists
def jaccard(A, O):
    set1 = set(A)
    set2 = set(O)
    return len(set1.intersection(set2))/len(set1.union(set2))

def common_words(A, O):
    return len(set(A).intersection(set(O)))

def reply_fraction(A, O):
    set1 = set(A)
    set2 = set(O)
    return len(set1.intersection(set2))/len(set1)

def op_fraction(A, O):
    set1 = set(A)
    set2 = set(O)
    return len(set1.intersection(set2))/len(set2)

# tokenizing and cleaning of comments text, replacing quotes and links with special tokens
def tokenize(comment):
    comment = comment.replace('>', 'QUOTE')
    comment = re.sub(r'http\S+', 'LINK', comment)
    comment = comment.replace('(LINK', 'LINK')
    tokens = word_tokenize(comment)
    return tokens
# identity function (just return)
def id(obj):
    return obj
# function to remove stop words from tokenized
def remove_stop_words(tokens):
    return [word for word in tokens if word.lower() not in stop_words]
# function to only retrieve the stopw ords from tokenized
def only_stop_words(tokens):
    return [word for word in tokens if word.lower() in stop_words]
# separate a root-path unit into root reply and full path
def separate(rooted_path_unit):
    return rooted_path_unit['comments'][0]['body'], '\n\n'.join([comment['body'] for comment in rooted_path_unit['comments']])

interplay_features = ['# common ', 'jaccard ', 'op frac ', 'reply frac ']
interplay_bases = ['all', 'content', 'stop']
interplay_cols = []
for feature in interplay_features:
    for base in interplay_bases:
        interplay_cols.append(feature + base)
INTERPLAY_COLS = interplay_cols

# load VAD and concreteness csv as pd, keep the mean columns for each word
CONCRETENESS = pd.read_csv('docs/concreteness.csv')[['Word', 'Rating.Mean']]
VAD = pd.read_csv('docs/VAD.csv')[['Word', 'V.Mean.Sum', 'A.Mean.Sum', 'D.Mean.Sum']]

ARG_COLS = ['# indefinite articles', '# definite articles', '# 1st person pronouns', '# 1st person plural pronouns',
            '# 2nd person pronouns', '# of links', '# of quotes', '# questions',
            'V', 'A', 'D', 'C',
            '# of sentences', '# of paragraphs', 'Flesch-Kincaid Readability']

def word_count(tokens):
    return len([w for w in tokens if w.isalnum()])

# return info for word category-based features in tokenized
def word_category_info(tokens):
    quote_link_counter = Counter(tokens)
    num_links = quote_link_counter['LINK']
    num_quotes = quote_link_counter['QUOTE']
    lowered = [token.lower() for token in tokens]
    counter = Counter(lowered)
    features_map = {'# indefinite articles':['a', 'an'], '# definite articles': ['the'], '# 1st person pronouns': ['i', 'me', 'my', 'mine', 'myself'],
                    '# 1st person plural pronouns': ['us', 'we', 'ours', 'ourselves', 'our'],
                    '# 2nd person pronouns': ['you', 'your', 'yours', 'yourself', 'yourselves'], '# questions': ['?']}

    count_map = {feature:sum([counter[token] for token in value]) for feature, value in features_map.items()}
    count_map['# of links'] = num_links
    count_map['# of quotes'] = num_quotes
    return count_map

# returns average V,A,D,C for content words in tokenized passage
def word_score_info(tokens):
    results = {feature:0.0 for feature in ['V', 'A', 'D','C']}
    tokens = remove_stop_words(tokens)
    lowered_tokens = [token.lower() for token in tokens]
    new_length = word_count(lowered_tokens)
    for token in lowered_tokens:
        if token in VAD['Word'].values:
            results['V'] += VAD.loc[VAD['Word'] == token, 'V.Mean.Sum'].item()
            results['A'] += VAD.loc[VAD['Word'] == token, 'A.Mean.Sum'].item()
            results['D'] += VAD.loc[VAD['Word'] == token, 'D.Mean.Sum'].item()
        if token in CONCRETENESS['Word'].values:
            results['C'] += CONCRETENESS.loc[CONCRETENESS['Word'] == token, 'Rating.Mean'].item()
    return {key:value/new_length for key, value in results.items()}

def num_sentences(text):
    return len(sent_tokenize(text))

def num_paragraphs(text):
    return len(text.split('\n\n'))

def flesch_kincaid_score(text):
    return fkscore(text).score['readability']

# entire text information for comments
def entire_text_features(text):
    results = {}
    results['# of sentences'] = num_sentences(text)
    results['# of paragraphs'] = num_paragraphs(text)
    results['Flesch-Kincaid Readability'] = flesch_kincaid_score(text)
    return results

def quarters(text):
    tokens = tokenize(text)
    return [list(array) for array in np.array_split(tokens, 4)]

# Persuaion utils

# CMV posts insert a footer at the end of the post, remove it and add title to the body of post
def remove_footer(submission):
    lines = [line for line in submission['selftext'].splitlines()
             if not line.lstrip().startswith('>') and not line.lstrip().startswith('_____')
             and "edit" not in " ".join(line.lower().split())
            ]
    return submission['title'] + "\n\n" + "\n".join(lines)

def remove_CMV(text):
    return text.replace('CMV:', '', 1)


from shapely.geometry import Point
import folium

class Roadmap():

    def __init__(self, dataset_str):
        # xypygamexy

        self.dataset_str = dataset_str
        self.map_props = get_map_props()

        self.lower_left = get_map_props()[dataset_str]['lower_left']
        self.upper_right = get_map_props()[dataset_str]['upper_right']

        try:  # movingpandas
            from movingpandas.geometry_utils import measure_distance_geodesic
            self.max_dis_y = measure_distance_geodesic(Point(self.lower_left[0], self.lower_left[1]),
                                                  Point(self.upper_right[0], self.lower_left[1]))
            self.max_dis_x = measure_distance_geodesic(Point(self.lower_left[0], self.lower_left[1]),
                                                  Point(self.lower_left[0], self.upper_right[1]))
            # print(f'max_x = {self.max_dis_x}, max_y = {self.max_dis_y}')
        except:
            # hardcode
            if dataset_str == 'NCSU':
                self.max_dis_y = 3255.4913305859623
                self.max_dis_x = 2718.3945272795013
            elif dataset_str == 'purdue':
                self.max_dis_y = 1671.8995666382975
                self.max_dis_x = 1221.4710883988212
            elif dataset_str == 'KAIST':
                self.max_dis_y = 2100.207579392558
                self.max_dis_x = 2174.930950809533

    def lonlat2pygamexy(self, lon, lat):
        '''
        pygamexy.
        yx
        np.array
        '''

        x = - self.max_dis_x * (lat - self.upper_right[1]) / (self.upper_right[1] - self.lower_left[1])
        y = self.max_dis_y * (lon - self.lower_left[0]) / (self.upper_right[0] - self.lower_left[0])
        return x, y

    def pygamexy2lonlat(self, x, y):
        lon = y * (self.upper_right[0] - self.lower_left[0]) / self.max_dis_y + self.lower_left[0]
        lat = - x * (self.upper_right[1] - self.lower_left[1]) / self.max_dis_x + self.upper_right[1]
        return lon, lat

def get_map_props():
    map_props = {
        'NCSU':
            {
                'lower_left': [-78.6988, 35.7651],  # lon, lat
                'upper_right': [-78.6628, 35.7896]
            },
        'purdue':
            {
                'lower_left': [-86.93, 40.4203],
                'upper_right': [-86.9103, 40.4313]
            },
        'KAIST':
            {
                'lower_left': [127.3475, 36.3597],
                'upper_right': [127.3709, 36.3793]
            }
    }
    return map_props

def traj_to_timestamped_geojson(index, trajectory, num_uav, num_agent, color, only_UVs=False):  # indextrajindexenum

    point_gdf = trajectory.df.copy()
    point_gdf["previous_geometry"] = point_gdf["geometry"].shift()
    point_gdf["time"] = point_gdf.index  # datetimeindex
    point_gdf["previous_time"] = point_gdf["time"].shift()

    features = []
    # for Point in GeoJSON type
    for _, row in point_gdf.iterrows():
        if only_UVs and index >= num_agent: break
        corrent_point_coordinates = [row["geometry"].xy[0][0], row["geometry"].xy[1][0]]
        current_time = [row["time"].isoformat()]

        # ra = {'uav': 3, 'car': 5, 'human': 0.5}  #
        ra = {'uav': 5, 'car': 7, 'human': 2}  #
        op = {'uav': 1, 'car': 1, 'human': 1}  # opacity0.2

        if index < num_uav:  # UAV
            radius, opacity = ra['uav'], op['uav']
        elif num_uav <= index < num_agent:  # CAR
            radius, opacity = ra['car'], op['car']
        else:  # human
            radius, opacity = ra['human'], op['human']

        # for Point in GeoJSON type  (Temporally Deprecated)
        features.append(  #
            {
                "type": "Feature",
                "geometry": {
                    "type": "Point",
                    "coordinates": corrent_point_coordinates,
                },
                "properties": {
                    "times": current_time,
                    "icon": 'circle',  # point
                    "iconstyle": {
                        'fillColor': color,
                        'fillOpacity': opacity,  #
                        'stroke': 'true',
                        'radius': radius,
                        'weight': 1,
                    },
                    "style": {  # line
                        "color": color,
                        "opacity": opacity
                    },
                    "code": 11,

                },
            }
        )
    return features

def folium_draw_circle(map, pos, color, radius, weight):  #
    folium.vector_layers.Circle(
        location=pos,  #
        radius=radius,  #  m
        color=color,  #
        # fill=True,  #
        # fill_color='#%02X%02X%02X' % (0, 0, 0),  #
        # fillOpacity=1,  # Fill opacity
        weight=weight  #
    ).add_to(map)

def folium_draw_CircleMarker(map, pos, color, radius):  #
    folium.CircleMarker(
        location=pos,
        radius=radius,
        color=color,
        stroke=False,
        fill=True,
        fill_opacity=1,
        opacity=1,
        popup="{} ".format(radius),
        tooltip=str(pos),
    ).add_to(map)

def get_border(ur, lf):
    upper_left = [lf[0], ur[1]]
    upper_right = [ur[0], ur[1]]
    lower_right = [ur[0], lf[1]]
    lower_left = [lf[0], lf[1]]

    coordinates = [
        upper_left,
        upper_right,
        lower_right,
        lower_left,
        upper_left
    ]

    geo_json = {"type": "FeatureCollection",
                "properties": {
                    "lower_left": lower_left,
                    "upper_right": upper_right
                },
                "features": []}

    grid_feature = {
        "type": "Feature",
        "geometry": {
            "type": "Polygon",
            "coordinates": [coordinates],
        }
    }

    geo_json["features"].append(grid_feature)

    return geo_json


import json

import boto3
import ulid as ulid
from Logger import Logger
from constants import *
from FileManager import FileManager
from DataframeManager import DataframeManager

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')

def results(status, job_execution_id, processed, failed):
    return {
        'statusCode': status,
        'body': json.dumps({
            'job_execution_id': job_execution_id,
            'processed_files': [f's3://{f.bucket_name}/{f.key}' for f in processed],
            'failed_files': [f's3://{f.bucket_name}/{f.key}' for f in failed]
        })
    }

def main(event, context):
    logger = Logger(__name__)
    job_execution_id = ulid.ulid()
    file_manager = FileManager(s3, s3_client)
    dataframe_manager = DataframeManager()

    logger.info('start processing')
    logger.info(f'execution id={job_execution_id}')
    landed_files_by_type = file_manager.find_landed_files()
    logger.info(f'landed {len(landed_files_by_type)} file(s)')
    process_files = dataframe_manager.process_landed_files(landed_files_by_type, job_execution_id, file_manager)
    return results(200, job_execution_id, process_files, [])

if __name__ == '__main__':
    main(None, None)


from django.conf import settings
from django.db import models

class Location(models.Model):
    id = models.AutoField(primary_key=True)
    name = models.TextField()
    description = models.TextField()

def get_default_new_item_location():
    try:
        return Location.objects.get(name=settings.NEW_ITEM_DEFAULT_LOCATION)
    except Location.DoesNotExist:
        return Location.objects.create(
            name=settings.NEW_ITEM_DEFAULT_LOCATION,
            description="Default new item location",
        )


import numpy as np
from gym import utils

from ex2.envs.env_utils import get_asset_xml
from ex2.envs.twod_mjc_env import TwoDEnv

INIT_POS = np.array([0.15,0.15])
TARGET = np.array([0.15, -0.15])
DIST_THRESH = 0.12

class TwoDMaze(TwoDEnv, utils.EzPickle):
    def __init__(self, verbose=False):
        self.verbose = verbose
        utils.EzPickle.__init__(self)
        TwoDEnv.__init__(self, get_asset_xml('twod_maze.xml'), 2, xbounds=[-0.3,0.3], ybounds=[-0.3,0.3])

    def _step(self, a):
        self.do_simulation(a, self.frame_skip)
        ob = self._get_obs()
        pos = ob[0:2]
        if pos[0]>0.0 and pos[1]<-.08:
            dist = np.sum(np.abs(pos-TARGET))
            reward = - (dist) * 5
        else:
            reward = -1

        if self.verbose:
            print(pos, reward)
        done = False
        return ob, reward, done, {}

    def reset_model(self):
        qpos = self.init_qpos + self.np_random.uniform(size=self.model.nq, low=-0.01, high=0.01)
        qvel = self.init_qvel + self.np_random.uniform(size=self.model.nv, low=-0.01, high=0.01)
        self.set_state(qpos, qvel)
        return self._get_obs()

    def _get_obs(self):
        return np.concatenate([self.model.data.qpos]).ravel() - INIT_POS

    def viewer_setup(self):
        v = self.viewer


class UtakaException(Exception):
	def __init__(self, args, httpStatus=200):
		self.args = args
		self.httpStatus = httpStatus

	def __str__(self):
		return str(httpStatus) + ": " + str(self.args)

	def toxml(self, debug=False):
		import xml.dom.minidom
		doc = xml.dom.minidom.Document()
		errorEl = doc.createElement("Error")
		for key, val in self.args.iteritems():
			keyEl = doc.createElement(str(key))
			keyEl.appendChild(doc.createTextNode(str(val)))
			errorEl.appendChild(keyEl)
		if debug:
			debugEl = doc.createElement("Debug")
			import traceback
			tracebackEl = doc.createElement("Traceback")
			tracebackEl.appendChild(doc.createTextNode(str(traceback.format_exc())))
			debugEl.appendChild(tracebackEl)
			if 'devArgs' in self.__dict__.keys():
				for key, val in self.devArgs.iteritems():
					keyEl = doc.createElement(str(key))
					keyEl.appendChild(doc.createTextNode(str(val)))
					debugEl.appendChild(keyEl)
			errorEl.appendChild(debugEl)
		doc.appendChild(errorEl)
		return doc.toxml('utf-8')


"""
models.py
Contains models for the coinex API
"""

from decimal import *
import coinex_api
from datetime import datetime

# set the decimal precision to 8
getcontext().prec = 8

class Balance:
    """
    A container for a balance of a given currency
    Attributes:
        currency: the Currency
        amount: a Decimal, the amount of currency
    Balance.get_own() : get a list of all own balances
    """

    next_id = 0

    def __init__(self, currency, amount, held=Decimal(0)):
        """
        Create a new Balance object.
        currency: either a currency object or an id
        amount: the amount, either a numeric or a string that parses to numeric
        held: an optional Decimal of how much is being held
        """
        self.held = Decimal(held)
        self.id = Balance.next_id
        Balance.next_id += 1
        if isinstance(currency, int):
            cur = Currency.get(currency)
            if cur is None:
                raise ValueError("currency id was not valid")
            self.currency = cur
        elif isinstance(currency, Currency):
            self.currency = currency
        else:
            raise ValueError(
                "currency was not valid, should be int or Currency"
            )
        self.amount = Decimal(amount)

    def __eq__(self, other):
        return isinstance(other, Balance) and other.id == self.id

    def __ne__(self, other):
        return not self.__eq__(other)

    @classmethod
    def get_own(cls):
        """
        Get all own balances
        """
        bals = coinex_api.balances()
        ret = []
        for bal in bals:
            curr = Currency.get(bal['currency_id'])
            amt = Decimal(bal['amount']) / pow(10, 8)
            held = Decimal(bal['held']) / pow(10, 8)
            b = Balance(curr, amt, held=held)
            ret.append(b)
            registry.put(b)
        return ret

class Currency:
    """
    A container for a currency
    Attributes:
        id: the currency ID
        abbreviation: the short name (ie 'BTC')
        name: the long name (ie 'Bitcoin')
    Currency.get(id_)  : get a currency that corresponds with the given ID
    Currency.get_all() : get all currencies available
    """

    _loaded = False

    def __init__(self, currency_id, abbreviation, name):
        self.id = int(currency_id)
        self.abbreviation = str(abbreviation)
        self.name = str(name)

    def __eq__(self, other):
        return isinstance(other, Currency) and other.id == self.id

    def __ne__(self, other):
        return not (self.__eq__(other))

    @classmethod
    def _refresh(cls):
        """
        Refresh all currencies, reloading from the API
        """
        registry.delete_all(cls)
        curs = coinex_api.currencies()
        for cur in curs:
            curncy = Currency(cur['id'], cur['name'], cur['desc'])
            registry.put(curncy)
        cls._loaded = True

    @classmethod
    def get(cls, id_):
        if not cls._loaded:
            cls._refresh()
        id_ = int(id_)
        return registry.get(Currency, id_)

    @classmethod
    def get_all(cls):
        """
        Returns a generator for all objects
        """
        if not cls._loaded:
            cls._refresh()
        return registry.get_all(Currency)

class Exchange:
    """
    A container for an exchange
    Attributes:
        id: the trade_pair_id
        from_currency: the Currency from which this trades (ie Bitcoin)
        to_currency: the Currency to which this trades (ie Mooncoin)
        get_orders() : get a list of all orders
        get_recent_trades() : get a list of recently executed trades
    Exchange.get(int_): get the Exchange for this ID
    Exchange.get_all(): get all Exchanges available
    """

    _loaded = False

    def __init__(self, trade_pair_id, from_currency, to_currency):
        """
        Construct a new exchange.
        from_currency: the Currency from which this is being transferred,
                       or the id of it
        to_currency: the currency to which this is being transferred,
                     or the id of it
        orders: a list of orders made on this exchange
        """
        self.id = int(trade_pair_id)

        if isinstance(from_currency, int):
            cur = Currency.get(from_currency)
            if cur is None:
                raise ValueError("The from id is invalid")
            self.from_currency = cur
        elif isinstance(from_currency, Currency):
            self.from_currency = from_currency
        else:
            raise ValueError("The from_currency was invalid")

        if isinstance(to_currency, int):
            cur = Currency.get(to_currency)
            if cur is None:
                raise ValueError("The to id is invalid")
        elif isinstance(to_currency, Currency):
            self.to_currency = to_currency
        else:
            raise ValueError("The to_currency was invalid")

    def get_orders(self):
        """
        Load / get all orders for the current exchange
        """
        # remove old ones if needed
        ords = coinex_api.orders(self.id)
        ret = []
        for order in ords:
            o = Order(API_resp=order)
            registry.put(o)
            ret.append(o)
        return ret

    def get_recent_trades(self):
        """
        Load / get all recent trades for this exchange
        """
        ords = coinex_api.last_trades(self.id)
        ret = []
        for order in ords:
            o = Order(API_resp=order)
            registry.put(o)
            ret.append(o)
        return ret

    def get_highest_bid(self):
        """
        Get order of the highest price that someone is bidding
        (willing to buy for)
        """
        ords = self.get_orders()
        ords = [ordr for ordr in ords if ordr.bid is True]
        best = max(ords, key=lambda x: x.rate)
        return best

    def get_lowest_ask(self):
        """
        Get order of the lowest price that someone is asking
        (willing to sell for)
        """
        ords = self.get_orders()
        ords = [ordr for ordr in ords if ordr.bid is False]
        best = min(ords, key=lambda x: x.rate)
        return best

    def get_best_offer(self, target_cur):
        """
        Get the best offer (order) for the given currency.
        NOTE: this may be highest bid or lowest ask
        """
        if target_cur == self.to_currency:
            return self.get_lowest_ask()
        elif target_cur == self.from_currency:
            return self.get_highest_bid()
        raise ValueError(
            'Unsupported currency for this exchange ' +
            target_cur.abbreviation
        )

    @classmethod
    def _refresh(cls):
        """
        Refresh the registry's exchanges
        """
        if (cls._loaded):
            registry.delete_all(Exchange)
        excs = coinex_api.trade_pairs()
        for exc in excs:
            exchange = Exchange(
                exc['id'],
                Currency.get(exc['market_id']),
                Currency.get(exc['currency_id'])
            )
            registry.put(exchange)

        cls._loaded = True

    @classmethod
    def get(cls, id_):
        if not cls._loaded:
            cls._refresh()
        return registry.get(Exchange, id_)

    @classmethod
    def get_all(cls):
        if not cls._loaded:
            cls._refresh()
        return registry.get_all(Exchange)

class Order:
    """
    A container for an order
    Attributes:
        order_id: the order id
        exchange: the Exchange for this order
        bid: true if this is bid(buy), false for ask(sell)
        rate: Decimal rate at from_currency per to_currency
        amount: the amount of this Order
        filled: the amount of this order that has been filled
        cancelled: true if this order is cancelled
        complete: true if this order is completed
    Order.get_own() : get all own orders
    """

    def __init__(self,
                 API_resp=None,
                 order_id=None,
                 exchange=None,
                 bid=None,
                 amount=None,
                 filled=None,
                 cancelled=None,
                 complete=None,
                 rate=None,
                 created_at=None,
                 completed_at=None):
        """
        Construct a new order
        NOTE: either pass the response from coinex_api into API_resp or
        manually add each property
        """
        if API_resp is not None:
            order = API_resp
            self.id = order['id']
            self.exchange = Exchange.get(API_resp['trade_pair_id'])
            self.amount = Decimal(order['amount']) / pow(10, 8)
            self.rate = Decimal(order['rate']) / pow(10, 8)
            self.bid = order['bid']
            # if this is a pending order do this
            if 'filled' in order:
                self.filled = Decimal(order['filled']) / pow(10, 8)
                self.cancelled = order['cancelled']
                self.complete = order['complete']
                self.created_at = datetime.strptime(
                    order['created_at'],
                    '%Y-%m-%dT%H:%M:%S.%fZ'
                )
                self.complete_at = None
            # else this order was already done, has some different keys
            else:
                self.filled = self.amount
                self.cancelled = False
                self.complete = True
                self.created_at = None
                self.completed_at = datetime.strptime(
                    order['created_at'],
                    '%Y-%m-%dT%H:%M:%S.%fZ'
                )
        else:
            self.id = int(order_id)
            self.exchange = exchange
            self.bid = bool(bid)
            self.amount = amount
            self.rate = Decimal(rate)
            self.filled = filled
            self.cancelled = cancelled
            self.complete = complete
            self.created_at = created_at
            self.completed_at = completed_at

    @classmethod
    def get_own(cls):
        """
        Get all own orders
        """
        ords = coinex_api.open_orders()
        ret = []
        for ordr in ords:
            exc = Exchange.get(ordr['trade_pair_id'])
            amt = Decimal(ordr['amount']) / pow(10, 8)
            rate = Decimal(ordr['rate']) / pow(10, 8)
            ret.append(
                Order(
                    API_resp=ordr
                )
            )
        return ret

    def get_compliment(self, transac_fee=Decimal('0.002'), max_amt=None):
        """
        Get the compliment for this order which, when submitted,
        will fulfill the other order.
        """
        amt = (self.amount - self.filled) / Decimal(1 - transac_fee)
        if max_amt is not None:
            amt = min(max_amt, amt)
        other = Order(
            order_id=-1,
            exchange=self.exchange,
            amount=amt,
            bid=not self.bid,
            rate=self.rate
        )
        return other

    def submit(self):
        """
        Submit this order to the coinex api.
        Resets this object's properties to reflect
        the new ones reported by coinex_api, and
        also returns the parsed JSON returned by coinex
        """
        ordr = coinex_api.submit_order(
            trade_pair_id=self.exchange.id,
            amount=self.amount,
            bid=self.bid,
            rate=self.rate
        )
        self.__init__(API_resp=ordr)
        return ordr

class Wallet:
    """
    A container to represent total currency held
    """

    next_id = 0

    def __init__(self):
        self.id = Wallet.next_id
        Wallet.next_id += 1

    @classmethod
    def get_balances(cls):
        return Balance.get_own()

    @classmethod
    def get_orders(cls):
        return Order.get_own()

class Registry:
    """
    Maintains a registry of all models
    """

    def __init__(self):
        self._dct = {}

    def get(self, model, id_):
        """
        Get a model from the registry by ID
        model: the class of model to get
        id_: the id of the model to get
        """
        if model in self._dct and id_ in self._dct[model]:
            return self._dct[model][id_]
        else:
            return None

    def get_all(self, cls):
        """
        returns a list for all the objects in the registry
        """
        ret = []
        dct = self._dct[cls]
        if dct is not None:
            for key in dct:
                ret.append(dct[key])
        return ret

    def put(self, model):
        """
        Put a model into the registry
        """
        cls = model.__class__
        if not (cls in self._dct):
            self._dct[cls] = {}
        self._dct[cls][model.id] = model

    def delete(self, obj, model=None):
        """
        Delete from the registry
        obj: either int, the object's id, or the model itself
        model: if an int supplied, this is the class of the model to delete
        returns the model that was deleted, or null if none was found
        """
        if isinstance(obj, int):
            if model in self._dct:
                if obj in self._dct[model]:
                    ret = self._dct[model][obj]
                    self._dct[model][obj] = None
                    return ret
        else:
            cls = obj.__class__
            if cls in self._dct:
                if obj.id in self._dct[cls]:
                    ret = self._dct[cls][obj.id]
                    self._dct[cls][obj.id] = None
                    return ret
        return None

    def delete_all(self, cls):
        """
        Delete all of a certain class from the registry
        cls: the class to remove
        """
        self._dct[cls] = {}

registry = Registry()


 # -*- coding: utf-8 -*-
from django import template
from web.models import Project
import re

register = template.Library()

def carousel(value, arg):
    projects_list = Project.objects.filter(image__gt='').filter(incarousel=True)
    return projects_list

def truncatewords_by_chars(value, arg):
    """Truncate the text when it exceeds a certain number of characters.
    Delete the last word only if partial.
    Adds '...' at the end of the text.

    Example:

        {{ text|truncatewords_by_chars:25 }}
    """
    try:
        length = int(arg)
    except ValueError:
        return value

    if len(value) > length:
        if value[length:length + 1].isspace():
            return value[:length].rstrip() + '...'
        else:
            return value[:length].rsplit(' ', 1)[0].rstrip() + '...'
    else:
        return value

def show_person_box(person,width):
    return {'person': person,
            'width':width}

def add_http(value):
    """Add http protocol when is missing in a URL."""
    if re.match('http://\w*', value):
        return value
    else:
        return 'http://'+value

def show_title(current_page,args):
    if current_page == '':
       current_page = 'Home'
    return 'The CulturePlex ─ '+current_page

register.filter('show_title', show_title)
register.filter('add_http', add_http)
register.filter('carousel', carousel)
register.filter('truncatewords_by_chars', truncatewords_by_chars)
register.inclusion_tag('person_box.html')(show_person_box)


from flask import current_app
from ldap3 import ObjectDef
from ..sanity import _check_dn_exists

def check_userbase_exists(fix=True):
    """
    Check the User base DN exists, and if not, create it.
    """
    conn = current_app.ldap3_login_manager.connection
    ou = ObjectDef("organizationalunit", conn)
    userbase = "%s,%s" % (
        current_app.config["LDAP_USER_DN"],
        current_app.config["LDAP_BASE_DN"],
    )
    _check_dn_exists(userbase, ou, fix)

def check_group_base_exists(fix=True):
    """
    Check the Group base DN exists, and if not, create it.
    """
    conn = current_app.ldap3_login_manager.connection
    ou = ObjectDef("organizationalunit", conn)
    groupbase = "%s,%s" % (
        current_app.config["LDAP_GROUP_DN"],
        current_app.config["LDAP_BASE_DN"],
    )
    _check_dn_exists(groupbase, ou, fix)