import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import pickle
from torch.utils import data as data2
import torch.optim as optim
import numpy as np
from torch.autograd import Variable
import os, sys
import copy
import math
import time

from distill_network import *
import PIL.Image as Image
import utils
import logging
import logging.config

init_opt = 'similar_init'
# config logging in python code
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# create a file handler
handler = logging.FileHandler('distill_main_{}.log'.format(init_opt))# set RotatingileHandler and use handler.doRollover() to create a new logging file each time)
handler.setLevel(logging.INFO)

# create a logging format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)

# add the file handler to the logger
logger.addHandler(handler)

# time.clock() => time.process_time(). Deprecated in Python 3.3 and will be removed from 3.8
try:
    clock = time.process_time
except AttributeError:
    clock = time.clock
# later shall use `clock()`

# define distill loss
class DistillLoss(nn.Module):

    def __init__(self):
        super(DistillLoss,self).__init__()

    def forward(self, input, target):

        # TODO implement the distill loss here

        y_shape = y.size()[1]
        x_added_dim = x.unsqueeze(1)
        x_stacked_along_dimension1 = x_added_dim.repeat(1,NUM_WORDS,1)
        diff = torch.sum((y - x_stacked_along_dimension1)**2,2)
        totloss = torch.sum(torch.sum(torch.sum(diff)))
        return totloss

# parameters for slide-learn
logger.info("set params")

batch_size=23
num_epochs = 200
learning_rate = 0.001
log_interval = 10
save_interval  = 200
seq_len = 75

distill = True

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# def load_dataloaders():

#     # load data
#     batch_size=23
#     [train_ids, train_labels, test_ids, test_labels] = pickle.load(open('slide_6_10.pkl', 'rb'))
#     # print("loading data")
#     # print(len(train_ids), len(train_labels), len(test_ids)) # 1173 1173 253
#     training_dataset = Dataset(train_ids, train_labels)
#     X_train = training_dataset.get_X()
#     Y_train = training_dataset.get_y()
#     # print("example Dataset")
#     # print(training_dataset[0][0].size()) # tuple e.g. (torch.Size([1, 6, 10, 75]), 0)
#     # from iCaRL print(X_train.shape, Y_train.shape, X_test.shape) #(50000, 3, 32, 32) (50000,) (10000, 3, 32, 32)
#     test_dataset = Dataset(test_ids, test_labels)
#     X_test = test_dataset.get_X()
#     Y_test = test_dataset.get_y()
#     # print(X_train.shape)
#     # print(len(X_train))
#     # print(Y_train[0])
#     # print(Y_train.shape)
#     # print(X_train.shape, Y_train.shape, X_test.shape)
#     train_loader = data2.DataLoader(training_dataset, batch_size=batch_size)
#     test_loader = data2.DataLoader(test_dataset, batch_size=batch_size)

#     return train_loader, test_loader

# def slide_learn(iteration):

#     dirName = 'slide_info_' + str(iteration)
#     try:
#         # Create target Directory
#         os.mkdir(dirName)
#     except FileExistsError:
#         print('Re-writing the touch')
#     dirName = dirName + '/'

#     train_loader, test_loader = load_dataloaders()

#     model = CNN_LSTM(seq_len).to(device)

#     # optimizer
#     optimizer = optim.Adam(model.parameters(), lr=learning_rate)

#     '''
#     def get_total_loss(networks):
#     frcnn_xe_loss = tf.reduce_mean([net.compute_frcnn_crossentropy_loss() for net in networks])
#     tf.summary.scalar('loss/frcnn/class', frcnn_xe_loss)
#     frcnn_bbox_loss = tf.reduce_mean([net.compute_frcnn_bbox_loss() for net in networks])
#     tf.summary.scalar('loss/frcnn/bbox', frcnn_bbox_loss)
#     l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
#     tf.summary.scalar('loss/weight_decay', l2_loss)
#     total_loss = frcnn_xe_loss + frcnn_bbox_loss + l2_loss

#     if args.distillation:
#         distillation_xe_loss = tf.reduce_mean([net.compute_distillation_crossentropy_loss() for net in networks])
#         tf.summary.scalar('loss/distillation/class', distillation_xe_loss)
#         distillation_bbox_loss = tf.reduce_mean([net.compute_distillation_bbox_loss() for net in networks])
#         tf.summary.scalar('loss/distillation/bbox', distillation_bbox_loss)
#         total_loss += distillation_xe_loss + distillation_bbox_loss
#     tf.summary.scalar('loss/total', total_loss)
#     return total_loss
#     '''

#     if distill:
#         criterion = nn.CrossEntropyLoss() + DistillLoss()

#     else:
#         criterion = nn.CrossEntropyLoss()

#     # set initial hidden state
#     (h_ini, c_ini) = (torch.zeros(2, batch_size, 50).to(device) , torch.zeros(2, batch_size, 50).to(device))

#     epoch_lists = []
#     model.train()
#     for epoch in range(1, num_epochs + 1):
#         for batch_idx, (data, target) in enumerate(train_loader):

#             data = np.expand_dims(data, axis=1)
#             data = torch.FloatTensor(data)
#             #print(target.size())
#             data, target = data.to(device), target.to(device)

#             data, target = Variable(data), Variable(target)
#             #print('Data size:', data.size())

#             optimizer.zero_grad()

#             # init hidden states
#             model.init_hidden(h_ini, c_ini)

#             output = model(data)
#             #print(target.size(), output.size())

#             loss = F.nll_loss(output, target)
#             loss.backward()
#             optimizer.step()
#             #if batch_idx % log_interval == 0:
#         epoch_lists.append(loss.item())
#         if epoch % save_interval == 0:
#             torch.save(model.state_dict(), dirName + 'model_epoch_' + str(epoch) +'.ckpt')

#     # save epochs
#     pickle.dump(epoch_lists, open(dirName + 'loss.pkl', 'wb'))

#     ## check for accuracy

#     results = []
#     model.eval()
#     test_loss = 0
#     correct = 0
#     for data, target in train_loader:

#         data = np.expand_dims(data, axis=1)
#         data = torch.FloatTensor(data)
#         data, target = data.to(device), target.to(device)
#         data, target = Variable(data, volatile=True), Variable(target)
#         output = model(data)
#         test_loss += criterion(
#             output, target).item()  # sum up batch loss
#         pred = output.data.max(
#             1, keepdim=True)[1]  # get the index of the max log-probability
#         correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

#     test_loss /= len(train_loader.dataset)
#     results.append( 100.0 * correct.item() / len(train_loader.dataset) )

#     test_loss = 0
#     correct = 0
#     for data, target in test_loader:

#         data = np.expand_dims(data, axis=1)
#         data = torch.FloatTensor(data)
#         data, target = data.to(device), target.to(device)
#         data, target = Variable(data, volatile=True), Variable(target)
#         output = model(data)
#         test_loss += criterion(
#             output, target).item()  # sum up batch loss
#         pred = output.data.max(
#             1, keepdim=True)[1]  # get the index of the max log-probability
#         correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

#     test_loss /= len(test_loader.dataset)

#     results.append( 100.0 * correct / len(test_loader.dataset) )

#     pickle.dump(results, open(dirName + 'results.pkl', 'wb'))

# slide_learn(400)

###########
# Distillation network
###########

def load_datasets():

    # load data
    [train_ids, train_labels, test_ids, test_labels] = pickle.load(open('slide_6_10_{}_classwise.pkl'.format(init_opt), 'rb'))
    # print("loading data")
    # print(len(train_ids), len(train_labels), len(test_ids)) # 1173 1173 253
    # interpret the data class by class
    logger.info("train_ids {} {}".format(len(train_ids), train_ids))
    X_train = []
    Y_train = []
    X_test = []
    Y_test = []
    for i in range(len(train_ids)):
        print("prepare data for class %d" %(i))
        training_dataset = Dataset(train_ids[i], train_labels[i])
        X_train.append(training_dataset.get_X())
        Y_train.append(training_dataset.get_y())
        # print("example Dataset")
        # print(training_dataset[0][0].size()) # tuple e.g. (torch.Size([1, 6, 10, 75]), 0)
        # from iCaRL print(X_train.shape, Y_train.shape, X_test.shape) #(50000, 3, 32, 32) (50000,) (10000, 3, 32, 32)
        test_dataset = Dataset(test_ids[i], test_labels[i])
        X_test.append(test_dataset.get_X())
        Y_test.append(test_dataset.get_y())
        # print(X_train.shape)
        # print(len(X_train))
        # print(Y_train[0])
        # print(Y_train.sh?ape)
    X_train = np.array(X_train)
    X_test = np.array(X_test)
    Y_train = np.array(Y_train)
    Y_test = np.array(Y_test)
    # print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape) # (23, 50, 6, 10, 75) (23, 50) (23, 12, 6, 10, 75) (23, 12)
    logger.info("load datasets {} {} {} {}".format(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape))
    X_val = 0
    Y_val = 0
    return X_train, Y_train, X_val, Y_val, X_test, Y_test

# parameters for distill-learning

iteration = 400
iteration_finetune = 300

lr = 0.001
schedules = range(50, iteration, 50)
gamma = 0.5
momentum = 0.9
decay = 0.0001
batchsize=23
num_class = 23
init_num_class = 3 # so that the rest 20 classes can be evenly distributed over periods
cur_num_class = init_num_class
new_num_class = 5
# np.random.seed(100)
# class_order = np.random.permutation(num_class)
class_order = np.arange(num_class)
logger.info("class order {}".format(class_order))

memory_K = 300 # (50 per class * 6 classes) #2000 # TODO: tune later
T = 2
dist_ratio = 0.5
stop_acc = 0.998

period_train = (num_class-init_num_class)//new_num_class
memory_size = memory_K//new_num_class # how many samples to load each time
class_old = np.array([], dtype=int)
memory_images = np.zeros(shape=(0, memory_size, 1, 6, 10, 75), dtype = np.uint8)
memory_labels = np.zeros(shape=(0, memory_size), dtype=int)
acc_nvld_basic = np.zeros((period_train))
acc_nvld_finetune = np.zeros((period_train))
crossentropy = nn.CrossEntropyLoss()

# get feature dim
# skip since we are not using pre-trained resnet
input_size = (1, 6, 10, 75)
# newshape=(-1,)+input_size "(-1, 1, 6, 10)"

# load the dataset
images_train, labels_train, images_val, labels_val, images_test, labels_test = load_datasets()
num_train_per_class = images_train.shape[1]
# logger.info("num_train_per_class %d " % num_train_per_class)
# get model state
model_path = 'model/model_slide_e2e_{}_{}.pth'.format(init_opt,cur_num_class+new_num_class)
flag_model = os.path.exists(model_path)

# gradually increase cur_num_class
for period in range(period_train):
    logger.info('-----------------------------------')
    logger.info('period = %d' % (period))
    if period == 0:
        class_novel = class_order[:(cur_num_class+new_num_class)]
    else:
        class_novel = class_order[cur_num_class:(cur_num_class+new_num_class)]
    logger.info("new classes: {}".format(class_novel))
    # organize the dataset for this period
    # TODO: instead of select sequentially, use representative classes for the init training
    images_new_train = np.reshape(images_train[class_novel], newshape=(-1, 1, 6, 10, 75)) # one more dim on time compared to orginal img
    labels_new_train = np.reshape(labels_train[class_novel], newshape=(-1))
    images_new_test = np.reshape(images_test[class_novel], newshape=(-1, 1, 6, 10, 75))
    labels_new_test = np.reshape(labels_test[class_novel], newshape=(-1))

    num_class_old = class_old.shape[0]

    if period == 0:
        # use the init data to train
        images_combined_train = images_new_train
        labels_combined_train = labels_new_train
        # create the model
        net = Network(num_class=(cur_num_class+new_num_class), num_new_class=0)
        net.to(device)

    else:
        # combine the representatives in memory and newly added data to train
        images_combined_train = np.concatenate((images_new_train, np.reshape(memory_images, newshape=(-1, 1, 6, 10, 75))), axis=0)
        labels_combined_train = np.concatenate((labels_new_train, np.reshape(memory_labels, newshape=(-1))), axis=0)
        # create a new model and make use of previously trained model by state_dict
        net = Network(num_class=num_class_old, num_new_class=new_num_class)
        # logger.info("num_class_old %d, new_num_class %d" %(num_class_old, new_num_class))
        if period == 1:
            last_net = Network(num_class=num_class_old, num_new_class=0)
        else:
            last_net = Network(num_class=num_class_old-new_num_class, num_new_class=new_num_class)
        last_net.load_state_dict(torch.load(model_path))
        last_net.eval()
        pretrained_dict = last_net.state_dict()
        net_dict = net.state_dict()

        # print("last_net dict")
        # print([key for key, value in pretrained_dict.items()])
        # print("net_dict")
        # print([key for key, value in net_dict.items()])

        # filter the pretrained dict, drop the last linear weight for net (and subnet if it exists)
        inherit_layers = ['cnn', 'lstm'] # drop the last linear layer
        pretrained_dict = {k:v for k, v in pretrained_dict.items() if any(layer in k for layer in inherit_layers)}
        # print("filtered pretrained_dict")
        # print([key for key, value in pretrained_dict.items()])

        # 1. filter out unnecessary keys
        pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in net_dict}
        # 2. overwrite entries in the existing state dict
        net_dict.update(pretrained_dict)
        # 3. load the new state dict
        net.load_state_dict(net_dict)
        net.to(device)

    # reorganize the datasets
    images_nvld_test = images_test[np.concatenate((class_old, class_novel), axis=0)]
    images_nvld_test = np.reshape(images_nvld_test, newshape= (-1, 1, 6, 10, 75))
    labels_nvld_test = labels_test[np.concatenate((class_old, class_novel), axis=0)]
    labels_nvld_test = np.reshape(labels_nvld_test, newshape=(-1))

    # skip data augmentation part

    logger.info("training size = %d"%(labels_combined_train.shape[0]))

    # training
    lrc = lr
    logger.info("current lr = %f" % (lrc))
    acc_training = []
    softmax = nn.Softmax(dim=1).to(device)

    if net.subnet:
        net_old = net.subnet
    else:
        net_old = copy.deepcopy(net)

    # training over iterations/ bactches
    for iter in range(iteration):

        # learning rate
        if iter in schedules:
            lrc *= gamma
            # print("type iter", type(iter)) # int
            logger.info("iteration {}, current lr= {}".format(iter, lrc))

        # optimizer
        #optimizer = optim.Adam(net.parameters(), lr=lrc, momentum=momentum, \
        #   weight_decay=decay, nesterov=True)
        optimizer = optim.Adam(net.parameters(), lr=lrc)

        # train
        # print("begin train")
        # print(labels_combined_train.shape[0])
        idx_train = np.random.permutation(labels_combined_train.shape[0])
        # print("idx_train")
        # print(idx_train)
        # raise ValueError("stop here")
        loss_avg = 0
        loss_cls_avg = 0
        loss_dist_avg = 0
        acc_avg = 0
        num_exp = 0
        tstart = time.process_time()
        batchnum_train = math.floor(labels_combined_train.shape[0]/batchsize)
        correct = 0
        results = []
        # load model
        if period == 0 and flag_model:
            logger.info("load model %s for period 0" % model_path)
            net.load_state_dict(torch.load(model_path))
            net = net.to(device)
            # not the case for CNN-LSTM unless u load the model before
            # TODO use the state_dict to load partial model for either first model or subnet
        # print("batchnum_train", batchnum_train)
        # print("idx_train", idx_train)
        net.train()
        for batch in range(batchnum_train):
            if period == 0 and flag_model: # no need training
                num_exp = 1
                break

            if batch == batchnum_train - 1:
                idx = idx_train[batch * batchsize:] # last batch, take all the rest
            else:
                idx = idx_train[batch * batchsize : (batch+1)*batchsize]
            bs = len(idx) # same as batchsize unless it is the last batch
            img = images_combined_train[idx]
            lab = labels_combined_train[idx]

            # TODO: change onehot encoding to current number of classes only
            # lab_onehot = utils.one_hot(lab, num_class)
            lab_onehot = utils.one_hot(lab, (cur_num_class+new_num_class))

            # transform
            # TODO double check the data format
            # see Network: batch_size, timesteps, C, H, W, sequence_size = x.size() requires 6D
            img = np.expand_dims(img, axis=1)
            img = torch.FloatTensor(img).to(device)
            # print("img input example", img.shape) # torch.Size([23, 1, 1, 6, 10, 75])
            # print(img)
            lab_onehot = torch.from_numpy(lab_onehot).float().to(device)

            optimizer.zero_grad()

            # init hidden space for lstm
            (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device) , torch.zeros(2, bs, 50).to(device))
            net.init_hidden(h_ini, c_ini)

            output = net(img)
            # print("output")
            # print(output)
            # try to use nll loss, same as slide_learn
            lab = torch.from_numpy(lab).to(device)
            # print("lab")
            # print(lab)

            # raise ValueError("stop here to check")

#            indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
#            indices = indices.to(device)
#            # if period > 0:
#            #     # print the variables for debugging
#            #     print("lab")
#            #     print(lab)
#            #     print("lab_onehot", lab_onehot.size())
#            #     print(lab_onehot)
#            #     print("output", output.size())
#            #     print(output)
#            #
#            #     print("indicesi", indices.size())
#            #     print(indices)
#            prob_cls = softmax(torch.index_select(output, 1, indices))
#            lab_onehot = torch.index_select(lab_onehot, 1, indices)
#            # if period >0:
#            #     print("prob_cls", prob_cls.size(), "lab_onehot", lab_onehot.size())
#            #     print("combined old & new indices")
#            #     print(indices)

            loss_cls = F.nll_loss(output, lab)
            # loss_cls = F.binary_cross_entropy(prob_cls, lab_onehot) # TODO: binary? # classification loss
            # print("batch", batch, "classification loss ", loss_cls)
            # distllation loss for only old class data !
            if period > 0:

                indices = torch.LongTensor(class_old).to(device)
                dist = torch.index_select(output, 1, indices)
                dist = softmax(dist/T)
                # same initialization procedure as net
                (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device), torch.zeros(2, bs, 50).to(device))
                net_old.init_hidden(h_ini, c_ini)
                output_old = net_old(img)
                ouput_old = torch.index_select(output_old, 1, indices)
                lab_dist = softmax(Variable(output_old, requires_grad=False))

                # # TODO: RH: what is the difference, one is class_old, one is enumerate(lab) if la in class_old
                # if not flag_dist_all: # TODO haven't set the flag before
                #     # only for old class data
                #     indices = [id for id, la in enumerate(lab) if la in class_old]
                #     indices = torch.LongTensor(indices)
                #     indices = indices.cuda()
                #     dist = torch.index_select(dist, 0, indices)
                #     lab_dist = torch.index_select(lab_dist, 0, indices)

                # TODO：should only calculate distill loss for those of labels of old classes

                # print("dist example", dist.size())
                # print(dist)
                # print("lab_dist example", lab_dist.size())
                # print(lab_dist)
                loss_dist = F.binary_cross_entropy(dist, lab_dist)  # distill loss
            else:
                loss_dist = 0
            # print("batch", batch, "distillation loss", loss_dist)
            loss = loss_cls + dist_ratio * loss_dist

            loss_avg += loss.item()
            loss_cls_avg += loss_cls.item()
            if period == 0:
                loss_dist_avg += 0
            else:
                loss_dist_avg += loss_dist.item()

            # acc = np.sum(np.equal(np.argmax(prob_cls.cpu().data.numpy(), axis=-1), np.argmax(lab_onehot.cpu().data.numpy(), axis=-1)))
            # acc_avg += acc
            # num_exp += np.shape(lab)[0]

            pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
            acc = pred.eq(lab.data.view_as(pred)).long().cpu().sum()
            acc_avg += acc
            num_exp += bs
            # In PyTorch, we need to set the gradients to zero before starting to do backpropragation
            # because PyTorch accumulates the gradients on subsequent backward passes.

            loss.backward()
            optimizer.step()

            # # add random noise to gradients / weights, not implemented now
            # if gredient_noise_ratio > 0:
            #     for p in net.parameters():
            #         p.data.sub_(gredient_noise_ratio * lrc * torch.from_numpy(
            #             (np.random.random(np.shape(p.data.cpu().data.numpy())) - 0.5)*2).float().cuda())

        loss_avg /= num_exp
        loss_cls_avg /= num_exp
        loss_dist_avg /= num_exp
        acc_avg /= num_exp
        acc_training.append(acc_avg)
        tend = time.process_time()
        tcost = tend - tstart

        logger.info('Training Period: %d \t Iter: %d \t time = %.1f \t loss = %.6f \t acc = %.4f' % (period, iter, tcost, loss_avg, acc_avg))

        # test all (novel + old) classes based on logists
        if period > -1:
            idx_test = np.random.permutation(labels_nvld_test.shape[0])
            loss_avg = 0
            acc_avg = 0
            num_exp = 0
            tstart = time.process_time()

            batchnum_test = math.ceil(labels_nvld_test.shape[0] / batchsize)
            for bi in range(batchnum_test):
                if bi == batchnum_test - 1:
                    idx = idx_test[bi * batchsize:]
                else:
                    idx = idx_test[bi * batchsize:(bi + 1) * batchsize]
                bs = len(idx)
                img = images_nvld_test[idx]
                lab = labels_nvld_test[idx]
                lab_onehot = utils.one_hot(lab, num_class)

                # normalization
                img = np.expand_dims(img, axis=1)
                img = torch.FloatTensor(img).to(device)

                # init hidden space for lstm
                (h_ini, c_ini) = (torch.zeros(2, bs, 50).to(device) , torch.zeros(2, bs, 50).to(device))
                net.init_hidden(h_ini, c_ini)

                output = net(img)
                indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                indices = indices.to(device)
                output = torch.index_select(output, 1, indices)
                output = softmax(output)
                output = output.cpu().data.numpy()
                lab_onehot = lab_onehot[:, np.concatenate((class_old, class_novel), axis=0)]

                acc = np.sum(np.equal(np.argmax(output, axis=-1), np.argmax(lab_onehot, axis=-1)))
                acc_avg += acc
                num_exp += np.shape(lab)[0]

            acc_avg /= num_exp
            tend = time.process_time()
            tcost = tend - tstart
            logger.info('Testing novel+old Period: %d \t Iter: %d \t time = %.1f \t\t\t\t\t\t acc = %.4f' % (period, iter, tcost, acc_avg))
            acc_nvld_basic[period] = acc_avg

        # raise ValueError("stop here to check")

        if len(acc_training)>90 and acc_training[-1]>stop_acc and acc_training[-5]>stop_acc:
            logger.info('training loss converged')
            break

    # raise ValueError("stop here to check")

    # save model for each period, update model_path
    #if period == 0 and (not flag_model):
    model_path = 'model/model_slide_e2e_{}_{}.pth'.format(init_opt,cur_num_class+new_num_class)
    logger.info('save model: %s' % model_path)
    torch.save(net.state_dict(), model_path)

    cur_num_class += new_num_class

   # balanced finetune

    net_old = copy.deepcopy(net)

    # finetune
    if period>0:
        # lrc = lr
        lrc = lr*0.1 # small learning rate for finetune
        print('current lr = %f' % (lrc))
        softmax = nn.Softmax(dim=-1).cuda()

        acc_finetune_train = []
        for iter in range(iteration_finetune):

            # learning rate
            if iter in schedules:
                lrc *= gamma
                print('current lr = %f'%(lrc))

            # criterion = nn.CrossEntropyLoss()
            optimizer = torch.optim.SGD(net.parameters(), lr=lrc, momentum=momentum,
                                        weight_decay=decay, nesterov=True)

            # finetune train
            # RH: different from the training mainly at the random permutation step?
            idx_finetune_novel = np.random.permutation(labels_novel_train.shape[0])
            idx_finetune_novel = idx_finetune_novel[:memory_size*num_class_novel]
            idx_finetune_old = np.arange(start=labels_novel_train.shape[0], stop=labels_combined_train.shape[0])
            idx_finetune = np.concatenate((idx_finetune_novel, idx_finetune_old), axis=0)
            np.random.shuffle(idx_finetune)

            loss_avg = 0
            acc_avg = 0
            num_exp = 0
            tstart = time.clock()

            batchnum_train = math.ceil(idx_finetune.shape[0] // batchsize)
            for bi in range(batchnum_train):
                if bi == batchnum_train - 1:
                    idx = idx_finetune[bi * batchsize:]
                else:
                    idx = idx_finetune[bi * batchsize:(bi + 1) * batchsize]
                img = images_combined_train[idx]
                lab = labels_combined_train[idx]
                lab_onehot = utils.one_hot(lab, num_class)

                # transform
                if flag_augmentation_e2e == False:
                    img = utils.img_transform(img, 'train')
                img = torch.from_numpy(img).float()
                img = img.cuda()
                lab_onehot = torch.from_numpy(lab_onehot)
                lab_onehot = lab_onehot.float()
                lab_onehot = lab_onehot.cuda()

                # print("Outside: input size", img.size(), "output_size", lab.size())
                output = net.forward(img)

                # classification loss
                indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                indices = indices.cuda()
                prob_cls = torch.index_select(output, 1, indices)
                prob_cls = softmax(prob_cls)
                lab_onehot = torch.index_select(lab_onehot, 1, indices)
                loss_cls = F.binary_cross_entropy(prob_cls, lab_onehot)

                # distillation loss for all classes (maybe the author only distillates for novel classes)
                if period>0:
                    indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                    indices = indices.cuda()
                    dist = torch.index_select(output, 1, indices)
                    dist = softmax(dist/T)

                    output_old = net_old.forward(img)
                    output_old = torch.index_select(output_old, 1, indices)
                    lab_dist = Variable(output_old, requires_grad=False)
                    lab_dist = softmax(lab_dist/T)

                    loss_dist = F.binary_cross_entropy(dist, lab_dist)
                else:
                    loss_dist = 0

                loss = loss_cls + dist_ratio*loss_dist

                loss_avg += loss.item()
                acc = np.sum(np.equal(np.argmax(prob_cls.cpu().data.numpy(), axis=-1), np.argmax(lab_onehot.cpu().data.numpy(), axis=-1)))
                acc_avg += acc
                num_exp += np.shape(lab)[0]

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                # add random noise to gradients / weights
                if gredient_noise_ratio > 0:
                    for p in net.parameters():
                        p.data.sub_(gredient_noise_ratio * lrc * torch.from_numpy(
                            (np.random.random(np.shape(p.data.cpu().data.numpy())) - 0.5) * 2).float().cuda())

            loss_avg /= num_exp
            acc_avg /= num_exp
            acc_finetune_train.append(acc_avg)
            tend = time.clock()
            tcost = tend - tstart
            print('Finetune Training Iter: %d \t time = %.1f \t loss = %.6f \t acc = %.4f'%(iter, tcost, loss_avg, acc_avg))

            # test all (novel + old) classes based on logists
            if period > 0:
                # images_nvld_test = images_test[np.concatenate((class_old, class_novel), axis=0)]
                # images_nvld_test = np.reshape(images_nvld_test, newshape=(-1, 3, 32, 32))
                # labels_nvld_test = labels_test[np.concatenate((class_old, class_novel), axis=0)]
                # labels_nvld_test = np.reshape(labels_nvld_test, newshape=(-1))
                idx_test = np.random.permutation(labels_nvld_test.shape[0])
                loss_avg = 0
                acc_avg = 0
                num_exp = 0
                tstart = time.clock()

                batchnum_test = math.ceil(labels_nvld_test.shape[0] // batchsize)
                for bi in range(batchnum_test):
                    if bi == batchnum_test - 1:
                        idx = idx_test[bi * batchsize:]
                    else:
                        idx = idx_test[bi * batchsize:(bi + 1) * batchsize]
                    img = images_nvld_test[idx]
                    lab = labels_nvld_test[idx]
                    lab_onehot = utils.one_hot(lab, num_class)

                    if flag_augmentation_e2e == False:  # old transform
                        img = utils.img_transform(img, 'test')
                    img = torch.from_numpy(img).float()
                    img = img.cuda()

                    # # normalization
                    output = net.forward(img)
                    indices = torch.LongTensor(np.concatenate((class_old, class_novel), axis=0))
                    indices = indices.cuda()
                    output = torch.index_select(output, 1, indices)
                    output = softmax(output)
                    output = output.cpu().data.numpy()
                    lab_onehot = lab_onehot[:, np.concatenate((class_old, class_novel), axis=0)]

                    acc = np.sum(np.equal(np.argmax(output, axis=-1), np.argmax(lab_onehot, axis=-1)))
                    acc_avg += acc
                    num_exp += np.shape(lab)[0]

                acc_avg /= num_exp
                tend = time.clock()
                tcost = tend - tstart
                print('Finetune Testing novel+old Period: %d \t Iter: %d \t time = %.1f \t\t\t\t\t\t acc = %.4f' % (period, iter, tcost, acc_avg))
                acc_nvld_finetune[period] = acc_avg

            if len(acc_finetune_train) > 20 and acc_finetune_train[-1] > stop_acc and acc_finetune_train[-5] > stop_acc:
                print('training loss converged')
                break

    if period>0:
        print('------------------- result ------------------------')
        print('Period: %d, basic acc = %.4f, finetune acc = %.4f' % (period, acc_nvld_basic[period], acc_nvld_finetune[period]))
        print('---------------------------------------------------')

    if period == period_train-1:
        print('------------------- ave result ------------------------')
        print('basic acc = %.4f, finetune acc = %.4f' % (np.mean(acc_nvld_basic[1:], keepdims=False), np.mean(acc_nvld_finetune[1:], keepdims=False)))
        print('---------------------------------------------------')

    # new after fine-tune training

    # memory management [random select]
    # reduce old memory
    memory_size = memory_K // (num_class_novel + num_class_old)
    if period >0:
        memory_images = memory_images[:, :memory_size]
        memory_labels = memory_labels[:, :memory_size]
    # add new memory
    memory_new_images = np.zeros((num_class_novel, memory_size, 3, 32, 32), dtype=np.uint8)
    memory_new_labels = np.zeros((num_class_novel, memory_size), dtype=np.int32)
    # random selection
    for k in range(num_class_novel):

        img = images_train[class_novel[k], np.random.permutation(500)[:memory_size]]
        memory_new_images[k] = img
        memory_new_labels[k] = np.tile(class_novel[k], (memory_size))

    # herding
    # to do

    if period > 0:
        memory_images = np.concatenate((memory_images, memory_new_images), axis=0)
        memory_labels = np.concatenate((memory_labels, memory_new_labels), axis=0)
    else:
        memory_images = memory_new_images
        memory_labels = memory_new_labels

    class_old = np.append(class_old, class_novel, axis=0)
    # TODO update the memory
    logger.info("update the memory")

# memory management [random select]
    # reduce old memory
    memory_size = min(num_train_per_class, memory_K // (new_num_class + num_class_old))
    if period >0:
        memory_images = memory_images[:, :memory_size]
        memory_labels = memory_labels[:, :memory_size]
    # add new memory
    memory_new_images = np.zeros((new_num_class, memory_size, 1, 6, 10, 75), dtype=np.uint8)
    memory_new_labels = np.zeros((new_num_class, memory_size), dtype=np.int32)
    # random selection
    for k in range(new_num_class):

        img = images_train[class_novel[k], np.random.permutation(num_train_per_class)[:memory_size]]
        img = np.expand_dims(img, axis=1) # broadcast array from shape (51, 6, 10, 75) to shape (51, 1, 6, 10, 75)
        memory_new_images[k] = img
        memory_new_labels[k] = np.tile(class_novel[k], (memory_size))

    # herding
    # to do

    if period > 0:
        memory_images = np.concatenate((memory_images, memory_new_images), axis=0)
        memory_labels = np.concatenate((memory_labels, memory_new_labels), axis=0)
    else:
        memory_images = memory_new_images
        memory_labels = memory_new_labels

    class_old = np.append(class_old, class_novel, axis=0)


space_unit = 20
font_size = int(4*space_unit)
staff_gap = space_unit
staff_width = int(space_unit/10)
ledger_width = int(2.6*staff_width)
ledger_radius = int(0.9*space_unit)
bar_width = int(space_unit/4)
double_bar_width = 2*bar_width
stem_width = int(space_unit/5)
stem_len = 3*space_unit
beam_width = int(staff_gap/2)
beam_len = int(2*space_unit)
part_beam_len = int(space_unit/1.2)

_symbols = ['beam', 'chord', 'rest', 'bar', 'text', 'slur', 'crescendo', 'clef']#, 'key', 'time']
_positions = ['top', 'bot', 'left', 'rite']
_attributes = ['c', 't', 'k','p','d', 'o', 'dir', 'num'] + _positions
##############clef,time,key,pitch,duration,onset,direction,number,positions

_flags = ['system', 'staff', 'measure'] + _symbols + _attributes
_hierarchy = [('system','staff'), ('staff','measure'), ('measure', 'beam'), ('measure', 'beam'), ('measure', 'chord'), ('measure', 'clef'), ('beam', 'chord'), ('measure','rest')]  #, ('clef','key'), ('clef', 'time')

inter_sym_dis = 3*space_unit
inter_stem_dis = space_unit*1.5
inter_beam_dis = int(space_unit/1.5)
inner_sym_dis = space_unit*2.5

left_margin = 5*space_unit
rite_margin = 5*space_unit
top_margin = 8*space_unit
bot_margin = 8*space_unit
img_height = top_margin + bot_margin + 4*staff_gap
center_height = img_height/2
img_width = 0 #unset yet
staff_top = top_margin
staff_bot = staff_top + 4*staff_gap
staff_left = left_margin
staff_rite = 0 #unset yet

#for pitches
pitch_index = {
    'c':0,
    'd':2,
    'e':4,
    'f':5,
    'g':7,
    'a':9,
    'b':11,
}

index_pitch = {
    0:'c',
    2:'d',
    4:'e',
    5:'f',
    7:'g',
    9:'a',
    11:'b',
}

#for symbol offsets [i,j]
solid_offset = {
    "treble_clef":[staff_gap + int(-2*font_size),int(-font_size/2.5)],
    "whole_note_head":[int(-2*font_size),int(-font_size/5)],
    "open_note_head":[int(-2*font_size),int(-font_size/7)],
    "solid_note_head":[int(-2*font_size),int(-font_size/7)],
    "single_aug_dot":[int(-2*font_size),0],
    "double_aug_dot":[0,0],
    "single_flag_up":[int(-2*font_size),0],
    "single_flag_down":[int(-2*font_size),0],
    "double_flag_up":[int(-2*font_size),0],
    "double_flag_down":[int(-2*font_size),0],
    "triple_flag_up":[int(-2*font_size),int(-font_size/80)],
    "triple_flag_down":[int(-2*font_size),int(-font_size/80)],
    "single_sharp":[int(-2*font_size),int(-font_size/4)],
    "single_flat":[int(-2*font_size),int(-font_size/4)],
    "single_natural":[int(-2*font_size),int(-font_size/4)],
    "double_sharp":[int(-2*font_size),int(-font_size/4)],
    "double_flat":[int(-2*font_size),int(-font_size/4)],
    "double_natural":[int(-2*font_size),int(-font_size/4)],
    "time_0":[int(-2*font_size),int(-font_size/7)],
    "time_1":[int(-2*font_size),int(-font_size/7)],
    "time_2":[int(-2*font_size),int(-font_size/7)],
    "time_3":[int(-2*font_size),int(-font_size/7)],
    "time_4":[int(-2*font_size),int(-font_size/7)],
    "time_5":[int(-2*font_size),int(-font_size/7)],
    "time_6":[int(-2*font_size),int(-font_size/7)],
    "time_7":[int(-2*font_size),int(-font_size/7)],
    "time_8":[int(-2*font_size),int(-font_size/7)],
    "time_9":[int(-2*font_size),int(-font_size/7)]
}
solid_symbol = list(solid_offset.keys())

#for isolated symbol #do we need this?
#iso_symbol = {"rest", "text"}

#for prim-to-prim distance
note_head_to_stem = space_unit/1.9
note_head_to_sat = space_unit#/1.2

#for vertical alignment
vertically_alignable = ['note_head','stem','flag','aug','sharp','flat','natural']


# Copyright 2019 3YOURMIND GmbH

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import unittest
import sys

from tests import fixtures

from django_migration_linter import MigrationLinter, Cache, DEFAULT_CACHE_PATH

if sys.version_info >= (3, 3):
    import unittest.mock as mock
else:
    import mock

class CacheTest(unittest.TestCase):
    MIGRATION_FILE = os.path.join(fixtures.ALTER_COLUMN_PROJECT, 'test_app', 'migrations', '0001_initial.py')

    def test_cache_normal(self):
        cache_file = os.path.join(DEFAULT_CACHE_PATH, 'test_project_add_not_null_column.pickle')
        if os.path.exists(cache_file):
            os.remove(cache_file)
        linter = MigrationLinter(fixtures.ADD_NOT_NULL_COLUMN_PROJECT)

        with mock.patch.object(MigrationLinter, 'get_sql', wraps=linter.get_sql)as sql_mock:
            linter.lint_all_migrations()
            self.assertEqual(sql_mock.call_count, 2)

        cache = Cache(
            fixtures.ADD_NOT_NULL_COLUMN_PROJECT,
            DEFAULT_CACHE_PATH
        )
        cache.load()

        self.assertEqual(cache['3ef74e7f3e53e273e2fc95379248d58d']['result'], 'OK')
        self.assertEqual(cache['e1e312b6d08ecbe017c25c58fc2be257']['result'], 'ERR')
        self.assertListEqual(
            cache['e1e312b6d08ecbe017c25c58fc2be257']['errors'],
            [{'err_msg': 'RENAMING tables', 'code': 'RENAME_TABLE', 'table': None, 'column': None}]
        )

        # Start the Linter again -> should use cache now.
        linter = MigrationLinter(fixtures.ADD_NOT_NULL_COLUMN_PROJECT)

        with mock.patch.object(MigrationLinter, 'get_sql', wraps=linter.get_sql) as sql_mock:
            linter.lint_all_migrations()
            self.assertEqual(sql_mock.call_count, 0)

        self.assertTrue(linter.has_errors)

    def test_cache_ignored(self):
        cache_file = os.path.join(DEFAULT_CACHE_PATH, 'test_project_ignore_migration.pickle')
        if os.path.exists(cache_file):
            os.remove(cache_file)
        linter = MigrationLinter(fixtures.IGNORE_MIGRATION_PROJECT)

        with mock.patch.object(MigrationLinter, 'get_sql', wraps=linter.get_sql) as sql_mock:
            linter.lint_all_migrations()
            self.assertEqual(sql_mock.call_count, 2)

        cache = Cache(
            fixtures.IGNORE_MIGRATION_PROJECT,
            DEFAULT_CACHE_PATH
        )
        cache.load()

        self.assertEqual(cache['63230606af0eccaef7f1f78c537c624c']['result'], 'OK')
        self.assertEqual(cache['5c5ca1780a9f28439c1defc1f32af894']['result'], 'IGNORE')

        # Start the Linter again -> should use cache now.
        linter = MigrationLinter(fixtures.IGNORE_MIGRATION_PROJECT)

        with mock.patch.object(MigrationLinter, 'get_sql', wraps=linter.get_sql) as sql_mock:
            linter.lint_all_migrations()
            self.assertEqual(sql_mock.call_count, 0)

    def test_cache_ignored_command_line(self):
        cache_file = os.path.join(DEFAULT_CACHE_PATH, 'test_project_ignore_migration.pickle')
        if os.path.exists(cache_file):
            os.remove(cache_file)
        linter = MigrationLinter(fixtures.IGNORE_MIGRATION_PROJECT,
                                 ignore_name_contains='0001')

        with mock.patch.object(MigrationLinter, 'get_sql', wraps=linter.get_sql) as sql_mock:
            linter.lint_all_migrations()
            self.assertEqual(sql_mock.call_count, 1)

        cache = Cache(
            fixtures.IGNORE_MIGRATION_PROJECT,
            DEFAULT_CACHE_PATH
        )
        cache.load()

        self.assertNotIn('63230606af0eccaef7f1f78c537c624c', cache)
        self.assertEqual(cache['5c5ca1780a9f28439c1defc1f32af894']['result'], 'IGNORE')

    def test_cache_modified(self):
        cache_file = os.path.join(DEFAULT_CACHE_PATH, 'test_project_alter_column.pickle')
        if os.path.exists(cache_file):
            os.remove(cache_file)
        linter = MigrationLinter(fixtures.ALTER_COLUMN_PROJECT)
        linter.lint_all_migrations()
        cache = Cache(
            fixtures.ALTER_COLUMN_PROJECT,
            DEFAULT_CACHE_PATH
        )
        cache.load()

        self.assertEqual(cache['8589aa107b6da296c4b49cd2681d2230']['result'], 'OK',
                         'If this fails, tearDown might have failed to remove '
                         'the modification from tests/test_project_fixtures/'
                         'test_project_alter_column/test_app/migrations/0001_initial.py'
                         )
        self.assertEqual(cache['8f54c4a434cfaa9838e8ca12eb988255']['result'], 'ERR')
        self.assertListEqual(
            cache['8f54c4a434cfaa9838e8ca12eb988255']['errors'],
            [{u'err_msg': u'ALTERING columns (Could be backward compatible. '
                          u'You may ignore this migration.)',
              u'code': u'ALTER_COLUMN',
              u'table': u'test_app_a',
              u'column': None}
             ]
        )

        # Modify migration
        backup_migration_file = self.MIGRATION_FILE + "_backup"
        shutil.copy2(self.MIGRATION_FILE, backup_migration_file)
        with open(self.MIGRATION_FILE, "a") as f:
            f.write("# modification at the end of the file")

        # Start the Linter again -> Cache should look different now
        linter = MigrationLinter(fixtures.ALTER_COLUMN_PROJECT)
        linter.lint_all_migrations()
        cache = Cache(
            fixtures.ALTER_COLUMN_PROJECT,
            DEFAULT_CACHE_PATH
        )
        cache.load()
        shutil.copy2(backup_migration_file, self.MIGRATION_FILE)
        os.remove(backup_migration_file)

        self.assertNotIn('8589aa107b6da296c4b49cd2681d2230', cache)
        self.assertEqual(cache['fbee628b1ab4bd1c14f8a4b41123e7cf']['result'], 'OK')


import tables
def crossproduct(words):
    attr=[]
    for i in range(len(words)):
        if words[i]=='FROM':
            table1=words[i+1]
        if words[i]=='AND':
            table2=words[i+1]
        if words[i]=="GET" or words[i]=="SELECT" or words[i]=="RETRIEVE":
            while(words[i+1]!="FROM"):
                attr.append(words[i+1])
                i+=1
    if (tables.finderror(table1,attr))==2:
        print(table1+' NOT FOUND')
        return 0
    if (tables.finderror(table2,attr))==2:
        print(table2+' NOT FOUND')
        return 0
    if tables.finderror(table1, attr) == 0:
        print("Attribute not found in "+table1)
    elif tables.finderror(table2, attr) == 0:
        print("Attribute not found in "+table2)
    else:
        print("SELECT "+','.join(attr)+" FROM "+table1+' JOIN '+table2)
    return 0


import math
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from util.nero import Nero
from util.data import normalize_data

# from tqdm import tqdm
tqdm = lambda x: x

class SimpleNet(nn.Module):
    def __init__(self, depth, width):
        super(SimpleNet, self).__init__()

        self.initial = nn.Linear(784, width, bias=False)
        self.layers = nn.ModuleList([nn.Linear(width, width, bias=False) for _ in range(depth-2)])
        self.final = nn.Linear(width, 1, bias=False)

    def forward(self, x):
        x = self.initial(x)
        x = F.relu(x) * math.sqrt(2)
        for layer in self.layers:
            x = layer(x)
            x = F.relu(x) * math.sqrt(2)
        return self.final(x)

def train_network(train_loader, test_loader, depth, width, init_lr, decay, break_on_fit=True, target_scale=1.0):

    model = SimpleNet(depth, width).cuda()
    optim = Nero(model.parameters(), lr=init_lr)
    lr_lambda = lambda x: decay**x
    lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optim, lr_lambda)

    train_acc_list = []
    train_acc = 0

    for epoch in tqdm(range(100)):
        model.train()

        for data, target in train_loader:
            data, target = (data.cuda(), target.cuda())
            data, target = normalize_data(data, target)

            y_pred = model(data).squeeze()
            loss = (y_pred - target_scale*target).norm()

            model.zero_grad()
            loss.backward()
            optim.step()

        lr_scheduler.step()

        model.eval()
        correct = 0
        total = 0

        for data, target in train_loader:
            data, target = (data.cuda(), target.cuda())
            data, target = normalize_data(data, target)

            y_pred = model(data).squeeze()
            correct += (target.float() == y_pred.sign()).sum().item()
            total += target.shape[0]

        train_acc = correct/total
        train_acc_list.append(train_acc)

        if break_on_fit and train_acc == 1.0: break

    model.eval()
    correct = 0
    total = 0

    for data, target in test_loader:
        data, target = (data.cuda(), target.cuda())
        data, target = normalize_data(data, target)

        y_pred = model(data).squeeze()
        correct += (target.float() == y_pred.sign()).sum().item()
        total += target.shape[0]

    test_acc = correct/total

    return train_acc_list, test_acc, model


from urllib.parse import parse_qs

from snow.exceptions import StreamExhausted

from ..core import GetRequest

class StreamLike(GetRequest):
    exhausted = False

    def __init__(self, *args, chunk_size=500, **kwargs):
        super(StreamLike, self).__init__(*args, **kwargs)
        self._chunk_size = chunk_size
        self._fields = None

    @property
    def _page_size(self):
        if self._offset + self._chunk_size >= self._limit:
            return self._limit - self._offset

        return self._chunk_size

    def _prepare_next(self, links):
        if "next" in links:
            url_next = str(links["next"]["url"])
            offset_next = int(parse_qs(url_next).get("sysparm_offset")[0])

            if offset_next >= self._limit:
                raise StreamExhausted
        else:
            raise StreamExhausted

        self._offset = offset_next

    async def read(self, **kwargs):
        response, content = await self.send(**kwargs)

        try:
            self._prepare_next(response.links)
        except StreamExhausted:
            self.exhausted = True

        yield content


# Definition for singly-linked list.
class ListNode(object):
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution(object):
    def addTwoNumbers(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        stackl1, stackl2 = [], []
        while l1 != None:
        	stackl1.append(l1.val)
        	l1=l1.next
        while l2 != None:
        	stackl2.append(l2.val)
        	l2=l2.next
        carry=0
        res=None
        while len(stackl2)>0 or len(stackl1)>0 or carry>0:
        	sum=carry
        	if len(stackl1)>0:
        		sum += stackl1.pop()
        	if len(stackl2)>0:
        		sum += stackl2.pop()
        	node=ListNode(sum%10)
        	node.next=res
        	res=node
        	carry=sum//10
        return res

def list2ln(listin):
	head, cur = None, None
	for i in listin:
		if head==None:
			head=ListNode(i)
			cur=head
		else:
			cur.next=ListNode(i)
			cur = cur.next
	return head

res=Solution().addTwoNumbers(list2ln([7,2,4,3]), list2ln([5,6,4]))
while res:
	print(res.val, end='->')
	res=res.next
print()

445. Add Two Numbers II/Accepted-246ms-96834740.py


# import the pygame module, so you can use it
import pygame
class ui_id:
    def __init__(self, **kwds):
        self.__dict__.update(kwds)

Hot = ui_id(Owner=None, Item=None, Index=None)
Active = ui_id(Owner=None, Item=None, Index=None)

# define a main function
def main():

    # initialize the pygame module
    pygame.init()
    pygame.font.init()

    # load and set the logo
    logo = pygame.image.load("logo32x32.png")
    pygame.display.set_icon(logo)
    pygame.display.set_caption("minimal program")

    screen_width = 1700
    screen_height = 1200

    curwindowxpos = 100
    curwindowypos = 100
    curwindowwidth = 400
    curwindowheight = 400

    bg_image = pygame.image.load("carrearend3.jpg")
    bg_image = pygame.transform.scale(bg_image, (3300,2191))

    bg_image2 = pygame.image.load("carrearend2.jpg")
    bg_image2 = pygame.transform.scale(bg_image2, (1450,815))

    # create a surface on screen that has the size of screen_width x screen_height
    screen = pygame.display.set_mode((screen_width,screen_height))

    licplate_left = 510
    licplate_top = 460
    licplace_width = 520
    licplace_heigth = 111
    eurosymbol_width = 40
    eurosymbol_height = licplace_heigth
    licplaterect = pygame.Rect(licplate_left, licplate_top, licplace_width, licplace_heigth)
    licplatecountryrect = pygame.Rect(licplate_left, licplate_top, eurosymbol_width, eurosymbol_height)
    licplatefont = pygame.font.Font("MANDATOR.ttf", 108)
    eurosymbolfont = pygame.font.Font("MANDATOR.ttf", 24)
    eurosymboltext = "GB"

    #pygame.font.SysFont('Verdana', 75, False)
    licplatetext = "L89HRH"

    OneRect = pygame.Rect(0, 0, 20, 20)
    TwoRect = pygame.Rect(20, 20, 20, 20)
    # define a variable to control the main loop
    running = True

    clock = pygame.time.Clock()
    fps = 30
    active = False
    done = False
    # main loop
    while running:
        # event handling, gets all event from the eventqueue
        for event in pygame.event.get():
            if event.type == pygame.MOUSEBUTTONDOWN:
                if licplaterect.collidepoint(event.pos):
                    # Toggle the active variable.
                    active = not active
                else:
                    active = False
                # Change the current color of the input box.
            if event.type == pygame.KEYDOWN:
                if active:
                    if event.key == pygame.K_RETURN:
                        print(licplatetext)
                        licplatetext = ''
                    elif event.key == pygame.K_BACKSPACE:
                        licplatetext = licplatetext[:-1]
                    else:
                        licplatetext += event.unicode
            #mainwindow.mousedown(event.pos)
            #
            #elif event.type == pygame.MOUSEBUTTONUP:
            #    if event.button == 1:
            #        mainwindow.mouseup()
            #
            #elif event.type == pygame.MOUSEMOTION:
            #    mainwindow.mousemotion(event.pos)

            # only do something if the event is of type QUIT
            if event.type == pygame.QUIT:
                # change the value to False, to exit the main loop
                running = False
        clock.tick(fps)

        screen.blit(bg_image, (-800, -800))
        pygame.draw.rect(screen, (244,200,66), licplaterect, 0)
        pygame.draw.rect(screen, (0,0,255), licplatecountryrect, 0)
        licplatetextsurface1 = licplatefont.render(licplatetext[:-3], False, (0, 0, 0))
        licplatetextsurface2 = licplatefont.render(licplatetext[-3:], False, (0, 0, 0))
        licplatetextsurface3 = licplatefont.render(licplatetext[:-3], False, (200,200,200))
        licplatetextsurface4 = licplatefont.render(licplatetext[-3:], False, (200,200,200))
        licplatetextsurface5 = licplatefont.render(licplatetext[:-3], False, (38,38,38))
        licplatetextsurface6 = licplatefont.render(licplatetext[-3:], False, (38,38,38))

        eurosymboltextsurface = eurosymbolfont.render(eurosymboltext, False, (230,255,0))
        licplatetextsurface1_left = licplate_left+eurosymbol_width+((licplace_width-eurosymbol_width)/2)-licplatetextsurface1.get_width()-10
        licplatetextsurface2_left = licplate_left+eurosymbol_width+((licplace_width-eurosymbol_width)/2)+10
        licplatetextsurface1_top = licplate_top+(licplace_heigth/2)-(licplatetextsurface1.get_height()/2)

        screen.blit(licplatetextsurface3,(licplatetextsurface1_left-1, licplatetextsurface1_top-1))
        screen.blit(licplatetextsurface4,(licplatetextsurface2_left-1, licplatetextsurface1_top-1))

        screen.blit(licplatetextsurface5,(licplatetextsurface1_left+1, licplatetextsurface1_top+1))
        screen.blit(licplatetextsurface6,(licplatetextsurface2_left+1, licplatetextsurface1_top+1))

        screen.blit(licplatetextsurface1,(licplatetextsurface1_left, licplatetextsurface1_top))
        screen.blit(licplatetextsurface2,(licplatetextsurface2_left, licplatetextsurface1_top))

        eurosymboltext_left = int(licplate_left+(eurosymbol_width/2)-(eurosymboltextsurface.get_width()/2))
        eurosymboltext_top = int(licplate_top+(licplace_heigth/2)+10)
        eurosymbolcircle_center_top = int(licplate_top+(licplace_heigth/2)-10-(licplace_heigth/4))
        eurosymbolcircle_center_left = int(licplate_left+(eurosymbol_width/2)-10-(eurosymbol_width/4))
        screen.blit(eurosymboltextsurface, (eurosymboltext_left, eurosymboltext_top))
        pygame.draw.circle(screen, (230,255,0), (licplate_left+20, licplate_top+35), int((eurosymbol_width/2)-6), 1)
        pygame.draw.rect(screen, (0,0,0), licplaterect, 2)

        pygame.display.flip()

# run the main function only if this module is executed as the main script
# (if you import this as a module then nothing is executed)
if __name__=="__main__":
    # call the main function
    main()


from flask_restful import Resource, request
import pandas as pd, json, os

raw_file = 'api/static/data/raw.csv'
final_file = 'api/static/data/final.csv'

class Filter(Resource):
    def get(self):
        if os.path.isfile(final_file):
            data = pd.read_csv(final_file)
        else:
            data = pd.read_csv(raw_file)

        if 'case_id' in data.columns and 'task' in data.columns and 'timestamp' in data.columns:
            # Semua event yang unik
            all_event = list(set(data['task']))

            # List dari semua id
            id_list = list(set(data['case_id']))
            # Mengambil data per id
            data_per_id = {}
            for id in id_list:
                if id not in data_per_id:
                    data_per_id[id] = data[data['case_id'] == id]
            # List start event dan end event
            start_list = []
            end_list = []
            for key, dt in data_per_id.items():
                for x in range(len(dt.index)):
                    if x == 0:
                        start_list.append(dt['task'].iloc[x])
                    elif dt['case_id'].iloc[x-1] != dt['case_id'].iloc[x]:
                        start_list.append(dt['task'].iloc[x])
                    elif x == len(dt.index)-1:
                        end_list.append(dt['task'].iloc[x])
                    elif dt['case_id'].iloc[x+1] != dt['case_id'].iloc[x]:
                        end_list.append(dt['task'].iloc[x])
            start_list = list(set(start_list))
            end_list = list(set(end_list))

            return json.dumps(
                {
                    'data': {
                        'start': start_list,
                        'end': end_list,
                        'all': all_event
                    },
                    'message': 'Succesfully load the data',
                    'status': 'success'
                }
            )

        else:
            return json.dumps(
                {
                    'data':'',
                    'message': 'Data is not finished',
                    'status': 'error'
                }
            )
        pass

    def post(self):
        args = request.get_json(force=True)
        if args != None:
            if os.path.isfile(final_file):
                data = pd.read_csv(final_file)
            else:
                data = pd.read_csv(raw_file)

            try:
                start = args['data']['start']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                # List dari semua id
                id_list = data['case_id'].unique()

                # Mengambil data per id
                data_per_id = dict()
                for id in id_list:
                    if id not in data_per_id:
                        data_per_id[id] = data[data['case_id'] == id]

                # List of selected start event
                list_of_data = []
                for key, dt in data_per_id.items():
                    for ev in start:
                        if dt['task'].iloc[0] == ev:
                            list_of_data.append(dt)

                data = pd.concat(list_of_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering start event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)

            try:
                end = args['data']['end']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                # List dari semua id
                id_list = data['case_id'].unique()

                # Mengambil data per id
                data_per_id = dict()
                for id in id_list:
                    if id not in data_per_id:
                        data_per_id[id] = data[data['case_id'] == id]

                # List selected end event
                list_of_data = []
                for key, dt in data_per_id.items():
                    for ev in end:
                        if dt['task'].iloc[-1] == ev:
                            list_of_data.append(dt)

                data = pd.concat(list_of_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering end event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)

            try:
                all = args['data']['all']

                # Kolom untuk sort data ulang
                data['Sort'] = list([x for x in range(0,len(data['case_id']))])

                list_data = []
                for idx, ev in enumerate(data['task']):
                    for evt in all:
                        if ev == evt:
                            list_data.append(data.iloc[idx])

                data = pd.DataFrame(list_data)
                data.sort_values('Sort', inplace=True)
                data.drop('Sort', axis=1, inplace=True)
                data.to_csv(final_file, index=False)

                return json.dumps(
                    {
                        'data': '',
                        'message': 'Success filtering all event',
                        'status': 'success'
                    }
                )
            except KeyError as ke:
                print(ke)
        else:
            return json.dumps(
                {
                    'data': '',
                    'message': 'Failed filtering the data',
                    'status': 'error'
                }
            )


import sys
from PIL import Image, ImageTk
import tkinter as tk
import argparse
import datetime
import cv2
import os
from tkinter.filedialog import askopenfilename
#from tkinter import filedialog
from tkinter import PhotoImage
from shutil import copyfile
import shutil

class Application:
    def __init__(self, output_path = "./"):

        self.vs = cv2.VideoCapture(0)  # capture video frames, 0 is your default video camera
        self.output_path = output_path  # store output path
        self.current_image = None  # current image from the camera

        self.root = tk.Tk()  # initialize root window
        self.root.title("Intellera")  # set window title
        # self.destructor function gets fired when the window is closed
        """"#self.root.protocol('WM_DELETE_WINDOW', self.destructor)"""

        self.panel = tk.Label(self.root)  # initialize image panel
        self.panel.pack(padx=5, pady=5)

        label = tk.Label(self.root, text="Intellera")
        label.config(font=("BankGothic LT BT", 40))
        label.pack()

        label1 = tk.Label(self.root, text="<-Take Photo          Browse Photo->")
        label1.config(font=("BankGothic LT BT", 15))
        label1.pack()

        # create a button, that when pressed, will take the current frame and save it to file
        btn = tk.Button(self.root, text="Snapshot!", command=self.take_snapshot, bg="red")
        btn.config(height=2, width=12)
        btn.pack(side=tk.RIGHT, expand = True, fill = tk.BOTH)

        btn2 = tk.Button(self.root, text="Browse", command=self.browsecsv, bg="#f9f485")
        btn2.config(height=2, width=12)
        btn2.pack(side=tk.RIGHT, expand = True, fill = tk.BOTH)

        """btn3 = tk.Button(self.root, text="Refresh", command=restart_program, bg="#ff9e0a")
        btn3.config(height=2, width=12)
        btn3.pack()"""

        # start a self.video_loop that constantly pools the video sensor
        # for the most recently read frame
        self.video_loop()

    def video_loop(self):
        """ Get frame from the video stream and show it in Tkinter """
        ok, frame = self.vs.read()  # read frame from video stream
        if ok:  # frame captured without any errors
            cv2image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)  # convert colors from BGR to RGBA
            self.current_image = Image.fromarray(cv2image)  # convert image for PIL
            imgtk = ImageTk.PhotoImage(image=self.current_image)  # convert image for tkinter
            self.panel.imgtk = imgtk  # anchor imgtk so it does not be deleted by garbage-collector
            self.panel.config(image=imgtk)  # show the image
        self.root.after(30, self.video_loop)  # call the same function after 30 milliseconds

    def take_snapshot(self):
        cam = cv2.VideoCapture(0)
        frame = cam.read()[1]
        cv2.imwrite(filename='C:/101-classification/examples/img.jpg', img=frame)
        os.system("check.bat")
        #self.video_loop()

    def destructor(self):
        #Destroy the root object and release all resources
        print("[INFO] closing...")
        self.root.destroy()
        self.vs.release()  # release web camera
        cv2.destroyAllWindows()  # it is not mandatory in this application

    def browsecsv(self):

        #os.remove("C:/101-classification/examples/img.jpg")
        ftypes = [('Images', "*.jpg*")]
        ttl = "Title"
        dir1 = 'C:\\'

        fileName = askopenfilename(filetypes=ftypes, initialdir=dir1, title=ttl)
        #os.remove("C:/101-classification/examples/img.jpg")
        shutil.copy(fileName,"C:/101-classification/examples/")
        #path="C:/101-classification/"
        os.chdir("C:/101-classification/")
        #fileName.save("{}").format(fileName)
        #os.rename(fileName, "img.jpg")
        #os.system("check.bat")
        os.system("python classify.py -m fashion.model -l mlb.pickle -i"+'"'+fileName+'"')
        #os.system("copy"+fileName+"C:/101-classification/examples")

#if __name__=='__main__':
    #main()
"""def restart_program():

    python = sys.executable
    os.execl(python, python, *sys.argv)"""

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-o", "--output", default="./",
    help="path to output directory to store snapshots (default: current folder")
args = vars(ap.parse_args())

# start the app
print("[INFO] starting...")
pba = Application(args["output"])
pba.root.mainloop()


# -*- coding: utf-8 -*-

"""
This script implements the fourth use case:
Attention points per brand.
"""

import pandas as pd
import requests, json
import numpy as np
import sys

class DictanovaAPIAuth(requests.auth.AuthBase):
	"""Attaches Dictanova Bearer Authentication to the given Request object."""

	def __init__(self, id, secret):
		self.apiclient_id = id
		self.apiclient_secret = secret
		self._token = None

	def __eq__(self, other):
		return all([
			self.apiclient_id == getattr(other, 'apiclient_id', None),
			self.apiclient_secret == getattr(other, 'apiclient_secret', None)
		])

	def __ne__(self, other):
		return not self == other

	def __call__(self, r):
		r.headers['Authorization'] = self.get_token()
		return r

	def get_token(self):
		# Get authentication token
		if self._token is None:
			payload = {
				"clientId": self.apiclient_id,
				"clientSecret": self.apiclient_secret
			}
			r = requests.post("https://api.dictanova.io/v1/token", json=payload)
			self._token = r.json()
		# Always use the one in cache
		return "Bearer %s" % self._token["access_token"]

if __name__ == "__main__":
	# Prepare Auth handler with API client id and secret
	# https://docs.dictanova.io/docs/authentication-and-security
	clientId, clientSecret = open("../credentials", "r").readline().strip().split(";")
	dictanova_auth = DictanovaAPIAuth(clientId, clientSecret)

	#################################### TOP NEGATIVE OPINIONS FROM DETRACTORS PER BRAND
	query = {
		"type": "COUNT",
		"field": "createdAt",
		"query": {
			"field": "metadata.recommande",
			"operator": "EQ",
			"value": "Ne recommande pas"
		},
		"dimensions": [{
				"field": "metadata.marque",
				"group": "DISTINCT"
			}, {
				"field": "TERMS",
				"group": "DISTINCT",
				"limit": 10
			}, {
				"field": "TERMS_POLARITY",
				"group": "NEGATIVE"
			}
		]
	}
	print("Query:")
	print(json.dumps(query, indent=4, sort_keys=False))
	# Requests
	r = requests.post(
		"https://api.dictanova.io/v1/aggregation/datasets/5b2286583a35940001399b1a/documents",
		json=query,
		auth=dictanova_auth)
	print(r)

	##################################################################### PREPARE RESULTS
	# Build dataframe
	df = pd.io.json.json_normalize(r.json()["periods"][0]["values"])
	df = df[df["volume"] > 0] # remove null
	df["brand"] = df["dimensions"].apply(lambda x: x[0])
	df["opinion"] = df["dimensions"].apply(lambda x: x[1])
	df.drop(columns=["dimensions"], inplace=True)
	df.sort_values(by=["brand", "value"], axis="index", ascending=False, inplace=True)

	##################################################################### DISPLAY RESULTS
	# Word cloud
	# pip3 install wordcloud
	import math
	from wordcloud import WordCloud
	import matplotlib.pyplot as plt
	from matplotlib import colors
	onlyred = colors.ListedColormap(['orangered'])
	brands = list(df["brand"].unique())
	row = math.ceil(math.sqrt(len(brands)))
	col = math.floor(math.sqrt(len(brands)))
	fig, axis = plt.subplots(row, col)
	fig.tight_layout()
	for i, brand in enumerate(brands):
		# Pretty print results
		print("Attention points for brand '%s':" % brand)
		print(df[df["brand"]==brand])
		# Compute word distribution for wordcloud
		wc_freq = {row.opinion:row.volume for row in df[df["brand"]==brand].itertuples()}
		wc = WordCloud(
			prefer_horizontal=1,
			font_path='Geomanist-Regular.otf',
			background_color="white",
			colormap=onlyred)
		wordcloud = wc.fit_words(wc_freq)
		# Add to subfig
		r = i//col
		c = i%col
		axis[r][c].imshow(wordcloud)
		axis[r][c].set_title(brand)
		axis[r][c].axis("off")
	# Render
	plt.show()


# Copyright (c) 2021, VMRaid and Contributors
# License: MIT. See LICENSE

import json

import vmraid
from vmraid import _
from vmraid.integrations.doctype.braintree_settings.braintree_settings import (
	get_client_token,
	get_gateway_controller,
)
from vmraid.utils import flt

no_cache = 1

expected_keys = (
	"amount",
	"title",
	"description",
	"reference_doctype",
	"reference_docname",
	"payer_name",
	"payer_email",
	"order_id",
	"currency",
)

def get_context(context):
	context.no_cache = 1

	# all these keys exist in form_dict
	if not (set(expected_keys) - set(list(vmraid.form_dict))):
		for key in expected_keys:
			context[key] = vmraid.form_dict[key]

		context.client_token = get_client_token(context.reference_docname)

		context["amount"] = flt(context["amount"])

		gateway_controller = get_gateway_controller(context.reference_docname)
		context["header_img"] = vmraid.db.get_value(
			"Braintree Settings", gateway_controller, "header_img"
		)

	else:
		vmraid.redirect_to_message(
			_("Some information is missing"),
			_("Looks like someone sent you to an incomplete URL. Please ask them to look into it."),
		)
		vmraid.local.flags.redirect_location = vmraid.local.response.location
		raise vmraid.Redirect

@vmraid.whitelist(allow_guest=True)
def make_payment(payload_nonce, data, reference_doctype, reference_docname):
	data = json.loads(data)

	data.update({"payload_nonce": payload_nonce})

	gateway_controller = get_gateway_controller(reference_docname)
	data = vmraid.get_doc("Braintree Settings", gateway_controller).create_payment_request(data)
	vmraid.db.commit()
	return data


class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution(object):

    def buildTree(self, inorder, postorder):
        if not postorder:
            return None
        root = TreeNode(postorder[-1])  # 创建树
        n = inorder.index(root.val)
        # inorder：左根右
        # postorder：左右根
        # 拿到根index ：n，:n都是左边值
        root.left = self.buildTree(inorder[:n], postorder[:n])
        # postorder：n:-1是右边值
        root.right = self.buildTree(inorder[n + 1:], postorder[n:-1])
        return root


'''         Dictionary          '''

def query(user_input):
    if user_input == 0:
        hun = input("What is the hungarian version of the word?:\n ")
        eng = input("What is the english version of the word?:\n ")
        add_word(hun, eng)
    elif user_input == 1:
        question = int(input("If the word is an english word -press \"0\", or if it\'s a hungarian word, - press \"1\" again.\n "))
        second_level(question)
    else:
        print("invalid input. Please enter a \"0\" or a \"1\".")
    return

def second_level(question):
    if question == 0:
        eng = input("What is this english word you want to translate?:\n ")
        translate_to_hun(eng)
    if question == 1:
        hun = input("What is this hungarian word you want to translate?:\n")
        translate_to_eng(hun)

def add_word(hun_word, eng_word):
    with open("dictionary.txt", "a") as f:
        f.write(hun_word + " : " + eng_word + "\n")
    return

def translate_to_hun(eng_word):
    my_dict = import_dict()
    for i in my_dict:
        if my_dict[i] == eng_word:
            print(i)
            return
    print("Invalid or not in dictionary. Feel free to enter new values.")

def translate_to_eng(hun_word):
    my_dict = import_dict()
    for i in my_dict:
        if i == hun_word:
            print(my_dict[i])
            return
    print("Invalid or not in dictionary. Feel free to enter new values.")

def import_dict():
    my_dict = { }
    with open("dictionary.txt") as f:
        for line in f:
            line = line.strip()
            words = line.split(" : ")
            my_dict[words[0]] = words[1]
    return my_dict

user_input = int(input("Please press \"0\" and enter if you want to add something to the dictionary, or if you want to translate press \"1\" and enter.:\n "))

query(user_input)


import re
import textwrap

from collections import OrderedDict
try:
    from collections.abc import Mapping, Sequence as SequenceABC
except ImportError:
    from collections import Mapping, Sequence as SequenceABC

from glycopeptidepy.structure.sequence import ProteinSequence
from glypy.utils.base import opener

class FastaHeader(Mapping):
    """Hold parsed properties of a FASTA sequence's
    definition line.

    This object supports the :class:`Mapping` interface, and
    keys may be accessed by attribute access notation.

    Attributes
    ----------
    defline : str
        The textual representation of definition line which
        lead to the collection of attributes stored therein.
    """
    def __init__(self, mapping, original=None):
        self._mapping = mapping
        if original is None:
            self.defline = self._make_defline()
        else:
            self.defline = original

    def __getitem__(self, key):
        return self._mapping[key]

    def __iter__(self):
        return iter(self._mapping)

    def items(self):
        return self._mapping.items()

    def keys(self):
        return self._mapping.keys()

    def values(self):
        return self._mapping.values()

    def __len__(self):
        return len(self._mapping)

    def __contains__(self, key):
        return key in self._mapping

    def __getattr__(self, key):
        if key == "_mapping":
            raise AttributeError(key)
        try:
            return self._mapping[key]
        except KeyError:
            raise AttributeError(key)

    def __str__(self):
        return self.defline

    def __repr__(self):
        return "{self.__class__.__name__}({self._mapping})".format(self=self)

    def _make_defline(self):
        raise NotImplementedError()

    def __hash__(self):
        return hash(self.defline)

    def __eq__(self, other):
        return str(self) == str(other)

    def __ne__(self, other):
        return not (self == other)

    def __dir__(self):
        base = set(dir(super(FastaHeader, self)))
        keys = set(self._mapping.keys())
        return list(base | keys)

class UnparsableDeflineError(ValueError):
    """Indicate that a definition line could
    not be parsed by the current parser.
    """
    pass

class DefLineParserBase(object):
    """A base class for definition line parser, providing
    some common machinery in the :meth:`__call__` method.

    This class requires that the subclass provide a :meth:`parse`
    method to convert a string (without a ">" prefix). and return
    an instance of :class:`.OrderedDict`.
    """
    def __call__(self, defline):
        if defline.startswith(">"):
            defline = defline[1:]
        return self._make_header(self.parse(defline), defline)

    def _make_header(self, mapping, defline):
        return FastaHeader(mapping, defline)

    def parse(self, defline):
        raise NotImplementedError()

class CallableDefLineParser(DefLineParserBase):
    """Parse a definition line using an arbitrary
    provided :class:`.Callable`.

    Attributes
    ----------
    callable_obj : Callable
    """
    def __init__(self, callable_obj):
        if not callable(callable_obj):
            raise TypeError("Must provide a callable object.")
        self.callable_obj = callable_obj

    def parse(self, defline):
        result = self.callable_obj(defline)
        if isinstance(result, Mapping):
            return OrderedDict(result.items())
        elif isinstance(result, SequenceABC):
            return OrderedDict(enumerate(result))
        else:
            raise UnparsableDeflineError(
                "Failed to parse {!r} using {!r} (return value: {!r})".format(
                    defline, self, result))

def split_on_whitespace(string):
    return re.split("\s+", string)

space_delim_parser = CallableDefLineParser(split_on_whitespace)

class RegexDefLineParser(DefLineParserBase):
    """Parse a definition line using a provided
    regular expression.

    If :attr:`pattern` contains named groups, this will
    produce a :class:`OrderedDict` where the group names
    are keys, otherwise their index will be used as the
    key.

    Attributes
    ----------
    pattern : re.pattern
    """
    def __init__(self, pattern):
        self.pattern = re.compile(pattern)
        self._is_keyed = "?P<" in pattern

    def parse(self, defline):
        match = self.pattern.search(defline)
        if match:
            if not self._is_keyed:
                return OrderedDict(enumerate(match.groups()))
            else:
                return (match.groupdict())
        else:
            raise UnparsableDeflineError(
                "Failed to parse {!r} using {!r}".format(
                    defline, self))

class DispatchingDefLineParser(DefLineParserBase):
    """Attempt to parse a definition line using one of many
    :class:`DeflineParserBase` instances.

    Attributes
    ----------
    graceful : bool
        If :attr:`graceful`, failure to parse a defline
        with all of :attr:`parsers` results in a :class:`FastaHeader`
        no properties other than *defline*
    header_type : type
        The type to use to contain the result of parsing.
        Defaults to :class:`FastaHeader`.
    parsers : list
        A :class:`list` of :class:`DeflineParserBase` instances
    """
    def __init__(self, parsers, graceful=False, header_type=FastaHeader):
        self.parsers = list(parsers)
        self.graceful = graceful
        self.header_type = header_type

    def _make_header(self, mapping, defline):
        return self.header_type(mapping, defline)

    def parse(self, defline):
        for parser in self.parsers:
            try:
                parts = parser.parse(defline)
                return parts
            except UnparsableDeflineError:
                continue
        else:
            if self.graceful:
                return FastaHeader({}, defline)
            else:
                raise UnparsableDeflineError(
                    "Failed to parse {:r} using {:r}".format(
                        defline, self))

uniprot_regex = r"(?P<db>[a-z^\|]+)\|(?P<accession>[A-Z0-9\-]+)\|(?P<name>\S*)(?:\s(?P<description>.+))?"
uniprot_parser = RegexDefLineParser(uniprot_regex)

partial_uniprot_regex = r"(?P<accession>[A-Z0-9]+)\|(?P<name>\S+)"
partial_uniprot_parser = RegexDefLineParser(partial_uniprot_regex)

default_parser = DispatchingDefLineParser([uniprot_parser, partial_uniprot_parser, space_delim_parser])

class FastaFileParser(object):
    def __init__(self, path, defline_parser=default_parser):
        self.state = "defline"
        self.handle = opener(path)
        self.defline = None
        self.sequence_chunks = []
        self.defline_parser = defline_parser
        self._generator = None

    def process_result(self, d):
        return d

    def __iter__(self):
        return self

    def __next__(self):
        if self._generator is None:
            self._generator = self._parse_lines()
        return next(self._generator)

    def next(self):
        return self.__next__()

    def _parse_lines(self):
        for line in self.handle:
            line = line.strip()
            if self.state == 'defline':
                if line.startswith(">"):
                    self.defline = line[1:]
                    self.state = "sequence"
                else:
                    continue
            else:
                if not re.match(r"^(\s+|>)", line):
                    self.sequence_chunks.append(line)
                else:
                    if self.defline is not None:
                        try:
                            yield self.process_result({
                                "name": self.defline_parser(self.defline),
                                "sequence": ''.join(self.sequence_chunks)
                            })
                        except KeyError as e:
                            print(e)
                            pass
                    self.sequence_chunks = []
                    self.defline = None
                    self.state = 'defline'
                    if line[0] == '>':
                        self.defline = re.sub(r"[\n\r]", "", line[1:])
                        self.state = "sequence"

        if len(self.sequence_chunks) > 0:
            try:
                yield self.process_result(
                    {
                        "name": self.defline_parser(self.defline),
                        "sequence": ''.join(self.sequence_chunks)
                    })
            except Exception as e:
                pass

class ProteinFastaFileParser(FastaFileParser):

    def __init__(self, path, defline_parser=default_parser):
        super(ProteinFastaFileParser, self).__init__(path, defline_parser)

    def process_result(self, d):
        p = ProteinSequence(d['name'], d['sequence'], annotations=dict(d['name']))
        return p

class FastaFileWriter(object):

    def __init__(self, handle):
        self.handle = handle

    def write(self, defline, sequence):
        defline = str(defline)
        if not defline.startswith(">"):
            defline = ">" + defline
        self.handle.write(defline)
        self.handle.write("\n")
        self.handle.write(sequence)
        self.handle.write("\n\n")

    def writelines(self, iterable):
        for defline, seq in iterable:
            self.write(defline, seq)

class ProteinFastaFileWriter(FastaFileWriter):

    def write(self, protein):
        defline = ">%s" % protein.name
        seq = '\n'.join(textwrap.wrap(protein.get_sequence(), 80))
        super(ProteinFastaFileWriter, self).write(defline, seq)

    def writelines(self, iterable):
        for protein in iterable:
            self.write(protein)

def read(f, defline_parser=default_parser, reader_type=ProteinFastaFileParser):
    return reader_type(f, defline_parser=defline_parser)


import os
import json
import argparse

class find_tweets:
    def __init__(self, j, q):
        #clear the results file

        self.json_dir = j
        self.query_file = q
        self.result_ids = []
        self.tweets = self.read_in_tweets()
        self.read_in_query_results()
        data = []
        for tid in self.result_ids:
            if tid in self.tweets:
                data.append({tid:self.tweets[tid]})

        with open('found_tweets.json', 'a') as f:
            json.dump(data,f)

    def read_in_tweets(self):
        files = os.listdir(self.json_dir)
        for file in files:
            path = os.path.join(self.json_dir, file)
            data = []
            with open(path) as f:
                data = json.load(f)
            return data

    def read_in_query_results(self):
        with open(self.query_file) as f:
            lines = f.readlines()
            for line in lines:
                prob = float(line.split()[4])
                if prob > 10.0:
                    self.result_ids.append(line.split()[2])

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--json_dir", help="json directory of files to convert")
    parser.add_argument("--query_file", help="query results file (default var/results/file.res")
    args = parser.parse_args()
    find_tweets(args.json_dir, args.query_file)


'''
This problem was asked by Stripe.

Given an array of integers, find the first missing positive integer in linear
time and constant space. In other words, find the lowest positive integer that
does not exist in the array. The array can contain duplicates and negative
numbers as well.

For example, the input [3, 4, -1, 1] should give 2. The input [1, 2, 0]
should give 3.

You can modify the input array in-place.
'''

def lowest_positive(nums):
    lowest = nums[0]
    j = 0
    # Find the first number greater than 0
    for i in range(len(nums)):
        if nums[i] > 0:
            lowest = nums[i]
            j = i
            break
    # Use the number found to get the lowest positive number in the array
    for i in range(j, len(nums)):
        # 1 is the lowest number we care about, we can stop if we find it
        if nums[i] == 1:
            lowest = 1
            break
        if nums[i] < lowest and nums[i] > 0:
            lowest = nums[i]
    # Increment 'lowest' until the missing positive number is found
    while True:
        if lowest in nums:
            lowest += 1
        else:
            return lowest

if __name__ == '__main__':
    print(lowest_positive([3, 4, -1, 1]))
    print(lowest_positive([1, 2, 0]))