# -*- coding: utf-8 -*-
"""
Created on Mon May 14 16:29:50 2018

@author: wfok007
"""
#from __future__ import print_function, division
#import matplotlib
#matplotlib.use('agg')

import torch, math
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
import matplotlib.pyplot as plt

import time
import os
from transform import  save, fromFloat_to_uint8
from runningAvg import averager
from decompress import decompress_mask, addAxis, fast_transfer, gpuDecompress, gpuDecompressMask, gpuNormalise
from NNfunctions import interpolate3D, metaGen, freeze_layer

def createCyclical(T, learning_rate, version, min_fraction=1/3):

    '''
    This function creates a lookup table. The rows are values of sin(t), and the
    columns are values of a linear learning rate function

    Inputs:
        T = is the period of the cycle
        learning rate = max rate
        version = high to low or low to high
        min_fraction = a fraction that calculates the slowest allowable learning rate

    Outputs:
        lookup = dictionary / the table
        storedKeys = dictionary key
        lrs = values of rates (in fraction) in a single complete cycle
    '''

    # no endpoint as we cycle all items anyway

    if version =='highLow':

        lrs = [np.linspace(1,min_fraction,int(T/2),
                           endpoint=False),
                np.linspace(min_fraction,1,int(T/2), endpoint=False)]
    elif version =='lowHigh':
        lrs = [np.linspace(min_fraction,1,int(T/2),
                           endpoint=False),
                np.linspace(1,min_fraction,int(T/2), endpoint=False)]

    lrs = np.round(np.hstack(lrs), decimals=2).tolist()
    print ('maximum learning rate {}'.format(learning_rate*lrs[0]))
    print ('minium learning rate {}'.format(learning_rate*lrs[int(T/2)]))
    print ('number of f updates or bites %d' %len(lrs))

    # mapping sin value to index to imporve on the flat part of the sin curve

    lookup = dict()
    for i in range(T):
        # as many decimals places as possible to get unique keys
        s = np.sin(np.pi*(i/T))

        lookup[s] = i

    storedKeys = list(lookup.keys())
    return lookup, storedKeys, lrs

def triangular(i):
    """
    this function helps to look up the right fraction to apply to
    the learning rate at any point in the cycle. Sine is handy for it is
    an even function.

    Inputs:
        i = counter created by pytorch
    output:
        the fraction at which the i point in time in a cycle.

    """

    s = np.abs(np.sin(np.pi*(i/T)))
    difference = [abs(k-s) for k in storedKeys]
    keyS =  storedKeys[difference.index(min(difference))]
    index = lookup[keyS]
    return lrs[index]

class Net(nn.Module):
    '''
    The main part of the network

    Inputs:
        original_model = pre-train model
        od = input depth
        oh =input height
        ow = input width
        multiple = fraction that adjusts the number of filters in the network
        kernels = number of stacks of inputs after intensity equalization with various kernels' sizes

    more pre-trained resnet can be found
    https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
    '''
    def __init__(self, original_model, od, oh, ow, multiple, kernels=2):
        super().__init__()
        # get the architecture of the pre-trained network
        children = list(original_model.children())

        self.kernels = kernels
        self.conv1 = nn.Sequential(*children[0:3]) # skip pooling
        self.conv2 = nn.Sequential(*children[4])
        self.conv3 = nn.Sequential(*children[5])
        self.conv4 = nn.Sequential(*children[6])
        self.conv5 = nn.Sequential(*children[7])

        self.d5 = nn.Sequential(nn.Conv3d(int(512*multiple), int(256*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(256*multiple)),
                                  nn.ReLU())

        self.h4 = nn.Sequential(nn.Conv3d(int(512*multiple),int(256*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(256*multiple)),
                                  nn.ReLU())

        self.d4 = nn.Sequential(nn.Conv3d(int(256*multiple), int(128*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(128*multiple)),
                                  nn.ReLU())

        self.h3 = nn.Sequential(nn.Conv3d(int(256*multiple), int(128*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(128*multiple)),
                                  nn.ReLU())

        self.d3 = nn.Sequential(nn.Conv3d(int(128*multiple), int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())

        self.h2 = nn.Sequential(nn.Conv3d(int(128*multiple),int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())

        self.d2 = nn.Sequential(nn.Conv3d(int(64*multiple), int(32*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(32*multiple)),
                                  nn.ReLU())

        self.s5 = nn.Sequential(nn.Conv3d(int(512*kernels), int(512*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(512*multiple)),
                                  nn.ReLU())
        self.s4 = nn.Sequential(nn.Conv3d(int(256*kernels), int(256*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(256*multiple)),
                                  nn.ReLU())
        self.s3 = nn.Sequential(nn.Conv3d(int(128*kernels), int(128*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(128*multiple)),
                                  nn.ReLU())
        self.s2 = nn.Sequential(nn.Conv3d(int(64*kernels), int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())
        self.s1 = nn.Sequential(nn.Conv3d(int(64*kernels), int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())
        self.s0 = nn.Sequential(nn.Conv3d(int(64*multiple), int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())

        self.h0 = nn.Sequential(nn.Conv3d(int(128*multiple), int(64*multiple), kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1),bias=False),
                                  nn.BatchNorm3d(int(64*multiple)),
                                  nn.ReLU())

        self.nin = nn.Conv3d(int(32*multiple)+2, 1, kernel_size=(3,3,3), stride=(1,1,1), padding=(1,1,1))

        self.inter1 = interpolate3D(88,14,14)
        self.inter2 = interpolate3D(88,28,28)
        self.inter3 = interpolate3D(88,56,56)
        self.inter4 = interpolate3D(od,oh,ow)
        self.inter5 = interpolate3D(88, 576, 576)

    def forward(self, x):
        x1 = self.conv1(x)
        x2 = self.conv2(x1)
        x3 = self.conv3(x2)
        x4 = self.conv4(x3)
        x5 = self.conv5(x4)

        x = x[:,0,:,:] # get rid of the 3 channels
        x = x.unsqueeze(0)
        x = x.view(x.shape[0], self.kernels, 88, x.shape[2], x.shape[3])

        xs = [x1, x2, x3, x4, x5]
        xs = [i.unsqueeze(0) for i in xs]
        xs = [i.view(self.kernels, 88, i.shape[2], i.shape[3], i.shape[4]) for i in xs]

        xs = [i.permute(0,2,1,3,4) for i in xs]
        x1, x2, x3, x4, x5 = xs

        # fragmented footprint, need to bring dimensions together first
        xs = [i.contiguous() for i in xs]
        xs = [i.view(1, -1, i.shape[2], i.shape[3], i.shape[4]) for i in xs]
        x1, x2, x3, x4, x5 = xs

        print ('x1 shape{}'.format(x1.shape))
        print ('x2 shape{}'.format(x2.shape))
        print ('x3 shape{}'.format(x3.shape))
        print ('x4 shape{}'.format(x4.shape))
        print ('x5 shape{}'.format(x5.shape))

#
        x15 = self.s5(x5)
        x14 = self.s4(x4)
        x13 = self.s3(x3)
        x12 = self.s2(x2)
        x11 = self.s1(x1)

        print ('x15 shape{}'.format(x15.shape))
        print ('x14 shape{}'.format(x14.shape))
        print ('x13 shape{}'.format(x13.shape))
        print ('x12 shape{}'.format(x12.shape))
        print ('x11 shape{}'.format(x11.shape))

        x25 = self.d5(x15)

        x25 = self.inter1(x25)

        x24 = self.h4(torch.cat((x14, x25), 1))

        x34 = self.d4(x24)

        x34 = self.inter2(x34)

        x23 = self.h3(torch.cat((x13, x34), 1))
        x33 = self.d3(x23)
        x33 = self.inter3(x33)

        print ('x25 shape{}'.format(x25.shape))
        print ('x24 shape{}'.format(x24.shape))
        print ('x34 shape{}'.format(x34.shape))
        print ('x23 shape{}'.format(x23.shape))
        print ('x33 shape{}'.format(x33.shape))

        x22 = self.h0(torch.cat((x12, x33), 1))
        x32 = self.s0(x22)
        x21 = self.h2(torch.cat((x11, x32), 1))
        x21 = self.d2(x21)

        x0 = self.inter4(x21)
        x00 = torch.cat((x0, x), 1)
        seg = self.inter5(self.nin(x00))

        print ('x22 shape{}'.format(x22.shape))
        print ('x32 shape{}'.format(x32.shape))
        print ('x21 shape{}'.format(x21.shape))
        print ('x0 shape{}'.format(x0.shape))
        print ('x00 shape{}'.format(x00.shape))

        return seg

def flip(arr):
    """
    toggle between 0 and 1
    """

    return arr*(-1) + 1

def torch_flip(arr):
    '''
    a = torch.tensor([[1, 0], [0, 1],[1,1], [0,0]])

    this line is wrong, torch sum is a reduced sum not elementwise sum
    return torch.sum(torch.mul(arr, -1), 1, keepdim=True)

    cannot do this as well
    c = torch.sum(b, torch.ones(4,2))
    '''
    return torch.mul(arr, -1) + 1
#
def cal_Dice_Similarity_Coefficient(pred_mask, mask):

    '''
    This function uses numpy to calculate Dice coefficient

    inputs:
        pred_mask int 32 0 or 1 (depth, height, width)
        mask/ ground truth float 32 0 or 255 (same dimension) '''

    print ('numpy dice ')
    mask = mask/255

    true_pos = np.sum(np.multiply(pred_mask, mask))
    true_pos = true_pos / 88
    flipped_mask = flip(mask)
    false_pos = np.sum(np.multiply(flipped_mask, pred_mask))
    false_pos = false_pos / 88
    flipped_pred = flip(pred_mask)
    false_neg = np.sum(np.multiply(mask, flipped_pred))
    false_neg = false_neg/ 88
    n = 2*true_pos
    dice = n/(n+false_neg+false_pos)
    print ('true pos %f'%true_pos)
    print ('false pos %f' %false_pos)
    print ('false_neg %f' %false_neg)
    print ('n %f'%n)
    print ('Dice similarity coefficient %f' %dice)
    return dice

class non_differentiable_Dice_coef(torch.nn.Module):

    """
    This function uses pytorch to calculate Dice coefficient

    """
    def __init__(self):
        super().__init__()
    def forward(self, output, target, threshold=0.5, returnArr=True):

        pred_mask = nn.functional.sigmoid(output)
        # no need to store the gradient because this is not used in the objective function
        pred_mask = torch.autograd.Variable(pred_mask.gt(threshold),requires_grad=False)
        # make sure the variable is of the right type
        pred_mask = pred_mask.type(torch.cuda.FloatTensor)

        true_pos_arr = torch.mul(pred_mask, target)
        true_pos = torch.sum(true_pos_arr)
        true_pos = true_pos / 88
        flipped_mask = torch_flip(target)

        false_pos_arr = torch.mul(flipped_mask, pred_mask)
        false_pos = torch.sum(false_pos_arr)
        false_pos = false_pos / 88
        flipped_pred = torch_flip(pred_mask)

        false_neg_arr = torch.mul(target, flipped_pred)
        false_neg = torch.sum(false_neg_arr)
        false_neg = false_neg/ 88

        n = torch.mul(true_pos,2)
        dice = torch.div(n, (n+false_neg+false_pos))

        if returnArr:
            # get the item, otherwise we have a pytorch tensor (scalar)
            return dice.item(), true_pos_arr.cpu().detach().numpy(),\
                false_pos_arr.cpu().detach().numpy(), false_neg_arr.cpu().detach().numpy()
        else:
            return dice.item()

class Dice_Loss(torch.nn.Module):
    """
    This function calculates Dice loss which is used in the objective function.
    """
    def __init__(self):
        super().__init__()
    def forward(self, output, target, smooth=1):
        pred_mask = nn.functional.sigmoid(output)
        a = torch.mul(pred_mask, target)

        depth = a.shape[2]
        b = a.view(depth, -1)

        c = torch.sum(b, dim=1)

        c = 2*c
        # just in case there is not 1 in the prediction or target
        c = c + smooth

        pred_square = torch.mul(pred_mask, pred_mask)
        target_square = torch.mul(target, target)

        pred_square = pred_square.view(depth, -1)
        target_square = target_square.view(depth, -1)
        ps = torch.sum(pred_square, dim=1)
        ts = torch.sum(target_square, dim=1)

        denominator = ps+ts
        denominator = denominator + smooth
        # minimize, so 1-
        slice_by_slice_loss = 1- torch.div(c, denominator)
        return torch.sum(slice_by_slice_loss)

def initialise_averager(model):
    """
    this function initializes a average for every trainable parameters
    This average takes the mean of a minibatch.
    Inputs:
        model = network

    output:
        dictionary of parameters and their averages
    """

    grad_averager = {}
    for name, parameter in model.named_parameters():
        # because of pre-training, get trainable parameters
        if parameter.requires_grad:
            grad_averager[name] = averager()
    return grad_averager

def validation(i, gaa_mean, gaa_std,generator_valid, model, criterion, accuracy, prog_dir):
    """
    This function validates the network whenever it is called. It randomly selects
    one sample in the validation dataset to valid the network.
    It is possible to valid more than one sample but in the interest of time, only one
    sample is used for validation.

    The current step is i

    gaa_mean, gaa_std = average intensity and standard deviation used for normalization

    generator_valid = a generator that holds the validation dataset
    model = network
    criterion = Dice loss
    accuracy = Dice coefficient
    prog_dir = storage directory

    Outputs:
        loss = Dice loss
        coef = Dice coefficient

    """

    print ('--------------------validation------------------')
    full_image, full_mask, image, mask  = next(generator_valid)
    image1 , target = prepareInputsForGPU(image, full_mask, gaa_mean, gaa_std)

    # no need to back-propagate
    with torch.no_grad():
        output = model(image1)
        loss = criterion(output, target)
        coef, true_pos_arr, false_pos_arr, false_neg_arr = accuracy(output, target, returnArr=True)
        sig = nn.functional.sigmoid(output)
        sig = sig.cpu().detach().numpy()
        pred_mask = sig[0,0,...]

    del output, sig, image1, target
    val_loss = loss.item()
    print ('validation loss %f; dice coef: %f' %(val_loss, coef))

    # use a default cutoff of 0.5
    pred_mask = pred_mask > 0.5
    pred_mask = pred_mask *1
    # save validation predictions and mistakes for visualization/ inspection
    temp_img = fromFloat_to_uint8(full_image)

    save(temp_img, pred_mask, 'validation_'+ str(i), prog_dir)

    # need to decompress the index to get mask
    mask = decompress_mask(full_mask,88, 576, 576)
    save(temp_img, mask, 'validation_groundTruth_'+str(i), prog_dir)
#
    save(temp_img, true_pos_arr[0,0,...], 'validation_true_pos_'+str(i), prog_dir)
    save(temp_img, false_pos_arr[0,0,...], 'validation_false_pos_'+str(i), prog_dir)
    save(temp_img, false_neg_arr[0,0,...], 'validation_false_neg_'+str(i), prog_dir)

    return loss.item(), coef

def plot_norm(i, normHist, holder, prog_dir):
    """
    This function plots the changing norm of the error derivatives
    Inputs:
        i = current step
        normHist = past norm magnitude
        holder = the step at which the norm was recorded
        prog_dir = save to this directory
    """

    names = list(normHist.keys())
    plt.figure()

    for name in names:
        if name[0] == 'd':
            plt.plot(adjustForMinibatch(holder), normHist[name], label=name)
    plt.xlabel('number of times weights being updated')
    plt.legend()
    plt.savefig(os.path.join(prog_dir, 'normD_'+str(i)+'.png'))
    plt.close()

    plt.figure()
    for name in names:
        if name[0] == 's':
            plt.plot(adjustForMinibatch(holder), normHist[name], label=name)
    plt.xlabel('number of times weights being updated')
    plt.legend()
    plt.savefig(os.path.join(prog_dir, 'normS_'+str(i)+'.png'))
    plt.close()

    plt.figure()

    for name in names:
        if name[0] != 's' and name[0] != 'd':
            plt.plot(adjustForMinibatch(holder), normHist[name], label=name)
    plt.xlabel('number of times weights being updated')
    plt.legend()
    plt.savefig(os.path.join(prog_dir, 'norm_'+str(i)+'.png'))
    plt.close()

def addCheckpoints(checkpoint_hist):
    """
    this function adds a vertical line on a figure at locations specified in
    checkpoint_hist
    """
    if len(checkpoint_hist) >0:
        for xc in adjustForMinibatch(checkpoint_hist):
            plt.axvline(x=xc, linestyle='--')

def adjustForMinibatch(trainHolder, minibatch=5):
    """
    As the step number is based on the number of times a case is fed to the network,
    not the number of times network updates itself, this function converts
    "feeding" to "updates"

    """
    return [t/ minibatch for t in trainHolder]
def plot_loss(i, trainHolder, trainLoss, validHolder, validLoss, name, checkpoint_hist, prog_dir):
    """
    This function puts the Dice loss over past weight updates on a chart.

    i = current step
    trainHolder = step at which loss was recorded during training
    trainLoss = the recorded loss
    validHolder = step at which loss was recorded during validation
    validLoss = recording
    name = output filename
    checkpoint_hist = steps at which weights/ checkpoints were taken
    prog_dir = storage directory

    """

    plt.figure()
    plt.scatter(adjustForMinibatch(trainHolder), trainLoss, label='train')
    plt.scatter(adjustForMinibatch(validHolder), validLoss, label='validation')
    addCheckpoints(checkpoint_hist)
    plt.xlabel('number of times weights being updated')
    plt.ylabel(name)
    plt.legend()
    plt.savefig(os.path.join(prog_dir, name+str(i)+'.png'))
    plt.close()

def plot_learningRate(i, normHolder, checkpoint_hist, lr_history):
    """
    This function puts the learning rates during.

    i = current step
    normHolder = step at which rate was recorded during training
    checkpoint_hist = steps at which weights/ checkpoints were taken
    lr_history = past rate values

    """

    plt.figure()
    plt.plot(adjustForMinibatch(normHolder), lr_history)
    addCheckpoints(checkpoint_hist)
    plt.xlabel('number of times weights being updated')
    plt.ylabel('learning rate')
    plt.savefig(os.path.join(prog_dir, 'learningRate'+str(i)+'.png'))
    plt.close()

def init_weights(m):
    """
    apply weight initialization scheme to a model with a particular layer type
    """

    if type(m) == nn.Conv3d:
        nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')

def createModel(od, oh, ow, multiple):
    """
    This block creates a model, then initialize its weights and put it onto the GPU
    Inputs:
        od, oh, ow, multiple = network specifications
    Output
        model
    """

    original_model = torchvision.models.resnet34(pretrained=True)

    model = Net(original_model, od, oh, ow, multiple)
    model.apply(init_weights)

    # freeze the resNet feature map parts
    for layer in [model.conv1, model.conv2, model.conv3, model.conv4, model.conv5]:
        freeze_layer(layer)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print (device)
    model.to(device)

    return model

def load_stat(hpc=r'Z:\mpi_heart' , name ='kernel_overall_mean_std.npz'):
    """
    load the statistics of training data
    return the mean and standard derivation (numpy array)
    """

    npzfile = np.load(os.path.join(hpc, name))
    mean = npzfile['mean']
    std = npzfile['std']
    return mean, std

def prepareInputsForGPU(image, full_mask,gaa_mean, gaa_std):

    """
    This packs a series of operations that transfer input to GPU and then format and normalizes it.
    Inputs:
        image, full_mask = MR images and their masks
        gaa_mean, gaa_std = input statistics

    Outputs:
        image and target for neural network on GPU

    """

    # transfer input to GPU
    gImage = fast_transfer(image)
    gImage = gpuDecompress(gImage)

    gMask = fast_transfer(full_mask, isImage=False)
    gMask = gpuDecompressMask(gMask, 88, 576, 576)
    image1, target = gpuNormalise(gImage, gMask, gaa_mean, gaa_std)

    return image1, target
if __name__ == '__main__':

    plt.close('all')

    prog_dir = r'Z:\mpi_heart\prog'
    training_dir = r'C:\Users\wfok007\femuring\imaging\AtriaSeg_2018_training'
    output_dir = r'C:\Users\wfok007\femuring\imaging\AtriaSeg_2018_training\transformation\ResNet'
    debug_dir = r'C:\Users\wfok007\femuring\imaging\AtriaSeg_2018_training\debug'
    hpc = r'Z:\mpi_heart'
    weight_dir = 'Z:\mpi_heart\weight_checkpoints'

    mean, std = load_stat()

    num_ranks = 125
    num_ranks_train = 375
    od, oh, ow = 88 , 112, 112
    multiple = 0.25
    model = createModel(od, oh,ow, multiple)

    # put the statistics on the GPU
    aa_mean, aa_std = addAxis(mean, std)
    gaa_mean, gaa_std = fast_transfer(aa_mean), fast_transfer(aa_std)

    # initialize the data for validation and training
    generator_valid = metaGen('valid', num_ranks, hpc)
    generator_train = metaGen('train', num_ranks_train, hpc)

    criterion = Dice_Loss()
    accuracy = non_differentiable_Dice_coef()

    learning_rate = math.pow(10, -1.5)
    T = 50
    lookup, storedKeys, lrs = createCyclical(T, learning_rate, 'highLow', min_fraction=1/10)

    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
                          lr=learning_rate)
    # useful for cyclical learning rate
    scheduler = lr_scheduler.LambdaLR(optimizer, triangular)

    grad_averager = initialise_averager(model)

    running_loss = 0.0
    running_coef = 0.0
    printout_freq = 100
    MAX_EPOCH = 10
    minibatch = 5
    saving_freq = 200
    validation_freq = 300

    trainLoss = []
    trainCo = []

    validLoss = []
    validCo = []

    normHist = {}
    trainHolder = []
    validHolder = []
    normHolder = []
    lr_history = []
    checkpoint_hist = []

    # grab and calculate running averages on the gradient
    for name, parameter in model.named_parameters():
        # because of pre-training, get trainable parameters
        if parameter.requires_grad:
            normHist[name] = list()

    pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print ('total number of parameters %d' %pytorch_total_params)

    # restore to the last checkpoint
    river = 0
    for i in range(river):
        if i % minibatch  == 0  and i >= minibatch:
            scheduler.step()
    print ('scheduler at the right step')

    start_time = time.time()
    for i, (full_image, full_mask, image, mask) in enumerate(generator_train):
        #print ('maximum allocated %f GB' %(torch.cuda.max_memory_allocated()/math.pow(10,9)))
        # restore to checkpoint
        if i == river:
            print ('i at the right step of %d' %river)
        elif i > river:

            image1 , target = prepareInputsForGPU(image, full_mask, gaa_mean, gaa_std)
            # zero the parameter gradients
            optimizer.zero_grad()

            output = model(image1)

            coef, true_pos_arr, false_pos_arr, false_neg_arr = accuracy(output, target, returnArr=True)
            loss = criterion(output, target)
            del target, output # cut memory usage

            loss.backward()

            # grab and calculate running averages on the gradient
            for name, parameter in model.named_parameters():
                # because of pre-training, get trainable parameters
                if parameter.requires_grad:

                    current_grad = parameter.grad.clone()
                    grad_averager[name].update(current_grad)

            if i % minibatch  == 0  and i >= minibatch:
                # only count the step for the scheduler whenever an update was made
                scheduler.step()
                for name, parameter in model.named_parameters():
                    if parameter.requires_grad:
                        parameter.grad = grad_averager[name].past
                        normHist[name].append(torch.norm(parameter.grad).cpu().numpy().tolist())

                normHolder.append(i)
                optimizer.step()
                print ('[%d] update weights time elasped: %f' %(i, time.time()-start_time))
                lr_history.append(scheduler.get_lr()[0])
                grad_averager = initialise_averager(model)

            # print statistics
            running_loss += loss.item()
            running_coef += coef
            del loss

            if i % printout_freq == 0 and i >= printout_freq-1:    # print every 2000 mini-batches
                avgLoss = running_loss / printout_freq
                avgAcc = running_coef / printout_freq
                print('[%d] loss: %f ; dice coef: %f' %(i, avgLoss, avgAcc ))
                trainLoss.append(avgLoss)
                trainCo.append(avgAcc)
                trainHolder.append(i)
                running_loss = 0.0
                running_coef = 0.0

            if i % saving_freq ==0 and i >= saving_freq-1:
                torch.save(model.state_dict(), os.path.join(weight_dir, 'weights'+str(i)))
                checkpoint_hist.append(i)

            del image1

            if i% validation_freq == 0 and i >= validation_freq-1:
                los,co = validation(i,gaa_mean, gaa_std, generator_valid, model, criterion, accuracy,prog_dir)
                validCo.append(co)
                validLoss.append(los)
                validHolder.append(i)

                plot_loss(i, trainHolder, trainLoss,validHolder, validLoss, 'loss', checkpoint_hist, prog_dir)
                plot_loss(i, trainHolder, trainCo, validHolder, validCo,'dice_coeff', checkpoint_hist, prog_dir)

                plot_norm(i, normHist, normHolder, prog_dir)
                plot_learningRate(i, normHolder, checkpoint_hist, lr_history)

        print ('[%d] maximum allocated %f GB' %(i, torch.cuda.max_memory_allocated()/math.pow(10,9)))

        if i > MAX_EPOCH: # terminate training if steps exceed a maximum value
            break

    print('Finished Training')


#!/usr/bin/python3
from src.form_ass.modules.create_array import get_create_array
from src.form_ass.modules.create_dict import get_create_dict

import logging
import argparse
from argparse import ArgumentParser

logger = logging.getLogger(__name__)

def create_array():
    logger.info('Start creating array')
    description = '''Start creating array'''
    logger.info(description)
    parser = ArgumentParser(prog='create_array.py', description=description)
    parser.add_argument(
        '--variable_name', '-vn', required=True, help='Name of the finished array'
    )

    args = parser.parse_args()
    get_create_array(args.variable_name)
    logger.info('Done')

def create_dict():
    logger.info('Start creating dict')
    description = '''Start creating dict'''
    logger.info(description)
    parser = ArgumentParser(prog='create_dict.py', description=description)
    parser.add_argument(
        '--variable_name', '-vn', required=True, help='Name of the finished dict'
    )
    parser.add_argument(
        '--sorted_lists',
        '-sorted',
        '-s',
        default=False,
        action=argparse.BooleanOptionalAction,
    )
    parser.add_argument(
        '--every_other',
        '-eo',
        default=False,
        action=argparse.BooleanOptionalAction,
    )
    parser.add_argument(
        '--beside',
        '-b',
        default=False,
        action=argparse.BooleanOptionalAction,
    )

    args = parser.parse_args()
    get_create_dict(
        args.variable_name, args.sorted_lists, args.every_other, args.beside
    )
    logger.info('Done')


from fastapi import APIRouter, status, Request, Header
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from app.api.api_htmx.deps import CommonDeps, LoggedInDeps
from app.api.api_V1 import daily_overview as api_daily
from app.api.api_htmx import utils
from datetime import date, timedelta,datetime

router = APIRouter()
templates = Jinja2Templates("app/templates")

@router.get(
    "/",
    response_class=HTMLResponse,
    status_code=status.HTTP_200_OK,
)
def get_index(deps:CommonDeps):
    context = {
        "request": deps['request'],
        "hx_request": deps['hx_request'],
        "user": deps['user'],
        "date": deps['client_date']
    }

    return templates.TemplateResponse("index/index.html", context, headers={'HX-Redirect': '/'})

@router.get(
    "/calorie_progress"
)
async def calorie_progress(deps:LoggedInDeps, total=False):
    offsets = await utils.calorie_progress_data(deps=deps, total=total)

    context = {
        "request": deps['request'],
        "hx_request": deps['hx_request'],
        "profile": deps['profile'],
        "offsets": offsets
    }
    return templates.TemplateResponse("index/calorie_progress.html", context)

@router.get(
    "/navbar",
    response_class=HTMLResponse,
    status_code=status.HTTP_200_OK,
)
def get_navbar(deps:CommonDeps):
    context = {
        "request": deps['request'],
        "hx_request": deps['hx_request'],
        "user": deps['user']
    }

    return templates.TemplateResponse("nav.html", context)

@router.get(
    "/not_found",
)
def get_not_found(deps:CommonDeps):
    context = {
        "request": deps['request'],
        "hx_request": deps['hx_request'],
        "user": deps['user']
    }

    return templates.TemplateResponse("404.html", context)


from typing import List

from mongoengine import DoesNotExist

from speid.models import Event, Transaction
from speid.tasks import celery
from speid.types import Estado, EventType

@celery.task
def retry_incoming_transactions(speid_ids: List[str]) -> None:
    for speid_id in speid_ids:
        transaction = Transaction.objects.get(speid_id=speid_id)
        transaction.events.append(Event(type=EventType.retry))
        transaction.confirm_callback_transaction()
        transaction.save()

@celery.task(bind=True)
def process_outgoing_transactions(self, transactions: list):
    for request_dic in transactions:
        try:
            transaction = Transaction.objects.get(
                speid_id=request_dic['speid_id']
            )
        except DoesNotExist:
            continue

        action = request_dic['action']
        if action == Estado.succeeded.value:
            new_estado = Estado.succeeded
            event_type = EventType.completed
        elif action == Estado.failed.value:
            new_estado = Estado.failed
            event_type = EventType.error
        else:
            raise ValueError('Invalid event type')

        if transaction.estado == new_estado:
            continue
        else:
            transaction.estado = new_estado
        transaction.set_state(transaction.estado)
        transaction.events.append(
            Event(type=event_type, metadata=str('Reversed by recon task'))
        )
        transaction.save()


# -*- coding: utf-8 -*-
"""
Created on Mon Mar  6 13:35:26 2023

@author: RENEEEEEEE

To Run: uvicorn api-interface:app --reload
"""

# imports
from fastapi import FastAPI
from pydantic import BaseModel
import sqlite3
from contextlib import closing

# init backend db
projectDb = "project.db"

# init app
app = FastAPI()

# Define image subsection / mask area
# formatted for cv2.rectangle
class Subdivision(BaseModel):
    id: int
    start_point: tuple[int, int] = []
    end_point: tuple[int, int] = []

# add new segment data to image
@app.post("/api/segment-image/{image_id}")
async def add_segmentation(image_id : int, subdivision : Subdivision):
    return {"im ID": image_id, "sub" : subdivision}

# add new segment data to image
@app.put("/api/segment-image/{image_id}")
async def update_segmentation(image_id : int, subdivision : Subdivision):
    return {"im ID": image_id, "sub" : subdivision}

# initiate database with correct tables
@app.get("/api/init-db")
async def check_tables():
    with closing(sqlite3.connect(projectDb)) as connection:
        with closing(connection.cursor()) as cursor:

            # check if table already exists
            sql_query = "SELECT name FROM sqlite_master WHERE type='table';"
            cursor.execute(sql_query)
            res = cursor.fetchall()
            if res == []:
                cursor.execute("CREATE TABLE segments (image_id INTEGER, segment_id INTEGER, start_point BLOB, end_point BLOB)")
                return({"status": "table added"})
            else:
                return({"status": "table already exists"})


from sqlalchemy import Table, Column, Integer, String, DateTime, ForeignKey, LargeBinary, MetaData, Boolean, Interval

metadata = MetaData()

Jobs = Table('jobs', metadata,
    Column('jobid', Integer, primary_key=True),
    Column('torrentid', Integer, ForeignKey('torrents.torrentid')),
    Column('added', DateTime(False)),
    Column('bid', Integer),
    Column('downloaded', Integer),
    Column('size', Integer),
    Column('speed', Integer),
    Column('eta', Interval),
    Column('completed', DateTime(False)),
    Column('userid', Integer, ForeignKey('users.userid')),
    Column('downloadid', Integer, ForeignKey('downloads.downloadid')),
)

Downloads = Table('downloads', metadata,
    Column('downloadid', Integer, primary_key=True),
    Column('start_time', DateTime(False)),
    Column('ip', String),
    Column('link', String, nullable=False),
)

Torrents = Table('torrents', metadata,
    Column('torrentid', Integer, primary_key=True),
    Column('name', String),
    Column('torrent', LargeBinary),
    Column('magnet_link', String),
    Column('infohash', String, unique=True)
)

Users = Table('users', metadata,
    Column('userid', Integer, primary_key=True),
    Column('email', String),
    Column('hash', String),
    Column('salt', String),
    Column('credit', Integer),
)


import re

f = open("input.txt", "r")
lines = f.readlines()
arr = []
for line in lines:
  arr.append(line.strip())

numbers_to_draw = [x for x in arr[0].split(",")]

boards = []
board = []
for row in arr[2:]:
  if row == '':
    boards.append(board)
    board = []
    continue
  m = re.match(r"(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)", row)
  if m == None:
    raise Exception
  board.append([*m.groups()])
boards.append(board)

def is_winner(board, drawn_numbers) -> bool:
  def winner(board, drawn_numbers):
    for row in board:
      winner = True
      for val in row:
        if val not in drawn_numbers:
          winner = False
      if winner:
        return True
  horizontal = winner(board, drawn_numbers)
  vertical = winner([[row[i] for row in board] for i in range(len(board[0]))], drawn_numbers)
  return True if horizontal or vertical else False

for i in range(len(numbers_to_draw)):
  for board in boards:
    if is_winner(board, numbers_to_draw[:i+1]):
      unmarked_nums_sum = 0
      for row in board:
        for val in row:
          if val not in numbers_to_draw[:i+1]:
            unmarked_nums_sum += int(val)
      print(unmarked_nums_sum*int(numbers_to_draw[i]))
      # Part 1: sys.exit(0) here
      boards = [b for b in boards if b is not board]


import pandas as pd

# Read the CSV file
df = pd.read_csv('lists/final_data.csv', names=["BASE_LINE_ANGLE", "TOP_MARGIN", "LINE_SPACING", "SLANT_ANGLE", "WORD_SPACING", "LETTER_SIZE", "Extroversion","Introversion","Emotional_stability","Neuroticism", "Openness_to_experience","Conscientiousness","Agreeableness", "file_name"])

# Define the target variables
target_variables = ["Extroversion", "Introversion", "Emotional_stability", "Neuroticism", "Openness_to_experience", "Conscientiousness", "Agreeableness"]

# Get the ranges for WORD_SPACING and LINE_SPACING for each target variable
for target in target_variables:
    target_data = df[df[target] == 1]
    word_spacing_range = (target_data["WORD_SPACING"].min(), target_data["WORD_SPACING"].max())
    line_spacing_range = (target_data["LINE_SPACING"].min(), target_data["LINE_SPACING"].max())
    print(f"Range for {target}: WORD_SPACING {word_spacing_range}, LINE_SPACING {line_spacing_range}")


import asyncio
import re

from aiogram import Bot, types, Dispatcher
from aiogram.dispatcher.webhook import get_new_configured_app
from aiohttp.web import run_app

import Toloka
import config

loop = asyncio.get_event_loop()

bot = Bot(config.TOKEN, loop=loop)
dp = Dispatcher(bot, loop=loop)

search_regexp = re.compile(r'/search\s(\w+)', flags=re.IGNORECASE)

def stringify_toloka_search_result(result: Toloka.TolokaSearchResult) -> str:
    return f'ID: {result.id}' \
           f'\nLink: {result.link}' \
           f'\nTitle: {result.title}' \
           f'\nForum name: {result.forum_name}' \
           f'\nForum parent: {result.forum_parent}' \
           f'\nComments: {result.comments}' \
           f'\nSize: {result.size}' \
           f'\nSeeders: {result.seeders}' \
           f'\nLeechers: {result.leechers}' \
           f'\nComplete: {result.complete}'

@dp.message_handler(commands=['start', 'help'])
async def send_welcome(m: types.Message):
    await m.reply('Hello! I can search Toloka.to and show you results with links'
                  '\nExample search:'
                  '\n`/search Interstellar`',
                  parse_mode=types.ParseMode.MARKDOWN, disable_web_page_preview=True)

@dp.message_handler(commands=['search'], regexp=search_regexp.pattern)
async def search_toloka(m: types.Message):
    query = search_regexp.match(m.text).group(1)
    results = await Toloka.search(query)

    if results:
        await m.reply(f'Results for {query}:', reply=False)

        for result in results:
            await m.reply(stringify_toloka_search_result(result), reply=False)

        await m.reply('End of list', reply=False)

    else:
        await m.reply(f'No results for {query}')

@dp.inline_handler(lambda query: query.query)
async def inline_search_toloka(q: types.InlineQuery):
    results = await Toloka.search(q.query, limit=50)
    inline_results = []

    if results:
        for n, result in enumerate(results):
            result: Toloka.TolokaSearchResult

            input_content = types.InputTextMessageContent(stringify_toloka_search_result(result))
            inline_result = types.InlineQueryResultArticle(id=str(n),
                                                           title=result.title,
                                                           input_message_content=input_content,
                                                           description=f'{result.size}, {result.seeders} S | {result.leechers} L | {result.complete} C')
            inline_results.append(inline_result)
            if n >= 50:
                break

    else:
        input_content = types.InputTextMessageContent(f'Nothing found for {q.query}')
        inline_result = types.InlineQueryResultArticle(id='1',
                                                       title='Nothing found',
                                                       input_message_content=input_content,
                                                       description=f'Nothing found for {q.query}')
        inline_results.append(inline_result)

    try:
        await bot.answer_inline_query(q.id, inline_results)
    except:
        pass

async def on_startup(*args, **kwargs):
    await bot.delete_webhook()
    await bot.set_webhook(config.WEBHOOK_URL)

async def on_shutdown(*args, **kwargs):
    await bot.delete_webhook()

if __name__ == '__main__':
    app = get_new_configured_app(dp, config.WEBHOOK_URL_PATH)

    app.on_startup.append(on_startup)
    app.on_shutdown.append(on_shutdown)

    run_app(app, port=config.WEBAPP_PORT)


from django.db import models

SEASON_CHOICES = (
    ('winter', 'Зима'),
    ('summer', 'Лето'),
)

class Image(models.Model):
    image = models.ImageField(upload_to='images/')
    description = models.TextField(max_length=500)

    def __str__(self):
        return self.description

class TravelCard(models.Model):
    name = models.CharField(
        max_length=100
    )
    small_description = models.CharField(
        max_length=500
    )
    season = models.CharField(
        max_length=10,
        choices=SEASON_CHOICES
    )
    description = models.TextField(
        max_length=5000
    )
    image = models.ForeignKey(
        Image,
        on_delete=models.SET_NULL,
        null=True,
        blank=True
    )

    def __str__(self):
        return self.name


import unittest
from unittest import TestCase
from business.table.table import Table
from business.role.game_master import GameMaster
from business.role.basic_player import BasicPlayer
from business.user.player import Player
from business.scenario.scenario import Scenario
from business.character.character import Character

class TestTable(TestCase):

    def test_add_gamemaster_case1(self):
        #case1 : empty table
        #GIVEN
        player = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='ninho'
                        , age=9
                        , game_master=GameMaster(username='ninho')
                        , basic_player=BasicPlayer(username='ninho'))
        player.save()
        scenario1 = Scenario(name='test'
                            , description='this is a test'
                            , username='ninho')
        player.game_master.add_scenario(scenario1)
        id_scenario = scenario1.id
        #table 7 must be empty and active in the db for the test
        table = Table(half_day=1, active = True, id=7)
        #WHEN
        added = table.add_gamemaster(player, id_scenario)
        #THEN
        self.assertTrue(added)
        #clean db
        Player.delete('ninho')

    def test_add_gamemaster_case2(self):
        #case2 : table has already a gamemaster
        #GIVEN
        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky1'
                        , age=9
                        , game_master=GameMaster(username='jacky1')
                        , basic_player=BasicPlayer(username='jacky1'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky2'
                        , age=9
                        , game_master=GameMaster(username='jacky2')
                        , basic_player=BasicPlayer(username='jacky2'))
        player1.save()
        player2.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky1')

        scenario1p2 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky2')

        player1.game_master.add_scenario(scenario1p1)
        player2.game_master.add_scenario(scenario1p2)

        table = Table(half_day=1, active = True, id=8)
        table.add_gamemaster(player1, scenario1p1.id)

        #WHEN
        added = table.add_gamemaster(player2, scenario1p2.id)

        #THEN
        self.assertFalse(added)

        #clean db
        Player.delete('jacky1')
        Player.delete('jacky2')

    def test_add_gamemaster_case3(self):
        #case3 : player is registered the same halfday
        #GIVEN

        player = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky3'
                        , age=9
                        , game_master=GameMaster(username='jacky3')
                        , basic_player=BasicPlayer(username='jacky3'))

        player.save()

        scenario1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky3')
        player.game_master.add_scenario(scenario1)

        #table 8 and 9 must be empty and active in the db for the test :
        #by default, they are.
        table1 = Table(half_day=1, active = True, id=8)
        table2 = Table(half_day=1, active = True, id=9)

        table1.add_gamemaster(player, scenario1.id)

        #WHEN
        added = table2.add_gamemaster(player, scenario1.id)
        #THEN
        self.assertFalse(added)
        #clean db
        Player.delete('jacky3')

    def test_add_basic_player_case1(self):
        #case1 : empty table
        #must be false because a player can join a table only if there is a game master
        #GIVEN
        player = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky4'
                        , age=9
                        , game_master=GameMaster(username='jacky4')
                        , basic_player=BasicPlayer(username='jacky4'))

        player.save()

        character1 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky4')

        player.basic_player.add_character(character1)
        #table 9 must be empty and active in the db for the test : by default, it is.
        table = Table(half_day=1, active = True, id=9)
        #WHEN
        added = table.add_basicplayer(player, character1.id)
        #THEN
        self.assertFalse(added)

    def test_add_basic_player_case2(self):
        #case2 : player registered the same halfday
        #GIVEN
        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky5'
                        , age=9
                        , game_master=GameMaster(username='jacky5')
                        , basic_player=BasicPlayer(username='jacky5'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky6'
                        , age=9
                        , game_master=GameMaster(username='jacky6')
                        , basic_player=BasicPlayer(username='jacky6'))

        player3 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky7'
                        , age=9
                        , game_master=GameMaster(username='jacky7')
                        , basic_player=BasicPlayer(username='jacky7'))

        player1.save()
        player2.save()
        player3.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky5')

        scenario1p3 = Scenario(name='test2'
                            , description='this is a test'
                            , username='jacky7')

        character1p2 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky6')
        character2p2 = Character(name='hugobosss',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky6')

        player1.game_master.add_scenario(scenario1p1)

        player2.basic_player.add_character(character1p2)
        player2.basic_player.add_character(character2p2)

        #tables 23 and 24 must be empty and active in the db for the test : by default, it is.
        table1 = Table(half_day=2, active = True, id=23)
        table2 = Table(half_day=2, active = True, id=24)

        table1.add_gamemaster(player1, scenario1p1.id)
        table1.add_basicplayer(player2, character1p2.id)
        table2.add_gamemaster(player3, scenario1p3.id)

        #WHEN
        added = table2.add_basicplayer(player2, character2p2.id)
        #THEN
        self.assertFalse(added)

        #clean db
        Player.delete('jacky4')
        Player.delete('jacky5')
        Player.delete('jacky6')
        Player.delete('jacky7')

    def test_add_basic_player_case3(self):
        #case 3: the number of basic player is 4
        #GIVEN

        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky8'
                        , age=9
                        , game_master=GameMaster(username='jacky8')
                        , basic_player=BasicPlayer(username='jacky8'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky9'
                        , age=9
                        , game_master=GameMaster(username='jacky9')
                        , basic_player=BasicPlayer(username='jacky9'))

        player3 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky10'
                        , age=9
                        , game_master=GameMaster(username='jacky10')
                        , basic_player=BasicPlayer(username='jacky10'))

        player4 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky11'
                        , age=9
                        , game_master=GameMaster(username='jacky11')
                        , basic_player=BasicPlayer(username='jacky11'))

        player5 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky12'
                        , age=9
                        , game_master=GameMaster(username='jacky12')
                        , basic_player=BasicPlayer(username='jacky12'))

        player6 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky13'
                        , age=9
                        , game_master=GameMaster(username='jacky13')
                        , basic_player=BasicPlayer(username='jacky13'))

        player1.save()
        player2.save()
        player3.save()
        player4.save()
        player5.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky8')

        character1p2 = Character(name='huross1',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky9')

        character1p3 = Character(name='huross2',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky10')

        character1p4 = Character(name='huross3',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky11')

        character1p5 = Character(name='huross4',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky12')

        character1p6 = Character(name='huross5',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky13')

        table = Table(half_day=2, active = True, id=25)
        table.add_gamemaster(player1, scenario1p1.id)
        table.add_basicplayer(player2, character1p2.id)
        table.add_basicplayer(player3, character1p3.id)
        table.add_basicplayer(player4, character1p4.id)
        table.add_basicplayer(player5, character1p5.id)

        #WHEN
        added = table.add_basicplayer(player2, character1p6.id)

        #THEN
        self.assertFalse(added)

        #clean db
        Player.delete('jacky8')
        Player.delete('jacky9')
        Player.delete('jacky10')
        Player.delete('jacky11')
        Player.delete('jacky12')
        Player.delete('jacky13')

    def test_add_basic_player_case4(self):
        #case 4: the player can be added
        #GIVEN

        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky14'
                        , age=9
                        , game_master=GameMaster(username='jacky14')
                        , basic_player=BasicPlayer(username='jacky14'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky15'
                        , age=9
                        , game_master=GameMaster(username='jacky15')
                        , basic_player=BasicPlayer(username='jacky15'))

        player1.save()
        player2.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky14')

        character1p2 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky15')

        player1.game_master.add_scenario(scenario1p1)
        player2.basic_player.add_character(character1p2)

        #tables 26 must be empty and active in the db for the test : by default, it is.
        table = Table(half_day=2, active = True, id=26)
        table.add_gamemaster(player1, scenario1p1.id)
        #WHEN
        added = table.add_basicplayer(player2, character1p2.id)
        #THEN
        self.assertTrue(added)
        #clean db
        Player.delete('jacky14')
        Player.delete('jacky15')

    def test_rm_player_case1(self):
        #case1 : the user is not in the table
        #GIVEN

        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky16'
                        , age=9
                        , game_master=GameMaster(username='jacky16')
                        , basic_player=BasicPlayer(username='jacky16'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky17'
                        , age=9
                        , game_master=GameMaster(username='jacky17')
                        , basic_player=BasicPlayer(username='jacky17'))

        player1.save()
        player2.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky16')

        character1p2 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky17')

        player1.game_master.add_scenario(scenario1p1)
        player2.basic_player.add_character(character1p2)

        #tables 27 must be empty and active in the db for the test : by default, it is.
        table = Table(half_day=2, active = True, id=27)
        table.add_gamemaster(player1, scenario1p1.id)
        table.add_basicplayer(player2, character1p2.id)
        username = 'jojokiki2'
        #WHEN
        removed = table.rm_player(username)
        #THEN
        self.assertFalse(removed)
        #clean db
        Player.delete('jacky16')
        Player.delete('jacky17')

    def test_rm_player_case2(self):
        #case2 : delete a basic player
        #GIVEN

        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky18'
                        , age=9
                        , game_master=GameMaster(username='jacky18')
                        , basic_player=BasicPlayer(username='jacky18'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky19'
                        , age=9
                        , game_master=GameMaster(username='jacky19')
                        , basic_player=BasicPlayer(username='jacky19'))

        player1.save()
        player2.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky18')

        character1p2 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky19')

        player1.game_master.add_scenario(scenario1p1)
        player2.basic_player.add_character(character1p2)

        #tables 28 must be empty and active in the db for the test : by default, it is.
        table = Table(half_day=2, active = True, id=28)
        table.add_gamemaster(player1, scenario1p1.id)
        table.add_basicplayer(player2, character1p2.id)
        username = 'jacky19' #the basic player
        #WHEN
        removed = table.rm_player(username)
        #THEN
        self.assertTrue(removed)
        #clean db
        Player.delete('jacky18')
        Player.delete('jacky19')

    def test_rm_player_case2(self):
        #case3 : delete a game master
        #GIVEN

        player1 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky20'
                        , age=9
                        , game_master=GameMaster(username='jacky20')
                        , basic_player=BasicPlayer(username='jacky20'))

        player2 = Player(firstname='Jean'
                        , lastname='Hill'
                        , username='jacky21'
                        , age=9
                        , game_master=GameMaster(username='jacky21')
                        , basic_player=BasicPlayer(username='jacky21'))

        player1.save()
        player2.save()

        scenario1p1 = Scenario(name='test'
                            , description='this is a test'
                            , username='jacky20')

        character1p2 = Character(name='huross',
                              level=5, race='bard',
                              equipment='amulet',
                              skill='battleaxes',
                              username='jacky21')

        player1.game_master.add_scenario(scenario1p1)
        player2.basic_player.add_character(character1p2)

        #tables 29 must be empty and active in the db for the test : by default, it is.
        table = Table(half_day=2, active = True, id=29)
        table.add_gamemaster(player1, scenario1p1.id)
        table.add_basicplayer(player2, character1p2.id)
        username = 'jacky20' #the basic player
        #WHEN
        removed = table.rm_player(username)
        #THEN
        self.assertTrue(removed)
        #clean db
        Player.delete('jacky20')
        Player.delete('jacky21')

    def test_activate_table(self):
        #GIVEN
        #tables 33 must be empty and desactivated in the db for the test : by default, it is.
        table = Table(half_day = 2, active = False, id = 33)
        #WHEN
        table.active_table()
        #then
        self.assertTrue(table.active)
        #desactive it back
        table.desactive_table()

    def test_desactive_table_case1(self):
        #case1 : table is empty
        #GIVEN
        #tables 44 must be empty and activated in the db for the test : by default, it is.
        table = Table(halfday = 3, active = True, id = 44)
        #WHEN
        table.desactive_table()
        #then
        self.assertFalse(table.active)
        #active it back
        table.active_table()

    def test_desactive_table_case1(self):
        #case2 : the table is not empty
        #tables 45 must be empty and activated in the db for the test : by default, it is.

        pass

if __name__ == '__main__':
    #please, reinitialize the database before the test
    unittest.main()


available_items = ['phones', 'fruits', 'laptops', 'micro-controllers']

item_sub_categories = {
    'phones': ['samsung', 'apple', 'redmi'],
    'fruits': ['banana', 'pear', 'oranges'],
    'laptops': ['acer', 'hp', 'dell'],
    'micro-controllers': ['arduino', 'esp32'],
}

price_listing = {
    'samsung': 1000,
    'apple': 1500,
    'redmi': 850,
    'banana': 20,
    'pear': 25,
    'oranges': 15,
    'acer': 5000,
    'hp': 3500,
    'dell': 3800,
    'arduino': 70,
    'esp32': 50
}


# -*- coding: utf-8 -*-
import os

from brain.brain import BASE_DIR
from brain.db import Database

import unittest
from unittest import TestCase

class TestDatabase(TestCase):
    def setUp(self):
        self.db = Database('test.db')
        self.db.connect()
        self.cr = self.db.conn.cursor()

    def test_connect(self):
        # sqlite creates a db file when it connects
        db = Database('connect_test.db')
        db.connect()
        db.conn.close()

        self.assertTrue(os.path.isfile(db.db_file))
        os.remove(db.db_file)

    def test_create_table(self):
        self.db.create_table()

        sql = """
            SELECT * FROM logs
        """

        self.cr.execute(sql)
        res = self.cr.fetchall()

        self.assertTrue(isinstance(res, list))

    def tearDown(self):
        self.db.conn.close()
        db_path = os.path.join(BASE_DIR, self.db.db_file)
        if os.path.isfile(db_path):
            os.remove(db_path)

if __name__ == '__main__':
    unittest.main()


from django.shortcuts import render, HttpResponse
from main import models

# Create your views here.
def index(request):
    models.Person.objects.all()

    cnt = models.Person.objects.count()
    print('....')
    if cnt > 0:
        name = models.Person.objects.all()[0].name
    return HttpResponse('hello, gushi!' + ' ' + str(cnt) + ' ' + name )

def index2(request):
    context = {}
    context['hello'] = 'hello, gushi!'
    context['hello'] = 'Activity is the only road to knowedge. Ceorge Bernard shaw, British dramalist .'
    import datetime
    now =  datetime.datetime.now()
    context['time'] = now
    return render(request, 'index.html', context)


def solutions(n, lost, reserve):
    set_reserve = set(reserve) - set(lost)
    set_lost = set(lost) - set(reserve)
    # d = dict()
    print(set_reserve, set_lost)

    for i in set_reserve:
        print(i)
        if (i-1) in set_lost:
            set_lost.remove(i-1)
        elif (i+1) in set_lost:
            set_lost.remove(i+1)

    return n - len(set_lost)

n = 5
lost = [2, 4]
reserve = [1, 3, 5]

print(solutions(5, lost, reserve))

    # d = dict()

    # duplicated = list(set(lost) & set(reserve))
    # for i in range(0, n+2):
    #     if i in duplicated:
    #         d[i] = 1
    #     elif i in lost:
    #         d[i] = 0
    #     elif i in reserve:
    #         d[i] = 2
    #     else:
    #         d[i] = 1

    # for l in lost:
    #     if d[l] == 1:
    #         pass
    #     else:
    #         if d[l-1] == 2:
    #             d[l] += 1
    #             d[l-1] -= 1
    #         elif d[l+1] == 2:
    #             d[l]+= 1
    #             d[l+1] -= 1

    # answer = 0
    # for i in range(1, n+1):
    #     if d[i] > 0:
    #         answer += 1

    # return answer


import json
from typing import List

import requests
from chainchomplib import LoggerInterface
from chainchomplib.configlayer.model.ChainfileModel import ChainfileModel

class ChainlinkStartHandler:

    PORT = 4410
    STRING_FOR_PREVIOUS = 'previous'
    STRING_FOR_NEXT = 'next'

    @staticmethod
    def contact_remote_chainlinks(chainfile_model: ChainfileModel, urls_done: dict) -> dict:
        ChainlinkStartHandler.__contact_remote_links(
            chainfile_model.next_links,
            urls_done,
            chainfile_model.get_serialized(),
            ChainlinkStartHandler.STRING_FOR_NEXT
        )
        ChainlinkStartHandler.__contact_remote_links(
            chainfile_model.previous_links,
            urls_done,
            chainfile_model.get_serialized(),
            ChainlinkStartHandler.STRING_FOR_PREVIOUS
        )
        return urls_done

    @staticmethod
    def contact_local_adapter(chainfile_model: ChainfileModel):
        url = f"http://localhost:{ChainlinkStartHandler.PORT}/chainfile/local/receive"
        params = {'chainfile': chainfile_model.get_serialized()}
        try:
            response = requests.post(url=url, data=json.dumps(params), timeout=1)
        except Exception:
            return False
        if response.status_code == 200:
            return True

        if response.status_code == 403 or response.status_code == 500:
            LoggerInterface.error(f'Error during transmission: {response.reason}')

        return False

    @staticmethod
    def __contact_remote_links(
            list_of_links: List[str],
            urls_done: dict,
            serialized_model: dict,
            contact_type: str
    ):
        for link_and_url in list_of_links:
            if link_and_url not in urls_done['url_tries'].keys():
                urls_done['url_tries'][link_and_url] = 0
            if urls_done['url_tries'][link_and_url] >= 10 and link_and_url not in urls_done[contact_type]:
                urls_done[contact_type].append(link_and_url)
            if link_and_url in urls_done[contact_type]:
                continue
            request_url = f"http://{link_and_url.split('::')[0]}:{ChainlinkStartHandler.PORT}/chainfile/remote/receive"
            params = {
                'chainfile': serialized_model,
                'is_next': contact_type == ChainlinkStartHandler.STRING_FOR_NEXT,
                'is_previous': contact_type == ChainlinkStartHandler.STRING_FOR_PREVIOUS,
                'name_of_called_link': link_and_url.split('::')[1]
            }
            try:
                response = requests.post(url=request_url, data=json.dumps(params), timeout=1)
            except requests.exceptions.ConnectTimeout:
                urls_done['url_tries'][link_and_url] += 1
                continue

            if response.status_code == 200:
                urls_done[contact_type].append(link_and_url)

            if response.status_code == 403 or response.status_code == 500:
                LoggerInterface.error(f'Error during transmission: {response.reason}')

            urls_done['url_tries'][link_and_url] += 1


# Task1A
input_file_1a = open("Input1a.txt", "r")
input_data_1a = input_file_1a.readline().split(" ")
dimensions_1a = list(map(int, input_data_1a))
input_lines_1a = input_file_1a.read().split("\n")
matrix_data_1a = []

for line_1a in input_lines_1a:
    line_values_1a = line_1a.split(" ")
    values_1a = list(map(int, line_values_1a))
    matrix_data_1a.append(values_1a)

n_1a = dimensions_1a[0]
matrix_1a = [[0] * (n_1a + 1) for i_1a in range(n_1a + 1)]
output_1a = ""
count_1a = 0

for entry_1a in matrix_data_1a:
    matrix_1a[entry_1a[0]][entry_1a[1]] = entry_1a[2]

for row_1a in matrix_1a:
    count_1a += 1
    for value_1a in row_1a:
        output_1a += f"{value_1a} "
    if count_1a != len(matrix_1a):
        output_1a += "\n"

output_file_1a = open("Output1a.txt", "w")
output_file_1a.write(output_1a)
output_file_1a.close()


import re
from mo_sql_parsing import parse as moz_parser
from mo_parsing.exceptions import ParseException
from dora_parser import logger

class Parser():
    """Parser Object"""

    @classmethod
    def clean(cls, query) -> str:
        """Remove query comments and line breaks
        :return: clean sql query
        """
        return re.sub(r'(--.*?\n)|(/\*(.|\n)*?\*/)', '', query)

    def __init__(self,query:str):
        """Initialize the parser class
        :param query: sql query that will be translate
        """
        self.query =  Parser.clean(query)
        try:
            self.tree = moz_parser(self.query)
        except ParseException as err:
            logger.error("Query not supported:\n--PARSER:%s",err)
            raise err

    def __repr__(self) -> repr:
        """Sql query is parsed with moz_sql_parser
        :return: tree
        """
        return str(self.tree)


__author__ = 'vin@misday.com'

import os, sys, urllib, urllib2, codecs, traceback, platform, re, threading, logging
from datetime import *
from pyvin.core import Processor
import bs4

reload(sys)
sys.setdefaultencoding('utf8')

sysstr = platform.system()

def get_timestamp():
    return datetime.now().strftime('%Y-%m-%d %H:%M:%S')

class Fetch:
    def __init__(self, log=None):
        if log:
            self.log = log
        else:
            self.log = logging.getLogger(Fetch.__name__)
        self.use_proxy = False;
        self.proxy = None

    def set_proxy(self, server, usr='', passwd=''):
        self.use_proxy = True;
        self.proxy_config = "http://%s:%s@%s" % (usr, passwd, server)
        self.proxy = {'http' : self.proxy_config}
        self.log.info('using proxy: %s' % (self.proxy_config))

    def get2(self, url):
        self.log.info('<GET2 %s>... ' % url)
        try:
            request = urllib2.Request(url)
            request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36')
            reader= urllib2.urlopen(request)
            response = reader.read()
            self.log.info('<GET2 %s>... OK' % url)
        except:
            self.log.warning('<GET2 %s>... failed' % url)
            traceback.print_exc()
            self.log.exception(traceback.format_exc())
            response = ''
        return response

    def get(self, url):
        self.log.info('<GET2 %s>... ' % url)
        try:
            wp = urllib.urlopen(url, proxies=self.proxy)
            response = wp.read()
            self.log.info('<GET2 %s>... OK' % url)
        except:
            self.log.warning('<GET2 %s>... failed' % url)
            traceback.print_exc()
            self.log.exception(traceback.format_exc())
            response = ''
        return response

    def retrieveFile(self, url, filename):
        self.log.info('<RETRIEVE %s --> %s>... ' % (url, filename))
        if os.path.exists(filename):
            self.log.info('<RETRIEVE %s --> %s>... exist, pass' % (url, filename))
            return
        try:
            urllib.URLopener(self.proxy).retrieve(url, filename)
            self.log.info('<RETRIEVE %s --> %s>... OK' % (url, filename))
        except:
            self.log.warning('<RETRIEVE %s --> %s>... failed' % (url, filename))
            traceback.print_exc()
            self.log.exception(traceback.format_exc())

    def wget(self, url, filename):
        self.log.info('<DOWNLOAD %s --> %s>... ' % (url, filename))
        if sysstr == 'Linux':
            try:
                cmd = 'wget %s -O %s' % (url, filename)
                # retry 10 times, timeout 120s
                args = '-c -T 120 -t 10'
                # quiet
                # args = '%s -nv' % args
                # args = '%s -q' % args
                # --no-clobber
                args = '%s -nc' % args
                # proxy
                if self.use_proxy:
                    args = '%s -e \"http_proxy=%s\"' % (args, self.proxy_config)
                cmd = '%s %s' % (cmd, args)
                # multi-thread
                # cmd = '%s &' % cmd
                os.system(cmd)
            except:
                self.log.warning('<DOWNLOAD %s --> %s>... failed' % (url, filename))
                traceback.print_exc()
                self.log.exception(traceback.format_exc())
        elif sysstr == 'Windows':
            self.log.warning('not support')
        else:
            self.log.warning('not support')

    def copyall(self, src, dst):
        if sysstr == 'Linux':
            os.system('cp -a %s/* %s/' % (src, dst))
        elif sysstr == 'Windows':
            os.system('xcopy /s/e/a/q/y %s\\* %s\\' % (src, dst))

class Spider(Processor):
    todo = []
    visited = []
    processings = []
    callbacks = {}

    (EVT_ON_ADD_URL,
     EVT_ON_REMOVE_URL,
     EVT_ON_URL_ERR,
     EVT_MAX_NUM) = range(Processor.EVT_MAX_NUM, Processor.EVT_MAX_NUM+4)

    def __init__(self, name='Spider', log=None):
        if log:
            self.log = log
        else:
            self.log = logging.getLogger(name)
        Processor.__init__(self, interval=1, log=self.log)
        self.init([Spider.EVT_ON_ADD_URL, Spider.EVT_ON_REMOVE_URL, Spider.EVT_ON_URL_ERR])
        self.bind(Processor.EVT_START, self.onStart)
        self.bind(Processor.EVT_LOOP, self.onLoop)
        self.bind(Processor.EVT_STOP, self.onStop)

        self.name = name
        self.fetch = Fetch(log=self.log)

        # will run how many thread, default is 1.
        self.thread_count_max = 2
        self.thread_count = 0

        self.lock = threading.Lock()

    def set_max_thread(self, count):
        self.thread_count_max = count

    def set_proxy(self, server, usr='', passwd=''):
        '''set proxy'''
        self.fetch.set_proxy(server, usr, passwd)

    def onStart(self, event):
        self.log.info('[%s] starting...' % self.name)
        return True

    def onLoop(self, event):
        count = self.url_count()
        if count > 0:
            # print 'url_count %d' % count
            if self.thread_count < self.thread_count_max: # there is
                url = self.pop_url()
                if url:
                    # print url
                    t = threading.Thread(target=self.processing, args=(url,), name=url)
                    t.start()
                    self.thead_count_increase()
            else:
                pass
            return True
        elif self.thread_count > 0: # waiting running thread
            return True
        else:
            return False

    def onStop(self, event):
        self.log.info('[%s] stopping...' % self.name)
        return True

    def processing(self, args):
        # print args
        url = args

        try:
            response = self.fetch.get(url)
            self._dispatch(url, response)
        except:
            traceback.print_exc()
            self.log.exception(traceback.format_exc())
            self.dispatch(Spider.EVT_ON_URL_ERR, url)
        else:
            self.release_url(url)

        self.thread_count_decrease()

    def _dispatch(self, url, response):
        '''dispatch fetched content to callbacks'''
        for u in self.callbacks.keys():
            if re.match(u, url, flags=0): #url.startswith(u): # TODO: replace with regular expression
                self.callbacks[u](url, response)

    def thead_count_increase(self):
        self.lock.acquire()
        self.thread_count += 1
        self.lock.release()

    def thread_count_decrease(self):
        self.lock.acquire()
        self.thread_count -= 1
        self.lock.release()

    def add_urls(self, urls):
        '''add url to queue'''
        self.lock.acquire()
        if len(urls) > 0:
            for url in urls:
                if url not in self.todo and url not in self.processings and url not in self.visited:
                    self.todo.append(url)
                    self.dispatch(Spider.EVT_ON_ADD_URL, url)
        self.lock.release()

    def pop_url(self):
        self.lock.acquire()
        if len(self.todo) > 0:
            url = self.todo.pop(0)
            self.processings.append(url)
        else:
            url = None
        self.lock.release()
        return url

    def release_url(self, url):
        self.lock.acquire()
        self.visited.append(url)
        self.processings.remove(url)
        self.dispatch(Spider.EVT_ON_REMOVE_URL, url)
        self.lock.release()

    def url_count(self):
        self.lock.acquire()
        count = len(self.todo)
        self.lock.release()
        return count

    def add_callbacks(self, callbacks={}):
        '''add callbacks for urls fetched'''
        # print type(callbacks)
        for u in callbacks.keys():
            self.callbacks[u] = callbacks[u]

    ###################################################################################################################

    # for content
    def clear_node(self, soup, node, att={}):
        tags = soup.findAll(node, attrs=att)
        if tags:
            # print '%s %d' % (node, len(tags))
            for tag in tags:
                tag.extract()

    def download(self, url, filename):
        Persist.ensurePath(filename)
        if sysstr == 'Linux':
            self.fetch.wget(url, filename)
        elif sysstr == 'Windows':
            self.fetch.retrieveFile(url, filename)
        else:
            self.log.warning('%s not supported now, will skip downloading %s' % (platform.system(), url))

class Persist():
    def __init__(self, filename, charset='utf-8', log=None):
        if log:
            self.log = log
        else:
            self.log = logging.getLogger(Persist.__name__)

        Persist.ensurePath(filename)
        self.log.info('<Store \'%s\'>' % filename)

        self.fp = fp = codecs.open(filename, 'w+', charset)

    def start_html(self):
        self.fp.write('\n')
        self.fp.write('\n')
        self.fp.write('<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />')
        self.fp.write('\n')

    def set_title(self, title):
        self.fp.write('
\n')
        self.fp.write(title)
        self.fp.write('
\n')

    def set_info(self, clazz):
        self.fp.write('
\n')
        self.fp.write(clazz)
        self.fp.write('
\n')

    def stop_html(self):
        self.fp.write('')

    def store(self, data):
        self.fp.write(data)

    def store_soup(self, soup, charset='utf-8'):
        content = str(soup.prettify())
        if len(charset):
            content = content.decode(charset)
        self.fp.write(content)

    def close(self):
        self.fp.close()

    @staticmethod
    def ensurePath(path):
        path = os.path.split(path)[0]
        if not os.path.exists(path):
            os.makedirs(path)

class SpiderStub:
    starts = []
    callbacks = {}

class SpiderSoup:
    @staticmethod
    def clearNode(soup, node, att={}):
        tags = soup.findAll(node, attrs=att)
        if tags:
            # print '%s %d' % (node, len(tags))
            for tag in tags:
                tag.extract()

    @staticmethod
    def clearAttr(soup, attrs, tag, att={}):
        tags = soup.findAll(tag, attrs=att)
        if tags:
            # print '%s %d' % (node, len(tags))
            for tag in tags:
                for attr in attrs:
                    del(tag[attr])

    @staticmethod
    def clearComments(soup):
        for child in soup.children:
            if isinstance(child, bs4.Comment):
                child.extract()

    @staticmethod
    def insertCss(soup, url):
        tag = soup.new_tag("link", href=url)
        tag['rel'] = 'stylesheet'
        tag['type'] = 'text/css'
        soup.head.append(tag)

    @staticmethod
    def insertScript(soup, url):
        tag = soup.new_tag("script")
        tag['src'] = url
        soup.head.append(tag)

def main():
    pass

if __name__ == '__main__':
    main()


# coding=UTF-8
import sys
import numpy as np
import matplotlib.pyplot as plt

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print('Usage: plot_scattered_path_out data.txt output_name')
        exit(1)
    else:
        path = sys.argv[1]
        output_name = sys.argv[2]
        gps_path = np.loadtxt(path)
        fig = plt.figure()
        plt.scatter(gps_path[:, 2], gps_path[:, 3], c=gps_path[:, gps_path.shape[1] - 1], cmap='autumn')
        plt.colorbar()
        plt.savefig(output_name + ".pdf", dpi=150)

        exit(1)


def premik(ukaz, x, y, smer):
    smeri = "NESW"
    premiki = [(0, -1), (1, 0), (0, 1), (-1, 0)]
    ismer = smeri.index(smer)
    if ukaz == "R":
        smer = smeri[(ismer + 1) % 4]
    elif ukaz == "L":
        smer = smeri[(ismer - 1) % 4]
    else:
        dx, dy = premiki[ismer]
        x += dx * ukaz
        y += dy * ukaz
    return x, y, smer

def izvedi(ime_datoteke):
    datoteka = open(ime_datoteke)
    x, y, smer = 0, 0, "N"
    stanje = [(x, y, smer),]
    for ukaz in datoteka:
        if ukaz.strip() == "DESNO":
            x, y, smer = premik("R", x, y, smer)
            stanje.append((x, y, smer))
        elif ukaz.strip() == "LEVO":
            x, y, smer = premik("L", x, y, smer)
            stanje.append((x, y, smer))
        else:
            ukaz = ukaz.split()
            x, y, smer = premik(int(ukaz[1]), x, y, smer)
            stanje.append((x, y, smer))
    datoteka.close()
    return stanje

def opisi_stanje(x, y, smer):
    smeri = "NESW"
    #ismeri pove na katerem mestu je "smer" v nizu smeri, potem pa glede na to določimo kateri elememnt iz niza vzeti
    ismer = smeri.index(smer)
    smeriIndikator = ["^", ">", "v", "<"]
    return "{x:>3.0f}:{y:<3.0f} {smer}".format(x=x, y=y, smer=smeriIndikator[ismer])

def prevedi(ime_vhoda, ime_izhoda):
    datoteka_izhod = open(ime_izhoda, "w")
    ukazi = izvedi(ime_vhoda)
    for ukaz in ukazi:
        x, y, smer = ukaz
        datoteka_izhod.write("{ukaz_izhoda}\n".format(ukaz_izhoda=opisi_stanje(x, y, smer)))
    datoteka_izhod.close()

def opisi_stanje_2(x, y, smer):
    smeri = "NESW"
    # ismeri pove na katerem mestu je "smer" v nizu smeri, potem pa glede na to določimo kateri elememnt iz niza vzeti
    ismer = smeri.index(smer)
    smeriIndikator = ["^", ">", "v", "<"]
    return "{smer}{x:>5}:{y})".format(x="("+str(x), y=y, smer=smeriIndikator[ismer])


#!/usr/bin/env python3
"""indi_big61_mirrorcover.py

One group - Main Control

Main Control
------------
ISwitchVector : Open, Close
    Want to make it busy when doing open or close
ITextVector : Opened, Closed, Error
    Opened - OK
    Closed - BUSY
    Error  - ALERT

Polling
-------
update : 1000ms
    Grabs the latest telemetry from the mirror covers and updates ITextVector
"""
# Python imports
import sys
from pathlib import Path

sys.path.insert(0, str(Path.cwd().parent))

# Github imports
from mtnpy import Kuiper
from pyindi.device import *

# Constants
MYDEVICE = 'Mirror Cover'
MAIN_CONTROL_GROUP = 'Main Control'

# State machine for mirror cover
class MirrorCover():
    def __init__(self):
        self._closing = self._opening = False
        self._state = None

    @property
    def closing(self):
        return self._closing

    @property
    def opening(self):
        return self._opening

    @property
    def state(self):
        return self._state

    @closing.setter
    def closing(self, value):
        self._closing = value
        self._opening = not value

    @opening.setter
    def opening(self, value):
        self._opening = value
        self._closing = not value

    @state.setter
    def state(self, value):
        self._state = value

    def reset(self):
        self._opening = False
        self._closing = False

    def busy(self):
        """Returns true if opening or closing to avoid button presses"""
        #return self._closing or self._opening
        if self._closing:
            return self._state != 'Closed'
        elif self._opening:
            return self._state != 'Opened'
        else:
            return False

# Globals
telescope = Kuiper()
mirror_cover = MirrorCover()

class Device(device):
    def ISGetProperties(self, device=None):
        """Builds and returns INDI properties for this device"""
        # Build properties
        commands_s = [
            ISwitch(
                'open',      # Name (for internal use)
                ISState.OFF, # INDI State (ISState.ON:ISState.OFF)
                'Open'       # Label (for display)
            ),
            ISwitch('close', ISState.OFF, 'Close')
        ]
        commands_svp = ISwitchVector(
            commands_s,        # The switches in the vector group
            MYDEVICE,          # Device to attach to
            'commands',        # Name (for internal use)
            IPState.IDLE,      # INDI state for leds
            ISRule.ATMOST1,    # INDI switch rule
            IPerm.RW,          # Type of property
            0,                 # Timeout
            'Commands',        # Label (for display)
            MAIN_CONTROL_GROUP # Group to attach to
        )
        states_t = [
            IText(
                'mirror_cover_state',
                '',
                'Mirror Cover State'
            )
        ]
        states_tvp = ITextVector(
            states_t,          # List of text properties
            MYDEVICE,          # Device to attach to
            'states',          # Name (for internal use)
            IPState.IDLE,      # INDI state
            IPerm.RO,          # INDI perm
            0,                 # Timeout
            None,              # Timestamp
            'States',          # Label (for display)
            MAIN_CONTROL_GROUP # Group to attach to
        )
        state_message_l = [
            ILight('idle', IPState.IDLE, 'Idle'),
            ILight(
                'mirror_cover_opening', IPState.IDLE, 'Mirror Cover Opening'
            ),
            ILight(
                'mirror_cover_closing', IPState.IDLE, 'Mirror Cover Closing'
            )
        ]
        state_message_lvp = ILightVector(
            state_message_l, MYDEVICE, 'state_message', IPState.IDLE, 0, None,
            'State Message', MAIN_CONTROL_GROUP
        )

        # Define properties
        self.IDDef(commands_svp)
        self.IDDef(state_message_lvp)
        self.IDDef(states_tvp)

        return

    def ISNewText(self, device, name, values, names):
        pass

    def ISNewNumber(self, device, name, values, names):
        pass

    def ISNewLight(self, device, name, values, names):
        pass

    def ISNewSwitch(self, device, name, values, names):
        # Figure out what switch vp was clicked on
        if name == 'commands':
            if mirror_cover.busy():
                # Won't let mirror covers be sent a command when busy
                # Busy means it is either opening or closing
                self.IDMessage('BUSY ignoring button press')
                return

            svp = self.IUUpdate(device, name, values, names)
            if svp['open'].value == 'On':
                # Open the mirror covers
                try:
                    ok = telescope.mirror_cover.command_open()
                    if not ok: raise

                except Exception:
                    svp.state = IPState.ALERT
                    svp['open'].value = 'Off'
                    self.IDMessage('Failed to open mirror covers')
                    self.IDSet(svp)
                    return

                # Handle command being fine
                svp.state = IPState.BUSY

                mirror_cover.opening = True
                # Find state message, reset lights, update
                try:
                    state_message_lvp = self.IUFind('state_message')
                except ValueError:
                    return
                reset_lights(state_message_lvp)
                state_message_lvp['mirror_cover_opening'].value = IPState.BUSY
                state_message_lvp.state = IPState.BUSY

            elif svp['close'].value == 'On':
                # Close the mirror covers
                try:
                    ok = telescope.mirror_cover.command_close()
                    if not ok: raise

                except Exception:
                    svp.state = IPState.ALERT
                    svp['close'].value = 'Off'
                    self.IDMessage('Failed to close mirror covers')
                    self.IDSet(svp)
                    return

                # Handle closing command ok
                svp.state = IPState.BUSY
                mirror_cover.closing = True
                # Find state message, reset lights, update
                try:
                    state_message_lvp = self.IUFind('state_message')
                except ValueError:
                    return
                reset_lights(state_message_lvp)
                state_message_lvp['mirror_cover_closing'].value = IPState.BUSY
                state_message_lvp.state = IPState.BUSY

            self.IDSet(svp)
            self.IDSet(state_message_lvp)

        return

    def ISNewLight(self, device, name, names, values):
        pass

    # Poll decorator
    # TODO make async
    @device.repeat(1000)
    def update(self):
        """Called after first getProperties is initiated then every x secs"""
        # Get the vp's for mirror cover
        try:
            states_tvp = self.IUFind('states')
            state_message_lvp = self.IUFind('state_message')
        except ValueError:
            # Could not find the vp's
            return

        # Get the data from mirror cover
        try:
            data = telescope.mirror_cover.request_state()
        except Exception:
            # Set IDLE for all vector properties for mirror cover
            states_tvp.state = IPState.IDLE
            state_message_lvp.state = IPState.IDLE
            self.IDSet(states_tvp)
            self.IDSet(state_message_lvp)
            mirror_cover.state = None
            return

        # Go through data and update properties
        update_properties(data, states_tvp)
        mirror_cover.state = data['mirror_cover_state']

        # Set ALERT if error in mirror cover data
        indi_states = {
            'Error': IPState.ALERT,
            'Opened': IPState.OK,
            'Closed': IPState.BUSY,
            'Partially Opened': IPState.BUSY
        }
        states_tvp.state = indi_states[data['mirror_cover_state']]

        self.IDSet(states_tvp)

        # Update state machine for open/close switches
        # I want users to know that its done opening or closing
        if not mirror_cover.busy():
            try:
                commands_svp = self.IUFind('commands')
            except ValueError:
                return

            commands_svp.state = IPState.IDLE # Reset to IDLE since done
            for c in commands_svp:
                c.value = 'Off'
            mirror_cover.reset() # Reset the opening and closing states
            reset_lights(state_message_lvp)
            state_message_lvp['idle'].value = IPState.OK
            state_message_lvp.state = IPState.OK
            self.IDSet(commands_svp)
            self.IDSet(state_message_lvp)

        return

def no_csp(value):
    """Removes space and case"""
    return value.lower().replace(' ', '_')

def format_boolean(value):
    """Return yes or no"""
    return 'Yes' if value else 'No'

def reset_lights(lvp, state=IPState.IDLE):
    """Resets the state of the light to state IDLE"""
    for l in lvp:
        l.value = state
    return

def update_properties(data, vp):
    """Updates the property values for vp selector"""
    for property in vp:
        # Only update if exists
        if data.get(property.name) is not None:
            value = data[property.name]

            # Format if bool
            if isinstance(value, bool):
                value = format_boolean(value)
            property.value = value

    # Set INDI state to OK since we got a response
    vp.state = IPState.OK

    return

driver = Device(name=MYDEVICE)
driver.start()


import logging
import os

# This is a minimal configuration to get you started with the Text mode.
# If you want to connect Errbot to chat services, checkout
# the options in the more complete config-template.py from here:
# https://raw.githubusercontent.com/errbotio/errbot/master/errbot/config-template.py

# BACKENDをTextからSlackに修正
BACKEND = 'Slack'

# 前述のAPI Tokenを入力
env_token = os.getenv("TOKEN")
BOT_IDENTITY = {
    'token': env_token
}

# adminにしたいユーザのアカウント名をタプルのかたちで入力（複数可）
BOT_ADMINS = ('@chapa')

BOT_DATA_DIR = r'data'
BOT_EXTRA_PLUGIN_DIR = r'plugins'

BOT_LOG_FILE = r'errbot.log'
BOT_LOG_LEVEL = logging.DEBUG


"""
Views for registration app

register: Allows user to pick a username and password to create an account.
          Uses the default UserCreationForm from Django
"""
from django.shortcuts import render, redirect
from django.contrib.auth.forms import UserCreationForm
from rest_framework import generics, permissions
from rest_framework.response import Response
from knox.models import AuthToken

from .serializers import RegisterSerializer, LoginSerializer
from userprofile.serializers import UserSerializer

class RegisterDetail(generics.GenericAPIView):
    serializer_class = RegisterSerializer

    def post(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.save()
        return Response({
            "user": UserSerializer(user, context=self.get_serializer_context()).data,
            "token": AuthToken.objects.create(user)[1]
        })

class LoginDetail(generics.GenericAPIView):
    serializer_class = LoginSerializer

    def post(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data
        return Response({
            "user": UserSerializer(user, context=self.get_serializer_context()).data,
            "token": AuthToken.objects.create(user)[1]
        })


import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

imgPath = "./testData/sudoku1.jpg"

# read the sudoku image as a gray scale image.
img = cv.imread(imgPath)
imgGray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
# "If the image cannot be read... the function returns an empty matrix" https://docs.opencv.org/
if imgGray is None:
    print("Error: could not read image from {}".format(imgPath))

# blur image
imgBlur = cv.GaussianBlur(imgGray, (25, 25), 0)
# perform threshold
imgThreshold = cv.adaptiveThreshold(imgBlur, 255, cv.ADAPTIVE_THRESH_MEAN_C, cv.THRESH_BINARY, 7, 3)
# invert the image
imgInverted = cv.bitwise_not(imgThreshold)
# dilate the image
kernel = np.ones((20, 20), np.uint8)
imgDilated = cv.dilate(imgInverted, kernel, iterations=1)

# plt.imshow(imgDilated, cmap="gray")
# plt.show()

# detect blob
params = cv.SimpleBlobDetector_Params()
params.minThreshold = 10
params.maxThreshold = 200
params.filterByCircularity = True
params.minCircularity = 0.1
detector = cv.SimpleBlobDetector_create(params)

keypoints = detector.detect(imgDilated)
im_with_keypoints = cv.drawKeypoints(img, keypoints, np.array([]), (0, 0, 255),cv.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
cv.imwrite("Keypoints.png", im_with_keypoints)


import numpy as np

TIE_MARK = -2
NO_WINNER_MARK = 0
ILLEGAL_MOVE_MARK = 2

class TicTacToeJudge(object):

	def __init__ (self, num_contigous_marks_to_win, player_marks):
		self.board = []
		self.n_winmarks = num_contigous_marks_to_win
		self.player_marks = player_marks

	def is_move_legal(self, new_board, old_board, player_mark):
		count = 0
		legal_move = True
		for index, x in np.ndenumerate(new_board):
			if x != old_board[index]:
				if x == player_mark:
					count += 1
				else :
					legal_move = False

		if count != 1:
			legal_move = False

		return legal_move

	def get_result(self, board):
		self.board = board
		winner = NO_WINNER_MARK
		for index, x in np.ndenumerate(self.board):
			if x == self.player_marks[0] or x == self.player_marks[1] :
				winner = self.get_winner(index)
				if winner != NO_WINNER_MARK:
					break

		# Tie Condition
		if sum(sum(board == 0)) == 0 and winner == NO_WINNER_MARK:
			winner = TIE_MARK

		return winner

	def check_index(self, i,j,shape_array):
		if i < 0 or j < 0:
			return False
		if i >= shape_array[0] or j >= shape_array[1]:
			return False
		return True

	def get_winner(self, index):
		x = self.board[index]
		i, j = index
		winner = 0
		shape = self.board.shape

		if not self.check_index(i-1, j-1, shape) or self.board[i-1,j-1] != x :
			winner = self.check_right_diagonal(index)
			if winner != 0:
				return winner
		if not self.check_index(i-1, j, shape) or self.board[i-1,j] != x :
			winner = self.check_vertical( index)
			if winner != 0:
				return winner
		if not self.check_index(i-1, j+1, shape) or self.board[i-1,j+1] != x :
			winner = self.check_left_diagonal(index)
			if winner != 0:
				return winner
		if not self.check_index(i, j-1, shape) or self.board[i,j-1] != x :
			winner = self.check_horizontal(index)

		return winner

	def check_right_diagonal(self, index):
		winner = 0
		i,j = index
		x = self.board[index]
		shape = self.board.shape
		for k in range(4):
			if self.check_index(i+k+1, j+k+1, shape) and self.board[i+k+1, j+k+1] == x :
				if k == self.n_winmarks-2:
					winner = x
			else :
				break
		return winner

	def check_vertical(self, index):
		winner = 0
		i,j = index
		x = self.board[index]
		shape = self.board.shape

		for k in range(4):
			if self.check_index(i+k+1, j, shape) and self.board[i+k+1, j] == x :
				if k == self.n_winmarks - 2:
					winner = x
			else :
				break

		return winner

	def check_left_diagonal(self, index):
		winner = 0
		i,j = index
		x = self.board[index]
		shape = self.board.shape

		for k in range(4):
			if self.check_index(i+k+1, j-k-1, shape) and self.board[i+k+1, j-k-1] == x :
				if k == self.n_winmarks-2:
					winner = x
			else :
				break

		return winner

	def check_horizontal(self, index):
		winner = 0
		i,j = index
		x = self.board[index]
		shape = self.board.shape

		for k in range(4):
			if self.check_index(i, j+k+1, shape) and self.board[i, j+k+1] == x :
				if k == self.n_winmarks-2:
					winner = x
			else :
				break

		return winner


from bs4 import BeautifulSoup
import requests

addr = "https://wwwh.smartevals.com/Reporting/Students/Wizard.aspx?Type=Classes"

r = requests.get(addr)

print(r.status_code)

soup = BeautifulSoup(r.text, 'lxml')
print(soup)
print(str(soup.find(id= "_ctl0_cphContent_grd1_DXDataRow0")))