# -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod
from collections import namedtuple
import os
import shutil
import subprocess
from tempfile import NamedTemporaryFile
from Bio import SeqIO
import itertools

class GeneFinder(object):
    __metaclass__ = ABCMeta

    executable = "gene_finder"
    name = "gene_finder"

    def __init__(self, data_dir, lib_dir):
        self.lib_dir = lib_dir
        self.data_dir = data_dir
        self.install()

    def execute(self, genome_name):
        self.genome_name = genome_name
        genome_path = os.path.join(self.data_dir, genome_name + ".fasta")
        args = self.get_args(genome_path)
        devnull = open(os.devnull, "w")

        subprocess.check_call(
            [os.path.join(self.lib_dir, self.executable)] + args,
            stdout=devnull, stderr=devnull)
        parsed_result_path = os.path.join(self.data_dir,
            "{0}_{1}_genes.fasta".format(genome_name, self.name))

        def rename(gene):
            gene.id = gene.name = gene.description = self.name
        result = itertools.imap(rename, self.parse_result())
        SeqIO.write(self.parse_result(), parsed_result_path, "fasta")
        return parsed_result_path

    __call__ = execute

    @abstractmethod
    def install(self):
        pass

    @abstractmethod
    def get_args(self, _genome_path):
        pass

    @abstractmethod
    def parse_result(self):
        pass

class GeneMarkCommonGC(GeneFinder):
    """
    ./gm -rn -m heuristic_path genome_path
    """
    executable = "gm"
    name = "gm_commonGC"

    def install(self):
        gm_key_path = os.path.join(self.lib_dir, "gm_key")
        if not os.path.isfile(os.path.expanduser("~/.gm_key")):
            shutil.copyfile(gm_key_path, os.path.expanduser("~/.gm_key"))

    def get_args(self, genome_path):
        gc = common_gc_content(SeqIO.parse(genome_path, "fasta"))
        gc = min(70, max(30, int(gc)))
        heuristic_dir = os.path.join(self.lib_dir, "heuristic_mat")
        heuristic_filename = os.path.join(heuristic_dir, "heu_11_{0}.mat".format(gc))
        return ["-rn", "-m", heuristic_filename, genome_path]

    def parse_result(self):
        result_path = os.path.join(self.data_dir, self.genome_name + ".fasta.rgn")
        if not os.path.isfile(os.path.expanduser(result_path)):
            return

        gm_result = SeqIO.parse(result_path, "fasta")

        for seq_record in gm_result:
            seq_record.description = ""
            yield seq_record

class GeneMarkEveryGC(GeneMarkCommonGC):
    """
   For every GC:
   ./gm -rn -m heuristic_for_the_gc_path contigs_with_the_gc_path
   """
    executable = "gm"
    name = "gm_everyGC"

    def execute(self, genome_name):
        genome_path = os.path.join(self.data_dir, genome_name + ".fasta")

        counter = [0]
        def rename(gene):
            gene.id = gene.description = gene.name = self.name + "_" + str(counter[0])
            counter[0] += 1
            return gene

        genes = []
        execute = super(GeneMarkEveryGC, self).execute
        for seq_record in SeqIO.parse(genome_path, "fasta"):
            if len(seq_record.seq) > 200:
                with NamedTemporaryFile(prefix=seq_record.id,
                    suffix=".fasta") as f:
                    SeqIO.write(seq_record, f, "fasta")
                    f.flush()
                    current_genes = SeqIO.parse(execute(f.name.replace(".fasta", "")),
                        "fasta")
                    genes.append(itertools.imap(rename, current_genes))

        parsed_result_path = os.path.join(self.data_dir,
            "{0}_{1}_genes.fasta".format(genome_name, self.name))
        SeqIO.write(itertools.chain(*genes), parsed_result_path, "fasta")
        return parsed_result_path

def common_gc_content(sequences):
    """
    Example:
    s = "AGCCT"
    gc_content = count_gc_content(s)
    print "GC content is %.2f%%"%gc_content

    Result:
    GC content is 60.00%
    """
    GC_count, length = 0, 0
    for seq_record in sequences:
        sequence = seq_record.seq
        GC_count += sequence.count("G") + sequence.count("C")
        length += len(sequence) - sequence.count("N")
    return round((1.0 * GC_count)/length * 100, 2)

def make_sam_out(predict_path, genes_path, genome_name, data_dir, method_name):
    devnull = open(os.devnull, "w")
    subprocess.call(['bwa', 'index', '-a', 'bwtsw', predict_path],
        stdout=devnull, stderr=devnull)
    subprocess.call(['bwa', 'index', '-a', 'bwtsw', genes_path],
        stdout=devnull, stderr=devnull)
    out_path_prefix = os.path.join(data_dir, genome_name) + '_'
    with open(out_path_prefix + method_name + '_on.sam', 'w') as fout:
        subprocess.call(['bwa', 'bwasw', predict_path, genes_path],
            stdout=fout, stderr=devnull)
    with open(out_path_prefix + 'on_' + method_name + '.sam', 'w') as fout:
        subprocess.call(['bwa', 'bwasw', genes_path, predict_path],
            stdout=fout, stderr=devnull)
    return out_path_prefix

def get_sam_map(sam_path):
    res = {}
    keys = set()
    with open(sam_path) as first:
        for line in first:
            if line.startswith('@'):
                pass
            else:
                sl = line.split()
                qname, rname, mapq = sl[0], sl[2], int(sl[4])
                keys.add(qname)
                if mapq >= 200: #mapq = -10*log(10, Pr{mapp. pos. is wrong}), if mapq = 200, Pr{..} = 10**(-20)
                    res[qname] = rname
    return res, len(keys)

def count_metrics(genes_sam_path, predict_sam_path):
    predict_on_genes, predict_count = get_sam_map(predict_sam_path)
    genes_on_predict, genes_count = get_sam_map(genes_sam_path)
    H = namedtuple('H', ['name', 'TP', 'FP', 'FN'])
    TP = len(predict_on_genes)
    H_predict_is_gene = H('predict is gene', TP, predict_count - TP,
        genes_count - len(set(predict_on_genes.values())))
    TP = len(genes_on_predict)
    H_gene_was_predicted = H('gene was predicted', TP, genes_count - TP,
        predict_count - len(set(genes_on_predict.values())))
    return H_predict_is_gene, H_gene_was_predicted

def compare_data(tools, genomes, data_dir):
    for genome_name, genes_path in genomes.iteritems():
        for tool in tools:
            path_result = tool.execute(genome_name)
            path_prefix = make_sam_out(path_result, genes_path, genome_name,
                data_dir, tool.name)
            hypothesises = count_metrics(path_prefix + tool.name + '_on.sam',
                path_prefix + 'on_' + tool.name + '.sam')
            for hypothesis in hypothesises:
                yield hypothesis.name, genome_name, tool.name, hypothesis.TP, \
                      hypothesis.FP, hypothesis.FN

if __name__ == "__main__":
    data_dir = 'Data'
    lib_dir = 'genemark_suite_linux_64/gmsuite'
    genomes = {'ECO2_seq': 'Data/eco_genes.fasta', 'ECO3_seq': 'Data/eco_genes.fasta',
               'ECO6_seq': 'Data/eco_genes.fasta', 'ECO7_seq': 'Data/eco_genes.fasta',
               'MRU5_seq': 'Data/mru_genes.fasta', 'MRU6_seq': 'Data/mru_genes.fasta',
               'MRU9_seq': 'Data/mru_genes.fasta', 'PHE4_seq': 'Data/phe_genes.fasta',
               'PHE6_seq': 'Data/phe_genes.fasta', 'PHE7_seq': 'Data/phe_genes.fasta'}
    gm_common_gc = GeneMarkCommonGC(data_dir, lib_dir)
    gm_every_gc = GeneMarkEveryGC(data_dir, lib_dir)
    tools = [gm_common_gc, gm_every_gc]
    with open('compare_out', 'w') as out:
        for data in compare_data(tools, genomes, data_dir):
            out.write(','.join([str(x) for x in data]) + '\n')


import pytest
from django.urls import reverse
from rest_framework import status

pytestmark = pytest.mark.django_db

def test_quiz_list(admin_client):
    url = reverse('api-quiz:list_quiz')
    response = admin_client.get(url, content_type="application/json")
    assert response.status_code == status.HTTP_200_OK

def test_quiz_api_detail(admin_client, quiz):
    url = reverse('api-quiz:retrieve_quiz', args=[quiz.id])
    response = admin_client.get(url)
    assert response.status_code == status.HTTP_200_OK
    assert response.json()['id'] == quiz.id

def test_quiz_api_update(admin_client, quiz):
    url = reverse('api-quiz:update_quiz', args=[quiz.id])
    response = admin_client.patch(url, data={'title': 'new_title'})
    assert response.status_code == status.HTTP_200_OK

def test_quiz_api_delete(admin_client, quiz):
    url = reverse('api-quiz:delete_quiz', args=[quiz.id])
    response = admin_client.delete(url)
    assert response.status_code == status.HTTP_204_NO_CONTENT


#from __future__ import print_function
"""Train  with PyTorch."""
"""
This code is forked and modified from 'https://github.com/kuangliu/pytorch-cifar'. Thanks to its contribution.
"""

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
from torch_geometric.data import DataListLoader,DataLoader
import statistics
import torchvision
import torchvision.transforms as transforms
from torch_geometric.nn import GCNConv, ChebConv,global_mean_pool,SplineConv,GraphConv, AGNNConv,TopKPooling,DataParallel,GATConv
#from pyts.image import RecurrencePlot
from torch_geometric.datasets import MNISTSuperpixels,Planetoid,TUDataset,PPI,Amazon,Reddit,CoraFull
import torch_geometric.transforms as T
from torch_geometric.nn import global_mean_pool as gap, global_max_pool as gmp
import numpy as np

from sklearn import metrics
import argparse
import os,sys
global resume
global ConCoeff

def ChooseModel(model_name,datasetroot,NetInfo):
    if len(NetInfo)==1:
        width=NetInfo[0]
    elif len(NetInfo)==2:
        width=NetInfo[0]
        weights=NetInfo[1]
    else:
        raise Exception("wrong weight info")

    if model_name=="GCN":
        net=GCN(datasetroot,width)
        if len(NetInfo)==2:
            state_dict = net.state_dict()
            for i in range(len(weights)-1):
                name='layers.{}.weight'.format(i)
                if i==0:
                    state_dict[name]=weights[i]
                else:
                    state_dict[name]=weights[i][:-1,:]
            net.load_state_dict(state_dict)

    elif model_name=="GAT":
        net=GAT(datasetroot,width)
    elif model_name=="SplineNet":
        net=SplineNet(datasetroot,width)
    elif model_name=="ChebConvNet":
        net=ChebConvNet(datasetroot,width)

    elif model_name=="AGNNNet":
        net=AGNNNet(datasetroot,width)

    else:
        raise Exception("model not support, Choose GCN, or SplineNet or ChebConvNet")

    return net

def TrainPart(modelName,datasetroot,start_epoch,num_epochs,trainloader,testloader,OptimizedNet,optimizerNew,criterionNew,NumCutoff,mark,SaveModule,model_to_save):
    best_acc=1
    TrainConvergence=[]
    TestConvergence=[]
    TestAccs=[]
    for epoch in range(start_epoch,num_epochs):
        if epoch%40==0 or epoch==num_epochs-1:
            global SVDOrNot
            SVDOrNot=[NumCutoff,"{}-{}".format(mark,epoch)]
            TrainLoss=train(trainloader,OptimizedNet,optimizerNew,criterionNew)

            """if epoch==40:
                NewNetworksize,NewNetworkWeight=RetainNetworkSize(OptimizedNet,params[2])[0:2]
                NetworkInfo=[NewNetworksize[0:-1],NewNetworkWeight]
                OptimizedNet=ChooseModel(modelName,datasetroot,NetworkInfo)
                OptimizedNet = OptimizedNet.to(device)
                WeightsContate=NewNetworkWeight

            elif epoch>40:
                for layer in range(len(NewNetworkWeight)):
                    NewNetworksize,NewNetworkWeight=RetainNetworkSize(OptimizedNet,params[2])[0:2]
                    WeightsContate[layer]=torch.cat((WeightsContate[layer],NewNetworkWeight[layer]),1)
                    [U,D,V]=torch.svd(WeightsContate[layer])
                    CutoffPoint=FindCutoffPoint(D,params[3])
                    NewNetworksize.append(CutoffPoint)
                    NewNetworkWeight.append(U[:,:CutoffPoint]@V[:CutoffPoint,:CutoffPoint])
                    NetworkInfo=[NewNetworksize[0:-1],NewNetworkWeight]
                    OptimizedNet=ChooseModel(modelName,datasetroot,NetworkInfo)
                    OptimizedNet = OptimizedNet.to(device)"""

        else:
            SVDOrNot=[]
            TrainLoss=train(trainloader,OptimizedNet,optimizerNew,criterionNew)

        [TestLoss,TestAcc]=test(testloader,OptimizedNet,criterionNew)
        print('\n Epoch: {},  tain loss: {:.4f}, test loss: {:.4f},test acc: {:.4f} \n'.format(epoch,TrainLoss,TestLoss,TestAcc))
        TrainConvergence.append(TrainLoss)
        TestConvergence.append(TestLoss)
        TestAccs.append(TestAcc)
               # save model
        if SaveModule and TestAcc< best_acc:
                state = {'net': OptimizedNet.state_dict,
                                'TrainConvergence': TrainConvergence,
                                'TestAcc': TestAcc,
                                'epoch': num_epochs,
                       }
                if not os.path.isdir('checkpoint'):
                    os.mkdir('checkpoint')
                torch.save(state, model_to_save)
                best_acc = TestAcc

                ## save recurrence plots
        """if epoch%20==0:
                save_recurrencePlots_file="../Results/RecurrencePlots/RecurrencePlots_{}_{}_BatchSize{}    \_ConCoeffi{}_epoch{}.png".format(dataset, model_name,params[0],params[1],epoch)
            save_recurrencePlots(net,save_recurrencePlots_file)"""
    del OptimizedNet
    return TrainConvergence, TestConvergence,TestAccs

def ContractionLayerCoefficients(num_features,*args):
    Numlayers,alpha=args
    width=[]
    tmpOld=np.random.randint(num_features*alpha,num_features)
    for k in range(Numlayers):
        tmpNew=np.random.randint(tmpOld*alpha,tmpOld)
        width.append(tmpNew)
        tmpOld=tmpNew
    return width

def ResumeModel(model_to_save):
    # Load checkpoint.
    #print('==> Resuming from checkpoint..')
    checkpoint = torch.load(model_to_save)
    net = checkpoint['net']
    TrainConvergence = checkpoint['TrainConvergence']
    Acc = checkpoint['TestAcc']
    start_epoch = checkpoint['epoch']
    return net,TrainConvergence,Acc,start_epoch

def FindCutoffPoint(DiagValues,coefficient):
    for i in range(DiagValues.shape[0]-1):
        if DiagValues[i]>DiagValues[i+1]*coefficient:
            CutoffPoint=i+1

    try:
        return CutoffPoint
    except:
        return DiagValues.shape[0]

"""def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.orthogonal_(m.weight)
        m.bias.data.fill_(0.01)"""

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('GCNConv') != -1:
        torch.nn.init.orthogonal_(m.weight.data)
        #torch.nn.init.xavier()
        m.bias.data.fill_(0.01)

def weight_reset(m):
    if isinstance(m, GCNConv) :
        m.reset_parameters()

def logging(message):
    global  print_to_logging
    if print_to_logging:
        print(message)
    else:
        pass

def print_nvidia_useage():
    global print_device_useage
    if print_device_useage:
        os.system('\n echo check gpu;nvidia-smi;echo check done \r')
    else:
        pass

def save_recurrencePlots(net,save_recurrencePlots_file):
    global save_recurrence_plots
    if save_recurrence_plots:
        for name,parameters in net.named_parameters():
            if "fc" in name and parameters.cpu().detach().numpy().ndim==2:
                hiddenState=parameters.cpu().detach().numpy()
                rp = RecurrencePlot()
                X_rp = rp.fit_transform(hiddenState)
                plt.figure(figsize=(6, 6))
                plt.imshow(X_rp[0], cmap='binary', origin='lower')
                plt.savefig(save_recurrencePlots_file,dpi=600)
            else:
                continue
    else:
        pass

      #net.apply(weight_reset)

class GCN(torch.nn.Module):
    def __init__(self,datasetroot,width):
        super(GCN,self).__init__()
        self.NumLayers=len(width)
        self.layers = nn.ModuleList()
        self.layers.append(GCNConv(datasetroot.num_features, width[0]))
        for i in range(self.NumLayers-1):
            layer=GCNConv(width[i],width[i+1])
            nn.init.xavier_uniform_(layer.weight)
            self.layers.append(layer)
        self.layers.append(GCNConv(width[-1], datasetroot.num_classes))

    def forward(self,data):
        x, edge_index,batch = data.x, data.edge_index,data.batch
        DiagElemnt=[]
        for layer in self.layers[:-1]:
            if len(SVDOrNot)==2:
                NumCutoff=SVDOrNot[0]
                mark=SVDOrNot[1]
                [U,D,V]=torch.svd(x)
                DiagElemnt.append(D[0:NumCutoff].detach().tolist())
                np.save(mark,DiagElemnt)
            x=layer(x, edge_index)
            x =x*torch.sigmoid(x)
        #x = F.dropout(x, training=self.training)
        x=self.layers[-1](x,edge_index)
        x = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)
        x = F.log_softmax(x,dim=1)
        return x

class GAT(torch.nn.Module):
    def __init__(self,datasetroot,width):
        super(GAT, self).__init__()
        self.NumLayers=len(width)
        self.layers = nn.ModuleList()
        self.layers.append(GATConv(datasetroot.num_features, width[0]))
        for i in range(self.NumLayers-1):
            layer=GATConv(width[i],width[i+1])
            nn.init.xavier_uniform_(layer.weight)
            self.layers.append(layer)
        self.layers.append(GATConv(width[-1], datasetroot.num_classes))

    def forward(self,data):
        x, edge_index,batch = data.x, data.edge_index,data.batch
        DiagElemnt=[]
        for layer in self.layers[:-1]:
            if len(SVDOrNot)==2:
                NumCutoff=SVDOrNot[0]
                mark=SVDOrNot[1]
                [U,D,V]=torch.svd(x)
                DiagElemnt.append(D[0:NumCutoff].detach().tolist())
                np.save(mark,DiagElemnt)
            x=layer(x, edge_index)
            x =x*torch.sigmoid(x)
        #x = F.dropout(x, training=self.training)
        x=self.layers[-1](x,edge_index)
        x = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)
        x = F.log_softmax(x,dim=1)
        return x

class AGNNNet(torch.nn.Module):
    def __init__(self,datasetroot,width):
        super(AGNNNet, self).__init__()
        self.NumLayers=len(width)
        self.layers = nn.ModuleList()
        self.lin1=torch.nn.Linear(datasetroot.num_features, width[0])
        for i in range(self.NumLayers-1):
            self.layers.append(torch.nn.Linear(width[i],width[i+1]))
        self.layers.append(torch.nn.Linear(width[-1], datasetroot.num_classes))
        self.prop1 = AGNNConv(requires_grad=True)
        self.prop2 = AGNNConv(requires_grad=True)
        self.layers = nn.ModuleList()

    def forward(self,data):
        x,edge_index=data.x,data.edge_index
        x = F.dropout(x, training=self.training)
        x=self.lin1(x)
        x = self.prop1(x, edge_index)
        x = self.prop2(x, edge_index)
        DiagElemnt=[]
        for layer in self.layers[:-1]:

            x=layer(x)
            x =x*torch.sigmoid(x)
            x = F.dropout(x, training=self.training)
        return F.log_softmax(x, dim=1)

class ChebConvNet(torch.nn.Module):
    def __init__(self,datasetroot,width):
        super(ChebConvNet,self).__init__()
        self.NumLayers=len(width)
        self.layers = nn.ModuleList()
        self.layers.append(ChebConv(datasetroot.num_features,width[0],K=1))
        for i in range(self.NumLayers-1):
            layer=ChebConv(width[i],width[i+1],K=1)
            nn.init.xavier_uniform_(layer.weight)
            self.layers.append(layer)
        self.layers.append(ChebConv(width[-1], datasetroot.num_classes,K=1))

    def forward(self,data):
        x, edge_index,batch = data.x, data.edge_index,data.batch
        DiagElemnt=[]
        for layer in self.layers[:-1]:
            if len(SVDOrNot)==2:
                NumCutoff=SVDOrNot[0]
                mark=SVDOrNot[1]
                [U,D,V]=torch.svd(x)
                DiagElemnt.append(D[0:NumCutoff].detach().tolist())
                np.save(mark,DiagElemnt)
            x=layer(x,edge_index)
            x =x*torch.sigmoid(x)
        x=self.layers[-1](x,edge_index)
        x = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)
        x = F.log_softmax(x,dim=1)
        return x

class SplineNet(torch.nn.Module):
    def __init__(self,datasetroot, width):
        super(SplineNet, self).__init__()
        self.NumLayers=len(width)
        self.layers = nn.ModuleList()
        self.layers.append(SplineConv(datasetroot.num_features, width[0], dim=1, kernel_size=2))
        for i in range(self.NumLayers-1):
            layer=SplineConv(width[i],width[i+1], dim=1, kernel_size=2)
            nn.init.xavier_uniform_(layer.weight)
            self.layers.append(layer)
        self.layers.append(SplineConv(width[-1], datasetroot.num_classes, dim=1, kernel_size=2))

    def forward(self,data):
        x, edge_index,edge_attr,batch = data.x, data.edge_index,data.edge_attr,data.batch
        DiagElemnt=[]
        for layer in self.layers[:-1]:
            if len(SVDOrNot)==2:
                NumCutoff=SVDOrNot[0]
                mark=SVDOrNot[1]
                [U,D,V]=torch.svd(x)
                DiagElemnt.append(D[0:NumCutoff].detach().tolist())
                np.save(mark,DiagElemnt)
            x=layer(x, edge_index,edge_attr)
            x =x*torch.sigmoid(x)
        x=self.layers[-1](x,edge_index)
        x = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)
        x = F.log_softmax(x,dim=1)
        return x

class topk_pool_Net(torch.nn.Module):
    def __init__(self,datasetroot, width):
        super(topk_pool_Net, self).__init__()
        self.NumLayers=len(width)
        self.conv1 = GraphConv(datasetroot.num_features, 128)
        self.pool1 = TopKPooling(128, ratio=0.8)
        self.conv2 = GraphConv(128, 128)
        self.pool2 = TopKPooling(128, ratio=0.8)
        self.conv3 = GraphConv(128, 128)
        self.pool3 = TopKPooling(128, ratio=0.8)
        self.layers = nn.ModuleList()
        self.layers.append(torch.nn.Linear(256,width[0]))
        for i in range(self.NumLayers-1):
            self.layers.append(torch.nn.Linear(width[i],width[i+1]))
        self.layers.append(torch.nn.Linear(width[-1], datasetroot.num_classes))

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch

        x = F.relu(self.conv1(x, edge_index))
        x, edge_index, _, batch, _, _ = self.pool1(x, edge_index, None, batch)
        x1 = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)

        x = F.relu(self.conv2(x, edge_index))
        x, edge_index, _, batch, _, _ = self.pool2(x, edge_index, None, batch)
        x2 = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)

        x = F.relu(self.conv3(x, edge_index))
        x, edge_index, _, batch, _, _ = self.pool3(x, edge_index, None, batch)
        x3 = torch.cat([gmp(x, batch), gap(x, batch)], dim=1)

        x = x1 + x2 + x3
        for layer in self.layers[:-1]:
            x=layer(x)
            x =x*torch.sigmoid(x)
            x = F.dropout(x, p=0.5, training=self.training)
        x = F.log_softmax(self.layers[-1](x),dim=1)
        return x

def ModelAndSave(dataset,modelName,train_dataset,params,num_epochs):
    model_to_save='./checkpoint/{}-{}-param_{}_{}_{}_{}-ckpt.pth'.format(dataset,modelName,params[0],params[1],params[2],params[3])
    if resume=="True" and os.path.exists(model_to_save):
        [net,TrainConvergence,Acc,start_epoch]=ResumeModel(model_to_save)
        if start_epoch>=num_epochs-1:
            pass

    else:
        width=ContractionLayerCoefficients(train_dataset.num_features,*params[1:3])
        NetworkInfo=[width]
        net =ChooseModel(modelName,train_dataset,NetworkInfo)
    return net, model_to_save

# Training

def RetainNetworkSize(net,ConCoeff):
    NewNetworksize=[]
    NewNetworkWeight=[]
    for layer_name, Weight in net.named_parameters():
        if ("weight" in layer_name) and ("layers" in layer_name):
            #print(layer_name)
            if Weight.dim()==3:
                Weight=Weight[0].view(Weight.size()[1],Weight.size()[2])
            [U,D,V]=torch.svd(Weight)
            #NumCutoff=SVDOrNot[0]
            #WeightsDynamics.append(D[:NumCutoff].tolist())
            CutoffPoint=FindCutoffPoint(D,ConCoeff)
            NewNetworksize.append(CutoffPoint)
            NewNetworkWeight.append(U[:,:CutoffPoint]@V[:CutoffPoint,:CutoffPoint])
            #NewNetworkWeight.append(U[:,:CutoffPoint]@torch.diag(D[:CutoffPoint])@V[:CutoffPoint,:CutoffPoint])

            """NewWeight= torch.mm(Weight,V[:,:CutoffPoint])
            net.conv2.weight = torch.nn.Parameter( NewWeight)"""
            #print("Original size is {},After SVD is {}".format(Weight.shape[1],CutoffPoint))
    return NewNetworksize,NewNetworkWeight

def train(trainloader,net,optimizer,criterion):
    net.train()
    train_loss = 0
    optimizer.zero_grad()
    for data in trainloader:
        data=data.to(device)
        optimizer.zero_grad()
        output=net(data)
        loss = criterion(output, data.y)
        loss.backward()
        train_loss+=data.num_graphs * loss.item()
        optimizer.step()

        """for layer_name, parameters in net.named_parameters():
             if "weight" in layer_name:
                net.conv.weight.div_(torch.norm(net.conv.weight, dim=2, keepdim=True)"""
        #test(testloader,net,criterion)
    return train_loss/len(train_dataset)

def test(testloader,net,criterion):
    net.eval()
    test_loss, accs= 0,0
    with torch.no_grad():
        for data in testloader:
            data=data.to(device)
            output= net(data)
            pred= output.max(1)[1]
            loss = criterion(output,data.y)
            test_loss+=loss.item()*data.num_graphs
            accs+=pred.eq(data.y.to(pred.device)).sum().item()

    #print('\n Test set: Average loss: {:.4f} \n'.format(test_loss[-1]))
    return test_loss/len(test_dataset),accs/len(test_dataset)

def TrainingNet(dataset,modelName,params,num_pre_epochs,num_epochs,NumCutoff,optimizerName,MonteSize,savepath):
    Batch_size=int(params[0])
    root='/git/data/GraphData/'+dataset
    TestAccs=[]
    for Monte_iter in range(MonteSize):
        # Data
        start_epoch = 0  # start from epoch 0 or last checkpoint epoch
        NewNetworkSizeAdjust=[]
        WeightsDynamicsEvolution=[]
        # model
        if dataset=='Cora' or dataset =='Citeseer' or dataset =='Pubmed':
            datasetroot= Planetoid(root=root, name=dataset, transform =T.NormalizeFeatures()).shuffle()
            trainloader = DataListLoader(datasetroot, batch_size=Batch_size, shuffle=True)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)
            criterion = nn.CrossEntropyLoss()

        elif dataset =="CoraFull":
            datasetroot = CoraFull(root=root,transform =T.NormalizeFeatures()).shuffle()
            trainloader = DataListLoader(datasetroot, batch_size=Batch_size, shuffle=True)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)

        elif dataset=="Amazon":
            datasetroot=Amazon(root, "Photo", transform=None, pre_transform=None)
            trainloader = DataListLoader(datasetroot, batch_size=Batch_size, shuffle=True)
            testloader = DataListLoader(datasetroot, batch_size=100, shuffle=False)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)

        elif dataset=='ENZYMES' or dataset=='MUTAG':
            datasetroot=TUDataset(root,name=dataset,use_node_attr=True)
            Num=len(datasetroot)// 10
            global train_dataset, test_dataset
            train_dataset = datasetroot[:Num]
            test_dataset = datasetroot[Num:]
            trainloader = DataLoader(train_dataset, batch_size=Batch_size)
            testloader = DataLoader(test_dataset, batch_size=60)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)

        elif dataset =="PPI":
            train_dataset = PPI(root, split='train')
            test_dataset = PPI(root, split='test')
            trainloader = DataLoader(train_dataset, batch_size=Batch_size, shuffle=True)
            testloader = DataLoader(test_dataset, batch_size=1, shuffle=False)
            [net,model_to_save]=ModelAndSave(dataset,modelName,train_dataset,params,num_epochs)
            criterion = torch.nn.BCEWithLogitsLoss()

        elif dataset =="Reddit":
            datasetroot=Reddit(root)
            trainloader = DataListLoader(datasetroot, batch_size=1, shuffle=True)
            testloader = DataListLoader(datasetroot, batch_size=2, shuffle=False)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)
            criterion = torch.nn.BCEWithLogitsLoss()

        elif dataset=='MNIST':
            datasetroot = MNISTSuperpixels(root=root, transform=T.Cartesian())
            trainloader = DataListLoader(datasetroot, batch_size=Batch_size, shuffle=True)
            testloader = DataListLoader(datasetroot, batch_size=100, shuffle=False)
            [net,model_to_save]=ModelAndSave(dataset,modelName,datasetroot,params,num_epochs)

        elif dataset=='CIFAR10':
            pass
        else:
            raise Exception("Input wrong datatset!!")

        FileName="{}-{}-param_{}_{}_{}_{}-monte_{}".format(dataset,modelName,params[0],params[1],params[2],params[3],Monte_iter)

        print('Let\'s use', torch.cuda.device_count(), 'GPUs!')
        global device
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        optimizer =optim.Adam(net.parameters(), lr=params[3], betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)
        criterion = nn.CrossEntropyLoss()
        net = net.to(device)

                #cudnn.benchmark = True
        logging('dataset:{}, Batch size: {}, Number of layers:{} ConCoeff: {}, LR:{}, MonteSize:{}'.format(dataset, params[0], params[1],params[2],params[3],Monte_iter))
        mark="{}/{}Convergence/DiagElement-{}".format(savepath,dataset,FileName)

        PreTrainConvergence,PreTestConvergence,PreTestAcc=TrainPart(modelName,datasetroot,start_epoch,num_pre_epochs,trainloader,testloader,net,optimizer,criterion,NumCutoff,mark,False,model_to_save)
        print('dataset: {}, model name: {}, Number epoches: {},  Pre-train error is: {}, Pre-test error is: {}, test acc is {}'.format(dataset,modelName,num_pre_epochs,PreTrainConvergence[-1],PreTestConvergence[-1],PreTestAcc[-1]))

        NewNetworksize,NewNetworkWeight=RetainNetworkSize(net,params[2])[0:2]
        NetworkInfo=[NewNetworksize[0:-1],NewNetworkWeight]
        OptimizedNet=ChooseModel(modelName,datasetroot,NetworkInfo)
        NewNetworksize.insert(0,datasetroot.num_features)
        NewNetworkSizeAdjust.append(NewNetworksize[0:-1])
        print(NewNetworkSizeAdjust)

        #OptimizedNet.apply(init_weights)
        #OptimizedNet = DataParallel(OptimizedNet)
        OptimizedNet = OptimizedNet.to(device)
        cudnn.benchmark = True
        criterionNew = nn.CrossEntropyLoss()
        if optimizerName =="SGD":
            optimizerNew = getattr(optim,optimizerName)(OptimizedNet.parameters(), lr=params[3], momentum=0.9, weight_decay=5e-4)
        elif optimizerName =="Adam":
            optimizerNew = getattr(optim,optimizerName)(OptimizedNet.parameters(), lr=params[3], betas=(0.9, 0.999), eps=1e-08, weight_decay=5e-4, amsgrad=False)

        TrainConvergence,TestConvergence,TestAcc=TrainPart(modelName,datasetroot,start_epoch,num_epochs,trainloader,testloader,OptimizedNet,optimizerNew,criterionNew,NumCutoff,mark,True,model_to_save)
        np.save("{}/{}Convergence/TrainConvergence-{}".format(savepath,dataset,FileName),TrainConvergence)
        np.save("{}/{}Convergence/TestConvergence-{}".format(savepath,dataset,FileName),TestConvergence)

        np.save("{}/{}Convergence/NewNetworkSizeAdjust-{}".format(savepath,dataset,FileName),NewNetworkSizeAdjust)

        #np.save(savepath+'TestConvergence-'+FileName,TestConvergence)
        #torch.cuda.empty_cache()

        print('dataset: {}, model name:{}, resized network size is {},  Number epoches:{},  Train error is: {}, Test error is: {}, test acc is {}\n'.format(dataset,modelName,NewNetworksize[0:-1],num_epochs,TrainConvergence[-1],TestConvergence[-1],TestAcc[-1]))
        TestAccs.append(TestAcc)
        np.save("{}/{}Convergence/MeanTestAccs-{}".format(savepath,dataset,FileName),TestAccs)
    print("The change of test error is:{}".format(TestAccs))
    print_nvidia_useage()

if __name__=="__main__":
    parser = argparse.ArgumentParser(description='PyTorch Training')
    parser.add_argument('--dataset',default='ENZYMES',type=str, help='dataset to train')
    parser.add_argument('--modelName',default='GCN',type=str, help='model to use')
    parser.add_argument('--LR', default=0.2, type=float, help='learning rate')
    parser.add_argument('--ConCoeff', default=0.99, type=float, help='contraction coefficients')
    parser.add_argument('--CutoffCoeff', default=0.1, type=float, help='contraction coefficients')
    parser.add_argument('--NumCutoff', default=5, type=float, help='contraction coefficients')
    parser.add_argument('--rho', type=float, default=1e-2, metavar='R',
                        help='cardinality weight (default: 1e-2)')
    parser.add_argument('--optimizer',default='SGD',type=str, help='optimizer to train')

    parser.add_argument('--num_pre_epochs', type=int, default=30, metavar='P',
                        help='number of epochs to pretrain (default: 3)')
    parser.add_argument('--num_epochs', type=int, default=100, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--MonteSize', default=1, type=int, help=' Monte Carlos size')
    parser.add_argument('--gpus', default="0", type=str, help="gpu devices")
    parser.add_argument('--BatchSize', default=512, type=int, help='batch size')
    parser.add_argument('--NumLayers', default=2, type=int, help='Number of layers')
    parser.add_argument('--PruningTimes', default=2, type=int, help='Pruning times')
    parser.add_argument('--savepath', type=str, default='Results', help='Path to save results')
    parser.add_argument('--return_output', type=str, default=False, help='Whether output')
    parser.add_argument('--resume', '-r', type=str,default=False, help='resume from checkpoint')
    parser.add_argument('--print_device_useage', type=str, default=False, help='Whether print gpu useage')
    parser.add_argument('--print_to_logging', type=str, default=True, help='Whether print')
    parser.add_argument('--save_recurrence_plots', type=str, default=False, help='Whether print')

    args = parser.parse_args()
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus
    print_to_logging=args.print_to_logging
    print_device_useage=args.print_device_useage
    return_output=args.return_output
    resume=args.resume
    save_recurrence_plots=args.save_recurrence_plots
    #params=[args.BatchSize,args.NumLayers,args.args.ConCoeff,args.CutoffCoeff]
    params=[args.BatchSize,args.NumLayers,args.ConCoeff,args.LR]

    TrainingNet(args.dataset,args.modelName,params,args.num_pre_epochs,args.num_epochs,args.NumCutoff,args.optimizer,args.MonteSize,args.savepath)


#!/usr/bin/python

import sys, sut, random
from optparse import OptionParser

def main(argv):
	parser = OptionParser()
	parser.add_option('-t', '--timeout', action="store", type="int", dest="timeout", default=30, help="time in seconds for testing")
	parser.add_option('-s', '--seed', action="store", type="int", dest="seed", default=10, help="seed for Python random.random object used for random number generation")
	parser.add_option('-d', '--depth', action="store", type="int", dest="depth", default=10, help="maximum length of a test")
	parser.add_option('-w', '--width', action="store", type="int", dest="width", default=10, help="maximum memory/BFS queue/other")
	parser.add_option('-f', '--fault', action="store_true", dest="fault", default=False, help="check for faults in the SUT")
	parser.add_option('-c', '--coverage', action="store_true", dest="coverage", default=False, help="produce a final coverage report")
	parser.add_option('-r', '--running', action="store_true", dest="running", default=False, help="produce running info on branch coverage")

	options, args = parser.parse_args()
	print("timeout:",options.timeout)
	print("seed:\t",options.seed)

	random.seed(options.seed)
	print("random:\t",random.random())

	if(options.fault):
		print("fault:\t enabled")

if __name__ == "__main__":
	main(sys.argv[1:])

"""
<timeout>: time in seconds for testing -- your tester must respect
this, within a small margin of error

<seed>: seed for Python Random.random object used for random number
generation in your code, if it is stochastic

<depth>: maximum length of a test generated by your algorithm

<width>: maximum memory/BFS queue/other parameter that is basically a
search width

<faults>: either 0 or 1 depending on whether your tester should check
for faults in the SUT; if true, you should save a test case for each
discovered failure (terminating the test that generated it), in the
current directory, as failure1.test failure2.test, etc.

<coverage>: either 0 or 1 depending on whether a final coverage report
should be produced, using TSTL's internalReport() function.  Your
tester is allowed to assume that coverage is being collected to guide
tests.

<running>: either 0 or 1 depending on whether running info on branch
coverage should be produced.
"""


import unittest
from POM.page import Wechat
from appium import webdriver
from HTMLTestRunner import HTMLTestRunner
from POM.logging_page import *
testcase_dir = 'E:\pycharm\Demo\First_Po\Testcase.py'
class Touch(unittest.TestCase):
    @classmethod
    def setUp(self):
        desired_caps = {
        'platformName' :'Android',
        'platformVersion': '4.4',
        'deviceName' : 'DU2SSE15C3115777',
        'appPackage' :'com.tencent.mm' ,# 'com.android.contacts'
        'appActivity': '.ui.LauncherUI',  # '.activities.DialtactsActivity'
        'unicodekeybord' : 'True',
        "resetKeyboard":"True"
        }
        self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps)
        # self.driver.implicitly_wait(30)
        self.driver.implicitly_wait(30)
    @classmethod
    def tearDown(self):
        self.driver.quit()
    def test000(self):
        x=loging(self.driver)
        #为登录页面断言
        self.assertEqual(self.driver.find_element_by_name('2971966792').text, '2971966792')
        #输入密码
        x.input_passw('lixiang123456')
        x.loging_enter()
        sleep(4)
        self.assertEqual(self.driver.find_element_by_name('Duang').text, 'Duang')
    def test001(self):
        print('正在测试test001')
        li=Wechat(self.driver)
        li.contacts('qichuangla')
        self.assertEqual(self.driver.find_element_by_name('Duang').text,'Duang')
    def test002(self):
        print('正在测试test001')
        li=Wechat(self.driver)
        li.discover()
        li.me()


#!/usr/bin/env python
# coding: utf-8

##############
# Grid ctrls #
##############

### Modules
import sharedValues
reload(sharedValues)
from sharedValues import MARGIN_VER, NET_WIDTH

# custom
from ..ui import userInterfaceValues
reload(userInterfaceValues)
from ..ui.userInterfaceValues import vanillaControlsSize

# standard
from vanilla import Group, CheckBox, TextBox, EditText, ColorWell
from AppKit import NSColor

### Constants
GRID_COLOR_ONE = (1, 0.4, 0.4, 1)
GRID_COLOR_TWO = (0.4, 0.64, 1, 1)
GRID_COLOR_THREE = (0.88, 0.4, 1, 1)
GRID_COLOR_FOUR = (0.4, 1, 0.64, 1)
GRID_COLOR_FIVE = (0.88, 1, 0.4, 1)
GRID_COLOR_INIT = [GRID_COLOR_ONE, GRID_COLOR_TWO, GRID_COLOR_THREE, GRID_COLOR_FOUR, GRID_COLOR_FIVE]

### Ctrls
class MultipleGridController(Group):

    def __init__(self, posSize, gridActive, ctrlsAmount, activeCtrls, offgridActive, callback):
        Group.__init__(self, posSize)
        assert activeCtrls <= ctrlsAmount

        self.ctrlHeight = posSize[3]
        self.gridActive = gridActive
        self.ctrlsAmount = ctrlsAmount
        self.activeCtrls = activeCtrls
        self.offgridActive = offgridActive
        self.gridIndexes = ['{:d}'.format(integer) for integer in range(1, ctrlsAmount+1)]
        self.callback = callback

        self.gridsDB = [{'horizontal': False, 'vertical': False, 'step': None, 'color': color} for color in GRID_COLOR_INIT]

        jumpin_Y = 4
        self.gridActiveCheck = CheckBox((0, jumpin_Y, NET_WIDTH*.6, vanillaControlsSize['CheckBoxRegularHeight']),
                                        "Show grids",
                                        value=self.gridActive,
                                        callback=self.gridActiveCheckCallback)

        for eachI in range(1, ctrlsAmount+1):
            jumpin_Y += vanillaControlsSize['EditTextRegularHeight'] + MARGIN_VER
            gridCtrl = SingleGridController((0, jumpin_Y, NET_WIDTH, vanillaControlsSize['EditTextRegularHeight']),
                                            index=eachI,
                                            isVertical=False,
                                            isHorizontal=False,
                                            step=None,
                                            gridColor=GRID_COLOR_INIT[eachI-1],
                                            callback=self.gridCtrlCallback)
            gridCtrl.enable(self.gridActive)
            setattr(self, 'grid{:0>2}'.format(eachI), gridCtrl)

        jumpin_Y += vanillaControlsSize['EditTextRegularHeight'] + MARGIN_VER
        self.showOffgridCheck = CheckBox((0, jumpin_Y, NET_WIDTH, vanillaControlsSize['CheckBoxRegularHeight']),
                                         "Show offgrid points",
                                         value=self.offgridActive,
                                         callback=self.showOffgridCheckCallback)

    def get(self):
        return self.gridActive, self.gridsDB, self.offgridActive

    def gridCtrlCallback(self, sender):
        ctrlIndex, ctrlDB = sender.get()
        self.gridsDB[ctrlIndex] = ctrlDB
        self.callback(self)

    def gridActiveCheckCallback(self, sender):
        self.gridActive = bool(sender.get())
        for eachI in range(1, self.ctrlsAmount+1):
            gridCtrl = getattr(self, 'grid{:0>2}'.format(eachI))
            gridCtrl.enable(self.gridActive)
        self.callback(self)

    def showOffgridCheckCallback(self, sender):
        self.offgridActive = bool(sender.get())
        self.callback(self)

class SingleGridController(Group):
    """this controller takes care of canvas grid drawing"""

    def __init__(self, posSize, index, isVertical, isHorizontal, step, gridColor, callback):
        Group.__init__(self, posSize)

        # from arguments to attributes
        self.ctrlX, self.ctrlY, self.ctrlWidth, self.ctrlHeight = posSize
        self.index = index
        self.isVertical = isVertical
        self.isHorizontal = isHorizontal
        self.step = step
        self.gridColor = gridColor
        self.callback = callback

        # ctrls
        jumpin_X = 12
        self.indexText = TextBox((jumpin_X, 0, 16, vanillaControlsSize['TextBoxRegularHeight']), '{:d})'.format(index))
        jumpin_X += self.indexText.getPosSize()[2]

        self.stepCtrl = EditText((jumpin_X, 0, 38, vanillaControlsSize['EditTextRegularHeight']),
                                 callback=self.stepCtrlCallback)
        jumpin_X += self.stepCtrl.getPosSize()[2] + 16

        self.isHorizontalCheck = CheckBox((jumpin_X, 0, 32, vanillaControlsSize['CheckBoxRegularHeight']),
                                          "H",
                                          value=self.isHorizontal,
                                          callback=self.isHorizontalCheckCallback)
        jumpin_X += self.isHorizontalCheck.getPosSize()[2]+2

        self.isVerticalCheck = CheckBox((jumpin_X, 0, 32, vanillaControlsSize['CheckBoxRegularHeight']),
                                        "V",
                                        value=self.isVertical,
                                        callback=self.isVerticalCheckCallback)
        jumpin_X += self.isVerticalCheck.getPosSize()[2]+10

        self.whichColorWell = ColorWell((jumpin_X, 0, 46, self.ctrlHeight),
                                        color=NSColor.colorWithCalibratedRed_green_blue_alpha_(*gridColor),
                                        callback=self.whichColorWellCallback)

    def enable(self, onOff):
        self.indexText.enable(onOff)
        self.stepCtrl.enable(onOff)
        self.isHorizontalCheck.enable(onOff)
        self.isVerticalCheck.enable(onOff)
        self.whichColorWell.enable(onOff)

    def get(self):
        return self.index-1, {'horizontal': self.isHorizontal, 'vertical': self.isVertical, 'step': self.step, 'color': self.gridColor}

    def stepCtrlCallback(self, sender):
        try:
            self.step = int(sender.get())
            self.callback(self)
        except ValueError as error:
            self.step = None
            self.stepCtrl.set('')

    def isHorizontalCheckCallback(self, sender):
        self.isHorizontal = bool(sender.get())
        self.callback(self)

    def isVerticalCheckCallback(self, sender):
        self.isVertical = bool(sender.get())
        self.callback(self)

    def whichColorWellCallback(self, sender):
        calibratedColor = sender.get()
        self.gridColor = (calibratedColor.redComponent(),
                          calibratedColor.greenComponent(),
                          calibratedColor.blueComponent(),
                          calibratedColor.alphaComponent())
        self.callback(self)


# Copyright (C) 2014,2015 VA Linux Systems Japan K.K.
# Copyright (C) 2014,2015 YAMAMOTO Takashi <yamamoto at valinux co jp>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron.plugins.ml2.drivers.openvswitch.agent.common import constants
from neutron.plugins.ml2.drivers.openvswitch.agent.openflow.ovs_ofctl \
    import br_dvr_process
from neutron.plugins.ml2.drivers.openvswitch.agent.openflow.ovs_ofctl \
    import ovs_bridge

class OVSPhysicalBridge(ovs_bridge.OVSAgentBridge,
                        br_dvr_process.OVSDVRProcessMixin):
    """openvswitch agent physical bridge specific logic."""

    # Used by OVSDVRProcessMixin
    dvr_process_table_id = constants.DVR_PROCESS_VLAN
    dvr_process_next_table_id = constants.LOCAL_VLAN_TRANSLATION

    def setup_default_table(self):
        self.install_normal()

    def provision_local_vlan(self, port, lvid, segmentation_id, distributed):
        table_id = constants.LOCAL_VLAN_TRANSLATION if distributed else 0
        if segmentation_id is None:
            self.add_flow(table=table_id,
                          priority=4,
                          in_port=port,
                          dl_vlan=lvid,
                          actions="strip_vlan,normal")
        else:
            self.add_flow(table=table_id,
                          priority=4,
                          in_port=port,
                          dl_vlan=lvid,
                          actions="mod_vlan_vid:%s,normal" % segmentation_id)

    def reclaim_local_vlan(self, port, lvid):
        self.delete_flows(in_port=port, dl_vlan=lvid)

    def add_dvr_mac_vlan(self, mac, port):
        self.install_output(table_id=constants.DVR_NOT_LEARN_VLAN,
            priority=2, eth_src=mac, port=port)

    def remove_dvr_mac_vlan(self, mac):
        # REVISIT(yamamoto): match in_port as well?
        self.delete_flows(table=constants.DVR_NOT_LEARN_VLAN,
                          dl_src=mac)


#FCN_16s
import cv2, os
import random
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from sklearn import preprocessing

from sklearn.model_selection import train_test_split, StratifiedKFold
## Import usual libraries
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
import keras, sys, time, warnings
from keras.models import *
from keras.layers import *
from sklearn.utils import shuffle
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import KFold

# Saving and loading model and weights
from keras.models import model_from_json
from keras.models import load_model

import pydensecrf.densecrf as dcrf

from pydensecrf.utils import compute_unary, create_pairwise_bilateral, \
    create_pairwise_gaussian, softmax_to_unary, unary_from_softmax

def give_color_to_seg_img(seg,n_classes):
    '''
    seg : (input_width,input_height,3)
    '''

    if len(seg.shape)==3:
        seg = seg[:,:,0]
    seg_img = np.zeros( (seg.shape[0],seg.shape[1],3) ).astype('float')
    colors = sns.color_palette("hls", n_classes)

    for c in range(n_classes):
        segc = (seg == c)
        seg_img[:,:,0] += (segc*( colors[c][0] ))
        seg_img[:,:,1] += (segc*( colors[c][1] ))
        seg_img[:,:,2] += (segc*( colors[c][2] ))

    return(seg_img)

def loadImage( path , width , height ):
    image = cv2.imread(path, 1)
    image = cv2.resize(image, ( width , height ))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    return image

def loadLabel( path , nClasses ,  width , height  ):
    label = np.zeros((height, width , nClasses ))
    image = cv2.imread(path, 1)
    image = cv2.resize(image, ( width,height ))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image_red = image[:, : , 0] # [width,height,channel]
    image_green = image[:, : , 1] # [width,height,channel]
    image_blue = image[:, : , 2] # [width,height,channel]
    for channel in range(nClasses):
        if channel == 0:
            label[: , : , channel ] = ((image_red == 255) & (image_green == 255) & (image_blue == 255)).astype(int) #name="generalface"
        elif channel == 1:
            label[: , : , channel ] = ((image_red == 0) & (image_green == 0) & (image_blue == 0)).astype(int) #name="background"
        else:
            continue
    return label

def Contigency(nClasses,labels, predicts):
    matrixs = []
    for channel in range(nClasses):
        TP = np.sum( (labels == channel) & (predicts == channel) )
        TN = np.sum( (labels != channel) & (predicts != channel) )
        FP = np.sum( (labels != channel) & (predicts == channel) )
        FN = np.sum( (labels == channel) & (predicts != channel) )

        matrix = []
        matrix.append(TP)
        matrix.append(TN)
        matrix.append(FP)
        matrix.append(FN)

        matrixs.append(matrix)

    return matrixs

def Validate(matrixs):
    ## mean Intersection over Union
    ## Mean IoU = TP/(FN + TP + FP)
    IoUs = []
    channel = 0
    for matrix in matrixs:
        TP = matrix[0]
        TN = matrix[1]
        FP = matrix[2]
        FN = matrix[3]
        IoU = TP/float(TP + FP + FN)
        acc = (TP+TN)/float(TP + TN + FP + FN)
        prec = TP/float(TP + FP)
        rec = TP/float(TP+FN)
        F1 = (2*prec*rec)/float(prec + rec)
        print("____________________________________________________________________")
        print("class {:02.0f}: #TP={:6.0f}, #TN={:6.0f}, #FP={:6.0f}, #FN={:5.0f}".format(channel,TP,TN,FP,FN))
        print("acc={:4.3f}, prec={:4.3f}, rec={:4.3f}, F1={:4.3f}, IoU={:4.3f}".format(acc,prec,rec,F1,IoU))
        channel = channel + 1
        IoUs.append(IoU)
    mIoU = np.mean(IoUs)
    print("_________________")
    print("Mean IoU: {:4.3f}".format(mIoU))

def CRFs(images, y_preds, n_classes):
    X_i = images.astype(np.uint8)
    Y_i = y_preds
    transpose_matrix = np.array(Y_i).transpose((2, 0, 1))
    unary = unary_from_softmax(transpose_matrix, 0.7)
    unary = np.ascontiguousarray(unary)
    d = dcrf.DenseCRF2D(X_i.shape[0], X_i.shape[1], n_classes)
    d.setUnaryEnergy(unary)
    d.addPairwiseGaussian(sxy=3, compat=3)
    d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=X_i, compat=10)
    Q = d.inference(20)
    arr = np.argmax(Q, axis=0)
    arr = arr.ravel().reshape((X_i.shape[0], X_i.shape[1]))
    result = np.zeros((X_i.shape[0], X_i.shape[1], n_classes ))
    for i in range(n_classes):
        result[:,:,i] = (arr==i).astype(int)

    return result

# load json and create model
json_file = open('/content/drive/My Drive/app/F7/output16/model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
global model
model = model_from_json(loaded_model_json)

# this is key : save the graph after loading the model
global graph
graph = tf.get_default_graph()

# load weights into new model
model.load_weights("/content/drive/My Drive/app/F7/output16/model.h5")
print("Loaded model from disk")

##### Read images in folder
location_image = "/content/drive/My Drive/app/F6/dataset1/images_prepped_test/"
location_segmentation = "/content/drive/My Drive/app/F6/dataset1/annotations_prepped_test/"

image_direction = os.listdir(location_image)
image_direction.sort()
segmentation_direction = os.listdir(location_segmentation)
segmentation_direction.sort()

##### Load and augment dataset
n_classes = 2
input_width = 224
input_height = 224

X = []
Y = []

for real_pic , seg_pic in zip(image_direction,segmentation_direction) :
    pre_X = loadImage(location_image + real_pic, input_width, input_height)
    pre_Y = loadLabel(location_segmentation + seg_pic, n_classes, input_width , input_height)
    X.append(pre_X)
    Y.append(pre_Y)

train_rate = 0.85
##### Separate data
X, Y = np.array(X) , np.array(Y)
X_val, y_val = shuffle(X,Y)

##### Validation
print ("=================================================")
print ("Validation before CRFs")
print ("-------------------------")
print ("X_test.shape = ", X_val.shape)
print ("y_test.shape = ", y_val.shape)

print ("-------------------------")
# Validate by contigency
y_pred = model.predict(X_val)
y_predi = np.argmax(y_pred, axis=3)
sh = give_color_to_seg_img(y_predi[0],n_classes)
plt.imshow(sh)
plt.show()
y_testi = np.argmax(y_val, axis=3)
matrixs = Contigency(n_classes, y_testi, y_predi)
Validate(matrixs)

print ("-------------------------")
print ("=================================================")
print ("Validation after CRFs")
print ("-------------------------")
results = []
for i in range(len(X_val)):
    images = X_val[i]
    labels = y_pred[i]
    result = CRFs(images,labels,n_classes)
    results.append(result)

results = np.asarray(results)
resultsi = np.argmax(results, axis=3)
sh = give_color_to_seg_img(resultsi[0],n_classes)
plt.imshow(sh)
plt.show()
matrixs = Contigency(n_classes, y_testi, resultsi)
Validate(matrixs)

print ("-------------------------")
print ("=================================================")


import igraph
from igraph import Graph, plot
import numpy as np
from typing import List, Tuple
from typing import Optional
import config
from api import get_friends
from api_models import User
from api import get_friends

def get_network(users_ids, as_edgelist=True):
    vertices = set()
    surnames = set()

    for user_id in users_ids:
        vertices.add(user_id)

        friends = get_friends(user_id, '')
        surnames_getting = get_friends(user_id, 'last_name')
        for i in surnames_getting['response']['items']:
            surname = i['last_name']
            surnames.add(surname)

        if not 'error' in friends:
            for user_friend in friends['response']['items']:
                vertices.add(user_friend)
    surnames = [sur for sur in surnames]

    vertices = [vert for vert in vertices]

    if as_edgelist:
        edges = []

        for user_id in vertices:
            friends = get_friends(user_id, '')

            if not 'error' in friends:
                for user_friend in friends['response']['items']:
                    if user_friend in vertices:
                        edges.append((vertices.index(user_id), vertices.index(user_friend)))

        return surnames, edges

    else:
        matrix = [[0 for _ in range(len(vertices))] for _ in range(len(vertices))]

        for user_id in vertices:
            friends = get_friends(user_id)

            if not 'error' in friends:
                for user_friend in friends['response']['items']:
                    if user_friend in vertices:
                        matrix[vertices.index(user_id)][vertices.index(user_friend)] = 1

def plot_graph(graph):
    vertices = graph[0]

    edges = graph[1]

    # Создание графа
    g = Graph(vertex_attrs={"label": vertices},
              edges=edges, directed=False)

    # Задаем стиль отображения графа
    N = len(vertices)
    visual_style = {}
    visual_style["layout"] = g.layout_fruchterman_reingold(
        maxiter=1000,
        area=N ** 3,
        repulserad=N ** 3)

    # Отрисовываем граф
    plot(g, **visual_style)

    communities = g.community_edge_betweenness(directed=False)
    clusters = communities.as_clustering()
    print(clusters)
    pal = igraph.drawing.colors.ClusterColoringPalette(len(clusters))
    g.vs['color'] = pal.get_many(clusters.membership)

user_ids = [150572566]
graph = get_network(user_ids)
plot_graph(graph)


#!/usr/bin/python3

import sys
from argparse import ArgumentParser

class grammar():

    rules = []
    max_length = 0
    count_operations = 0

    def __init__(self, verbose=False, filename=''):
        self.verbose = verbose
        self.filename = filename
        self.template = '{:>8} -> {:<12} | {:>%d} -> {:<%d}\n'
        if self.verbose:
            open(self.filename, 'w').close()
        self.read_rules()

    def read_rules(self):
        self.rules = []
        with open(self.__class__.__name__ + '.txt', 'r') as f_in:
            for line in f_in:
                line = line[:-1]
                if line == '':
                    continue
                else:
                    head, _, body = line.split(' ')
                    self.rules.append((head, body))

    def parse(self, number):
        pass

    def apply_rule(self, rule, string):
        new = string.replace(rule[0], rule[1], 1)

        if new != string:
            self.count_operations += 1
            if self.verbose:
                with open(self.filename, 'a') as f_out:
                    f_out.write((self.template % (self.max_length, self.max_length)).format(rule[0], rule[1], string, new))
            string = new

        return string

class grammar0(grammar):

    def parse(self, number):
        self.count_operations = 0
        self.max_length = 2 + number * 2
        string  = '[init]'

        string = self.apply_rule(self.rules[0], string)
        while (self.count_operations < number - 1):
            string = self.apply_rule(self.rules[-1], string)
        done = False
        while not done:
            done = True

            for rule in self.rules:
                    new = self.apply_rule(rule, string)
                    if (new != string):
                        done = False
                        string = new
                        break

        return all(symbol == '0' for symbol in string)

class grammar1(grammar):

    def parse(self, number):
        self.count_operations = 0
        self.max_length = 7 + number
        string = '[init]'

        if (number == 2):
            string = self.apply_rule(self.rules[0], string)
        else:
            string = self.apply_rule(self.rules[1], string)

            while (self.count_operations < number - 1):
                string = self.apply_rule(self.rules[-1], string)
            done = False
            while not done:
                done = True

                for rule in self.rules:
                    new = self.apply_rule(rule, string)
                    if (new != string):
                        done = False
                        string = new
                        break

        return all(symbol == '0' for symbol in string)

grammars = {
    0 : grammar0,
    1 : grammar1
}

def main():
    parser = ArgumentParser(prog = 'prime_grammar.py', description = 'Free (21 rules) and context sensitive (31 rules) grammars for prime numbers.\nParsing and derive.')
    parser.add_argument('-n', '--number', type = int, help = 'number to check for prime', default = -1)
    parser.add_argument('-t', '--type', type = int, help = 'grammar\'s type: 0 -- free; 1 -- context sensitive', default = 1)
    parser.add_argument('-o', '--output', type=str, help='file for output', default='log.txt')
    parser.add_argument('-v', '--verbose', help = 'verbose derive', action='store_true')

    args = parser.parse_args()

    g = grammars[args.type](args.verbose, args.output)

    if (args.number > 1):
        if (g.parse(args.number)):
            answer = 'is prime'
        else:
            answer = 'is not prime'
        print('{} {} | count of operations: {}'.format(args.number, answer, g.count_operations))
    else:
        number = 2
        while True:
            if (g.parse(number)):
                print('{} | count of operations: {}'.format(number, g.count_operations))
            number += 1

if (__name__ == "__main__"):
    main()


import pytest
import os

from trade_journal.trade_journal import TradeJournal
from pathlib import Path

@pytest.fixture
def tjO():
    '''Returns a trade_journal object for a Counter trade'''

    td = TradeJournal(url="../../data/testCounter.xlsx",
                      worksheet="trading_journal",
                      settingf="../../data/settings.ini")
    return td

@pytest.fixture
def clean_tmp():
    yield
    print("\nCleanup .xlsx file")
    os.remove("../../data/testCounter1.xlsx")

def test_tj_nonexists(clean_tmp):
    '''Instantiates a trade_journal object when the .xlsx pointed by url does not exist'''

    fname = Path("../../data/testCounter1.xlsx")

    td = TradeJournal(url=fname,
                      worksheet="trading_journal",
                      settingf="../../data/settings.ini")

    # check that fname has been initialized

    assert fname.exists() is True

def test_fetch_tradelist(tjO):
    trade_list = tjO.fetch_tradelist()

    assert len(trade_list.tlist) == 4

def test_write_tradelist(tjO):
    trade_list = tjO.fetch_tradelist()
    trade_list.analyze()

    tjO.write_tradelist(trade_list)


import numpy as np
import pandas as pd
import sqlalchemy
import psycopg2
import os

from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, inspect
from flask_sqlalchemy import SQLAlchemy

from flask import Flask, jsonify, render_template

engine = create_engine("postgresql+psycopg2://postgres:lovelife2@localhost:5432/Youtube_project")

# Reflect database into  a new model
Base = automap_base()
# Reflect the tables

Base.prepare(engine, reflect=True)

Trending = Base.Classes.youtube
session = Session(engine)

# Flask Setup
#################################################
app = Flask(__name__)

# dialect+driver://username:password@host:port/database
#app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql+psycopg2://postgres:postgres@localhost:5432/Youtube_project"
#db = SQLAlchemy(app)

#################################################
# Flask Routes
#################################################
@app.route("/")
def home():
    return (
        render_template("index.html")
    )

@app.route("/data")
def data():
   sel = [
   Trending.video_id,
   Trending.trending_date,
   Trending.category_id,
   Trending.views,
   Trending.category,
   Trending.likes,
   Trending.dislikes,
   Trending.comments_count
   ]
   results = session.query(*sel).all()
   season_data = []
   for video_id, trending_date, category_id, views, category, likes, dislikes, comments_count  in results:
       season_dict = {}
       season_dict["video_id"] = video_id
       season_dict["trending_date"] = trending_date
       season_dict["category_id"] = category_id
       season_dict["views"] = views
       season_dict["category"] = category
       season_dict["likes"] = likes
       season_dict["dislikes"] = dislikes
       season_dict["comments_count"] = comments_count
       season_data.append(season_dict)
   return jsonify(season_data)


#coding=utf-8

"""
    Faça um Programa que leia um vetor de 5 números inteiros, mostre a soma, a multiplicação e os números.
"""

numbers = []

for i in range(1, 6):
    num = int(input("Informe o {}º número: ".format(i)))
    numbers.append(num)

soma = 0
mult = 1

for x in numbers:
    soma += x
    mult *= x

print("\nNúmeros: {}".format(str(numbers).strip("[]")))
print("Soma dos número: {}\nMultiplicação dos números: {}".format(soma, mult))


from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',

    # url(r'^$', 'fishsite.views.home', name='home'),
    # Main sites
    url(r'^$', include('fishtraits.urls')),
    url(r'^admin/', include(admin.site.urls)),
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    url(r'^download/attr/$','fishtraits.views.download_attr'),
    url(r'^download/key/$',     'fishtraits.views.download_key'),
    url(r'^download/taxo/$','fishtraits.views.download_taxo'),

	url(r'^fishtraits/', include('fishtraits.urls', namespace="fishtraits")),
    url(r'^geodist/(?P<trait_id>\d+)/$', 'fishtraits.views.geodist'),
	url(r'^keyword_search/$', 'fishtraits.views.keyword_search', name='keyword_search'),

    # Load initial dataset
    # url(r'^import$', 'fishtraits.views.index'),

	url(r'^range/$', 'fishtraits.views.range'),

    url(r'^search/attr/$', 'fishtraits.views.search_attr', name='search_attr'),
    url(r'^search/key/$', 'fishtraits.views.search_key', name='search_key'),
    url(r'^search_lookup/$', 'fishtraits.views.search_lookup'),
    url(r'^search/map/$', 'fishtraits.views.search_map'),
    url(r'^search/taxo/$', 'fishtraits.views.search_taxo', name='search_taxo'),

	url(r'^result/attr/$',  'fishtraits.views.result_attr', name='result_attr'),
	url(r'^result/key/$',   'fishtraits.views.result_key',  name='result_key'),
    url(r'^result/taxo/$',  'fishtraits.views.result_taxo', name='result_taxo'),

    url(r'^taxonomy/$', 'fishtraits.views.taxonomy'),

	# django_publications app
	url(r'^publications/', include('publications.urls')),
    url(r'^admin/publications/publication/import_bibtex/$', 'publications.admin_views.import_bibtex'),

)


# -*-  coding:utf-8 -*-

import os

def cleanDir(dir):
    ''' 清空目录以及其下所有子目录和文件'''
    if not os.path.exists(dir): return
    if os.path.isdir(dir):
        paths = os.listdir(dir)
        for path in paths:
            filePath = os.path.join(dir, path)
            cleanDir(filePath)
        os.rmdir(dir)
    else:
        os.remove(dir)

def decodeZoneInfo(zoneId):
    ''' 对zoneId解码，获取国家、省份、城市信息

    Args:
        zoneId - 编码后的区域ID

    Returns:
        返回分解后的元组类型数据，（国家ID，省份ID，城市ID）
    '''
    if not zoneId or zoneId == -1: return None
    mask = 0xffff
    cityId = zoneId & mask
    provId = zoneId >> 16 & mask
    countryId = zoneId >> 32 & mask
    return (None if countryId == mask else countryId, None if provId == mask else provId, None if cityId == mask else cityId)

def encodeZoneInfo(countryId=0xffff, provId=0xffff, cityId=0xffff):
    '''
    将国家ID，省份ID，城市ID编码成一个8字节整数
        无效位添FFFF
    @param countryId:国家id
    @param provId:省id
    @param cityId:城市id
    '''

#    if not countryId or not provId or not cityId: return None
    zoneId = countryId << 32 | provId << 16 | cityId
    #zoneId = unpack('l', pack('L', zoneId & 0xffffffffffffffff))[0]
    return zoneId

def authZoneInfo(userZone, dataZone):
    '''
    验证用户是否对该数据拥有权限
    @param userZone:用户区域信息
    @param dataZone:数据区域信息
    '''
    return dataZone & userZone == dataZone

def authZoneInfo2(userZone, countryId=0xffff, provId=0xffff, cityId=0xffff):
    '''
    验证用户是否对该数据拥有权限
    @param userZone:用户区域信息
    @param countryId:国家id
    @param provId:省id
    @param cityId:城市id
    '''
    dataZone = encodeZoneInfo(int(countryId), int(provId), int(cityId))
    return dataZone & int(userZone) == dataZone

def getRequestHost(request,urlpath='download'):
    '''获取请求中的服务器地址
    Args:
        request:Web服务请求对象
    Return:
        (ip,port)
    '''
    from urlparse import urlsplit
    import urllib2
    ip = ''
    port = 80
    if request:
        referer = request.received_headers.get('referer')
        if referer:
            result = urlsplit(str(referer))
            netloc = result.netloc
            if netloc:
                host = netloc.split(':')
                ip = host[0]
                if len(host) == 2:
                    port = int(host[1])
        if not ip:
            host = request.host
            ip = host.host
            if host.port:
                port = host.port
        try:
            urllib2.urlopen('http://%s:%s/%s'%(ip,port,urlpath))
        except:
            host = request.host
            ip = host.host
            if host.port:
                port = host.port
    return ip,port


from torch.nn.modules.batchnorm import _BatchNorm
from torch.nn.parameter import Parameter
from torch.nn import functional as F

import torch
import torch.nn as nn

class SAN_V3(_BatchNorm):
    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, batch=-1):
        super().__init__(num_features, eps, momentum, affine)
        self.batch = batch

        z_dim = 8

        self.mean_cfc_1 = Parameter(torch.Tensor(num_features, 3, z_dim))
        self.mean_cfc_2 = Parameter(torch.Tensor(num_features, z_dim+1, 1))
        self.mean_cfc_1.data.fill_(0)
        self.mean_cfc_2.data.fill_(0)

        self.std_cfc_1 = Parameter(torch.Tensor(num_features, 3, z_dim))
        self.std_cfc_2 = Parameter(torch.Tensor(num_features, z_dim+1, 1))
        self.std_cfc_1.data.fill_(0)
        self.std_cfc_2.data.fill_(0)

        self.activation = nn.Sigmoid()
        setattr(self.mean_cfc_1, 'affine_gate', True)
        setattr(self.mean_cfc_2, 'affine_gate', True)
        setattr(self.std_cfc_1, 'affine_gate', True)
        setattr(self.std_cfc_2, 'affine_gate', True)

    def get_feat(self, feat):
        B, C, H, W = feat.size()

        feat = feat.view(B, C, -1)
        mean_i = feat.mean(dim=2, keepdim=True)
        var_i = feat.var(dim=2, keepdim=True)
        std_i = var_i.sqrt()

        feat_ones = torch.ones(B, C, 1).cuda()

        mean_feat = torch.cat((mean_i, std_i, feat_ones), dim=2)

        return torch.transpose(mean_feat, 0, 1)

    def forward(self, input):
        B, C, H, W = input.size()
        feat_ones = torch.ones(C, B, 1).cuda()

        feat = self.get_feat(input)

        mean_att_z = torch.bmm(feat, self.mean_cfc_1)
        mean_att_z = torch.cat((mean_att_z, feat_ones), dim=2)
        mean_att = torch.bmm(mean_att_z, self.mean_cfc_2)
        mean_att = torch.transpose(mean_att, 0, 1).squeeze()
        mean_att = self.activation(mean_att)

        std_att_z = torch.bmm(feat, self.std_cfc_1)
        std_att_z = torch.cat((std_att_z, feat_ones), dim=2)
        std_att = torch.bmm(std_att_z, self.std_cfc_2)
        std_att = torch.transpose(std_att, 0, 1).squeeze()
        std_att = self.activation(std_att)

        # Compute BN (B x C x None)
        if self.batch == -1 or self.batch == B:
            out = F.batch_norm(
                    input, self.running_mean, self.running_var, None, None,
                    self.training, self.momentum, self.eps)
        else:
            out = input.new(input.size())
            for i in range(0, B, self.batch):
                j = min(B, i + self.batch)
                out[i:j, :, :, :] = F.batch_norm(
                    input[i:j, :, :, :], self.running_mean, self.running_var, None, None,
                    self.training, self.momentum, self.eps)

        out.mul_(self.weight[None, :, None, None] * std_att[:, :, None, None])
        out.add_(self.bias[None, :, None, None] * mean_att[:, :, None, None])
        out = out.view(B, C, H, W)

        return out

    def extra_repr(self):
        return '{num_features}, eps={eps}, momentum={momentum}, affine={affine}, ' \
                'track_running_stats={track_running_stats}, ' \
                'batch={batch}'.format(**self.__dict__)


"""
    Объединение файлов Excel путем складывания значений во всех листах с выгрузкой результата в файл RESULT.xls.
    Форматирование в результирующем файле при этом теряется.

    Использование:
        Помещаем файл excel_merge.py в каталог с файлами *.xls, *.xlsx, *.ods и запускаем его
        В результате в текущем каталоге создастся файл RESULT.xls и автоматически откроется.

    Примечания:
        Во всех файлах должны совпадать имена Листов
        По умолчанию обработка ведется начиная со второго столбца (B) второй строки (Можно изменить в переменных
            X_MIN, X_MAX, Y_MIN, Y_MAX)

    Установка:
        Для использования данного скрипта, необходим Python3 и библиотека pyexcel
        Python3 можно скачать по ссылке: https://www.python.org/downloads/ затем установить
        Для установки расширения pyexcel выполнить в командной строке команду:
            pip install pyexcel pyexcel-xls pyexcel-xlsx pyexcel-ods

    Данный скрипт был протестирован на Python 3.6, pyexcel==0.6.2
"""

import os, sys
import pyexcel
import glob

curdir = os.path.abspath(os.path.dirname(__file__)) # Текущий каталог, где лежит этот файл

# === Настройки ===
WORKDIR = curdir # Рабочий каталог, он же текущий по умолчанию

if WORKDIR:
    os.chdir(WORKDIR) # Работа в текущем каталоге

INFILE = None # Начальный файл. Может быть None, тогда начальный файл считается первый по списку # os.path.join(curdir,'out','infile.xlsx')
OUTFILE = 'RESULT.xls' # Имя исходящего результирующего файла XLS (может находиться в другой папке) # os.path.join(curdir,'out','out.xls')

X_MIN = 2 # Номер столбца для начала обработки
X_MAX = 100000 # Максимальный Номер столбца для конца обработки
Y_MIN = 2 # Номер начальной строки
Y_MAX = 100000 # Максимальный Номер конечной строки

ISOPEN_FILE = True  # Открыть результирующий файл после обработки?
ISNOTCLOSE = True  # После обработки файла не закрывать окно скрипта

def parse_float(x):
    if x == '':
        res = 0
    else:
        try:
            res = float(x)
        except:
            res = None
    return res

def merge_all_sheets():
    """ Складывает все значения в файлах Excel в рабочей директории по всем листам и записывает результат в RESULT.xls
          Названия листов во всех файлах должны совпадать, так же как и диапазоны строк и столбцов """
    err_files = set()
    err_all = 0
    files_count = 0
    book_res = None  # Рабочий массив данных
    if INFILE:  # В рабочий массив загружаем данные из начального файла, если он задан
        book_res = pyexcel.get_book_dict(file_name=INFILE)

    files = glob.glob('*.xls*')
    files.extend(glob.glob('*.ods'))
    for fn in files:
        if not fn.startswith('~') and fn != OUTFILE and fn != INFILE:
            print(fn)
            files_count += 1
            if book_res is None:
                book_res = pyexcel.get_book_dict(file_name=fn)
            else:
                book_cur = pyexcel.get_book_dict(file_name=fn)
                for sheet in book_res:
                    #print(sheet)
                    for y, row in enumerate(book_res[sheet]):
                        #print(y, row)
                        for x, cell in enumerate(book_res[sheet][y]):
                            if y>=Y_MIN-1 and y<=Y_MAX-1 and x>=X_MIN-1 and x<=X_MAX-1:  # Ограничиваем обработку только определенным диапазоном
                                try:
                                    cell1 = parse_float(book_res[sheet][y][x])
                                    cell2 = parse_float(book_cur[sheet][y][x])
                                    if cell1 is not None and cell2 is not None:
                                        book_res[sheet][y][x] = cell1 + cell2
                                        #print(x, book_res[sheet][y][x], type(book_res[sheet][y][x]))
                                except:
                                    print('Ошибка! Файл: {} Лист: {} Строка: {} Колонка: {}'.format(fn, sheet, y, x))
                                    err_all += 1
                                    err_files.add(fn)
                    #print(book_res[sheet])
    print('\nОбработка завершена!\n  Всего обработано файлов: {}'.format(files_count))
    if err_all > 0:
        print('  Всего ошибок: {}'.format(err_all))
        print('  Ошибки в файлах: {}'.format(', '.join(err_files)))
    else:
        print('  Ошибок нет')
    pyexcel.save_book_as(bookdict=book_res, dest_file_name=OUTFILE)

    if ISOPEN_FILE:
        import os
        os.startfile(OUTFILE)

def merge_first_sheet():
    """ Объединяет значения в файлах Excel в рабочей директории и записывает результат в RESULT.xls
          Берутся только первые листы
    """
    err_files = set()
    err_all = 0
    files_count = 0
    sheet_res = None  # Рабочий массив данных
    if INFILE:  # В рабочий массив загружаем данные из начального файла, если он задан
        sheet_res = pyexcel.get_book_dict(file_name=INFILE)
        #sheet_name =

    files = glob.glob('*.xls*')
    sheets = []
    for fn in files:
        if not fn.startswith('~') and fn != OUTFILE and fn != INFILE:
            print(fn)
            files_count += 1
            if sheet_res is None:
                #sheet_res = pyexcel.get_sheet(file_name=fn)
                sheet_res = pyexcel.get_array(file_name=fn)
            else:
                sheet_cur = pyexcel.get_array(file_name=fn)
                # Определение названия листа (берется первый)
                sheet_name = list(pyexcel.get_book_dict(file_name=fn).keys())[0]
                #print(sheet_name)
                #print(sheet_res)
                for y, row in enumerate(sheet_res):
                    #print(y, row)
                    for x, cell in enumerate(sheet_res[y]):
                        if y>=Y_MIN-1 and y<=Y_MAX-1 and x>=X_MIN-1 and x<=X_MAX-1:  # Ограничиваем обработку только определенным диапазоном
                            try:
                                cell1 = parse_float(sheet_res[y][x])
                                cell2 = parse_float(sheet_cur[y][x])
                                if cell1 is not None and cell2 is not None:
                                    sheet_res[y][x] = cell1 + cell2
                                    #print(x, sheet_res[y][x], type(sheet_res[y][x]))
                            except:
                                print('Ошибка! Файл: {} Лист: {} Строка: {} Колонка: {}'.format(fn, sheet_name, y, x))
                                err_all += 1
                                err_files.add(fn)
                #print(book_res[sheet])
    print('\nОбработка завершена!\n  Всего обработано файлов: {}'.format(files_count))
    if err_all > 0:
        print('  Всего ошибок: {}'.format(err_all))
        print('  Ошибки в файлах: {}'.format(', '.join(err_files)))
    else:
        print('  Ошибок нет')

    pyexcel.save_as(array=sheet_res, dest_file_name=OUTFILE)

    if ISOPEN_FILE:
        import os
        os.startfile(OUTFILE)

def main():
    print('Версия Python:', sys.version)
    print('Текущая папка:', curdir)  # .decode('utf-8'),
    print('Рабочая папка:', WORKDIR)  # .decode('utf-8'),
    print()
    # Слить все файлы в рабочей директории со всеми листами
    merge_all_sheets()
    # Слить все файлы в рабочей директории только по первым листам
    #merge_first_sheet()

if __name__=='__main__':
    main()
    if ISNOTCLOSE:
        input()


#!/usr/bin/env python
"""
    Script that migrates the users from UMR 0.8.1 to 0.9.0
    It goes through all the users in the database, grabs
    their learned_language and the languages of their sources
    and adds these languages to the user_language table.

"""

import zeeguu.core
from zeeguu.core.model import User, UserLanguage, RSSFeedRegistration

session = zeeguu.core.db.session

counter = 0

all_users = User.query.all()

for user in all_users:
    learned_languages = []

    # Grab the learned language and add it to learned_languages
    learned_languages.append(user.learned_language)

    # Go through the user sources
    user_feed_registrations = RSSFeedRegistration.feeds_for_user(user)

    for feed_reg in user_feed_registrations:
        # Try catch is needed because apparently there are NoneType feeds in registrations..
        try:
            feed = feed_reg.rss_feed
            feed_language = feed.language
            if feed_language not in learned_languages:
                learned_languages.append(feed_language)
        except Exception as e:
            print(e)

    for language in learned_languages:
        user_language = UserLanguage.find_or_create(session, user, language)
        user_language.reading_news = True
        session.add(user_language)

    print(f'Added user_languages for user user {user}')
    session.commit()


# send and receive numpy arrays on ZeroMQ
import zmq
import numpy
import torch
import json

def send_array_and_str(socket, A, sentence, flags=0, copy=True, track=False):
    """send a numpy array with metadata"""
    md = dict(dtype = str(A.dtype),shape = A.shape)
    socket.send_string(sentence, flags|zmq.SNDMORE)
    socket.send_json(md, flags|zmq.SNDMORE)
    return socket.send(A, flags, copy=copy, track=track)

def recv_array_and_str(socket, flags=0, copy=True, track=False):
    """recv a numpy array and sentence"""
    sentence = socket.recv_string(flags=flags)
    md = socket.recv_json(flags=flags)
    msg = socket.recv(flags=flags, copy=copy, track=track)
    buf = memoryview(msg)
    A = numpy.frombuffer(buf, dtype=md['dtype'])
    return sentence, A.reshape(md['shape'])

def preprocess(text, tokenizer):
    '''tokenize text into subwords and convert to indices
    :param text str: text to be preprocessed
    :param tokenizer: BertTokenizer object
    :output: torch tensor of vocab ids
    '''
    tokenized_text = tokenizer.tokenize(text)
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    tokens_tensor = torch.tensor([indexed_tokens]).numpy()
    return tokens_tensor


"""empty message

Revision ID: b56e01bac4ee
Revises: 477bbbfb48d6
Create Date: 2018-12-28 11:14:36.963527

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'b56e01bac4ee'
down_revision = '477bbbfb48d6'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('uage', sa.Integer(), nullable=True))
    op.add_column('user', sa.Column('uemail', sa.String(length=200), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('user', 'uemail')
    op.drop_column('user', 'uage')
    # ### end Alembic commands ###