# Copyright (C) 2020 Jørgen S. Dokken
#
# This file is part of DOLFINX_MPC
#
# SPDX-License-Identifier:    LGPL-3.0-or-later
#
import dolfinx
import dolfinx.geometry as geometry
import dolfinx.mesh as dmesh
import dolfinx.fem as fem
import dolfinx.io
import dolfinx_mpc
import dolfinx_mpc.utils
import numpy as np
import ufl
from petsc4py import PETSc

comp_col_pts = geometry.compute_collisions_point
get_basis = dolfinx_mpc.cpp.mpc.get_basis_functions

def demo_stacked_cubes():
    with dolfinx.io.XDMFFile(dolfinx.MPI.comm_world,
                             "meshes/mesh3D.xdmf", "r") as xdmf:
        mesh = xdmf.read_mesh(name="Grid")
    mesh.create_connectivity_all()

    V = dolfinx.VectorFunctionSpace(mesh, ("Lagrange", 1))

    # Define boundary conditions (HAS TO BE NON-MASTER NODES)
    u_bc = dolfinx.function.Function(V)
    with u_bc.vector.localForm() as u_local:
        u_local.set(0.0)

    # Move top in y-dir
    V0 = V.sub(0).collapse()
    V1 = V.sub(2).collapse()
    V2 = V.sub(2).collapse()

    def top(x):
        return np.isclose(x[2], 2)
    fdim = mesh.topology.dim - 1
    top_facets = dmesh.locate_entities_geometrical(
        mesh, fdim, top, boundary_only=True)
    top_values = np.full(len(top_facets), 3, dtype=np.intc)
    mt = dolfinx.mesh.MeshTags(mesh, fdim,
                               top_facets,
                               top_values)

    ds = ufl.Measure("ds", domain=mesh, subdomain_data=mt, subdomain_id=3)
    g = dolfinx.Constant(mesh, (0, 0, -5*9.81e1))

    # Define boundary conditions (HAS TO BE NON-MASTER NODES)
    u_bc = dolfinx.function.Function(V)
    with u_bc.vector.localForm() as u_local:
        u_local.set(0.0)

    # Fix bottom in all directions
    def bottom(x):
        return np.isclose(x[2], 0)
    bottom_facets = dmesh.locate_entities_geometrical(
        mesh, fdim, bottom, boundary_only=True)
    bottom_dofs = fem.locate_dofs_topological(V, fdim, bottom_facets)
    bc_bottom = fem.DirichletBC(u_bc, bottom_dofs)

    # Fix top corner in x-direction
    zero = dolfinx.Function(V0)
    with zero.vector.localForm() as zero_local:
        zero_local.set(0.0)

    def first_corner(x):
        return np.isclose(x.T, [0, 0, 2]).all(axis=1)

    def second_corner(x):
        return np.isclose(x.T, [0, 1, 2]).all(axis=1)

    def third_corner(x):
        return np.isclose(x.T, [0, 1, 2]).all(axis=1)

    def fourth_corner(x):
        return np.isclose(x.T, [1, 1, 2]).all(axis=1)

    bcs = [bc_bottom]
    for cond in [first_corner, second_corner,
                 third_corner, fourth_corner]:
        for i, Vi in enumerate([V0, V1]):
            dofs = fem.locate_dofs_geometrical((V.sub(i), Vi), cond)
            bc = dolfinx.DirichletBC(zero, dofs, V.sub(i))
            bcs.append(bc)

    # Define variational problem
    u = ufl.TrialFunction(V)
    v = ufl.TestFunction(V)

    # Elasticity parameters
    E = 1.0e4
    nu = 0.1
    mu = dolfinx.Constant(mesh, E / (2.0 * (1.0 + nu)))
    lmbda = dolfinx.Constant(mesh, E * nu / ((1.0 + nu) * (1.0 - 2.0 * nu)))

    # Stress computation
    def sigma(v):
        return (2.0 * mu * ufl.sym(ufl.grad(v)) +
                lmbda * ufl.tr(ufl.sym(ufl.grad(v))) * ufl.Identity(len(v)))

    # Define variational problem
    u = ufl.TrialFunction(V)
    v = ufl.TestFunction(V)
    a = ufl.inner(sigma(u), ufl.grad(v)) * ufl.dx
    lhs = ufl.inner(dolfinx.Constant(mesh, (0, 0, 0)), v)*ufl.dx\
        + ufl.inner(g, v)*ds

    # Create MPC
    slaves = []
    masters = []
    coeffs = []
    offsets = [0]

    # Locate dofs on both interfaces
    def boundaries(x):
        return np.isclose(x[2], 1)
    facets = dmesh.locate_entities_geometrical(
        mesh, fdim, boundaries, boundary_only=True)
    # Slicing of list is due to the fact that we only require y-components
    interface_dofs = fem.locate_dofs_topological((V.sub(2), V2),
                                                 mesh.topology.dim-1,
                                                 facets)[:, 0]

    # Get some cell info
    # (range should be reduced with mesh function and markers)
    global_indices = V.dofmap.index_map.global_indices(False)
    num_cells = mesh.topology.index_map(mesh.topology.dim).size_local
    cell_midpoints = dolfinx.cpp.mesh.midpoints(mesh, mesh.topology.dim,
                                                range(num_cells))
    x_coords = V.tabulate_dof_coordinates()
    tree = geometry.BoundingBoxTree(mesh, mesh.topology.dim)
    mesh.create_connectivity(mesh.topology.dim, mesh.topology.dim-1)
    mesh.create_connectivity(mesh.topology.dim-1, mesh.topology.dim)
    cell_to_facet = mesh.topology.connectivity(mesh.topology.dim,
                                               mesh.topology.dim-1)
    for cell_index in range(num_cells):
        midpoint = cell_midpoints[cell_index]
        # NOTE, we should rather use mesh markers here to
        # determine if this could be a slave cell
        is_slave_cell = midpoint[2] < 1

        cell_dofs = V.dofmap.cell_dofs(cell_index)

        # Check if any dofs in cell is on interface
        has_dofs_on_interface = np.isin(cell_dofs, interface_dofs)
        if np.any(has_dofs_on_interface) and is_slave_cell:
            # Check if it is a slave node (x[1]>1)
            local_indices = np.flatnonzero(has_dofs_on_interface)
            for dof_index in local_indices:
                slave_coords = x_coords[cell_dofs[dof_index]]
                global_slave = global_indices[cell_dofs[dof_index]]
                if global_slave not in slaves:
                    slaves.append(global_slave)
                    # Need some parallel comm here
                    # Now find which master cell is close to the slave dof
                    possible_cells = comp_col_pts(tree, slave_coords)[0]
                    for other_cell in possible_cells:
                        if other_cell == cell_index:
                            continue
                        if cell_midpoints[other_cell][2] < 1:
                            continue
                        other_facets = cell_to_facet.links(other_cell)
                        has_boundary_facets = np.any(np.isin(other_facets,
                                                             facets))
                        if has_boundary_facets:
                            other_dofs = V.dofmap.cell_dofs(other_cell)
                            # Get basis values, and add coeffs of sub space 2
                            # to masters with corresponding coeff
                            basis_values = get_basis(V._cpp_object,
                                                     slave_coords, other_cell)
                            z_values = basis_values[:, 2]
                            z_dofs = V.sub(2).dofmap.cell_dofs(other_cell)
                            other_local_indices = np.flatnonzero(
                                np.isin(other_dofs, z_dofs))
                            for local_idx in other_local_indices:
                                if not np.isclose(z_values[local_idx], 0):
                                    masters.append(
                                        global_indices[other_dofs[local_idx]])
                                    coeffs.append(z_values[local_idx])

                            offsets.append(len(masters))

                            break

    slaves, masters, coeffs, offsets = (np.array(slaves), np.array(masters),
                                        np.array(coeffs), np.array(offsets))

    mpc = dolfinx_mpc.cpp.mpc.MultiPointConstraint(V._cpp_object, slaves,
                                                   masters, coeffs, offsets)

    # Setup MPC system
    A = dolfinx_mpc.assemble_matrix(a, mpc, bcs=bcs)
    b = dolfinx_mpc.assemble_vector(lhs, mpc)

    # Apply boundary conditions
    fem.apply_lifting(b, [a], [bcs])
    b.ghostUpdate(addv=PETSc.InsertMode.ADD_VALUES,
                  mode=PETSc.ScatterMode.REVERSE)
    fem.set_bc(b, bcs)

    # Solve Linear problem
    solver = PETSc.KSP().create(dolfinx.MPI.comm_world)
    solver.setType(PETSc.KSP.Type.PREONLY)
    solver.getPC().setType(PETSc.PC.Type.LU)
    solver.setOperators(A)
    uh = b.copy()
    uh.set(0)
    solver.solve(b, uh)
    uh.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                   mode=PETSc.ScatterMode.FORWARD)

    # Back substitute to slave dofs
    dolfinx_mpc.backsubstitution(mpc, uh, V.dofmap)

    # Create functionspace and function for mpc vector
    Vmpc_cpp = dolfinx.cpp.function.FunctionSpace(mesh, V.element,
                                                  mpc.mpc_dofmap())
    Vmpc = dolfinx.FunctionSpace(None, V.ufl_element(), Vmpc_cpp)

    # Write solution to file
    u_h = dolfinx.Function(Vmpc)
    u_h.vector.setArray(uh.array)
    u_h.name = "u_mpc"
    outfile = dolfinx.io.XDMFFile(dolfinx.MPI.comm_world,
                                  "results/cube_bench3d.xdmf", "w")
    outfile.write_mesh(mesh)
    outfile.write_function(u_h)

    # Transfer data from the MPC problem to numpy arrays for comparison
    A_mpc_np = dolfinx_mpc.utils.PETScMatrix_to_global_numpy(A)
    mpc_vec_np = dolfinx_mpc.utils.PETScVector_to_global_numpy(b)

    # Solve the MPC problem using a global transformation matrix
    # and numpy solvers to get reference values

    # Generate reference matrices and unconstrained solution
    A_org = fem.assemble_matrix(a, bcs)

    A_org.assemble()
    L_org = fem.assemble_vector(lhs)
    fem.apply_lifting(L_org, [a], [bcs])
    L_org.ghostUpdate(addv=PETSc.InsertMode.ADD_VALUES,
                      mode=PETSc.ScatterMode.REVERSE)
    fem.set_bc(L_org, bcs)

    # Create global transformation matrix
    K = dolfinx_mpc.utils.create_transformation_matrix(V.dim(), slaves,
                                                       masters, coeffs,
                                                       offsets)
    # Create reduced A
    A_global = dolfinx_mpc.utils.PETScMatrix_to_global_numpy(A_org)
    reduced_A = np.matmul(np.matmul(K.T, A_global), K)
    # Created reduced L
    vec = dolfinx_mpc.utils.PETScVector_to_global_numpy(L_org)
    reduced_L = np.dot(K.T, vec)
    # Solve linear system
    # d = np.linalg.solve(reduced_A, reduced_L)
    # Back substitution to full solution vector
    # uh_numpy = np.dot(K, d)

    # # Compare LHS, RHS and solution with reference values
    dolfinx_mpc.utils.compare_matrices(reduced_A, A_mpc_np, slaves)
    dolfinx_mpc.utils.compare_vectors(reduced_L, mpc_vec_np, slaves)

if __name__ == "__main__":
    demo_stacked_cubes()


class Solution:

    def solve(self, spcl_idx, nds, cost, _min):
        if spcl_idx == len(self._specials):
            for i, nd in enumerate(nds):
                cost += nd * self._prices[i]
            return cost

        spcl = self._specials[spcl_idx]
        n = 0
        flag = True
        while flag:
            n += 1
            for i, s in enumerate(spcl[:-1]):
                if s * n > nds[i]:
                    flag = False
                    break

        for i in range(n):
            new_nds = [nd - s * i for nd, s in zip(nds, spcl[:-1])]
            if i != 0:
                cost += spcl[-1]
            _min = min(self.solve(spcl_idx + 1, new_nds, cost, _min), _min)

        return _min

    def shoppingOffers(self, prices, specials, needs):
        self._prices = prices
        self._specials = specials
        self._needs = needs

        return self.solve(0, needs, 0, 10 ** 9)


from __future__ import absolute_import
from __future__ import unicode_literals

import unittest

import mock
from pytest import raises

from paasta_tools.deployd.common import BounceTimers
from paasta_tools.deployd.common import ServiceInstance
from paasta_tools.deployd.workers import PaastaDeployWorker
from paasta_tools.marathon_tools import DEFAULT_SOA_DIR

class TestPaastaDeployWorker(unittest.TestCase):
    def setUp(self):
        self.mock_inbox_q = mock.Mock()
        self.mock_bounce_q = mock.Mock()
        self.mock_metrics = mock.Mock()
        mock_config = mock.Mock(
            get_cluster=mock.Mock(return_value='westeros-prod'),
            get_deployd_worker_failure_backoff_factor=mock.Mock(return_value=30)
        )
        with mock.patch(
            'paasta_tools.deployd.workers.PaastaDeployWorker.setup', autospec=True
        ):
            self.worker = PaastaDeployWorker(1,
                                             self.mock_inbox_q,
                                             self.mock_bounce_q,
                                             mock_config,
                                             self.mock_metrics)

    def test_setup(self):
        with mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.load_marathon_config', autospec=True
        ), mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.get_marathon_client', autospec=True
        ):
            self.worker.setup()

    def test_setup_timers(self):
        mock_si = mock.Mock(bounce_timers=None,
                            service='universe',
                            instance='c137')
        ret = self.worker.setup_timers(mock_si)
        calls = [mock.call('bounce_length_timer',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137'),
                 mock.call().start(),
                 mock.call('processed_by_worker',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137'),
                 mock.call('setup_marathon_timer',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137')]
        self.mock_metrics.create_timer.assert_has_calls(calls)
        assert ret == BounceTimers(processed_by_worker=self.mock_metrics.create_timer.return_value,
                                   setup_marathon=self.mock_metrics.create_timer.return_value,
                                   bounce_length=self.mock_metrics.create_timer.return_value)

    def test_run(self):
        with mock.patch(
            'time.sleep', autospec=True
        ) as mock_sleep, mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.get_all_marathon_apps', autospec=True
        ) as mock_get_all_marathon_apps, mock.patch(
            'paasta_tools.deployd.workers.PaastaDeployWorker.setup_timers', autospec=True
        ) as mock_setup_timers, mock.patch(
            'paasta_tools.deployd.workers.deploy_marathon_service', autospec=True
        ) as mock_deploy_marathon_service, mock.patch(
            'time.time', autospec=True, return_value=1
        ):
            mock_sleep.side_effect = LoopBreak
            self.worker.marathon_client = mock.Mock()
            self.worker.marathon_config = mock.Mock()
            mock_deploy_marathon_service.return_value = (0, None)
            mock_si = mock.Mock(service='universe',
                                instance='c137',
                                failures=0)
            self.mock_bounce_q.get.return_value = mock_si
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert not mock_setup_timers.return_value.processed_by_worker.start.called
            assert not self.mock_inbox_q.put.called
            assert mock_setup_timers.return_value.bounce_length.stop.called

            mock_deploy_marathon_service.return_value = (0, 60)
            mock_setup_timers.return_value.bounce_length.stop.reset_mock()
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert mock_setup_timers.return_value.processed_by_worker.start.called
            self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe',
                                                                     instance='c137',
                                                                     bounce_by=61,
                                                                     watcher='Worker1',
                                                                     bounce_timers=mock_setup_timers.return_value,
                                                                     failures=0))
            assert not mock_setup_timers.return_value.bounce_length.stop.called

            mock_deploy_marathon_service.side_effect = Exception()
            mock_setup_timers.return_value.bounce_length.stop.reset_mock()
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert mock_setup_timers.return_value.processed_by_worker.start.called
            self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe',
                                                                     instance='c137',
                                                                     bounce_by=61,
                                                                     watcher='Worker1',
                                                                     bounce_timers=mock_setup_timers.return_value,
                                                                     failures=1))
            assert not mock_setup_timers.return_value.bounce_length.stop.called

class LoopBreak(Exception):
    pass


import torch
from torch.autograd import Variable
from torch.nn import Module
import h5py

import numpy as np

from allennlp.data.dataset import Dataset
from allennlp.data import Token, Vocabulary, Instance
from allennlp.data.fields import TextField
from allennlp.data.token_indexers.elmo_indexer import ELMoTokenCharactersIndexer
from allennlp.nn.util import remove_sentence_boundaries
from allennlp.modules.elmo import _ElmoBiLm

from chunking.data import variableFromSentence

options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
elmo_bilm = _ElmoBiLm(options_file, weight_file).cuda()

indexer = ELMoTokenCharactersIndexer()

__all__ = ['elmo_bilm', 'embed_sentence', 'ElmoEmbedder', 'variablesFromPairElmo', 'elmo_variable_from_sentence']

use_cuda = torch.cuda.is_available()

class ElmoEmbedder(Module):
    def __init__(self, elmo_bilm, special_tokens, device):
        super(ElmoEmbedder, self).__init__()
        self.elmo_bilm = elmo_bilm
        self.device = device
        self.special_tokens = special_tokens
        self.elmo_id_to_special_token = {tuple(token_to_elmo_id(tok).view(-1).data): tok_id for (tok_id, tok) in enumerate(special_tokens)}
        self.special_token_to_id = {tok: i for (i, tok) in enumerate(special_tokens)}
        self.dimension = len(special_tokens) + 1024
        self.cached_embedding = (-1, None)

    def forward(self, input, token_index):
        input_code = tuple([tuple(input[0][i].data) for i in range(input.shape[1])])
        token_code = tuple(input[0][token_index].data)
        if token_code in self.elmo_id_to_special_token:
            tok_id = self.elmo_id_to_special_token[token_code]
            result = Variable(torch.from_numpy(np.eye(self.dimension)[tok_id]).float())
        elif input_code == self.cached_embedding[0]:
            embedded = self.cached_embedding[1]
            result = torch.cat([Variable(torch.from_numpy(np.zeros(len(self.special_tokens))).float()),
                                Variable(embedded[token_index])])
        else:
            embedded = embed_numerical_sent(input, elmo_bilm, self.device)
            self.cached_embedding = (input_code, embedded)
            result = torch.cat([Variable(torch.from_numpy(np.zeros(len(self.special_tokens))).float()),
                                Variable(embedded[token_index])])
        if use_cuda:
            return result.cuda()
        else:
            return result

def character_ids_to_embeddings(character_ids, elmo_bilm, device):
    # returns (batch_size, 3, num_times, 1024) embeddings and (batch_size, num_times) mask
    if device >= 0:
        character_ids = character_ids.cuda(device=device)
    bilm_output = elmo_bilm(character_ids)
    layer_activations = bilm_output['activations']
    mask_with_bos_eos = bilm_output['mask']
    without_bos_eos = [remove_sentence_boundaries(layer, mask_with_bos_eos)
            for layer in layer_activations]
    # without_bos_eos is a 3 element list of (batch_size, num_times, dim) arrays
    activations = torch.cat([ele[0].unsqueeze(1) for ele in without_bos_eos], dim=1)
    mask = without_bos_eos[0][1]
    return activations, mask

def embed_numerical_sent(input_var, elmo_bilm, device):
    embeddings, mask = character_ids_to_embeddings(input_var, elmo_bilm, device)
    sent_embeddings = []
    for i in range(1):
        length = int(mask[i, :].sum())
        sentence_embedding = embeddings[i, :, :length, :].data.cpu().numpy()
        a = torch.from_numpy(sentence_embedding[0])
        b = torch.from_numpy(sentence_embedding[1])
        c = torch.from_numpy(sentence_embedding[2])
        result = (a + b + c) / 3.0
        sent_embeddings.append(result)
    return sent_embeddings[0]

def token_to_elmo_id(token):
    tokens = [Token(token)]
    field = TextField(tokens, {'character_ids': indexer})
    instance = Instance({"elmo": field})
    instances = [instance]
    dataset = Dataset(instances)
    vocab = Vocabulary()
    for instance in dataset.instances:
        instance.index_fields(vocab)
    #dataset.index_instances(vocab) # replaced by above, so that there's no progress bar
    return dataset.as_tensor_dict()['elmo']['character_ids']

def batch_to_ids(batch):
    """
    Given a batch (as list of tokenized sentences), return a batch
    of padded character ids.
    """
    instances = []
    for sentence in batch:
        tokens = [Token(token) for token in sentence]
        field = TextField(tokens, {'character_ids': indexer})
        instance = Instance({"elmo": field})
        instances.append(instance)

    dataset = Dataset(instances)
    vocab = Vocabulary()
    for instance in dataset.instances:
        instance.index_fields(vocab)
    #dataset.index_instances(vocab)        #replaced by above, so there's no progress bar
    return dataset.as_tensor_dict()['elmo']['character_ids']

def elmo_variable_from_sentence(sent):
    tokens = sent.split()
    tokens.append('eos')
    return batch_to_ids([tokens])

# Maps the source and target sentence into PyTorch vectors (of the token ids).
def variablesFromPairElmo(pair, output_lang):
    input_variable = elmo_variable_from_sentence(pair[0])
    target_variable = variableFromSentence(output_lang, pair[1])
    return (input_variable, target_variable)

def pad(tokens, desired_length, padder):
    padding = [padder] * (desired_length - len(tokens))
    return tokens + padding

def elmo_variable_from_sentences(sents):
    tokens = [sent.split() for sent in sents]
    max_size = max([len(x) for x in tokens]) + 1
    padded = [pad(tok, max_size, padder='eos') for tok in tokens]
    return batch_to_ids(padded)


import re

from datetime import datetime, timedelta

import string

from lxml import etree, objectify

class CT_Chart_Container(objectify.ObjectifiedElement):
    """
    ``<p:graphicFrame>`` element, which is a container for a table, a chart,
    or another graphical object.
    """
    DATATYPE_TABLE = 'http://schemas.openxmlformats.org/drawingml/2006/table'

    _chart_tmpl = (

      '<c:chartSpace %s>'
      '<c:date1904 val="0"/>'
      '<c:lang val="en-US"/>'
      '<c:roundedCorners val="0"/>'
      '<mc:AlternateContent xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006">'
      '<mc:Choice Requires="c14" xmlns:c14="http://schemas.microsoft.com/office/drawing/2007/8/2/chart">'
      '<c14:style val="102"/>'
      '</mc:Choice>'
      '<mc:Fallback>'
      '<c:style val="2"/>'
      '</mc:Fallback>'
      '</mc:AlternateContent>'
      '<c:chart>'
      '<c:autoTitleDeleted val="0"/>'
      '<c:plotArea>'
      '<c:layout/>'

      '</c:plotArea>'
      '<c:legend>'
      '<c:legendPos val="r"/>'
          '<c:layout/>'
          '<c:overlay val="0"/>'
        '</c:legend>'
        '<c:plotVisOnly val="1"/>'
        '<c:dispBlanksAs val="gap"/>'
        '<c:showDLblsOverMax val="0"/>'
      '</c:chart>'
      '<c:txPr>'
      '<a:bodyPr/>'
      '<a:lstStyle/>'
      '<a:p>'
          '<a:pPr>'
            '<a:defRPr sz="1800"/>'
          '</a:pPr>'
          '<a:endParaRPr lang="en-US"/>'
        '</a:p>'
      '</c:txPr>'
      '<c:externalData r:id="rId1">'
        '<c:autoUpdate val="0"/>'
      '</c:externalData>'
    '</c:chartSpace>'

       % nsdecls('c','a','r')
)

    _chart_tmpl1 = (

        '<c:chart %s>'
        '<c:autoTitleDeleted val="0"/>'
        '<c:plotArea>'
        '<c:lineChart>'

        '</c:lineChart>'
        '</c:plotArea>'
        '</c:chart>' % nsdecls('c')
    )

    @staticmethod
    def new_chart_wrapper():

        xml = CT_Chart_Container._chart_tmpl
        graphicFrame = oxml_fromstring(xml)

        objectify.deannotate(graphicFrame, cleanup_namespaces=True)
        return graphicFrame

    @staticmethod
    def new_chart(data,headings_xlsx):

        headings=[[]]
        for i,e in enumerate(headings_xlsx):
            headings[i].append(headings_xlsx[i])
            if i == len(headings_xlsx)-1:
                continue
            else:
                headings.append([])

        chartFrame =  CT_Chart_Container.new_chart_wrapper()

        # set type of contained graphic to table
        chartData = chartFrame['chart'].plotArea

        # add chart data element tree
        chart = CT_ChartCell.new_chart(data,headings,headings_xlsx)
        chartData.append(chart)
        chart =CT_plotArea.new_plot_vars()
        chartData.append(chart)

        chart =CT_plotArea.plot_val_vars()
        chartData.append(chart)
        objectify.deannotate(chartData, cleanup_namespaces=True)
        chartXML = etree.tostring(chartFrame, pretty_print=True,xml_declaration = True, encoding='UTF-8', standalone="yes")
        return chartXML

class CT_ChartCell(objectify.ObjectifiedElement):

    _tbl_tmpl = (
        '<dummy>'
        '</dummy>'
    )

    @staticmethod
    def new_chart(data,headings,headings_xlsx):

        #xml = new('c:lineChart')
        #grouping = sub_elm(xml,'c:grouping',val="standard")
        #varycolors = sub_elm(xml,'c:varyColors',val="0")

        xml = new('c:lineChart')
        grouping = sub_elm(xml,'c:grouping',val="standard")
        varycolors = sub_elm(xml,'c:varyColors',val="0")

        chart = xml

        for idx, val in enumerate(headings):

            if idx == 0:
                continue
            else:
                ref_series = "Sheet1!$"+ string.uppercase[idx]+ '$1'

                ref_cat    = 'Sheet1!$A$2:$A$'+ str(len(data[1])+1)

                ref_val   = "Sheet1!$"+ string.uppercase[idx]+ '$2' + ":$"  + string.uppercase[idx] + "$"+str(len(data[0])+1)

                tr = sub_elm(chart, 'c:ser')
                tx = sub_elm(tr,'c:idx',val=str(idx-1))
                tx = sub_elm(tr,'c:order',val=str(idx-1))

                # Category for Legend

                tx = sub_elm(tr,'c:tx')
                ostrRef = CT_ChartCell.strRef_node(val,ref_series)
                tx.append(ostrRef)

                marker = sub_elm(tr,'c:marker')
                sym    = sub_elm (marker,'c:symbol', val = "none")

                # X-axis Categories

                cat = sub_elm(tr,'c:cat')

                ostrRef = CT_ChartCell.strRef_node(data[0],ref_cat)
                cat.append(ostrRef)

                # Series Values - Y-Axis

                val = sub_elm(tr,'c:val')

                ostrRef = CT_ChartCell.numRef_node(data[idx],ref_val)
                val.append(ostrRef)

                marker = sub_elm(tr,'c:smooth', val = "0")

            # c:dLbls Settings for Line Chart

        odLbls = sub_elm(chart,'c:dLbls')
        legend = sub_elm(odLbls,'c:showLegendKey',val ="0")
        legend = sub_elm(odLbls,'c:showVal',val ="0")
        legend = sub_elm(odLbls,'c:showCatName',val ="0")
        legend = sub_elm(odLbls,'c:showSerName',val ="0")
        legend = sub_elm(odLbls,'c:showPercent',val ="0")
        legend = sub_elm(odLbls,'c:showBubbleSize',val ="0")

        mark   = sub_elm(chart,'c:marker', val="1")
        mark   =  sub_elm(chart,'c:smooth', val="0")
        mark   =  sub_elm(chart,'c:axId', val="172167552")
        mark   =  sub_elm(chart,'c:axId', val="172169088")

        # add specified number of rows and columns

        objectify.deannotate(chart, cleanup_namespaces=True)
        return chart

    @staticmethod
    def strRef_node(series,series_ref):
        """ create new strRef node c:strCache -> <c:ptCount> -> <c:pt idx =0> -> <c:v>
        """
        #xml = CT_ChartCell._tbl_tmpl
        #series= ['Series 1','Series 2','Series 3']
        #series_ref = 'Sheet1!$B$1'
        data = new('c:strRef')
        strRef= data
        f = sub_elm(strRef,'c:f')
        ostrCache = sub_elm(strRef,'c:strCache')
        optCount = sub_elm(ostrCache,'c:ptCount',val=str(len(series)))

        for idx, val in enumerate(series):
            opt = sub_elm(ostrCache,'c:pt',idx=str(idx))
            #ov = sub_elm(opt,'c:v')
            opt.v= val

        strRef.f = series_ref

        objectify.deannotate(data, cleanup_namespaces=True)
        return data
    @staticmethod
    def numRef_node(series,series_ref):
        """ create new  c:strCache -> <c:ptCount> -> <c:pt idx =0> -> <c:v>
        """
        #xml = CT_ChartCell._tbl_tmpl
        #series= ['Series 1','Series 2','Series 3']
        #series_ref = 'Sheet1!$B$1'
        data = new('c:numRef')
        strRef= data
        f = sub_elm(strRef,'c:f')
        ostrCache = sub_elm(strRef,'c:numCache')
        ostrCache.formatCode = 'General'
        optCount = sub_elm(ostrCache,'c:ptCount',val=str(len(series)))

        for idx, val in enumerate(series):
            opt = sub_elm(ostrCache,'c:pt',idx=str(idx))
            #ov = sub_elm(opt,'c:v')
            opt.v= val

        strRef.f = series_ref

        objectify.deannotate(data, cleanup_namespaces=True)
        return data

class CT_plotArea(objectify.ObjectifiedElement):
    """ <c:catAx>
        <c:axId val="172167552"/>
        <c:scaling>
          <c:orientation val="minMax"/>
        </c:scaling>
        <c:delete val="0"/>
        <c:axPos val="b"/>
        <c:majorTickMark val="out"/>
        <c:minorTickMark val="none"/>
        <c:tickLblPos val="nextTo"/>
        <c:crossAx val="172169088"/>
        <c:crosses val="autoZero"/>
        <c:auto val="1"/>
        <c:lblAlgn val="ctr"/>
        <c:lblOffset val="100"/>
        <c:noMultiLvlLbl val="0"/>
      </c:catAx>
      <c:valAx>
        <c:axId val="172169088"/>
        <c:scaling>
          <c:orientation val="minMax"/>
        </c:scaling>
        <c:delete val="0"/>
        <c:axPos val="l"/>
        <c:majorGridlines/>
        <c:numFmt formatCode="General" sourceLinked="1"/>
        <c:majorTickMark val="out"/>
        <c:minorTickMark val="none"/>
        <c:tickLblPos val="nextTo"/>
        <c:crossAx val="172167552"/>
        <c:crosses val="autoZero"/>
        <c:crossBetween val="between"/>
      </c:valAx>"""

    @staticmethod
    def new_plot_vars():
        ocatAx = new('c:catAx')
        ocatAx_l= sub_elm(ocatAx,'c:axId', val ="172167552")
        ocatAx_l= sub_elm(ocatAx,'c:scaling')
        ocatAx_l= sub_elm(ocatAx_l,'c:orientation', val ="minMax")
        ocatAx_l= sub_elm(ocatAx,'c:delete',val="0")
        ocatAx_l= sub_elm(ocatAx,'c:axPos',val="b")
        ocatAx_l= sub_elm(ocatAx,'c:majorTickMark',val="out")
        ocatAx_l= sub_elm(ocatAx,'c:minorTickMark',val="none")
        ocatAx_l= sub_elm(ocatAx,'c:tickLblPos',val="nextTo")
        ocatAx_l= sub_elm(ocatAx,'c:crossAx',val="172169088")
        ocatAx_l= sub_elm(ocatAx,'c:crosses',val="autoZero")
        ocatAx_l= sub_elm(ocatAx,'c:auto',val="1")
        ocatAx_l= sub_elm(ocatAx,'c:lblAlgn',val="ctr")
        ocatAx_l= sub_elm(ocatAx,'c:lblOffset',val="100")
        ocatAx_l= sub_elm(ocatAx,'c:noMultiLvlLbl',val="0")

        objectify.deannotate(ocatAx, cleanup_namespaces=True)
        return ocatAx

    @staticmethod
    def plot_val_vars():

        ovalAx = new('c:valAx')
        ovalAx_l= sub_elm(ovalAx,'c:axId', val ="172169088")
        ovalAx_l= sub_elm(ovalAx,'c:scaling')
        ovalAx_l= sub_elm(ovalAx_l,'c:orientation', val ="minMax")
        ovalAx_l= sub_elm(ovalAx,'c:delete',val="0")
        ovalAx_l= sub_elm(ovalAx,'c:axPos',val="l")
        ovalAx_l= sub_elm(ovalAx,'c:majorGridlines')
        ovalAx_l= sub_elm(ovalAx,'c:numFmt',formatCode="General" ,sourceLinked ="1")
        ovalAx_l= sub_elm(ovalAx,'c:majorTickMark',val="out")
        ovalAx_l= sub_elm(ovalAx,'c:minorTickMark',val="none")
        ovalAx_l= sub_elm(ovalAx,'c:tickLblPos',val="nextTo")
        ovalAx_l= sub_elm(ovalAx,'c:crossAx',val="172167552")
        ovalAx_l= sub_elm(ovalAx,'c:crosses',val="autoZero")
        ovalAx_l= sub_elm(ovalAx,'c:crossBetween',val="between")

        objectify.deannotate(ovalAx, cleanup_namespaces=True)
        return ovalAx


import pymysql

db = pymysql.connect('localhost', 'root', 'root', 'dada')
cursor = db.cursor()
name = "update"
id = 1
sql = """
    DELETE FROM py_user WHERE id = %d;
""" % (id)
try:
    cursor.execute(sql)
    db.commit()
    print("delete successful")
except:
    print("delete error")
    db.rollback()

finally:
    db.close()


import csv
import dssm as dssm
import words_to_matrix as wtm

dssm_model = dssm.Dssm()

dssm_model.compile(optimizer='adam',
                   loss='binary_crossentropy')

dssm_model.load_weights('./checkpoints/my_checkpoint')  # load weights from trained model

main_query = ''
data = []
with open('testnew.csv') as csvfile:
    csvreader = csv.reader(csvfile)
    cnt = 0
    for row in csvreader:
        query = [row[3]]
        title = [row[2]]
        if not main_query:
            main_query = query
        if main_query != query:
            print(dssm_model.predict(data))  # predict relevance (range [0;1]) from test data
            data.clear()
            main_query = query
        query_trigram = wtm.sentences_to_bag_of_trigrams(query)
        title_trigram = wtm.sentences_to_bag_of_trigrams(title)
        data.append((query_trigram, title_trigram))


from . import encoder
from . import decoder

# coordindates are scaled to this range within tile
extents = 4096

def decode(tile):
    vector_tile = decoder.TileData()
    message = vector_tile.getMessage(tile)
    return message

def encode(layers):
    vector_tile = encoder.VectorTile(extents)
    if (isinstance(layers, list)):
        for layer in layers:
            vector_tile.addFeatures(layer['features'], layer['name'])
    else:
        vector_tile.addFeatures(layers['features'], layers['name'])

    return vector_tile.tile.SerializeToString()


# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 08:36:13 2017
@author: Celfras--Levi
"""

from PyQt5.QtWidgets import QMainWindow,QWidget,QGridLayout,QGroupBox,QApplication,QPushButton,QHBoxLayout,QVBoxLayout,QTextEdit,QLineEdit,QLabel,QComboBox
from PyQt5.QtGui import QIcon,QPalette,QFont,QTextCursor
from PyQt5.QtCore import Qt,QObject,pyqtSignal,QThread,pyqtSlot,QPropertyAnimation
import sys
import time

from sys import path
path.append('.')
from catchtpdata import Readtpdata,Catchdata,inputdatatokey

global textnarray
textnarray=[]

class EmittingStream(QObject):
    textWritten = pyqtSignal(str)
    def flush(self):
        pass

    def write(self,text):
        self.textWritten.emit(str(text))

class KeyboardUI(QMainWindow):

    def __init__(self):
        super().__init__()
        self.keyboard()
        QThread.currentThread().setObjectName('main')
        self.__work_done=None
        self.__threads=None
        self.animation=None
        self.animation = None

    def __del__(self):
        sys.__stdout__ = sys.stdout                                  # 初始化标准输出
        sys.__stderr__ = sys.stderr                                  # 初始化标准输出'''

    def closeEvent(self,event):
        if self.animation is None:
            self.animation=QPropertyAnimation(self,"windowOpacity")
            self.animation.setDuration(300)
            self.animation.setStartValue(1)
            self.animation.setEndValue(0)
            self.animation.finished.connect(self.close)
            self.animation.start()
            event.ignore()

    def keyboard(self):
        self.setWindowTitle('KeyBoard Test Tool Beta1.0')

        self.savedStdout = sys.stdout
        self.savedStderr = sys.stderr
        sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)  # 重新定义系统标准输出
        sys.stderr = EmittingStream(textWritten=self.normalOutputWritten)  # 重新定义系统标准错误'''

        mainLayout=QGridLayout()
        serailLayout=QGridLayout()

        line1Layout=QHBoxLayout()
        line2Layout=QHBoxLayout()
        line3Layout=QHBoxLayout()
        line4Layout=QHBoxLayout()
        line5Layout=QHBoxLayout()
        line6Layout=QHBoxLayout()
        line7Layout=QHBoxLayout()

        self.Serial_chose_Widget=QGroupBox('')
        self.TxtWidget=QTextEdit()
        self.Txtline=QLineEdit()
        self.Txtline.setFont(QFont("Roman times",13))

        self.runbutton=QPushButton('RUN')

        self.serial_name_label=QLabel(' 串口')
        self.serial_name_widget=QComboBox()

        self.serial_name_widget.addItem("COM1")
        self.serial_name_widget.addItem("COM2")
        self.serial_name_widget.addItem("COM3")
        self.serial_name_widget.addItem("COM4")
        self.serial_name_widget.addItem("COM5")
        self.serial_name_widget.addItem("COM6")
        self.serial_name_widget.addItem("COM7")
        self.serial_name_widget.addItem("COM8")
        self.serial_name_widget.addItem("COM9")
        self.serial_name_widget.addItem("COM10")

        self.serail_baudrate_label=QLabel("波特率")
        self.serial_lineedit = QLineEdit()

        self.Serial_chose_Widget.setLayout(serailLayout)
        serailLayout.addWidget(self.serial_name_label,0,0,1,2)
        serailLayout.addWidget(self.serial_name_widget, 0,2,1,3)
        serailLayout.addWidget(self.serail_baudrate_label,1,0,1,2)
        serailLayout.addWidget(self.serial_lineedit,1,2,1,3)

        self.line1widget=QWidget()
        self.line1widget.setLayout(line1Layout)
        self.line2widget=QWidget()
        self.line2widget.setLayout(line2Layout)
        self.line3widget=QWidget()
        self.line3widget.setLayout(line3Layout)
        self.line4widget=QWidget()
        self.line4widget.setLayout(line4Layout)
        self.line5widget=QWidget()
        self.line5widget.setLayout(line5Layout)
        self.line6widget=QWidget()
        self.line6widget.setLayout(line6Layout)
        self.line7widget=QWidget()
        self.line7widget.setLayout(line7Layout)

        MainWindow=QWidget()
        MainWindow.setLayout(mainLayout)
        self.setCentralWidget(MainWindow)

        mainLayout.addWidget(self.Serial_chose_Widget,0,0,1,1)
        mainLayout.addWidget(self.TxtWidget,0,1,1,18)
        mainLayout.addWidget(self.runbutton,1,0,1,1)
        mainLayout.addWidget(self.Txtline,1,1,1,18)
        key_line1=['Esc','F1','F2','F3','F4','F5','F6','F7','F8','F9','F10','F11','F12','Home','End','Insert','Delete']
        key_line2=['~\n·\n','!\n1\n','@\n2\n','#\n3\n','$\n4\n','%\n5\n','^\n6\n','&&\n7\n','*\n8\n','(\n9\n',')\n0\n','-\n-\n','+\n=\n','\nBackspace\n']
        key_line3=['\nTab\n','\nQ\n','\nW\n','\nE\n','\nR\n','\nT\n','\nY\n','\nU\n','\nI\n','\nO\n','\nP\n','{\n[\n','}\n]\n','|\n\\\n']
        key_line4=['\nCaps Lock\n','\nA\n','\nS\n','\nD\n','\nF\n','\nG\n','\nH\n','\nJ\n','\nK\n','\nL\n',':\n;\n','"\n\'\n','\nEnter\n']
        key_line5=['\nShift\n','\nZ\n','\nX\n','\nC\n','\nV\n','\nB\n','\nN\n','\nM\n','<\n,\n','>\n.\n','?\n/\n','\nShift\n']
        key_line6=['\nFn\n','\nCtrl\n','\n▆▇\n▇▇','\nAlt\n','\n------------------------------------------------------------------\n','\nAlt\n','\nPrtScn\n','\nCtrl\n','\nPgUp\n','\n↑\n','\nPgDn\n']
        key_line7=['←','↓','→']

        key_line=[]
        key_line.append(key_line1)
        key_line.append(key_line2)
        key_line.append(key_line3)
        key_line.append(key_line4)
        key_line.append(key_line5)
        key_line.append(key_line6)
        key_line.append(key_line7)

        self.button=locals()

        positions1= [ j for j in range(17)]
        for position,key in zip(positions1,key_line1):
            self.button['button1_%s'%position]=QPushButton(key)
            self.button['button1_%s'%position].setFont(QFont("Roman times",8,QFont.Bold))
            #self.button['button1_%s'%position].setCheckable(True)
            #self.button['button1_%s'%position].setAutoExclusive(True)
            line1Layout.addWidget(self.button['button1_%s'%position],position)
        mainLayout.addWidget(self.line1widget,2,0,1,19)

        positions2= [ j for j in range(14)]
        for position,key in zip(positions2,key_line2):
            self.button['button2_%s'%position]=QPushButton(key)
            self.button['button2_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))
            line2Layout.addWidget(self.button['button2_%s'%position],position)
            line2Layout.setSpacing(8)
        mainLayout.addWidget(self.line2widget,3,0,1,19)

        positions3= [ j for j in range(14)]
        for position,key in zip(positions3,key_line3):
            self.button['button3_%s'%position]=QPushButton(key)
            self.button['button3_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))
            line3Layout.addWidget(self.button['button3_%s'%position],position)
            line3Layout.setSpacing(10)
        mainLayout.addWidget(self.line3widget,4,0,1,19)

        positions4= [ j for j in range(13)]
        for position,key in zip(positions4,key_line4):
            self.button['button4_%s'%position]=QPushButton(key)
            self.button['button4_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))
            line4Layout.addWidget(self.button['button4_%s'%position],position)
            line4Layout.setSpacing(13)
        mainLayout.addWidget(self.line4widget,5,0,1,19)

        positions5= [ j for j in range(12)]
        for position,key in zip(positions5,key_line5):
            self.button['button5_%s'%position]=QPushButton(key)
            self.button['button5_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))
            line5Layout.addWidget(self.button['button5_%s'%position],position)
            line5Layout.setSpacing(15)
        mainLayout.addWidget(self.line5widget,6,0,1,19)

        positions6= [ j for j in range(11)]
        for position,key in zip(positions6,key_line6):
            self.button['button6_%s'%position]=QPushButton(key)
            self.button['button6_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))
            if key=='\n------------------------------------------------------------------\n':
                self.button['button6_%s'%position].setStyleSheet('''
                QPushButton{
                color: rgb(211,211,211) ;
                }
                ''')

            line6Layout.addWidget(self.button['button6_%s'%position],position)
            line6Layout.setSpacing(12)
        mainLayout.addWidget(self.line6widget,7,0,1,19)

        positions7= [ j for j in range(3)]
        for position,key in zip(positions7,key_line7):
            self.button['button7_%s'%position]=QPushButton(key)
            self.button['button7_%s'%position].setFont(QFont("Roman times",9,QFont.Bold))

            line7Layout.addWidget(self.button['button7_%s'%position],position+8)
            line7Layout.setSpacing(12)
        mainLayout.addWidget(self.line7widget,8,14,1,5)

        self.move(300,150)
        self.show()

    def normalOutputWritten(self, text):
        cursor = self.TxtWidget.textCursor()
        cursor.movePosition(QTextCursor.End)
        cursor.insertText(text)
        self.TxtWidget.setTextCursor(cursor)
        self.TxtWidget.ensureCursorVisible()

def changebutton(data):

    linekey_1=['Esc','F1','F2','F3','F4','F5','F6','F7','F8','F9','F10','F11','F12','Home','End','Insert','Delete']
    linekey_2=['~','1','2','3','4','5','6','7','8','9','0','-','+','Backspace']
    linekey_3=['Tab','Q','W','E','R','T','Y','U','I','O','P','{','}','|']
    linekey_4=['Capslock','A','S','D','F','G','H','J','K','L',':','"','Enter']
    linekey_5=['Shift','Z','X','C','V','B','N','M','<','>','?','Right Shift']
    linekey_6=['Fn','Ctrl','Windows','Alt','Space','Right Alt','PrtScn','Right Ctrl','Pgup','↑','PgDn']
    linekey_7=['←','↓','→']

    linekey=[]
    linekey.append(linekey_1)
    linekey.append(linekey_2)
    linekey.append(linekey_3)
    linekey.append(linekey_4)
    linekey.append(linekey_5)
    linekey.append(linekey_6)
    linekey.append(linekey_7)

    if len(data)!=0:
        for d in data:
            row,col,state=d[0],d[1],d[2]
            #print('进入判断',d)
            if state==1:
                #print('button'+str(row+1)+'_'+str(col))
                ex.button['button'+str(row+1)+'_'+str(col)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(255,255,102);
                    }
                    ''')
                text=str(linekey[row][col])
                textnarray.append(text)

            else:
                ex.button['button'+str(row+1)+'_'+str(col)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211) ;
                    }
                    ''')

class Changebuttoncolor(QThread,QApplication):
    _signal=pyqtSignal(list)
    datalist=[]
    def __init__(self):
        super().__init__()
        self.working=True

    def changestart(self,printdata):
        self.start()
        #print(printdata)

    def run(self):
        temp=Catchdata()
        self.datalist=inputdatatokey(temp)
        if len(self.datalist)!=0:
            self._signal.emit(self.datalist)
            #print('数据已发送',self.datalist)
            '''self._signal.connect(changebutton)'''
            time.sleep(0.02)
        else:
            pass

if __name__=='__main__':

    app=0
    app=QApplication(sys.argv)
    app.setStyleSheet('''
        QPushButton{
        background-color: rgb(211,211,211) ;
        }
        ''')
    ex=KeyboardUI()
    ex.runbutton.setStyleSheet('''
        QPushButton{
        background-color: rgb(51,153,255) ;
        height:18px;
        border-style: inset;
        border-radius: 5px;
        border-color: beige;
        }
        QPushButton:pressed{
        background-color: rgb(220, 220, 0);
        border-style: inset;}'''
        )
    ex.show()

    while 1:
        try:
            threads=Changebuttoncolor()
            threads._signal.connect(changebutton)
            threads.start()
            time.sleep(0.17)
            #QApplication.processEvents()
            #time.sleep(0.08)
            line_txt=((str(textnarray).replace(',','')).replace(' ','')).replace('\'',' ')
            ex.Txtline.setText(line_txt)

            for col1 in range(17):
                ex.button['button1'+'_'+str(col1)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')
            for col2 in range(14):
                ex.button['button2'+'_'+str(col2)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')

            for col3 in range(14):
                ex.button['button3'+'_'+str(col3)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')

            for col4 in range(13):
                ex.button['button4'+'_'+str(col4)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')

            for col5 in range(12):
                ex.button['button5'+'_'+str(col5)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')

            for col6 in range(11):
                if col6== 4:
                    ex.button['button6'+'_'+str(col6)].setStyleSheet('''
                        QPushButton{
                        background-color: rgb(211,211,211);
                        color: rgb(211,211,211) ;
                        }
                        ''')
                else:
                    ex.button['button6'+'_'+str(col6)].setStyleSheet('''
                        QPushButton{
                        background-color: rgb(211,211,211);
                        }
                        ''')

            for col7 in range(3):
                ex.button['button7'+'_'+str(col7)].setStyleSheet('''
                    QPushButton{
                    background-color: rgb(211,211,211);
                    }
                    ''')

            QApplication.processEvents()

        except Exception:
            print('程序停止')
            break

    sys.exit(app.exec_())


# This program draws a polygon on a Canvas.
import tkinter

class MyGUI:
    def __init__(self):
        # Create the main window.
        self.main_window = tkinter.Tk()

        # Create the Canvas widget.
        self.canvas = tkinter.Canvas(self.main_window, width=160, height=160)

        # Draw a polygon.
        self.canvas.create_polygon(60, 20, 100, 20, 140, 60, 140, 100,
                                   100, 140, 60, 140, 20, 100, 20, 60)

        # Pack the canvas.
        self.canvas.pack()

        # Start the mainloop.
        tkinter.mainloop()

# Create an instance of the MyGUI class.
my_gui = MyGUI()


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 17 19:32:29 2018

@author: tzlmyq
"""

from queue import PriorityQueue
class MedianFinder(object):
    """
    Before starting to solve this problem, we have to understand what is the concept of median number -- If the size of the list is even, median is the mean of the two middle value, i.e., the mean of the minimum of the bigger half and the maximum of the smaller half
    """
    def __init__(self):
        """
        initialize your data structure here.
        """
        self.maxHeap = PriorityQueue()
        self.minHeap = PriorityQueue()

    def addNum(self, num):
        """
        :type num: int
        :rtype: void
        """
        self.maxHeap.put(-num)
        self.minHeap.put(-self.maxHeap.get())
        if self.minHeap.qsize() > self.maxHeap.qsize():
            self.maxHeap.put(-self.minHeap.get())

    def findMedian(self):
        """
        :rtype: float
        """
        if self.maxHeap.qsize() > self.maxHeap.qsize():
            temp = -self.maxHeap.get()
            self.maxHeap.put(-temp)
            return temp
        else:
            maxH = -self.maxHeap.get()
            self.maxHeap.put(maxH)
            minH = self.minHeap.get()
            self.minHeap.put(minH)
            return (minH - maxH)/2.0

m = MedianFinder()
m.addNum(1)
print(m.findMedian())
m.addNum(2)
print(m.findMedian())
m.addNum(3)
print(m.findMedian())


import pickle
import argparse
import sys
import os
import logging
from datetime import datetime
from itertools import chain, zip_longest

from .config import Config

logger = logging.getLogger("app")

def make_dirs(d):
    if not os.path.exists(d):
        os.makedirs(d)

def setup_loggers(output_dir):
    # create handlers
    app_fh = logging.FileHandler(os.path.join(output_dir, "app.log"))
    app_sh = logging.StreamHandler(sys.stdout)

    # create formatters and add to handlers
    file_formatter = logging.Formatter("%(levelname)s:%(message)s")
    stream_formatter = logging.Formatter("%(name)s:%(levelname)s:%(message)s")
    app_fh.setFormatter(file_formatter)
    app_sh.setFormatter(stream_formatter)

    # add handlers to loggers
    app_logger = logging.getLogger("app")
    app_logger.addHandler(app_fh)
    app_logger.addHandler(app_sh)

    # set log levels
    app_logger.setLevel('INFO')
    app_logger.propogate = False

def load_config_args(args):
    config = Config()
    if args.config:
        config.load(args.config)

    config.add_to_config("comment", args.comment)
    config.add_to_config("cuda_no", args.cuda_no)
    config['model_props']['type'] = args.model_type
    config.add_to_config("model_props", config['model_props'])
    return config

def setup_simulation(args):
    config = load_config_args(args)

    # create output directory
    t = datetime.today()
    output_dir = os.path.join(
        config["output"], f"{t.month}_{t.day}_{t.hour}_{t.minute}_{t.second}"
    )
    make_dirs(output_dir)

    setup_loggers(output_dir)
    logger.info(f"Created output directory {output_dir}")

    # save options
    config_path = os.path.join(output_dir, "config.toml")
    config.save(config_path)
    logger.info(f"Saved configuration options at {config_path}")

    return output_dir, config

def loadpkl(fname):
    with open(fname, 'rb') as f:
        return pickle.load(f)

def savepkl(fname, obj):
    with open(fname, 'wb') as f:
        pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL)

def flatten_1_deg(arr):
    return list(chain(*arr))


from datetime import datetime
from dateutil.tz import UTC
import pandas
import random
import requests
import unittest
import textwrap

from datamart_geo import GeoData
from datamart_profiler import process_dataset
from datamart_profiler.core import expand_attribute_name
from datamart_profiler import profile_types
from datamart_profiler import spatial
from datamart_profiler.spatial import LATITUDE, LONGITUDE, LatLongColumn
from datamart_profiler.temporal import get_temporal_resolution, parse_date

from .utils import DataTestCase, data

def check_ranges(min_, max_):
    def check(ranges):
        assert 2 <= len(ranges) <= 3
        for rg in ranges:
            assert rg.keys() == {'range'}
            rg = rg['range']
            assert rg.keys() == {'gte', 'lte'}
            gte, lte = rg['gte'], rg['lte']
            assert min_ <= gte <= lte <= max_

        return True

    return check

def check_geo_ranges(min_long, min_lat, max_long, max_lat):
    def check(ranges):
        assert len(ranges) == 3
        for rg in ranges:
            assert rg.keys() == {'range'}
            rg = rg['range']
            assert rg.keys() == {'type', 'coordinates'}
            assert rg['type'] == 'envelope'
            [long1, lat1], [long2, lat2] = rg['coordinates']
            assert min_lat <= lat2 <= lat1 <= max_lat
            assert min_long <= long1 <= long2 <= max_long

        return True

    return check

def check_plot(kind):
    def check(plot):
        return plot['type'] == kind

    return check

class TestNames(unittest.TestCase):
    def test_names(self):
        """Test expanding column names."""
        self.assertEqual(
            list(expand_attribute_name('Apt221bBakerStreet')),
            ['Apt', '221', 'b', 'Baker', 'Street'],
        )
        self.assertEqual(
            list(expand_attribute_name('place')),
            ['place'],
        )

class TestLatlongSelection(DataTestCase):
    def test_normalize_name(self):
        """Test normalizing column names."""
        self.assertEqual(
            spatial.normalize_latlong_column_name('latitude', LATITUDE),
            '',
        )
        self.assertEqual(
            spatial.normalize_latlong_column_name('Place_Latitude', LATITUDE),
            'place_',
        )
        self.assertEqual(
            spatial.normalize_latlong_column_name('start_Long_deg', LONGITUDE),
            'start__deg',
        )
        self.assertEqual(
            spatial.normalize_latlong_column_name('start_Lon_deg', LONGITUDE),
            'start__deg',
        )

    def test_pairing(self):
        """Test pairing latitude and longitude columns by name or matching pairs defined by the user."""
        pairs, (missed_lat, missed_long) = spatial.pair_latlong_columns(
            [
                LatLongColumn(0, 'Pickup_latitude', [1], None),
                LatLongColumn(1, 'lat', [7], None),
                LatLongColumn(2, 'dropoff_latitude', [2], None),
                LatLongColumn(3, 'latitude_place', [8], None),
                LatLongColumn(4, 'la_coord', [8], '1'),
            ],
            [
                LatLongColumn(5, 'long', [5], None),
                LatLongColumn(6, 'dropoff_Longitude', [3], None),
                LatLongColumn(7, 'pickup_longitude', [4], None),
                LatLongColumn(8, 'other_Longitude', [6], None),
                LatLongColumn(9, 'lo_coord', [8], '1'),
            ],
        )
        self.assertEqual(
            pairs,
            [
                (LatLongColumn(1, 'lat', [7], None),
                 LatLongColumn(5, 'long', [5], None)),
                (LatLongColumn(2, 'dropoff_latitude', [2], None),
                 LatLongColumn(6, 'dropoff_Longitude', [3], None)),
                (LatLongColumn(0, 'Pickup_latitude', [1], None),
                 LatLongColumn(7, 'pickup_longitude', [4], None)),
                (LatLongColumn(4, 'la_coord', [8], '1'),
                 LatLongColumn(9, 'lo_coord', [8], '1')),
            ],
        )
        self.assertEqual(
            missed_lat,
            ['latitude_place'],
        )
        self.assertEqual(
            missed_long,
            ['other_Longitude'],
        )

    def test_process(self):
        with data('lat_longs.csv', 'r') as data_fp:
            dataframe = pandas.read_csv(data_fp)
        metadata = process_dataset(
            dataframe,
        )
        # Check columns
        self.assertJson(
            [
                {k: v for k, v in c.items()
                 if k in ['name', 'structural_type', 'semantic_types']}
                for c in metadata['columns']],
            [
                {
                    'name': 'from latitude',
                    'structural_type': 'http://schema.org/Float',
                    'semantic_types': ['http://schema.org/latitude'],
                },
                {
                    'name': 'to long',
                    'structural_type': 'http://schema.org/Float',
                    'semantic_types': ['http://schema.org/longitude'],
                },
                {
                    'name': 'to lat',
                    'structural_type': 'http://schema.org/Float',
                    'semantic_types': ['http://schema.org/latitude'],
                },
                {
                    'name': 'from longitude',
                    'structural_type': 'http://schema.org/Float',
                    'semantic_types': ['http://schema.org/longitude'],
                },
                {
                    'name': 'unpaired lat',
                    'structural_type': 'http://schema.org/Float',
                    'semantic_types': [],
                },
            ]
        )
        # Check pairs
        self.assertJson(
            [{k: v for k, v in c.items() if k != 'ranges'} for c in metadata['spatial_coverage']],
            [
                {'type': 'latlong', 'column_names': ['to lat', 'to long'], 'column_indexes': [2, 1]},
                {'type': 'latlong', 'column_names': ['from latitude', 'from longitude'], 'column_indexes': [0, 3]},
            ],
        )

class TestDates(DataTestCase):
    def test_parse(self):
        """Test parsing dates."""
        self.assertEqual(
            parse_date('Monday July 1, 2019'),
            datetime(2019, 7, 1, tzinfo=UTC),
        )
        self.assertEqual(
            parse_date('20190702T211319Z'),
            datetime(2019, 7, 2, 21, 13, 19, tzinfo=UTC),
        )
        dt = parse_date('2019-07-02T21:13:19-04:00')
        self.assertEqual(
            dt.replace(tzinfo=None),
            datetime(2019, 7, 2, 21, 13, 19),
        )
        self.assertEqual(
            dt.astimezone(UTC),
            datetime(2019, 7, 3, 1, 13, 19, tzinfo=UTC),
        )

        # Check that unknown timezones are not accepted
        self.assertEqual(
            parse_date('2019-07-02 18:05 UTC'),
            datetime(2019, 7, 2, 18, 5, tzinfo=UTC),
        )
        self.assertEqual(
            parse_date('2019-07-02 18:05 L'),
            None,
        )

    def test_year(self):
        """Test the 'year' special-case."""
        dataframe = pandas.DataFrame({
            'year': [2004, 2005, 2006],
            'number': [2014, 2015, float('nan')],
        })
        metadata = process_dataset(dataframe)

        def year_rng(year):
            year = float(year)
            return {'range': {'gte': year, 'lte': year}}

        self.assertJson(
            metadata,
            {
                'nb_rows': 3,
                'nb_profiled_rows': 3,
                'types': ['numerical', 'temporal'],
                'attribute_keywords': ['year', 'number'],
                'columns': [
                    {
                        'name': 'year',
                        'structural_type': 'http://schema.org/Text',
                        'semantic_types': ['http://schema.org/DateTime'],
                        'unclean_values_ratio': 0.0,
                        'num_distinct_values': 3,
                        'mean': 1104508800.0,
                        'stddev': lambda n: round(n, 3) == 25784316.871,
                        'coverage': [
                            year_rng(1072915200.0),
                            year_rng(1104537600.0),
                            year_rng(1136073600.0),
                        ],
                        'temporal_resolution': 'year',
                    },
                    {
                        'name': 'number',
                        'structural_type': 'http://schema.org/Integer',
                        'semantic_types': [],
                        'missing_values_ratio': lambda n: round(n, 2) == 0.33,
                        'unclean_values_ratio': 0.0,
                        'num_distinct_values': 2,
                        'mean': 2014.5,
                        'stddev': 0.5,
                        'coverage': [
                            {'range': {'gte': 2014.0, 'lte': 2014.0}},
                            {'range': {'gte': 2015.0, 'lte': 2015.0}},
                        ],
                    },
                ],
            },
        )

class TestTemporalResolutions(unittest.TestCase):
    def test_pandas(self):
        def get_res(values):
            idx = pandas.Index(pandas.to_datetime(values))
            return get_temporal_resolution(idx)

        self.do_checks(get_res)

    def test_native(self):
        def get_res(values):
            values = [parse_date(d) for d in values]
            return get_temporal_resolution(values)

        self.do_checks(get_res)

    def do_checks(self, get_res):
        self.assertEqual(
            get_res([
                '2020-01-14T21:05:02',
                '2020-01-14T21:05:07',
                '2020-01-14T21:05:30',
                '2020-01-14T22:21:54',
                '2020-01-15T05:12:41',
                '2020-03-21T05:12:41',
            ]),
            'second',
        )
        self.assertEqual(
            get_res([
                '2020-01-14T21:05:02',
                '2020-01-14T21:05:07',
                '2020-01-14T21:05:30',
            ]),
            'second',
        )
        self.assertEqual(
            get_res([
                '2020-01-14T21:05:00Z',
                '2020-01-14T21:05:00Z',
                '2020-01-14T21:05:00Z',
                '2020-01-14T21:21:00Z',
                '2020-01-15T05:12:00Z',
                '2020-03-21T05:12:00Z',
                '2020-03-21T05:13:00Z',
            ]),
            'minute',
        )
        self.assertEqual(
            get_res([
                '2020-01-14T21:30:00',
                '2020-01-14T21:30:00',
                '2020-01-14T21:30:00',
                '2020-01-14T22:31:07',
                '2020-01-15T05:29:59',
                '2020-03-21T05:29:59',
            ]),
            'hour',
        )
        self.assertEqual(
            get_res([
                '2020-01-14',
                '2020-01-14',
                '2020-01-14',
                '2020-01-14',
                '2020-01-15',
                '2020-03-21',
            ]),
            'day',
        )
        self.assertEqual(
            get_res([
                '2020-01-14T05:12:41',
                '2020-01-14T05:12:41',
                '2020-01-14T05:12:41',
                '2020-01-14T05:12:41',
                '2020-01-15T05:13:03',
                '2020-01-16T05:12:41',
                '2020-03-21T05:12:41',
            ]),
            'day',
        )
        self.assertEqual(
            get_res([
                '2020-01-30',
                '2020-01-30',
                '2020-02-13',
                '2020-02-21',
                '2020-03-05',
                '2020-03-05',
                '2020-03-12',
            ]),
            'week',
        )
        self.assertEqual(
            get_res([
                '2020-02-06',
                '2020-02-06',
                '2020-03-07',
                '2020-05-07',
                '2020-06-06',
            ]),
            'month',
        )
        self.assertEqual(
            get_res([
                '2017-02-06',
                '2018-03-06',
                '2018-03-06',
                '2020-02-04',
            ]),
            'year',
        )
        self.assertEqual(
            get_res([
                '2017-01-01',
                '2018-04-01',
                '2018-07-01',
                '2018-12-31',
            ]),
            'quarter',
        )

class TestTypes(unittest.TestCase):
    def do_test(self, match, positive, negative):
        for elem in textwrap.dedent(positive).splitlines():
            if elem:
                self.assertTrue(match(elem),
                                "Didn't match: %s" % elem)
        for elem in textwrap.dedent(negative).splitlines():
            if elem:
                self.assertFalse(match(elem),
                                 "Shouldn't have matched: %s" % elem)

    def test_ints(self):
        positive = '''\
        12
        0
        +478
        -17
        '''
        negative = '''\
        1.7
        7A
        ++2
        --34
        +-7
        -+18
        '''
        self.do_test(
            profile_types._re_int.match,
            positive, negative,
        )
        self.assertFalse(profile_types._re_int.match(''))

    def test_floats(self):
        positive = '''\
        12.
        0.
        .7
        123.456
        +123.456
        +.456
        -.4
        .4e17
        -.4e17
        +8.4e17
        +8.e17
        '''
        negative = '''\
        1.7.3
        .7.3
        7.3.
        .
        -.
        +.
        7.A
        .e8
        8e17
        1.3e
        '''
        self.do_test(
            profile_types._re_float.match,
            positive, negative,
        )
        self.assertFalse(profile_types._re_float.match(''))

    def test_geo_combined(self):
        positive = '''\
        70 WASHINGTON SQUARE S, NEW YORK, NY 10012 (40.729753, -73.997174)
        R\u00C9MI'S HOUSE, BROOKLYN, NY (40.729753, -73.997174)
        '''
        negative = '''\
        70 Washington Square (40.729753, -73.997174)
        '''
        self.do_test(
            profile_types._re_geo_combined.match,
            positive, negative,
        )

class TestTruncate(unittest.TestCase):
    def test_simple(self):
        from datamart_profiler.core import truncate_string

        self.assertEqual(truncate_string("abc", 10), "abc")
        self.assertEqual(truncate_string("abcdefghij", 10), "abcdefghij")
        self.assertEqual(truncate_string("abcdefghijk", 10), "abcdefg...")
        self.assertEqual(truncate_string("abcdefghijklmnop", 10), "abcdefg...")

    def test_words(self):
        from datamart_profiler.core import truncate_string

        self.assertEqual(
            truncate_string("abcde fghijklmnopqrs tuvwxyzABCD EF", 30),
            "abcde fghijklmnopqrs...",
        )
        self.assertEqual(
            truncate_string("abcde fghijklmnopqrs tu vwxyzABCD EF", 30),
            "abcde fghijklmnopqrs tu...",
        )
        self.assertEqual(
            truncate_string("abc defghijklmnopqrstuvwxyzABCDEFGHI", 30),
            "abc defghijklmnopqrstuvwxyz...",
        )

class TestNominatim(DataTestCase):
    def test_profile(self):
        queries = {
            "70 Washington Square S, New York, NY 10012": [{
                'lat': 40.7294, 'lon': -73.9972,
            }],
            "6 MetroTech, Brooklyn, NY 11201": [{
                'lat': 40.6944, 'lon': -73.9857,
            }],
            "251 Mercer St, New York, NY 10012": [{
                'lat': 40.7287, 'lon': -73.9957,
            }],
        }

        def replacement(url, *, q):
            if not replacement.failed:  # Fail just once
                replacement.failed = True
                response = requests.Response()
                response.status_code = 500
                raise requests.HTTPError("Fake 500 error", response=response)
            return [queries[qe] for qe in q]
        replacement.failed = False

        old_query = spatial.nominatim_query
        old_min_batch_size = spatial.NOMINATIM_MIN_SPLIT_BATCH_SIZE
        spatial.nominatim_query = replacement
        spatial.NOMINATIM_MIN_SPLIT_BATCH_SIZE = 2
        try:
            with data('addresses.csv', 'r') as data_fp:
                metadata = process_dataset(
                    data_fp,
                    nominatim='http://nominatim/',
                    coverage=True,
                )
        finally:
            spatial.nominatim_query = old_query
            spatial.NOMINATIM_MIN_SPLIT_BATCH_SIZE = old_min_batch_size

        self.assertJson(
            metadata,
            {
                'size': 142,
                'nb_rows': 3,
                'nb_profiled_rows': 3,
                'types': ['spatial'],
                'attribute_keywords': ['place', 'loc'],
                'columns': [
                    {
                        'name': 'place',
                        'num_distinct_values': 3,
                        'structural_type': 'http://schema.org/Text',
                        'semantic_types': [],
                    },
                    {
                        'name': 'loc',
                        'structural_type': 'http://schema.org/Text',
                        'semantic_types': [
                            'http://schema.org/Text',
                            'http://schema.org/address',
                        ],
                    },
                ],
                'spatial_coverage': [
                    {
                        'type': 'address',
                        'column_names': ['loc'],
                        'column_indexes': [1],
                        'ranges': check_geo_ranges(-74.00, 40.69, -73.98, 40.73),
                    },
                ],
            },
        )

    def test_querying(self):
        queries = {
            'a': [{'lat': 11.0, 'lon': 12.0}],
            'b': [],
            'c': [{'lat': 31.0, 'lon': 32.0}],
        }
        old_query = spatial.nominatim_query
        spatial.nominatim_query = \
            lambda url, *, q: [queries[qe] for qe in q]
        try:
            res, empty = spatial.nominatim_resolve_all(
                'http://240.123.45.67:21',
                ['a', 'b', 'c', 'b', 'b', 'c'],
            )
            self.assertEqual(
                res,
                [
                    (11.0, 12.0),
                    (31.0, 32.0),
                    (31.0, 32.0),
                ],
            )
            self.assertEqual(empty, 6)

            res, empty = spatial.nominatim_resolve_all(
                'http://240.123.45.67:21',
                [
                    'a', 'b', 'c', 'b', 'b', 'c', 'a', 'b', 'c',
                    # Second batch
                    'b', 'b', 'c',
                ],
            )
            self.assertEqual(
                res,
                [
                    (11.0, 12.0),
                    (11.0, 12.0),
                    (31.0, 32.0),
                    (31.0, 32.0),
                    (31.0, 32.0),
                    # Second batch
                    (31.0, 32.0),
                ],
            )
            self.assertEqual(empty, 12)
        finally:
            spatial.nominatim_query = old_query

class TestGeo(DataTestCase):
    @classmethod
    def setUpClass(cls):
        cls.geo_data = GeoData.from_local_cache()
        cls.geo_data.load_areas([0, 1], bounds=True)

    def test_profile(self):
        with data('admins.csv', 'r') as data_fp:
            metadata = process_dataset(
                data_fp,
                geo_data=self.geo_data,
                coverage=True,
            )

        self.assertJson(
            metadata,
            {
                'size': 93,
                'nb_rows': 5,
                'nb_profiled_rows': 5,
                'types': ['spatial'],
                'attribute_keywords': ['zero', 'one'],
                'columns': [
                    {
                        'name': 'zero',
                        'structural_type': 'http://schema.org/Text',
                        'semantic_types': [
                            'http://schema.org/AdministrativeArea',
                            'http://schema.org/Enumeration',
                        ],
                        'num_distinct_values': 2,
                        'admin_area_level': 0,
                    },
                    {
                        'name': 'one',
                        'structural_type': 'http://schema.org/Text',
                        'semantic_types': [
                            'http://schema.org/AdministrativeArea',
                            'http://schema.org/Enumeration',
                        ],
                        'num_distinct_values': 5,
                        'admin_area_level': 1,
                    },
                ],
                'spatial_coverage': [
                    {
                        'type': 'admin',
                        'column_names': ['zero'],
                        'column_indexes': [0],
                        'ranges': [
                            {
                                'range': {
                                    'type': 'envelope',
                                    'coordinates': [
                                        [-61.79784095, 55.065334377],
                                        [55.8545028, -21.370782159],
                                    ],
                                },
                            },
                        ],
                    },
                    {
                        'type': 'admin',
                        'column_names': ['one'],
                        'column_indexes': [1],
                        'ranges': [
                            {
                                'range': {
                                    'type': 'envelope',
                                    'coordinates': [
                                        [-61.79784, 53.72778],
                                        [13.81686, 14.40811],
                                    ],
                                },
                            },
                        ],
                    },
                ],
            },
        )

class TestMedianDist(unittest.TestCase):
    def test_median_dist(self):
        points = []

        def make_grid(mx, my):
            for y in range(-100, 100):
                for x in range(-100, 100):
                    points.append((
                        mx + x + random.random() * 0.2,
                        my + y + random.random() * 0.2,
                    ))
        make_grid(0, 0)
        make_grid(500, 0)
        make_grid(-200, 300)

        self.assertAlmostEqual(
            spatial.median_smallest_distance(points),
            0.9,
            delta=0.05
        )


#!/usr/bin/python3
"""
Create view route for states
"""
from api.v1.views import app_views
from flask import Flask, jsonify, abort, request
from models import storage
from models.state import State

@app_views.route('/states', strict_slashes=False, methods=['GET'])
def list_states():
    """list states"""
    list_states = []
    get_state = storage.all(State)
    for k, v in get_state.items():
        list_states.append(v.to_dict())
    return jsonify(list_states)

@app_views.route('/states/<state_id>', strict_slashes=False, methods=['GET'])
def state(state_id=None):
    """list states"""
    get_state = {}
    if state_id:
        get_state = storage.get(State, state_id)

    if get_state is None:
        return abort(404)

    return jsonify(get_state.to_dict()), 200

@app_views.route('/states/<state_id>', strict_slashes=False,
                 methods=['DELETE'])
def del_state(state_id=None):
    """delete states"""

    get_state = storage.get(State, state_id)
    if get_state:
        storage.delete(get_state)
        storage.save()
        return jsonify({}), 200
    else:
        return abort(404)

@app_views.route('/states', strict_slashes=False, methods=['POST'])
def add_state():
    """add state"""
    if not request.get_json(silent=True):
        return jsonify({'error': 'Not a JSON'}), 400

    if 'name' not in request.json:
        abort(400, 'Missing name')

    new_s = State()
    data = dict(request.get_json(silent=True))
    new_s.name = data['name']

    storage.new(new_s)
    storage.save()
    storage.reload()

    return jsonify(new_s.to_dict()), 201

@app_views.route('/states/<state_id>', strict_slashes=False, methods=['PUT'])
def update_state(state_id=None):
    """update states"""

    data = request.get_json(silent=True)
    if not data:
        return jsonify({'error': 'Not a JSON'}), 400

    get_state = storage.get(State, state_id)

    if get_state:
        update_s = dict(request.get_json(silent=True))
        for k, v in update_s.items():
            setattr(get_state, k, v)

        storage.save()
    else:
        abort(404)

    return jsonify(get_state.to_dict()), 200


import time

from django.contrib.sites.models import Site
from django.core.paginator import PageNotAnInteger, EmptyPage, Paginator
from django.template.loader import render_to_string
from django.urls import reverse

from tortulka.settings import SEPARATOR
from .encryption_util import encrypt
from .models import Subscriber
from .tasks import send_news_email

def paginate(objects, size, request, context, var_name='object_list'):
    """Paginate objects provided by view.

    This function takes:
    * list of elements;
    * number of objects per page;
    * request object to get url parameters from;
    * context to set new variables into;
    * var_name - variable name for list of objects.

    It returns updated context object.
    """

    # apply pagination
    paginator = Paginator(objects, size)

    # try to get page number from request
    page = request.GET.get('page', 1)
    try:
        object_list = paginator.page(page)
    except PageNotAnInteger:
        # if page is not an integer, deliver first page
        object_list = paginator.page(1)
    except EmptyPage:
        # if page is out of range (e.g. 9999),
        # deliver last page of results
        object_list = paginator.page(paginator.num_pages)

    # set variables into context
    context[var_name] = object_list
    context['is_paginated'] = object_list.has_other_pages()
    context['page_obj'] = object_list
    context['paginator'] = paginator

    return context

def filtering(request, object_list):
    """Filter objects provided by view.

    This function takes:
    * request:
    * list of elements;

    It returns filtered Queryset.
    """

    # searching
    search_product = request.GET.get('q', '')
    if search_product:
        object_list = object_list.filter(
            title__icontains=search_product)

    # filter by category
    category_filter = request.GET.get('category', '')
    if category_filter == 'all':
        object_list = object_list
    elif category_filter:
        object_list = object_list.filter(category__slug=category_filter)

    # filter by tiers
    tiers_filter = request.GET.get('tiers', '')
    if tiers_filter == 'all':
        object_list = object_list
    elif tiers_filter:
        object_list = object_list.filter(category__slug=tiers_filter)

    # ordering
    order_by = request.GET.get('order_by', '')
    if order_by == 'newest':
        object_list = object_list.order_by('-added', 'id')
    elif order_by == 'oldest':
        object_list = object_list.order_by('added', 'id')
    elif order_by == 'abc':
        object_list = object_list.order_by('title', 'id')

    return object_list

def send_newsletter(subject, message):
    """
    Send letter to newsletter subscribers
    :param subject: str - subject of letter
    :param message:  str - message of letter
    :return: None
    """
    subscribers = list(Subscriber.objects.values_list('email', flat=True))
    for subscriber in subscribers:
        token = encrypt(subscriber + SEPARATOR + str(time.time()))
        url = reverse('unsubscribe')
        domain = Site.objects.get_current().domain
        confirmation_url = domain + url + '?token=' + token
        message = render_to_string('news_letter.html', {
            'message': message,
            'confirmation_url': confirmation_url
        })
        send_news_email.delay(subject, message, subscriber)

def send_subscription_email(email, url):
    """
    Send email to subscriber for confirmation subscription

    :param email: Subscriber email
    :param url: url to subscription confirmation page
    :return: None
    """
    token = encrypt(email + SEPARATOR + str(time.time()))
    confirmation_url = url + '?token=' + token
    mail_subject = 'Підтвердження підписки на розсилку'
    message = render_to_string('subscription.html', {
        'confirmation_url': confirmation_url
    })
    to_email = email
    send_news_email.delay(mail_subject, message, to_email)


import urllib
import os

from BeautifulSoup import BeautifulSoup

limesurvey_download_page = 'https://www.limesurvey.org/de/stable-release'
limesurvey_download_base_url = 'https://www.limesurvey.org'
archive_file_name = 'limesurvey.tar'

path = os.path.join("/tmp", archive_file_name)

page = urllib.urlopen(limesurvey_download_page).read()
soup = BeautifulSoup(page)

url = []
for div_class in soup.findAll('div', {"class": "uk-width-1-1"}):
    for a_class in div_class.findAll('a', {"class": "btn btn-success"}):
        url.append(a_class['href'])

for element in url:
    if 'targz' in element:
        limesurvey_download_base_url += element
        break

urllib.urlretrieve(limesurvey_download_base_url, path)


import csv
from os import sep
import pandas as pd
import time

def toNodeCSV(txt_path,csv_path_node):
    # 读取文件路径与目的文件路径
    txt_path = txt_path
    csv_path_node = csv_path_node

    #首先设置文件头
    with open(csv_path_node, 'w')  as csvfile_node:
        header = ['userid:ID',':LABEL']
        writer = csv.writer(csvfile_node)
        writer.writerow(header)
    csvfile_node.close()

    #读取文件， error_bad_lines可能是跳过报错行,header=None是说明txt文件中没有表头。sep='\s*'表示有多个空格作为间隔符
    #reader = pd.read_table(txt_path, iterator=True, encoding='utf8', error_bad_lines=False)
    reader = pd.read_table(txt_path, header=None, sep='\s*',iterator=True, encoding='utf8', error_bad_lines=False)
    #循环读取
    loop = True
    while loop:
        try:
            #每次读取5000个
            chunk = reader.get_chunk(50000)
            #print(chunk[0])
            #print(chunk[1])
            s1=chunk[0]
            s2=chunk[1]
            print("=====")
            chunk_new = pd.DataFrame()
            chunk_new[0] = pd.concat([s1,s2],axis=0)
            chunk_new[1]="USER"
            #print(chunk_new)
            #mode='a' 追加读取, index=False 舍去行数
            chunk_new.to_csv(csv_path_node, mode='a', index=False, header=None, encoding='utf_8_sig')
        except StopIteration:
            loop = False
            print("Iteration is stopped.")

def toRelationCSV(txt_path,csv_path_edge):
    # 读取文件路径与目的文件路径
    txt_path = txt_path
    csv_path_edge = csv_path_edge

    #首先设置文件头
    with open(csv_path_edge, 'w')  as csvfile_edge:
        header = [':START_ID',':END_ID',':TYPE']
        writer = csv.writer(csvfile_edge)
        writer.writerow(header)
    csvfile_edge.close()

    #读取文件， error_bad_lines可能是跳过报错行,header=None是说明txt文件中没有表头。sep='\s*'表示有多个空格作为间隔符
    #reader = pd.read_table(txt_path, iterator=True, encoding='utf8', error_bad_lines=False)
    reader = pd.read_table(txt_path, header=None, sep='\s*',iterator=True, encoding='utf8', error_bad_lines=False)
    #循环读取
    loop = True
    i = 0
    while loop:
        try:
            #每次读取5000个
            chunk = reader.get_chunk(10000000)
            #print(chunk)
            print(i)
            i=i+1
            chunk[2]="Follow"
            #print(chunk)

            new_chunk = chunk.drop_duplicates()

            #mode='a' 追加读取, index=False 舍去行数
            new_chunk.to_csv(csv_path_edge, mode='a', index=False, header=None, encoding='utf_8_sig')
        except StopIteration:
            loop = False
            print("Iteration is stopped.")

def removeDuplicate(csv_path_node, final_csv_path_node):
    # 读取文件路径与目的文件路径
    csv_path_node = csv_path_node
    final_csv_path_node = final_csv_path_node

    #首先为final csv设置文件头
    #with open(final_csv_path_node, 'w')  as csvfile_node:
    #    header = ['userid:ID',':LABEL']
    #    writer = csv.writer(csvfile_node)
    #    writer.writerow(header)
    #csvfile_node.close()

    #读取csv
    csvframe = pd.read_csv(csv_path_node, encoding='utf8')
    final_csv= csvframe.drop_duplicates()
    final_csv.to_csv(final_csv_path_node,header=True, index=False)

def removeDuplicate_limitedMemory(csv_path_node, final_csv_path_node):
    # 读取文件路径与目的文件路径
    csv_path_node = csv_path_node
    final_csv_path_node = final_csv_path_node

    #首先为final csv设置文件头
    with open(final_csv_path_node, 'w')  as csvfile_node:
        header = ['userid:ID',':LABEL']
        writer = csv.writer(csvfile_node)
        writer.writerow(header)
    csvfile_node.close()

    #读取csv
    csvreader = pd.read_csv(csv_path_node, header=None, iterator=True, encoding='utf8', error_bad_lines=False)
    #循环读取
    loop = True
    while loop:
        try:
            #每次读取100000个
            chunk = csvreader.get_chunk(90000000)
            new_chunk = chunk.drop_duplicates()
            print(new_chunk.head())
            print("=====")
            print(new_chunk.describe())
            #mode='a' 追加读取, index=False 舍去行数
            new_chunk.to_csv(final_csv_path_node, mode='a', index=False, header=None, encoding='utf_8_sig')
            #time.sleep(10)
        except StopIteration:
            loop = False
            print("removeDuplicate Iteration is stopped.")

def removeDuplicateEdge_limitedMemory(csv_path_edge, final_csv_path_edge):
    # 读取文件路径与目的文件路径
    csv_path_edge = csv_path_edge
    final_csv_path_edge = final_csv_path_edge

    #首先为final csv设置文件头
    with open(final_csv_path_edge, 'w')  as csvfile_edge:
        header = [':START_ID',':END_ID',':TYPE']
        writer = csv.writer(csvfile_edge)
        writer.writerow(header)
    csvfile_edge.close()

    #读取csv
    csvreader = pd.read_csv(csv_path_edge, header=None, iterator=True, encoding='utf8', error_bad_lines=False)
    #循环读取
    loop = True
    i = 0
    while loop:
        try:
            #每次读取100000个
            chunk = csvreader.get_chunk(200000000)
            new_chunk = chunk.drop_duplicates()
            print(new_chunk.head())
            print(i)
            i=i+1
            #print(new_chunk.describe())
            #mode='a' 追加读取, index=False 舍去行数
            new_chunk.to_csv(final_csv_path_edge, mode='a', index=False, header=None, encoding='utf_8_sig')
            #time.sleep(10)
            if i==1:
                print("end")
                break
        except StopIteration:
            loop = False
            print("removeDuplicateEdge Iteration is stopped.")

if __name__ == '__main__':
    txt_path = 'H:/Data/links-anon.txt'

    csv_path_node= 'H:/Data/node.csv'
    final_csv_path_node = "H:/data/noDuplicateNodes.csv"

    csv_path_edge = "H:/data/Relation.csv"

    final_csv_path_node2 = "H:/data/noDuplicateNodes2.csv"
    final_csv_path_node3 = "H:/data/noDuplicateNodes3.csv"
    final_csv_path_node4 = "H:/data/noDuplicateNodes4.csv"
    final_csv_path_node5 = "H:/data/noDuplicateNodes5.csv"
    final_csv_path_node6 = "H:/data/noDuplicateNodes6.csv"

    final_csv_path_edge1 = "H:/data/noDuplicateEdges1.csv"
    final_csv_path_edge2 = "H:/data/noDuplicateEdges2.csv"
    final_csv_path_edge3 = "H:/data/noDuplicateEdges3.csv"
    final_csv_path_edge5 = "H:/data/noDuplicateEdges5.csv"

    #toNodeCSV(txt_path,csv_path_node)
    #removeDuplicate_limitedMemory(final_csv_path_node5,final_csv_path_node6)
    #toRelationCSV(txt_path,csv_path_edge)
    removeDuplicateEdge_limitedMemory(final_csv_path_edge3,final_csv_path_edge5)