# Author: Pyeong Eun Kim
# Date: Dec.01.2020
# Title: Graph-based Deep Generative Model version 3 (single batch only)
# Affiliation: JLK Genome Research Centre

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributions.categorical import Categorical
import dgl
from functools import partial

class ScaffoldGNN(nn.Module):
    def __init__(self, num_auxiliary, num_prop_rounds=[3,3,2,2,2,2],
                 num_node_features=[8,4,3,2], num_edge_features=[4,5],
                 node_emb_dim=128, edge_emb_dim=128, edge_embedding_per_round=True):
        super(ScaffoldGNN, self).__init__()

        # set number of propagation of each propagating functions
        self.num_prop_rounds = {"whole":None, "scaffold":None,
                                "addnode":None, "addedge":None,
                                "selectnode":None, "selectisomer":None}
        for n, key in enumerate(self.num_prop_rounds.keys()):
            self.num_prop_rounds[key] = num_prop_rounds[n]

        # set all arguments to be shared
        self.num_auxiliary = num_auxiliary
        self.num_node_types = num_node_features[0]
        self.num_edge_types = num_edge_features[0]
        self.node_emb_dim = node_emb_dim
        self.edge_emb_dim = edge_emb_dim

        self.init_node_embedding_atom = nn.Embedding(num_node_features[0], node_emb_dim)
        self.init_node_embedding_chiral = nn.Embedding(num_node_features[1], node_emb_dim)
        self.init_node_embedding_formal_charge = nn.Embedding(num_node_features[2], node_emb_dim)
        self.init_node_embedding_aromatic = nn.Embedding(num_node_features[3], node_emb_dim)

        self.init_edge_embedding_bond = nn.Embedding(num_edge_features[0], edge_emb_dim)
        self.init_edge_embedding_stereo = nn.Embedding(num_edge_features[1], edge_emb_dim)

        """ENCODING PART"""
        # whole graph part
        self.whole_graph_prop = ConditionalProp(self.num_prop_rounds["whole"],
                                                num_auxiliary, node_emb_dim, edge_emb_dim) # node_emb_dim
        self.whole_graph_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=2*node_emb_dim)  # node_emb_dim*2
        self.fc_mu = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.fc_var = nn.Linear(node_emb_dim*2, node_emb_dim)

        """DECODING PART"""
        # scaffold graph part
        self.scaffold_graph_prop = ConditionalProp(self.num_prop_rounds["scaffold"],
                                                   num_auxiliary, node_emb_dim, edge_emb_dim)

        """DECODING ACTION DECISION"""
        # add node
        # we will concat auxiliary with latent vector when propagate scaffold graph
        num_auxiliary_decode = num_auxiliary + node_emb_dim
        self.add_nodes_prop = ConditionalProp(self.num_prop_rounds["addnode"],
                                              num_auxiliary_decode,
                                              node_emb_dim, edge_emb_dim)
        self.add_nodes_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)
        self.add_nodes_mlp = MLP(3, node_emb_dim + num_auxiliary_decode, node_emb_dim, self.num_node_types + 1, True)
        #self.add_nodes_softmax = nn.Softmax(dim=1)

        # init node
        self.init_nodes_mlp1 = nn.Embedding(self.num_node_types, node_emb_dim)
        self.init_nodes_mlp2 = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.init_nodes_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)

        # add edge
        self.add_edges_prop = ConditionalProp(self.num_prop_rounds["addedge"],
                                              num_auxiliary_decode,
                                              node_emb_dim, edge_emb_dim)
        self.add_edges_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)
        # in this case we did not use num_edge_types + 1
        # instead we used num_edge_type because we are not considering adding aromatic bonds
        # thus, the final logit would correspond to [single, double, triple, STOP]
        self.add_edges_mlp = MLP(3, node_emb_dim + num_auxiliary_decode, node_emb_dim, self.num_edge_types, True)
        #self.add_edges_softmax = nn.Softmax(dim=1)

        # init edge
        # use self.num_edge_types -1 because it corresponds to [single, double, triple]
        # from which aromatic is excluded.
        self.init_edges_mlp1 = nn.Embedding(self.num_edge_types - 1, node_emb_dim)
        self.init_edges_mlp2 = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.init_edges_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)

        # select dest node
        self.select_nodes_prop = ConditionalProp(self.num_prop_rounds["selectnode"],
                                                 num_auxiliary_decode,
                                                 node_emb_dim, edge_emb_dim)
        self.select_nodes_mlp = nn.Linear(node_emb_dim + node_emb_dim + num_auxiliary_decode, 1)

    def prepare_for_traininig(self):
        self.node_probability = []
        self.edge_probability = []
        self.dest_probability = []

    def whole_graph_encode(self, g_whole, conditions):
        """
        conditions: real conditions such as MW, logP, etc...
        """
        g_whole.ndata["node_features"] = torch.stack([
                                                self.init_node_embedding_atom(g_whole.ndata["atom_type"]),
                                                self.init_node_embedding_chiral(g_whole.ndata["chirality_type"]),
                                                self.init_node_embedding_formal_charge(g_whole.ndata["charge_type"]),
                                                self.init_node_embedding_aromatic(g_whole.ndata["aromatic_type"]),
                                                ], dim=1).sum(1)
        g_whole.edata["edge_features"] = torch.stack([
                                                self.init_edge_embedding_bond(g_whole.edata["bond_type"]),
                                                self.init_edge_embedding_stereo(g_whole.edata["stereo_type"])
                                                ], dim=1).sum(1)
        g_whole = self.whole_graph_prop(g_whole, conditions)
        # print("################after whole graph:", g_whole.edata["edge_features"].size())
        readout = self.whole_graph_readout(g_whole)
        mu = self.fc_mu(readout)
        logvar = self.fc_var(readout)
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        z = eps * std + mu

        return z, logvar, mu

    def scaffold_propagate(self, g_scaffold, conditions):
        g_scaffold.ndata["node_features"] = torch.stack([
                                                self.init_node_embedding_atom(g_scaffold.ndata["atom_type"]),
                                                self.init_node_embedding_chiral(g_scaffold.ndata["chirality_type"]),
                                                self.init_node_embedding_formal_charge(g_scaffold.ndata["charge_type"]),
                                                self.init_node_embedding_aromatic(g_scaffold.ndata["aromatic_type"]),
                                                ], dim=1).sum(1)
        g_scaffold.edata["edge_features"] = torch.stack([
                                                self.init_edge_embedding_bond(g_scaffold.edata["bond_type"]),
                                                self.init_edge_embedding_stereo(g_scaffold.edata["stereo_type"])
                                                ], dim=1).sum(1)
        g_scaffold = self.scaffold_graph_prop(g_scaffold, conditions)

        return g_scaffold

    def add_nodes_encode(self, g_scaffold, conditions, training=True):
        g_scaffold = self.add_nodes_prop(g_scaffold, conditions)
        readout = self.add_nodes_readout(g_scaffold)
        readout = torch.cat([readout, conditions])
        readout = self.add_nodes_mlp(readout)
        #readout = self.add_nodes_softmax(readout)
        if not training:
            # instead of sampling max, we sampled from distribution
            dist = F.softmax(readout, dim=0)
            dist = Categorical(dist)
            return dist.sample()
        else:
            return readout

    def init_nodes_encode(self, g_scaffold, categorical_node_feat):
        node_feat = self.init_nodes_mlp1(categorical_node_feat) #Gxembed
        readout = self.init_nodes_readout(g_scaffold)   # Gxembed
        readout = torch.cat([readout, node_feat])
        node_feat = self.init_nodes_mlp2(readout)
        return node_feat

    def add_edges_encode(self, g_scaffold, conditions, training=True):
        g_scaffold = self.add_edges_prop(g_scaffold, conditions)
        readout = self.add_edges_readout(g_scaffold)
        readout = torch.cat([readout, conditions])
        readout = self.add_edges_mlp(readout)
        #readout = self.add_edges_softmax(readout)
        if not training:
            # instead of sampling max, we sampled from distribution
            dist = F.softmax(readout, dim=0)
            dist = Categorical(dist)
            return dist.sample()
        else:
            return readout

    def init_edges_encode(self, g_scaffold, categorical_edge_feat):
        edge_feat = self.init_edges_mlp1(categorical_edge_feat) #Gxembed
        readout = self.init_edges_readout(g_scaffold)   # Gxembed
        readout = torch.cat([readout, edge_feat])
        edge_feat = self.init_edges_mlp2(readout)
        return edge_feat

    def select_nodes_encode(self, g_scaffold, conditions):
        src = g_scaffold.number_of_nodes() - 1
        possible_dest = range(src)
        g_scaffold = self.select_nodes_prop(g_scaffold, conditions)
        possible_dest_emb = g_scaffold.nodes[possible_dest].data["node_features"]
        src_emb_expand = g_scaffold.nodes[src].data["node_features"].expand(src, -1)
        condition_expand = conditions.expand(src, -1)
        # print(possible_dest_emb.size(), src_emb_expand.size(), condition_expand.size())
        logits = self.select_nodes_mlp(torch.cat([possible_dest_emb,
                                                  src_emb_expand,
                                                  condition_expand], dim=1).float()).squeeze(1)
        # print(logits.size())
        return logits

    def forward(self, g_whole, g_scaffold, conditions, actions):
        self.prepare_for_traininig()
        z, logvar, mu = self.whole_graph_encode(g_whole, conditions)
        g_scaffold = self.scaffold_propagate(g_scaffold, conditions)
        # print("after scaffold:",g_scaffold.edata["edge_features"].size())
        # concat conditions with latent vector for decoding process
        conditions = torch.cat([z, conditions])
        # print("conditions", conditions.size())
        add_node = True
        index = 0
        while add_node:
            # add node but which node?
            node_dist = self.add_nodes_encode(g_scaffold, conditions)  # if stop add_node = False (setting from dataset)
            self.node_probability.append(node_dist)
            if actions[index][0] == self.num_node_types:
                add_node = False
                continue
            else:
                g_scaffold.add_nodes(1)
                num_nodes = g_scaffold.number_of_nodes()
                # get node feature
                init_node = actions[index][0]
                new_node_feat = self.init_nodes_encode(g_scaffold, init_node)
                # print(new_node_feat.size(), g_scaffold.nodes[num_nodes - 1].data['node_features'].size())
                g_scaffold.nodes[num_nodes - 1].data['node_features'] = new_node_feat.unsqueeze(0)
            edge_index = 0
            add_edge = True
            #print(actions[index], actions)
            while add_edge and edge_index < len(actions[index][1]):
                edge_add_action = self.add_edges_encode(g_scaffold, conditions)
                self.edge_probability.append(edge_add_action)
                # print("%%%%%%%%%%%",actions[index], edge_index)
                if actions[index][1][edge_index] == self.num_edge_types - 1:
                    add_edge = False
                    continue
                else:
                    init_edge = actions[index][1][edge_index]  # edge types
                    # print("after scaffold:",g_scaffold.edata["edge_features"].size())
                    select_node = self.select_nodes_encode(g_scaffold, conditions)
                    # print("###############after scaffold:",g_scaffold.edata["edge_features"].size())
                    self.dest_probability.append(select_node)
                    src, dest = num_nodes - 1, actions[index][2][edge_index]  # edge dest
                    src_list = [src, dest]
                    dest_list = [dest, src]
                    g_scaffold.add_edges(src_list, dest_list)
                    edge_emb = self.init_edges_encode(g_scaffold, init_edge)
                    # print("###edge###",g_scaffold.edges[src_list, dest_list].data['edge_features'].size(),torch.stack([edge_emb, edge_emb], dim=0).size())
                    g_scaffold.edges[src_list, dest_list].data['edge_features'] = torch.stack([edge_emb, edge_emb], dim=0)
                edge_index += 1
            index += 1

        return z, logvar, mu, self.node_probability, self.edge_probability, self.dest_probability

    def sample(self):
        """
        # TO DO: implement sampling for validation
        """
        return None

class ConditionalProp(nn.Module):
    def __init__(self, num_prop_rounds, num_auxiliary,
                 node_emb_dim=128, edge_emb_dim=128):
        super(ConditionalProp, self).__init__()

        # number of propagation
        self.num_prop_rounds = num_prop_rounds
        # print("auxiliary number:", num_auxiliary)
        # make propagation layers corresponding to num rounds
        self.prop_layers = nn.ModuleList()
        for round in range(num_prop_rounds):
            self.prop_layers.append(SingleProp(num_auxiliary, node_emb_dim, edge_emb_dim))

    def forward(self, g, auxiliary):
        # condition should be list of conditions
        # node categorical to embed
        """
        # if feed g of whole molecule and scaffold, "node_features, and edge_features"
        # should be already embedded by shared embedding layer
        """
        # print("**********edge feature")
        # print(g.edata["edge_features"].size())
        original_edge = g.edata["edge_features"]
        g.edata["edge_features"] = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        for round in range(self.num_prop_rounds):
            g = self.prop_layers[round](g, auxiliary)
        g.edata["edge_features"] = original_edge
        return g

class MLP(nn.Module):
    """ MultilayerPerceptron (modified from https://github.com/weihua916/powerful-gnns/blob/master/models/mlp.py)
    Args:
        num_layers (int): number of layers in the neural networks. If num_layers=1, this reduces to linear model.
        input_dim (int): dimensionality of input features
        hidden_dim (int): dimensionality of hidden units at ALL layers
        output_dim (int): number of classes for prediction
        batch_norm (Bool): batch normalisation of after MLP (default to True)
    """
    def __init__(self, num_layers, input_dim, hidden_dim, output_dim, batch_norm=True):
        super(MLP, self).__init__()

        self.linear_or_not = True #default is linear model
        self.num_layers = num_layers

        if num_layers < 1:
            raise ValueError("number of layers should be positive!")
        elif num_layers == 1:
            #Linear model
            self.linear = nn.Linear(input_dim, output_dim)
        else:
            # Multi-layer model
            self.linear_or_not = False
            self.linears = torch.nn.ModuleList()
            self.batch_norms = torch.nn.ModuleList()

            self.linears.append(nn.Linear(input_dim, hidden_dim))
            for layer in range(num_layers - 2):
                self.linears.append(nn.Linear(hidden_dim, hidden_dim))
            self.linears.append(nn.Linear(hidden_dim, output_dim))

            for layer in range(num_layers - 1):
                if batch_norm:
                    self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
                else:
                    self.batch_norms.append(nn.Identity())
    def forward(self, x):
        if self.linear_or_not:
            #If linear model
            return self.linear(x)
        else:
            #If MLP
            h = x.float()
            for layer in range(self.num_layers - 1):
                # print(layer, h.size(), self.linears[layer])
                ## print(self.linears[layer](h))
                if len(h.size()) > 2:
                    h = F.relu(self.linears[layer](h))
                else:

                    h = F.relu(self.linears[layer](h))
            return self.linears[self.num_layers - 1](h)

class SingleProp(nn.Module):
    def __init__(self, num_auxiliary, node_emb_dim,
                 edge_emb_dim=None, num_mlp_layer=2,
                 activation_dim=None, hidden_dim=None,
                 batch_norm=True):
        super(SingleProp, self).__init__()
        # num_auxiliary can be number of conditions or dimension of latent vector
        # eg) 4 for conditions such as [logP_whole, mw_whole, logP_scaffold, mw_whole]
        # eg) 128 for latent vectors such as [0.1, 0.3, ..., 0.9, -1.1]
        self.num_auxiliary = num_auxiliary

        self.node_emb_dim = node_emb_dim

        if not edge_emb_dim:
            edge_emb_dim = node_emb_dim
        if not hidden_dim:
            # same as GIN paper
            hidden_dim = 2 * (node_emb_dim*2 + edge_emb_dim + self.num_auxiliary)

        # if custom activation dim exists apply; otherwise use default
        if activation_dim:
            self.activation_dim = activation_dim
        else:
            # same as paper <https://arxiv.org/abs/1905.13639> indicated
            self.activation_dim = node_emb_dim

        # input being [hv, hu, xuv]
        # dimension [node_hidden_dim, node_hidden_dim, edge_hidden_dim]

        self.message_func = MLP(num_layers=num_mlp_layer,
                                input_dim=node_emb_dim*2 + edge_emb_dim + self.num_auxiliary,
                                hidden_dim=hidden_dim,
                                output_dim=self.activation_dim,
                                batch_norm=False)
        self.reduce_func = partial(self.get_activation_node)
        self.node_update_func = nn.GRUCell(self.activation_dim, node_emb_dim)

        # initialise model parameters (Xaiver Uniform)
        #self.initialise_parameters()

    def initialise_parameters(self):
        """Reinitialise model parameters."""

        nn.init.xavier_uniform_(self.node_update_func.weight.data)
        nn.init.xavier_uniform_(self.message_func)
        if self.edge_embedding_per_round:
            nn.init.xavier_uniform_(self.edge_embedding.weight.data)

    def concat_msg(self, edges):
        """
            Concatenate features from source nodes with edge features and aggregate them in destination nodes.
            For an edge u->v, return concat([h_u, x_uv]).
            v is the destination node which would get the aggregated signals.
        """
        return {'m': torch.cat([edges.src['node_features'],  # feature of neighbor nodes
                                edges.data['edge_features']], # feature of edges connecting to neighbor nodes
                                dim=1)}

    def get_activation_node(self, nodes):
        """
            Concatenate features of each node with features of its neighboring nodes and edges connecting the node.
            Condition is already concatenated within the edge features.
            Then, we use MLP to calculate weighted sum of all messages, which is called activation of nodes.
        """
        node_old = nodes.data['node_features']
        m = nodes.mailbox['m'] # collect messages assigned by msg which is fed to update_all function.
        message = torch.cat([node_old.unsqueeze(1).expand(-1, m.size(1), -1), m], dim=2)
        # print("message_size()--------------------")
        # print(message.size(), m.size(), node_old.size())
        # print("--------------------")
        node_activation = self.message_func(message).sum(1)
        return {'a': node_activation}

    def forward(self, g, auxiliary):

        #edge_condition = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        #g.edata["edge_features"] = self.message_func(edge_condition).sum(1)
        #g.edata["edge_features"] = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        # print("feat size()--------------------")
        # print(g.edata["edge_features"].size(), auxiliary)
        # print("--------------------")
        # get activation of each nodes ("a")
        g.update_all(message_func=self.concat_msg,
                     reduce_func=self.get_activation_node)

        # update node features based on the activation of nodes
        g.ndata['node_features'] = self.node_update_func(g.ndata['a'], g.ndata['node_features'])

        return g

class GraphReadOut(nn.Module):
    def __init__(self, node_embed_dim, graph_embed_dim=None, activation="sigmoid", mode='mean'):
        super(GraphReadOut, self).__init__()

        assert mode in ['max', 'mean', 'sum'], \
            "Expect mode to be 'max' or 'mean' or 'sum', got {}".format(mode)
        assert activation in ['sigmoid', 'relu', 'tanh'], \
            "Expect activation to be 'sigmoid' or 'relu' or 'tanh', got {}".format(activation)
        self.mode = mode
        # Setting from the paper
        if graph_embed_dim:
            self.graph_embed_dim = graph_embed_dim
        else:
            self.graph_embed_dim = 2 * node_embed_dim

        if activation == "sigmoid":
            act = nn.Sigmoid()
        elif activation == "relu":
            act = nn.ReLU()
        elif activation == "tanh":
            act = nn.Tanh()
        # Embed graphs
        self.node_gating = nn.Sequential(
            nn.Linear(node_embed_dim, 1),
            act
        )
        self.node_to_graph = nn.Linear(node_embed_dim,
                                       self.graph_embed_dim)

    def forward(self, g):
        node_embeds = g.ndata['node_features']
        graph_embeds = self.node_gating(node_embeds) * self.node_to_graph(node_embeds)
        with g.local_scope():
            g.ndata['graph_features'] = graph_embeds
            if self.mode == 'max':
                graph_feats = dgl.max_nodes(g, 'graph_features')
            elif self.mode == 'mean':
                graph_feats = dgl.mean_nodes(g, 'graph_features')
            elif self.mode == 'sum':
                graph_feats = dgl.sum_nodes(g, 'graph_features')
        if graph_feats.size(0) == 1:
            return graph_feats.squeeze(0)  # if batch == 1 delete batch dimension
        else:
            return graph_feats


from logging import getLogger

import ckan.plugins as p
import ckan.plugins.toolkit as toolkit
from ckan.common import request
from routes.mapper import SubMapper
import ckan.model as model

import ckanext.interlinking.logic.action as action
import ckanext.interlinking.logic.auth as auth
import ckanext.interlinking.controllers.package as package_controller
import ckan.lib.i18n as i18n

import pprint
import json
log = getLogger(__name__)

class ReclinePreviewInterlinking(p.SingletonPlugin):
    """This extension previews resources using recline for interlinking purposes

        This extension implements several interfaces

      - ``IConfigurer`` to modify the configuration
      - ``IActions`` to define interlinking-specific API actions
      - ``IRoutes`` to create mapping to the required controllers
      - ``IAuthFunctions`` to define auth functions for the interlinking-specific API actions
      - ``ITemplateHelpers`` to define new template helpers

        Important note: For the moment since there is not way to select better preview, if recline preview is enabled recline-interlinking plugin
                        must be loaded with higher priority in development.ini
    """
    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.IActions)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IAuthFunctions)
    p.implements(p.IPackageController, inherit=True)
    p.implements(p.ITemplateHelpers)
    #p.implements(p.IResourcePreview, inherit=True)

    # IConfigurer 1/1 function (inherit=True)
    def update_config(self, config):
        ''' Set up the resource library, public directory and
        template directory for the preview
        '''
        toolkit.add_public_directory(config, 'theme/public')
        toolkit.add_template_directory(config, 'theme/templates')
        toolkit.add_resource('theme/public', 'ckanext-interlinking')
        toolkit.add_resource('theme/public/vendor/slickgrid', 'ckanext-interlinking-slickgrid')

    # IActions 1/1 function
    def get_actions(self):
        return {
                'interlinking_resource_create': action.interlinking_resource_create,
                'interlinking_resource_update': action.interlinking_resource_update,
                'interlinking_resource_delete': action.interlinking_resource_delete,
                'interlinking_resource_finalize': action.interlinking_resource_finalize,
                'interlinking_resource_search': action.interlinking_resource_search,
                'interlinking_get_reference_resources': action.interlinking_get_reference_resources,
                'interlinking_resource_download': action.interlinking_resource_download,
                'interlinking_check_interlink_complete': action.interlinking_check_interlink_complete,
                'interlinking_apply_to_all': action.interlinking_apply_to_all,
                'interlinking_star_search': action.interlinking_star_search,
                }

    # IRoutes 1/1 function (inherit=True)
    def before_map(self, mapper):
        mapper.connect(
                'resource_interlink',
                '/dataset/{id}/resource_interlink/{resource_id}',
                controller='ckanext.interlinking.controllers.package:InterlinkingController',
                action = 'resource_interlink')

        mapper.connect(
                'resource_interlink_inner',
                '/dataset/{id}/resource_interlink_inner/{resource_id}',
                controller='ckanext.interlinking.controllers.package:InterlinkingController',
                action = 'resource_datapreview')

        mapper.connect(
                       'interlinking_resource_download',
                       '/dataset/{id}/resource/interlinking/{resource_id}',
                       controller = 'ckanext.interlinking.controllers.package:InterlinkingController',
                       action = 'interlinking_resource_download')
        return mapper

    #IPackageController 1/1 function (inherit=True)
    def before_view(self, data_dict):
        # TODO: Need to cut intermediate interlinking resources here
        # so they are not visible in UI/other API functions
        for k,v in data_dict.iteritems():
            pass

            if k=='resources':
                new_res = []
                for res in v:
                    if not 'interlinking_resource' in res:
                        new_res.append(res)
                data_dict.update({k:new_res})
        data_dict.update({'num_resources':len(new_res)})
        return data_dict

    # IAuthFunctions 1/1 function
    def get_auth_functions(self):
        return {
                'interlinking_resource_create': auth.interlinking_resource_create,
                'interlinking_resource_update': auth.interlinking_resource_update,
                'interlinking_resource_delete': auth.interlinking_resource_delete,
                'interlinking_resource_finalize': auth.interlinking_resource_finalize,
                'interlinking_check_interlink_complete': auth.interlinking_check_interlink_complete,
                'interlinking_get_reference_resources': auth.interlinking_get_reference_resources,
                'interlinking_apply_to_all': auth.interlinking_apply_to_all,
                }
    # ITemplateHelpers 1/1 function
    def get_helpers(self):
        return {
                #'resource_edit': resource_edit,
                }

    #IResourcePreview
    """
    def can_preview(self, data_dict):
        # if the resource is in the datastore then we can preview it with recline
        if data_dict['resource'].get('datastore_active'):
            return True
        format_lower = data_dict['resource']['format'].lower()
        previewable = format_lower in ['csv', 'xls', 'tsv']
        return {
                'can_preview': previewable,
                'quality': 3
                }
    """
    def _get_context(self):
        return {
                'model':model,
                'session':model.Session,
                'ignore_auth':True,
                'api_version':3,
                }


num = int(input())
s = ""
ops = []
for i in range(num):
    op = input().split(" ")
    ops.append(op)
    if op[0] == 'T':
        s += op[1]
    elif op[0] == "Q":
        if len(s) >= int(op[1]):
            print(s[int(op[1])-1])
        else:
            print(s)
            print(op)
            print(ops)
    else:
        s = s[:len(s)-int(op[1])]


import edwards
from modular import ModInt
from group import Group, Point

# TODO:
#   - conversions to weierstrauss form

class montEdwardsCurve(edwards.EdwardsCurve):
    """
    A Montgomery curve over a field K is defined by the equation:
    By^2 = x^3 + Ax^2 + X for A, B in K.
    K is a finite prime field of order p.

    Change of parameters maps:
    tE(a, d) --> M(A, B): (2(a + d)/(a - d), 4/(a - d))
    B = 4 * inverse(ed.a - ed.d, ed.p)
    A = 2 * (ed.a + ed.d) * inverse(ed.a - ed.d, ed.p)

    Curve25519 is the curve
    """

    def __init__(self, A, B, p, r, gx, gy, ed):
        self.name = "Montgomery"
        self.c = ed
        self.p = p
        self.r = r
        self.A = A
        self.B = B
        self.base = montEdwardsPoint(self, gx, gy)

        self.one = self.c.one
        self.zero = self.c.zero
        self.i = montEdwardsPoint(self.zero, self.one)

        if not self.base._on_curve():
            raise Exception("base is not on curve")

    def point(self):
        return montEdwardsPoint(self, ModInt(self.p), ModInt(self.p))

class montEdwardsPoint(edwards.EdwardsPoint):
    def __init__(self, curve, x=ModInt(), y=ModInt()):
        self.c = curve
        self.x = x
        self.y = y

    def _on_curve(self):
        """
        Tests that point (a) lies on the Montgomery curve with specified
        parameters: By^2 = x^3 + Ax^2 + x
        """
        x, y = self.x, self.y
        P = self.c.p
        l, r, yy, xx, t1 = ModInt(P), ModInt(P), ModInt(P), ModInt(P), ModInt(P)
        yy.mul(y, y)
        xx.mul(x, x)
        t1.mul(xx, self.c.A)
        l.mul(self.c.B, yy)
        r.exp(x, ModInt(P, 3)).add(r, t1).add(r, x)
        print("l", l.v, r.v, self)
        return l.equal(r)

    def identity(self):
        self.set(self.c.i)
        return self

    def inverse(self, a):
        self.x.set(a.x)
        self.x.neg(a.y)
        return self

    def add(self, p, q):
        """
        Adds two points on the Montgomery curve together.
        Only certain points can be added together: p != +/- q
        So this addition law is not unified
        Computational cost:
        """
        print("p", p)
        x1, x2 = p
        y1, y2 = q

        if p == q or p == self.inverse(q):
            return self.double(p)

        if p == self.identity() and q == self.identity():
            return self.identity()
        elif p == self.identity():
            return q
        elif q == self.identity():
            return p

        delta = (y2 - y1) * inverse(x2 - x1, self.p)
        x3 = self.B * delta**2 - self.A - x1 - x2

        y3 = delta * (x1 - x3) - y1

        # firstnum = (2 * x1 + x2 + self.A) * (y2 - y1)
        # firstdenom = x2 - x1

        # secondnum = pow(y2 - y1, 3)
        # seconddenom = pow(x2 - x1, 3)
        # y3 = firstnum * inverse(firstdenom, self.p) - (self.B * secondnum) * inverse(seconddenom, self.p) - y1

        p = (x3 % self.p, y3 % self.p)
        print("\nincorrect answer", p)
        assert self.is_element(p)
        return p

    def double(self, p):
        """
        Doubles a point on the curve.
        Computational cost:
        """
        x1, y1 = p

        if p == self.identity():
            return self.identity()

        delta = (3 * x1**2 + 2 * self.A * x1 + 1) * inverse(2*self.B*y1, self.p)
        x3 = self.B * delta**2 - self.A - 2 *x1
        y3 = delta * (x1 - x3) - y1
        # num = (x1**2 - 1)**2
        # denom = (4*x1*(x1**2 + self.A * x1 + 1))
        # x3 = num * inverse(denom, self.p)

        # firstnum = (3 * x1 + self.A) * (3 * x1**2 + 2 * self.A*x1 +1)
        # firstdenom = 2 * self.B * y1

        # secondnum = self.B * pow(3 * x1**2 + 2 * self.A * x1 + 1, 3)
        # seconddenom = pow(2 * self.B * y1, 3)

        # y3 = firstnum * inverse(firstdenom, self.p) - secondnum * inverse(seconddenom, self.p) - y1

        p = (x3 % self.p, y3 % self.p)
        print("correct answer", p)
        assert self.is_element(p)
        return p

    def from_ep(self, a):
        return self

    def to_ep(self, a):
        return ed


#!/usr/bin/python3.5
# -*- coding: UTF-8 -*-

import subprocess
import re
import os
import argparse

parser = argparse.ArgumentParser(prog='find_so')
parser.add_argument('--so_db_dir', default='so', type=str, help='copy all valid so dir')
parser.add_argument('--so_dest_dir', default='so', type=str, help='copy needed so to dest dir')
parser.add_argument('--copy_2_dest', action='store_true', default=False, help='copy file to text, or copy so to dest_dir')
parser.add_argument('--source_file', type=str, default=None, help='binary or *.so')
args = parser.parse_args()

so_db_dir = args.so_db_dir
so_dest_dir = args.so_dest_dir
copy_2_dest = args.copy_2_dest
so_or_exectuable_path = args.source_file

def copy_all_so_2_db():
    if not os.path.exists(so_db_dir):
        os.mkdir(so_db_dir)

    fh = subprocess.Popen("ldd -r " + so_or_exectuable_path, stdout=subprocess.PIPE, shell=True)
    arr = fh.stdout.readlines()
    patten = re.compile(r'.+ => (.+) \(')

    for line in arr:
        content = line.decode('ascii')
        content = content.replace('\t', '')
        res = patten.match(content)
        if res:
            so_name_path = res.group(1)

            if os.path.islink(so_name_path):
                target_file = os.path.realpath(so_name_path)

                print('copy {} => {}'.format(target_file, so_db_dir))
                os.system('cp ' + target_file + ' ' + so_db_dir)
                print('copy link {} => {}'.format(so_name_path, so_db_dir))
                os.system('cp -d ' + so_name_path + ' ' + so_db_dir)
            else:
                print('copy {} => {}'.format(so_name_path, so_db_dir))
                os.system('cp ' + so_name_path + ' ' + so_db_dir)

def copy_need_2_dest():
    if not os.path.exists(so_dest_dir):
        os.mkdir(so_dest_dir)
    while True:
        fh = subprocess.Popen("ldd -r " + so_or_exectuable_path, stdout=subprocess.PIPE, shell=True)
        arr = fh.stdout.readlines()
        patten = re.compile(r'(.+) => not found')

        hand_len = 0
        for line in arr:
            content = line.decode('ascii')
            content = content.replace('\t', '')
            res = patten.match(content)
            if res:
                so_name = res.group(1)
                existed_so_name_path = so_db_dir + "/" + so_name
                if os.path.islink(existed_so_name_path):
                    target_file = os.path.realpath(existed_so_name_path)

                    print('copy {} => {}'.format(target_file, so_dest_dir))
                    os.system('cp ' + target_file + ' ' + so_dest_dir)
                    print('copy link {} => {}'.format(existed_so_name_path, so_dest_dir))
                    os.system('cp -d ' + existed_so_name_path + ' ' + so_dest_dir)
                else:
                    print('copy {} => {}'.format(existed_so_name_path, so_dest_dir))
                    os.system('cp ' + existed_so_name_path + ' ' + so_dest_dir)

                hand_len = hand_len + 1

        if not hand_len:
            break

if __name__ == "__main__":
    if copy_2_dest:
        copy_need_2_dest()
    else:
        copy_all_so_2_db()


import numpy as np
from distance import pdist
from sklearn.cluster import KMeans

def kmeans_clustering(all_features, vocab_size, epsilon, max_iter):
    """
    The function kmeans implements a k-means algorithm that finds the centers of vocab_size clusters
    and groups the all_features around the clusters. As an output, centroids contains a
    center of the each cluster.

    :param all_features: an N x d matrix, where d is the dimensionality of the feature representation.
    :param vocab_size: number of clusters.
    :param epsilon: When the maximum distance between previous and current centroid is less than epsilon,
        stop the iteration.
    :param max_iter: maximum iteration of the k-means algorithm.

    :return: an vocab_size x d array, where each entry is a center of the cluster.
    """

    # Your code here. You should also change the return value.

    n = all_features.shape[0]
    dim = all_features.shape[1]

    mean = np.mean(all_features, axis = 0)
    std = np.std(all_features, axis = 0)
    centers = np.random.randn(vocab_size,dim)*std + mean

    centers_old = np.zeros(centers.shape)
    centers_new = centers

    error = np.linalg.norm(centers_new - centers_old)

    distances = np.zeros((n,vocab_size))
    clusters = np.zeros(n)

    iteration = 0

    while error >= epsilon and iteration < max_iter :
        for i in range(vocab_size):
            distances[:,i] = np.linalg.norm(all_features - centers_new[i], axis = 1)
        clusters = np.argmin(distances,axis = 1)
        centers_old = centers_new
        for i in range(vocab_size):
            centers_new[i] = np.mean(all_features[clusters == i],axis = 0)

        error = np.linalg.norm(centers_new - centers_old)
        iteration = iteration + 1
    nans = []
    for i in range(vocab_size):
        if np.isnan(centers_new[i,0]):
            nans.append(i)
    centers_new = np.delete(centers_new, nans, axis=0)

    return centers_new


import argparse
import requests
from urllib import parse

DEFAULT_COLOR = "blue"
DEFAULT_TEXT = "foo"
DEFAULT_VALUE = "bar"
DEFAULT_FILE_NAME = "badge.svg"

parser = argparse.ArgumentParser()
parser.add_argument("--color", help="hex value of the badge color, eg.#ff4545, blue, red")
parser.add_argument("--text", help="the text displayed on the badge, avoid using '#'")
parser.add_argument("--value", help="the value displayed on the badge")
parser.add_argument("--file_name", help="the output filename")

args = parser.parse_args()

color = args.color if args.color else DEFAULT_COLOR
color = parse.quote(color)
text = args.text if args.text else DEFAULT_TEXT
text = text.replace("-","_")
value = args.value if args.value else DEFAULT_VALUE
value = value.replace("-","_")
file_name = args.file_name if args.file_name else DEFAULT_FILE_NAME

url = f"https://img.shields.io/badge/{text}-{value}-{color}"
content = requests.get(url).text
print(url)
with open(file_name, 'w') as writer:
    writer.write(content)


## Student Name: Harsh Siddhapura
## Student ID: 1230169813
## Date: 09/01/2023

import tkinter as tk
from tkinter import messagebox

# Step 1: Input Collection
def get_key_input():
    key = entry.get()
    if len(key) != 1:
        messagebox.showerror("Error", "Please press only one key.")
    else:
        return key

# Step 2: Processing
def ascii_mapping(key):
    return ord(key)

def convert_to_binary(ascii_value):
    return bin(ascii_value)[2:]

# Step 3: Output Display
def display_binary(binary_value):
    print(f"Binary representation: {binary_value}")
    binary_label.config(text=f"Binary representation: {binary_value}")

# Main Program
def process_input():
    key = get_key_input()
    if key:
        ascii_val = ascii_mapping(key)
        ascii_label.config(text=f"ASCII value of '{key}': {ascii_val}")
        print(f"ASCII value of '{key}': {ascii_val}")
        binary_val = convert_to_binary(ascii_val)
        display_binary(binary_val)

# Create the main window
root = tk.Tk()
root.title("Binary Converter")
root.configure(bg="yellow")

# Create GUI elements
instruction_label = tk.Label(root, text="Press a key:")
instruction_label.pack()

entry = tk.Entry(root)
entry.pack()

convert_button = tk.Button(root, text="Convert", command=process_input)
convert_button.pack()

ascii_label = tk.Label(root, text="")
ascii_label.pack()

binary_label = tk.Label(root, text="")
binary_label.pack()

# Start the GUI event loop
root.mainloop()

Sem-1/IFT510 - Architecture /Python Labs/Module-2/Lab-2/asciiToBinary-GUI.py


#Given 2 int values, return True if one is negative and one is positive. Except if the parameter "negative" is True, then return True only if both are negative.

def pos_neg(x, y, negative):

    if negative:
        return x < 0 and y < 0
    else:
        return (x < 0 or x > 0) and (y < 0 or y > 0)

x = pos_neg(1, -1, False)
y = pos_neg(-1, 1, False)
z = pos_neg(-4, -5, True)

print(x, y, z)


"""
Построение кучи
Sample Input 1:
6
0 1 2 3 4 5
Sample Output 1:
0
Sample Input 2:
6
7 6 5 4 3 2
Sample Output 2:
4
2 5
1 4
0 2
2 5
"""

res = []

def swap(i, j, arr):
    """
    Функция для обмена значений двух переменных
    :param i: i-ая позиция элемента
    :param j: j-ая позиция элемента
    :param arr: массив чисел
    """
    arr[i], arr[j] = arr[j], arr[i]

def construction(i, arr):
    """
    Функция для построения кучи
    :param i: i-ый элемент массива
    :param arr: массив чисел
    """
    if 2 * i + 2 < len(arr) and arr[2 * i + 2] < arr[2 * i + 1] and arr[i] > arr[i * 2 + 2]:
        res.append([i, i * 2 + 2])
        swap(i, i * 2 + 2, arr)
        construction(2 * i + 2, arr)
    elif 2 * i + 1 < len(arr) and arr[2 * i + 1] < arr[i]:
        res.append([i, i * 2 + 1])
        swap(i, i * 2 + 1, arr)
        construction(2 * i + 1, arr)

def main():
    n = int(input())
    arr = [int(el) for el in input().split()][:n]
    for i in range(len(arr) - 1, -1, -1):
        construction(i, arr)

    if not res:
        print(0)
    else:
        print(len(res))
        for i in range(len(res)):
            print(res[i][0], res[i][1])

if __name__ == '__main__':
    main()

Алгоритмы_ теория и практика. Структуры данных/(2.3.1)Heap_construction.py


from selenium.webdriver.chrome.options import Options
import json
import elastic
from pprint import pprint

def search_record(es_object, index_name, search):
    res = es_object.search(index=index_name, body=search)
    pprint(res)

if __name__ == '__main__':
  elastic_search = elastic.connect_elasticsearch()

  if elastic_search is not None:
      search_object = {'query': {'match': {'title': 'cube'}}}            ####search record with  search_text = "cube"
      search_record(elastic_search, 'amazon', json.dumps(search_object))

      search_by_same_prefix = {'query': {'prefix': {'title': 'he'}}}
      search_record(elastic_search, 'amazon', json.dumps(search_by_same_prefix))     ### search record with title having same prefix = "he" ; helmet,headphone

      search_by_price = {'_source': ['title'], 'query': {'range': {'price': {'gte': 2000}}}}
      search_record(elastic_search, 'amazon', json.dumps(search_by_price))   #### search record with price >2000


import sys
import os
from pathlib import Path
from PyQt5 import QtCore, Qt
from PyQt5 import QtGui
from PyQt5 import QtWidgets
import utility
import numpy as np
import math
import pandas
#import pyqtgraph
import sip

class gestureWidget(QtWidgets.QWidget):

    def __init__(self, *args, **kwargs):
        QtWidgets.QWidget.__init__(self, *args, **kwargs)
        self.gesture_data = None
        self.gesture_pos = 0
        self.gesture_prev = -1
        self.gesture_len = 0
        self.gesture_dict = {0:'metaphoric',1:'beats',2:'deictics',3:'iconic'}
        self.graph_width = 910
        self.graph_height = 200
        self.time = 0
        self.layout = QtWidgets.QVBoxLayout(self)
        self.createUI()

    def createUI(self):

        self.time = 0
        self.pause_signal = False
        # video position slider
        self.graph_widget = QtWidgets.QWidget()
        # Layout of Container Widget
        self.graph_layout = QtWidgets.QGridLayout()
        self.graph_widget.setLayout(self.graph_layout)

        self.scrollarea = QtWidgets.QScrollArea()
        self.scrollarea.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        self.scrollarea.setWidgetResizable(True)
        # self.scrollarea.setGeometry(10, 0, self.graph_width, self.graph_height * 3)
        self.scrollarea.setWidget(self.graph_widget)

        self.positionSlider = QtWidgets.QSlider(QtCore.Qt.Horizontal, self)
        self.positionSlider.setStyleSheet("QSlider::groove:horizontal {background-color:grey;}"
                                "QSlider::handle:horizontal {background-color:black; height:8px; width: 8px;}")
        # self.positionSlider.setMinimum(1)
        # self.positionSlider.setMaximum(9)
        # self.positionSlider.setTickInterval(1)
        # self.positionSlider.setSingleStep(1) # arrow-key step-size
        # self.positionSlider.setPageStep(1) # mouse-wheel/page-key step-size
        # self.positionSlider.setTickPosition(QtWidgets.QSlider.TicksBelow)

        #self.positionSlider.valueChanged.connect(self.update_eeg_graph)

        self.hbuttonbox = QtWidgets.QHBoxLayout()
        self.openbutton = QtWidgets.QPushButton("Open gesture")
        self.hbuttonbox.addWidget(self.openbutton)
        self.openbutton.clicked.connect(self.open_file)

        self.prevbutton = QtWidgets.QPushButton("Prev")
        self.hbuttonbox.addWidget(self.prevbutton)
        self.prevbutton.clicked.connect(self.get_prev)

        self.nextbutton = QtWidgets.QPushButton("Next")
        self.hbuttonbox.addWidget(self.nextbutton)
        self.nextbutton.clicked.connect(self.get_next)
        # self.comboLabel = QtWidgets.QLabel("Select Channels:")
        # self.combobox = QtWidgets.QComboBox(self)
        # self.comboboxDelegate = utility.SubclassOfQStyledItemDelegate()
        # self.combobox.setItemDelegate(self.comboboxDelegate)
        # self.combobox.setSizeAdjustPolicy(0)
        # self.hbuttonbox.addWidget(self.comboLabel)
        # self.hbuttonbox.addWidget(self.combobox)

        self.hbuttonbox.addStretch(1)
        self.layout.addWidget(self.scrollarea)
        self.layout.addWidget(self.positionSlider)
        self.layout.addLayout(self.hbuttonbox)

        self.positionSlider.valueChanged.connect(self.setGestureType)
        self.setLayout(self.layout)

    def setGestureType(self):
        if self.gesture_pos == self.gesture_len:
            self.pause_signal = False
            return
        self.time = self.positionSlider.value()
        #print(int(self.time/1000))
        self.pause_signal = False
        if int(self.time/1000) == self.gesture_data[self.gesture_pos,2]:
            self.pause_signal = True
            label = QtWidgets.QLabel("
"+str(self.gesture_data[self.gesture_pos,3])+"")
            self.graph_layout.addWidget(label,self.gesture_pos+1,0,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
            label = QtWidgets.QLabel("
"+self.gesture_dict[self.gesture_data[self.gesture_pos][1]]+"")
            self.graph_layout.addWidget(label,self.gesture_pos+1,1,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
            label = QtWidgets.QLabel("
"+str(self.gesture_data[self.gesture_pos,4:])+"")
            self.graph_layout.addWidget(label,self.gesture_pos+1,2,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
            self.gesture_prev = self.gesture_pos
            self.gesture_pos = self.gesture_pos+1
        else:
            if int(self.time/1000) > self.gesture_data[self.gesture_pos,2]:
                self.pause_signal = True
                label = QtWidgets.QLabel("
"+str(self.gesture_data[self.gesture_pos,3])+"")
                self.graph_layout.addWidget(label,self.gesture_pos+1,0,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
                label = QtWidgets.QLabel("
"+self.gesture_dict[self.gesture_data[self.gesture_pos][1]]+"")
                self.graph_layout.addWidget(label,self.gesture_pos+1,1,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
                label = QtWidgets.QLabel("
"+str(self.gesture_data[self.gesture_pos,4:])+"")
                self.graph_layout.addWidget(label,self.gesture_pos+1,2,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
                self.gesture_prev = self.gesture_pos
                self.gesture_pos = self.gesture_pos+1

    def open_file(self, filename=None):
        home = str(Path.home())
        filename, _ = QtWidgets.QFileDialog.getOpenFileName(self, "Open Gesture", home)
        if not filename:
            return
        self.gesture_data = pandas.read_csv(filename).as_matrix()
        self.gesture_data[:,2] = (self.gesture_data[:,2]/30).astype('int64')
        # for i in range(len(self.gesture_data)):
        #     self.gesture_data[i]=self.gesture_data[i].split(' ')
        #     self.gesture_data[i][0] = int(self.gesture_data[i][0])
        #     self.gesture_data[i][1] = int(int(self.gesture_data[i][1])/30)
        self.gesture_len = self.gesture_data.shape[0]
        label = QtWidgets.QLabel("
"+'Question'+"")
        self.graph_layout.addWidget(label,0,0,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
        label = QtWidgets.QLabel("
"+'Gesture'+"")
        self.graph_layout.addWidget(label,0,1,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)
        label = QtWidgets.QLabel("
"+'Scores'+"")
        self.graph_layout.addWidget(label,0,2,QtCore.Qt.AlignHCenter | QtCore.Qt.AlignTop)

    def get_next(self):
        try:
            self.gesture_prev += 1
            self.positionSlider.setValue(self.gesture_data[self.gesture_prev,2]*1000 - 100)
        except:
            self.gesture_prev -=1

    def get_prev(self):
        if self.gesture_prev == -1:
            self.positionSlider.setValue(0)
        else:
            self.positionSlider.setValue(self.gesture_data[self.gesture_prev,2]*1000 - 100)
            self.gesture_prev -= 1

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    w = gestureWidget()
    w.show()
    sys.exit(app.exec_())


"""
Play a wav file passed as argument to this program
Using non-blocking code, speed of the sound is too fast.  It's as if only the first half of the audio packets is output.
"""
import pyaudio
import wave
import sys
import numpy as np
import samplerate
import pycodec2

# selected sound card
sound_card = 0
AUDIO_SAMPLE_RATE = 48000  # must be a value supported by the sound card (48000)

# validation. If a wave file hasn't been specified, exit.
if len(sys.argv) < 2:
    print("Plays a wave file.\n\n" + \
          "Usage: %s filename.wav" % sys.argv[0])
    sys.exit(-1)

'''
************************************************************************
      This is the start of the "minimum needed to read a wave"
************************************************************************
'''
# open the file for reading.
wf = wave.open(sys.argv[1], 'rb')

# create an audio object
p = pyaudio.PyAudio()
c2 = pycodec2.Codec2(1200)
INT16_BYTE_SIZE = 2
PACKET_SIZE = c2.samples_per_frame() * INT16_BYTE_SIZE
STRUCT_FORMAT = '{}h'.format(c2.samples_per_frame())

# List existing sound cards for this hardware
for ii in range(p.get_device_count()):
    print(str(ii) + '\t' + p.get_device_info_by_index(ii).get('name'))

# open stream based on the wave object which has been input.
stream = p.open(format=p.get_format_from_width(wf.getsampwidth()),
                channels=wf.getnchannels(),
                rate=AUDIO_SAMPLE_RATE,
                output=True,
                output_device_index=sound_card)

# read data (based on the chunk size)
data = wf.readframes(c2.samples_per_frame())

# play stream (looping from beginning of file to the end)
while data != '':
    packet = np.frombuffer(data, dtype=np.int16)
    encoded = c2.encode(packet)
    packet = c2.decode(encoded)
    snd_data = np.asarray(samplerate.resample(packet, AUDIO_SAMPLE_RATE / wf.getframerate(), 'sinc_best'),
                          dtype=np.int16)
    # writing to the stream is what *actually* plays the sound.
    stream.write(snd_data)
    data = wf.readframes(c2.samples_per_frame())

# cleanup stuff.
stream.close()
p.terminate()


excepE = ['tower']
dest = {'x': 78, 'y': 39}
done = False

def getPos(obj):
    return obj.pos.x, obj.pos.y

def toPos(x, y):
    return {'x': x, 'y': y}

def rushTo(xy):
    if hero.isReady("jump"):
        hero.jumpTo(xy)
    hero.move(xy)

def exclude(units, exceps):
    return [u for u in units if u.type not in exceps]

def farthest(enemies):
    farthestD = 0
    for enemy in enemies:
        d = hero.distanceTo(enemy)
        if d > farthestD and enemy.health > 0:
            farthestD = d
            farthest = enemy
    return farthest

def nearby(enemies, dis):
    return sum([hero.distanceTo(e) < dis for e in enemies])

def selfDying(self):
    if self.health < self.maxHealth / 2:
        hero.command(self, "move", {'x': self.pos.x - 1, 'y': self.pos.y})

def evade():
    x, y = getPos(hero)
    orbs = hero.findEnemyMissiles()
    if len(orbs):
        orb = hero.findNearest(orbs)
        if hero.distanceTo(orb) < 3:
            if y > 14:
                hero.moveXY(x, y - 7)
            else:
                hero.moveXY(x, y + 7)

def lowHP():
    friends = hero.findFriends()
    lowest = 9999
    dying = None
    if len(friends):
        for friend in friends:
            hp = friend.health
            if hp < friend.maxHealth / 3 and hp < lowest and hp > 0:
                lowest = hp
                dying = friend
    return dying

def soldierAtk(soldier):
    selfDying(soldier)
    enemies = hero.findByType("ogre")
    if not len(enemies):
        enemy = soldier.findNearestEnemy()
    else:
        enemy = enemies[0]
    if enemy:
        hero.command(soldier, "attack", enemy)
    else:
        hero.command(soldier, "move", soldier.pos)

def archerAtk(archer):
    enemies = hero.findByType("chieftain")
    if not len(enemies):
        enemy = archer.findNearestEnemy()
    else:
        enemy = enemies[0]
    if enemy and archer.pos.x > 53 and archer.pos.y < 40:
        hero.command(archer, "attack", enemy)
    elif hero.time > 6:
        hero.command(archer, "move", dest)

def riderAtk(rider):
    enemies = hero.findByType("robot-walker")
    if len(enemies):
        hero.command(rider, "move", {'x': enemies[0].pos.x / 2 + enemies[1].pos.x / 2,
                                     'y': enemies[0].pos.y / 2 + enemies[1].pos.y / 2})

def palaAtk(pala):
    selfDying(pala)
    dying = lowHP()
    enemies = hero.findByType("witch")
    if not len(enemies):
        enemy = pala.findNearestEnemy()
    else:
        enemy = enemies[0]
    if dying and pala.canCast('heal'):
        hero.command(pala, "cast", 'heal', dying)
    else:
        hero.command(pala, "move", dest)
    if pala.canCast('heal') and hero.health < 2 * hero.maxHealth / 3:
        hero.command(pala, "cast", 'heal', hero)
    if pala.pos == dest:
        hero.command(pala, "shield")

def summon(soldier):
    while hero.gold >= hero.costOf(soldier):
        hero.summon(soldier)

def command():
    for unit in hero.findFriends():
        t = unit.type
        if t == 'griffin-rider':
            riderAtk(unit)
        elif t == 'soldier':
            soldierAtk(unit)
        elif t == 'paladin':
            palaAtk(unit)
        elif t == 'archer':
            archerAtk(unit)

def atk():
    if hero.gold > hero.costOf('griffin-rider'):
        hero.summon('griffin-rider')
    command()

    enemies = hero.findByType("robot-walker")
    if len(enemies):
        pass
    else:
        enemy = hero.findNearest([e for e in hero.findEnemies() if e.type not in ['ice-yak', 'cow']])
        if enemy:
            command()
            if hero.canCast("chain-lightning", enemy) and hero.time > 7:
                hero.cast("chain-lightning", enemy)
        elif not len(enemies):
            hero.move({'x': 78, 'y': 14})
    return True

def run():
    while True:
        evade()
        atk()

run()


import librosa
from keras.models import load_model
from sklearn.metrics import accuracy_score
from clean import  envelope
from kapre.time_frequency import STFT, Magnitude, ApplyFilterbank, MagnitudeToDecibel
from sklearn.preprocessing import LabelEncoder
import numpy as np
from glob import glob
import os
from tqdm import tqdm
import pandas as pd
def make_prediction(modelFile,logDir,testFile,timeIn,sampleRate,thresHold):

    model = load_model(modelFile,
        custom_objects={'STFT':STFT,
                        'Magnitude':Magnitude,
                        'ApplyFilterbank':ApplyFilterbank,
                        'MagnitudeToDecibel':MagnitudeToDecibel})
    wav_paths = glob('{}/*'.format(testFile), recursive=True)
    wav_paths = sorted([x.replace(os.sep, '/') for x in wav_paths if '.wav' in x])

    labels = [os.path.split(x)[0].split('/')[-1] for x in wav_paths]
    le = LabelEncoder()
    results = []
    label_pred = []
    for z, wav_fn in tqdm(enumerate(wav_paths), total=len(wav_paths)):
        wav,rate = librosa.load(wav_fn,sampleRate)
        mask = envelope(wav, rate, threshold=thresHold)
        clean_wav = wav[mask]
        step = int(sampleRate*timeIn)
        batch = []
        for i in range(0, clean_wav.shape[0], step):
            sample = clean_wav[i:i+step]
            sample = sample.reshape(-1, 1)
            if sample.shape[0] < step:
                tmp = np.zeros(shape=(step, 1), dtype=np.float32)
                tmp[:sample.shape[0],:] = sample.flatten().reshape(-1, 1)
                sample = tmp
            batch.append(sample)
        X_batch = np.array(batch, dtype=np.float32)
        y_pred = model.predict(X_batch)
        y_mean = np.mean(y_pred, axis=0)
        y_pred = np.argmax(y_mean)
        label_pred.append(classes[y_pred])
        print('Actual class: {}, Predicted class: {}'.format(wav_fn, classes[y_pred]))
        results.append(y_mean)

    np.save(os.path.join('logs', logDir), np.array(results))
    return label_pred

if __name__ == '__main__':
    classes = sorted(os.listdir('clean'))
    modelFile='models/lstm.h5'
    logDir='y-pred'
    testFile='TestPredict'
    timeIn=1.0
    sampleRate=16000
    thresHold=0.0005
    label_pred=make_prediction(modelFile,logDir,testFile,timeIn,sampleRate,thresHold)
    df=pd.read_csv('Test.csv')
    df['y_pred']=label_pred
    df.to_csv('Pred.csv', index=False)
    pre= pd.read_csv('Pred.csv')
    acc_score = accuracy_score(y_true=pre.label, y_pred=pre.y_pred)
    print(acc_score)


import pandas as pd
import yfinance as yf
from datetime import datetime, timedelta, timezone
import random

def last_trading_day():
    rightnow = datetime.now(timezone(timedelta(hours=-5), 'EST'))

    # US Markets close at 4pm, but afterhours trading ends at 8pm.
    # yFinance stubbornly only gives the day's data after 8pm, so we will wait until 9pm to pull data from
    # the current day.
    market_close = rightnow.replace(hour=21, minute=0, second=0, microsecond=0)
    if rightnow < market_close:
        DELTA = 1
    # If it is saturday or sunday
    elif rightnow.weekday() >= 5:
        DELTA = 1
    else:
        DELTA = 0

    start_date = (datetime.now() - timedelta(days=15)).strftime("%Y-%m-%d")
    end_date = (datetime.now() - pd.tseries.offsets.BDay(DELTA)).strftime("%Y-%m-%d")
    MarketIndex = "^GSPC" # We can use the S&P 500's data to see the last day where we have data

    market_hist = yf.Ticker(MarketIndex).history(start=start_date, end=end_date).filter(like="Close").dropna()

    latest_day = market_hist.index[-1]
    return latest_day.strftime("%Y-%m-%d")

# get_stock_beta produces the beta of a specified ticker
# Inputs:
# stock_returns_series is of type Series, represents the column of a dataframe of the returns of the price

def get_weight_list(k):
    # We will pick k random numbers from a range of 250 numbers
    # random.sample is O(n), so we want to minimize this number
    random_num_list = random.sample(range(250), k)

    listsum = sum(random_num_list)
    for i in range(len(random_num_list)):
        random_num_list[i] /= listsum

    return random_num_list

def get_stock_beta(stock_returns_series, marketVar, market_hist):
    binary_portfolio = market_hist[["Returns"]].copy()
    binary_portfolio["stock"] = stock_returns_series
    return (binary_portfolio.cov() / marketVar)['stock'].loc["Returns"]

#get_all_betas takes a list of tickers and outputs a list of all the betas of every
# ticker in the list.

def get_all_betas(ticker_list, ticker_hist, marketVar, market_hist):
    betas = {}
    for ticker in ticker_list:
        ticker_returns = ticker_hist[ticker][['Close']].pct_change()*100
        betas[ticker] = get_stock_beta(ticker_returns['Close'], marketVar, market_hist)
    return betas

def get_option_interest(ticker):
    stock = yf.Ticker(ticker)
    options = stock.option_chain(stock.options[0])
    calls = pd.DataFrame().append(options.calls)
    puts = pd.DataFrame().append(options.puts)
    option_interest = calls.loc[calls['inTheMoney']]['openInterest'].sum() + puts.loc[puts['inTheMoney']]['openInterest'].sum()
    return option_interest

def get_daily_volume(ticker, start, end, ticker_hist):
    filled_in_hist = ticker_hist.copy()
    filled_in_hist.fillna(0, inplace=True)
    volume = ticker_hist[ticker].loc[pd.to_datetime(start) : pd.to_datetime(end)].Volume.mean()
    return volume

def import_options(tickerlist, opt_start, opt_end, tickerhist):
    option_interest_dict = {}
    for ticker in tickerlist:
        try:
            open_interest = get_option_interest(ticker)
            daily_vol = get_daily_volume(ticker, opt_start, opt_end, tickerhist)
            option_interest_dict[ticker] = open_interest / daily_vol
        # yFinance's historical data is sometimes incomplete. This try except will catch any tickers that have insufficient data and drops them from our analysis.
        except IndexError as error:
            print(f'Dropped {ticker} - no data')
    return option_interest_dict

def generate_risky_portfolio(tickerlist: list, totalspend: int):
    ticker_hist = yf.download(
                tickers = " ".join(tickerlist),
                # Download Data From the past 6 months
                period = "6mo",
                interval = "1d",
                group_by = 'ticker',
                threads = True,
            )

    # Data Cleanup
    ticker_hist.dropna(how='all', inplace=True)
    ticker_hist.fillna(method='ffill', inplace=True)
    ticker_hist.fillna(method='bfill', inplace=True)

    start_date = (datetime.now() - timedelta(days=60)).strftime("%Y-%m-%d")
    end_date = last_trading_day()

    MarketIndex = "^GSPC" # We will compare each stock's covariance to the S&P 500

    market_hist = yf.Ticker(MarketIndex).history(start=start_date, end=end_date).filter(like="Close")
    market_hist["Returns"] = market_hist["Close"].pct_change()*100

    market_var = market_hist["Returns"].var()

    beta_dict = get_all_betas(tickerlist, ticker_hist, market_var, market_hist)
    beta_df = pd.DataFrame.from_dict(beta_dict, orient='index')
    beta_df.columns = ['Beta']
    beta_df.index.name = 'Ticker'

    #Sorts the dictionary from lowest beta to highest beta
    sorted_beta_df = beta_df.sort_values('Beta').copy()

    if len(sorted_beta_df) > 10:
        #Creates a dataframe with the 10 lowest betas and 10 highest betas.
        lower_bound = sorted_beta_df.iloc[:10]
        upper_bound = sorted_beta_df.iloc[-10:]
        # Gets the final stock list we want

        #Calculates the mean of the upper and lower list of betas
        avg_upper = upper_bound['Beta'].mean()
        avg_lower = lower_bound['Beta'].mean()

        # Compares the averages of the two and makes the one with the higher absolute value a
        # candidate for our final list of stocks.
        if abs(avg_upper) >= abs(avg_lower):
            beta_final_ticker_list = list(upper_bound.index)
        else:
            beta_final_ticker_list = list(lower_bound.index)
    else:
        beta_final_ticker_list = list(sorted_beta_df.index)

    # Options Analysis

    opt_start = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
    opt_end = end_date

    option_interest_dict = import_options(tickerlist, opt_start, opt_end, ticker_hist)

    if len(option_interest_dict.keys()) >= len(beta_final_ticker_list):
        # Creates a dataframe that corresponds tickers with the option interest over the daily volume
        option_interest_df = pd.DataFrame.from_dict(option_interest_dict, orient='index')
        option_interest_df.columns = ['Option Interest / dailyVol']
        sorted_option_interest_df = option_interest_df.sort_values('Option Interest / dailyVol').copy()
        sorted_option_interest_df.dropna(inplace=True)
        final_option_df = sorted_option_interest_df.tail(10)
        opt_tickers = list(final_option_df.index)
        print('Found an option portfolio')
    else:
        option_interest_df = False
        print('Error - Insufficient options data for inputted tickers, skipping options analysis...')

    # Monte Carlo Analysis
    trials = 250

    # We will collect the risk coefficients of all portfolios in this dictionary
    # Ultimately, we are comparing the best stocks obtained by the beta and options method,
    # and will take the best one from whichever method.
    risk_coefficients = {}

    # Don't mind the O(4n^2), it's O(e^2ln(2n)) :^)
    for i in range(trials):
        portfolio = pd.DataFrame(index=pd.date_range(start=start_date, end=end_date))
        portfolio_shares = {}
        weight_list = get_weight_list(len(beta_final_ticker_list))
        total = 0
        for i in range(len(beta_final_ticker_list)):
            allocation = weight_list[i] * totalspend
            # Simulate purchasing $100,000 worth of shares at the beginning of the time period
            initial_ticker_price = ticker_hist[beta_final_ticker_list[i]].Close.iloc[0]
            shares = allocation / initial_ticker_price

            # Alongside this, create a portfolio of shares with the same weightings, but priced in the present.
            present_ticker_price = ticker_hist[beta_final_ticker_list[i]].Close.loc[end_date]
            shares_present = allocation / present_ticker_price
            portfolio_shares[beta_final_ticker_list[i]] = shares_present
            total += present_ticker_price * shares_present

            # Simulate Purchasing of the shares and perform analysis on the past data
            portfolio[beta_final_ticker_list[i]] = shares * ticker_hist[beta_final_ticker_list[i]].loc[pd.to_datetime(start_date) : pd.to_datetime(end_date)].Close
        portfolio.dropna(how='all', inplace=True)
        portfolio.fillna(method='ffill', inplace=True)
        portfolio.fillna(method='bfill', inplace=True)
        portfolio['Total Value'] = portfolio.sum(axis=1)
        portfolio['Percent Return'] = portfolio['Total Value'].pct_change() * 100

        # Calculate risk coefficient
        std = portfolio['Percent Return'].std()
        avg_return = portfolio['Percent Return'].mean()
        risk_coefficient = (avg_return * std) ** 2

        # Save into risk_coefficients dict
        risk_coefficients[risk_coefficient] = portfolio_shares

        if option_interest_df is not False:
            portfolio = pd.DataFrame(index=pd.date_range(start=start_date, end=end_date))
            portfolio_shares = {}
            weight_list = get_weight_list(len(opt_tickers))
            total = 0
            for i in range(len(opt_tickers)):
                allocation = weight_list[i] * totalspend
                # Simulate purchasing $100,000 worth of shares at the beginning of the time period
                initial_ticker_price = ticker_hist[opt_tickers[i]].Close.iloc[0]
                shares = allocation / initial_ticker_price

                # Alongside this, create a portfolio of shares with the same weightings, but priced in the present.
                present_ticker_price = ticker_hist[opt_tickers[i]].Close.loc[end_date]
                shares_present = allocation / present_ticker_price
                portfolio_shares[opt_tickers[i]] = shares_present
                total += present_ticker_price * shares_present

                # Simulate Purchasing of the shares and perform analysis on the past data
                portfolio[opt_tickers[i]] = shares * ticker_hist[opt_tickers[i]].loc[pd.to_datetime(start_date) : pd.to_datetime(end_date)].Close
            portfolio.dropna(how='all', inplace=True)
            portfolio.fillna(method='ffill', inplace=True)
            portfolio.fillna(method='bfill', inplace=True)
            portfolio['Total Value'] = portfolio.sum(axis=1)
            portfolio['Percent Return'] = portfolio['Total Value'].pct_change() * 100

            # Calculate risk coefficient
            std = portfolio['Percent Return'].std()
            avg_return = portfolio['Percent Return'].mean()
            risk_coefficient = (avg_return * std) ** 2

            # Save into risk_coefficients dict
            risk_coefficients[risk_coefficient] = portfolio_shares

    optimal_weight = risk_coefficients[max(risk_coefficients.keys())]

    # Create final output portfolio
    final_stocks_df = pd.DataFrame.from_dict(optimal_weight, orient='index')
    final_stocks_df.reset_index(inplace=True)
    final_stocks_df.columns = ['Ticker', 'Shares']
    final_stocks_df.set_index('Ticker', inplace=True)
    return (final_stocks_df, end_date)


import re
import sys
from textwrap import dedent

import netaddr
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.orm import validates
from netaddr import AddrFormatError, IPAddress

from pyfarm.core.enums import AgentState, STRING_TYPES, PY3
from pyfarm.core.config import read_env_number, read_env_int, read_env_bool
from pyfarm.master.application import db, app
from pyfarm.models.core.functions import repr_ip
from pyfarm.models.core.mixins import (
    ValidatePriorityMixin, UtilityMixins, ReprMixin)
from pyfarm.models.core.types import (
    id_column, IPv4Address, IDTypeAgent, IDTypeTag, UseAgentAddressEnum,
    AgentStateEnum)
from pyfarm.models.core.cfg import (
    TABLE_AGENT, TABLE_SOFTWARE, TABLE_TAG, TABLE_AGENT_TAG_ASSOC,
    MAX_HOSTNAME_LENGTH, MAX_TAG_LENGTH, TABLE_AGENT_SOFTWARE_ASSOC,
    TABLE_PROJECT_AGENTS, TABLE_PROJECT)

PYFARM_REQUIRE_PRIVATE_IP = read_env_bool("PYFARM_REQUIRE_PRIVATE_IP", False)
REGEX_HOSTNAME = re.compile("^(?!-)[A-Z\d-]{1,63}(?<!-)"
                            "(\.(?!-)[A-Z\d-]{1,63}(?<!-))*\.?$"
                            , re.IGNORECASE)

AgentSoftwareAssociation = db.Table(
    TABLE_AGENT_SOFTWARE_ASSOC, db.metadata,
    db.Column("agent_id", IDTypeAgent,
              db.ForeignKey("%s.id" % TABLE_AGENT), primary_key=True),
    db.Column("software_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_SOFTWARE), primary_key=True))

AgentTagAssociation = db.Table(
    TABLE_AGENT_TAG_ASSOC, db.metadata,
    db.Column("agent_id", IDTypeAgent,
              db.ForeignKey("%s.id" % TABLE_AGENT), primary_key=True),
    db.Column("tag_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_TAG), primary_key=True))

AgentProjects = db.Table(
    TABLE_PROJECT_AGENTS, db.metadata,
    db.Column("agent_id", IDTypeAgent,
              db.ForeignKey("%s.id" % TABLE_AGENT), primary_key=True),
    db.Column("project_id", db.Integer,
              db.ForeignKey("%s.id" % TABLE_PROJECT), primary_key=True))

class AgentTaggingMixin(object):
    """
    Mixin used which provides some common structures to
    :class:`.AgentTag` and :class:`.AgentSoftware`
    """
    if not PY3:
        NUMERIC_TYPES = (int, long)
    else:
        NUMERIC_TYPES = int

    @validates("tag", "software")
    def validate_string_column(self, key, value):
        """
        Ensures `value` is a string or something that can be converted
        to a string.
        """
        if isinstance(value, self.NUMERIC_TYPES):
            value = str(value)
        elif not isinstance(value, STRING_TYPES):
            raise ValueError("expected a string for `%s`" % key)

        return value

class Agent(db.Model, ValidatePriorityMixin, UtilityMixins, ReprMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`hostname`
            * :attr:`ip`
            * :attr:`port`

    """
    __tablename__ = TABLE_AGENT
    __table_args__ = (UniqueConstraint("hostname", "ip", "port"), )
    STATE_DEFAULT = "online"
    REPR_COLUMNS = (
        "id", "hostname", "state", "ip", "remote_ip", "port", "cpus",
        "ram", "free_ram")
    REPR_CONVERT_COLUMN = {
        "ip": repr_ip,
        "remote_ip": repr_ip,
        "state": repr}
    MIN_PORT = read_env_int("PYFARM_AGENT_MIN_PORT", 1024)
    MAX_PORT = read_env_int("PYFARM_AGENT_MAX_PORT", 65535)
    MIN_CPUS = read_env_int("PYFARM_AGENT_MIN_CPUS", 1)
    MAX_CPUS = read_env_int("PYFARM_AGENT_MAX_CPUS", 256)
    MIN_RAM = read_env_int("PYFARM_AGENT_MIN_RAM", 16)
    MAX_RAM = read_env_int("PYFARM_AGENT_MAX_RAM", 262144)

    # quick check of the configured data
    assert MIN_PORT >= 1, "$PYFARM_AGENT_MIN_PORT must be > 0"
    assert MAX_PORT >= 1, "$PYFARM_AGENT_MAX_PORT must be > 0"
    assert MAX_PORT >= MIN_PORT, "MIN_PORT must be <= MAX_PORT"
    assert MIN_CPUS >= 1, "$PYFARM_AGENT_MIN_CPUS must be > 0"
    assert MAX_CPUS >= 1, "$PYFARM_AGENT_MAX_CPUS must be > 0"
    assert MAX_CPUS >= MIN_CPUS, "MIN_CPUS must be <= MAX_CPUS"
    assert MIN_RAM >= 1, "$PYFARM_AGENT_MIN_RAM must be > 0"
    assert MAX_RAM >= 1, "$PYFARM_AGENT_MAX_RAM must be > 0"
    assert MAX_RAM >= MIN_RAM, "MIN_RAM must be <= MAX_RAM"

    id = id_column(IDTypeAgent)

    # basic host attribute information
    hostname = db.Column(db.String(MAX_HOSTNAME_LENGTH), nullable=False,
                         doc=dedent("""
                         The hostname we should use to talk to this host.
                         Preferably this value will be the fully qualified
                         name instead of the base hostname alone."""))
    ip = db.Column(IPv4Address, nullable=True,
                   doc="The IPv4 network address this host resides on")
    remote_ip = db.Column(IPv4Address, nullable=True,
                          doc="the remote address which came in with the "
                              "request")
    use_address = db.Column(UseAgentAddressEnum, nullable=False,
                            default="remote",
                            doc="The address we should use when communicating "
                                "with the agent")
    ram = db.Column(db.Integer, nullable=False,
                    doc="The amount of ram installed on the agent in megabytes")
    free_ram = db.Column(db.Integer, nullable=False,
                         doc="The amount of ram which was last considered free")
    cpus = db.Column(db.Integer, nullable=False,
                     doc="The number of cpus installed on the agent")
    port = db.Column(db.Integer, nullable=False,
                     doc="The port the agent is currently running on")
    time_offset = db.Column(db.Integer, nullable=False, default=0,
                            doc="the offset in seconds the agent is from "
                                "an official time server")

    # host state
    state = db.Column(AgentStateEnum, default=AgentState.ONLINE,
                      nullable=False,
                      doc=dedent("""
                      Stores the current state of the host.  This value can be
                      changed either by a master telling the host to do
                      something with a task or from the host via REST api."""))

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float,
                               default=read_env_number(
                                   "PYFARM_AGENT_RAM_ALLOCATION", .8),
                               doc=dedent("""
                               The amount of ram the agent is allowed to
                               allocate towards work.  A value of 1.0 would
                               mean to let the agent use all of the memory
                               installed on the system when assigning work."""))

    cpu_allocation = db.Column(db.Float,
                               default=read_env_number(
                                   "PYFARM_AGENT_CPU_ALLOCATION", 1.0),
                               doc=dedent("""
                               The total amount of cpu space an agent is
                               allowed to process work in.  A value of 1.0
                               would mean an agent can handle as much work
                               as the system could handle given the
                               requirements of a task.  For example if an agent
                               has 8 cpus, cpu_allocation is .5, and a task
                               requires 4 cpus then only that task will run
                               on the system."""))

    # relationships
    tasks = db.relationship("Task", backref="agent", lazy="dynamic",
                            doc=dedent("""
                            Relationship between an :class:`Agent`
                            and any :class:`pyfarm.models.Task`
                            objects"""))
    tags = db.relationship("Tag", secondary=AgentTagAssociation,
                            backref=db.backref("agents", lazy="dynamic"),
                            lazy="dynamic",
                            doc="Tags associated with this agent")
    software = db.relationship("Software",
                               secondary=AgentSoftwareAssociation,
                               backref=db.backref("agents", lazy="dynamic"),
                               lazy="dynamic",
                               doc="software this agent has installed or is "
                                   "configured for")
    projects = db.relationship("Project",
                               secondary=AgentProjects,
                               backref=db.backref("agents", lazy="dynamic"),
                               lazy="dynamic",
                               doc="The project or projects this agent is "
                                   "associated with.  By default an agent "
                                   "which is not associated with any projects "
                                   "will be a member of all projects.")

    @classmethod
    def validate_hostname(cls, key, value):
        """
        Ensures that the hostname provided by `value` matches a regular
        expression that expresses what a valid hostname is.
        """
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @classmethod
    def validate_resource(cls, key, value):
        """
        Ensure the `value` provided for `key` is within an expected range as
        specified in `agent.yml`
        """
        min_value = getattr(cls, "MIN_%s" % key.upper())
        max_value = getattr(cls, "MAX_%s" % key.upper())

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

    @classmethod
    def validate_ip_address(cls, key, value):
        """
        Ensures the :attr:`ip` address is valid.  This checks to ensure
        that the value provided is:

            * not a hostmask
            * not link local (:rfc:`3927`)
            * not used for multicast (:rfc:`1112`)
            * not a netmask (:rfc:`4632`)
            * not reserved (:rfc:`6052`)
            * a private address (:rfc:`1918`)
        """
        if not value:
            return

        try:
            ip = netaddr.IPAddress(value)

        except (AddrFormatError, ValueError) as e:
            raise ValueError(
                "%s is not a valid address format: %s" % (value, e))

        if not app.config.get("DEV_ALLOW_ANY_AGENT_ADDRESS", False):
            if PYFARM_REQUIRE_PRIVATE_IP and not ip.is_private():
                raise ValueError("%s is not a private ip address" % value)

            if not app.config.get("DEV_ALLOW_ANY_AGENT_ADDRESS", False) and \
                not all([
                    not ip.is_hostmask(), not ip.is_link_local(),
                    not ip.is_loopback(), not ip.is_multicast(),
                    not ip.is_netmask(), not ip.is_reserved()
                ]):
                raise ValueError("%s is not a usable ip address" % value)

        return value

    @validates("ip")
    def validate_address_column(self, key, value):
        """validates the ip column"""
        return self.validate_ip_address(key, value)

    @validates("hostname")
    def validate_hostname_column(self, key, value):
        """validates the hostname column"""
        return self.validate_hostname(key, value)

    @validates("ram", "cpus", "port")
    def validate_resource_column(self, key, value):
        """validates the ram, cpus, and port columns"""
        return self.validate_resource(key, value)

    def serialize_column(self, column):
        """serializes a single column, typically used by a dictionary mixin"""
        if isinstance(column, IPAddress):
            return str(column)
        return column


import random
from itertools import count
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

filepath = "/tmp/csv-file.csv"
index = count()

def animate(i):
    data = pd.read_csv(filepath)
    x = data['xvalues']
    y1 = data['total_1']
    y2 = data['total_2']

    plt.cla()
    plt.plot(x, y1, label="Channel 1")
    plt.plot(x, y2, label="Channel 2")

    plt.legend(loc="upper left")
    plt.tight_layout()

ani = FuncAnimation(plt.gcf(),
                    animate,
                    interval=500,
                    cache_frame_data=False)
plt.show()


#
# This file is part of r-ar-verificator.
#
# r-ar-verificator is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# r-ar-verificator is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with r-ar-verificator (file COPYING in the main directory). If not, see
# http://www.gnu.org/licenses/.
"""

"""

# Required Inputs for testing:
import spacy
from typing import List, Union, Any

# spaCy vectorizer and similarity function
nlp = spacy.load("en_core_web_md")

def activity_similarity_check_one_spacy(sentences_descr: List[str], sentences_log: List[str]) -> \
        tuple[list[Any], list[Union[str, list[str]]], list[Union[str, list[str]]], list[Union[int, Any]], list[
            Union[str, list[str]]], list[Union[int, Any]], list[Union[int, Any]]]:
    # Returns:
    # similarity_matrix
    # most_similar_sentences
    # corresponding_sentence_description
    # corresponding_sentence_log
    # corresponding_similarity_scores
    return __similarity_check_spacy(sentences_descr=sentences_descr, sentences_log=sentences_log)

# Approach of V2
def resource_activity_similarity_check_two_spacy(resources_descr: List[str], resources_matched_log: List[str]):

    # Calculate similarity
    resources_checked_descr = []
    resources_checked_log = []
    similarity_scores = []
    index = len(resources_descr)
    for i in range(index):
        # Transform resource of description and log into vector
        doc_descr = nlp(resources_descr[i])
        doc_log = nlp(resources_matched_log[i])

        # Similarity result
        similarity = doc_descr.similarity(doc_log)
        similarity_scores.append(similarity)
        # Resource checked log
        resources_checked_log.append(resources_matched_log[i])
        # Resource checked description
        resources_checked_descr.append(resources_descr[i])

    # Return similarity score and tuple of
    return resources_checked_log, resources_checked_descr, similarity_scores

def __similarity_check_spacy(sentences_descr: List[str], sentences_log: List[str]) -> \
        tuple[list[Any], list[Union[str, list[str]]], list[Union[str, list[str]]], list[Union[int, Any]], list[
            Union[str, list[str]]], list[Union[int, Any]], list[Union[int, Any]]]:
    # length of amount descr
    similarity_matrix = []

    for sent_log in sentences_log:
        sim_list = []
        doc_log = nlp(sent_log)
        for sent_descr in sentences_descr:
            doc_descr = nlp(sent_descr)
            sim_list.append(doc_log.similarity(doc_descr))
        similarity_matrix.append(sim_list)

    # 3. Refine this output process:
    most_similar_sentences = []
    corresponding_sentence_log = []
    corresponding_id_log = []
    corresponding_sentence_description = []
    corresponding_id_description = []
    corresponding_similarity_scores = []

    # Count to get the current process event from event log which is compared
    count_row = 0
    # Go through the similarity matrix
    for row in similarity_matrix:
        count_column = 0
        similarity_index = 0
        similarity_value = 0
        # Go through for every sentence the similarity to every sentence in the event log
        for value_sim in row:
            # Store the position and value of the most similar sentence
            if value_sim >= similarity_value:
                similarity_value = value_sim
                similarity_index = count_column
            count_column = count_column + 1

        # Add results
        most_similar_sentences.append(sentences_descr[similarity_index])
        corresponding_sentence_description.append(sentences_descr[similarity_index])
        corresponding_id_description.append(similarity_index + 1)
        corresponding_similarity_scores.append(similarity_value)

        corresponding_sentence_log.append(sentences_log[count_row])
        corresponding_id_log.append(count_row + 1)
        # Increase count for description list
        count_row = count_row + 1

    # Matrix of similarity scores, list of similar sentences, list of highest similarity score
    return similarity_matrix, most_similar_sentences, corresponding_sentence_description, corresponding_id_description, corresponding_sentence_log, corresponding_id_log, corresponding_similarity_scores


import click
from sliceofml.api import API

from sliceofml.apikey import (
    read_credentials,
    prompt_api_details,
    request_access_token,
    write_netrc,
)
from sliceofml.config import TWITTER_API
from sliceofml.display import Display

@click.group()
def cli():
    """

    Slice of ML is your little 🍰 of ML.
    A tiny, terminal, twitter client that fetches the top ML tweets of the past
    24 hours or 7 days!

    """

@cli.command("login")
@click.option("--relogin", "-r", is_flag=True, help="Force a relogin.")
def login(relogin):
    apikey_configured = read_credentials(TWITTER_API) is not None
    if relogin:
        apikey_configured = False
    if not apikey_configured:
        try:
            (client_id, client_secret, app_name) = prompt_api_details()
            token = request_access_token(client_id, client_secret)
            write_netrc(TWITTER_API, app_name, token)
            click.echo("You've sucessfully logged in ✅")
        except Exception:
            click.echo("Failed to fetch your token, check your credentials! ❌")
    else:
        click.echo(
            "You're already logged in! 🔑 \nTry --relogin to update your credentials!"
        )

@cli.command("slice")
@click.option(
    "--daily",
    "frequency",
    flag_value="daily",
    default=True,
    help="Fetch the Top ML tweets for the past 24 hours.",
)
@click.option(
    "--weekly",
    "frequency",
    flag_value="weekly",
    help="Fetch the Top ML tweets for the past 7 days.",
)
def slice(frequency):
    display = Display()
    credentials = read_credentials(TWITTER_API)
    if credentials is None:
        display.error("Please login before running this command! ❌")
        return
    tweets = API(credentials[0], credentials[1], TWITTER_API).query(frequency)
    display.tweetsAsTable(tweets, frequency)


# Given a binary tree, find the maximum path sum.
# The path may start and end at any node in the tree.

# For example:
# Given the below binary tree,

#        1↵      / ↵     2   3↵

# Return6.
class TreeNode:
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

#
#
# @param root TreeNode类
# @return int整型
#

class Solution:
    def maxPathSum(self, root):
        stack = [root]
        while(stack):
            node = stack.pop()
            if ((not node.left) and (not node.right)):
                node.maxUp = node.val
                node.maxAround = node.val
            else:
                if ((node.left and hasattr(node.left, 'maxUp')) or (node.right and hasattr(node.right, 'maxUp'))):
                    if (node.left):
                        leftUp = node.left.maxUp
                        leftAround = node.left.maxAround
                        if (not node.right):
                            node.maxUp = max(node.val, leftUp + node.val)
                            node.maxAround = max(leftAround, node.maxUp)
                    if (node.right):
                        rightUp = node.right.maxUp
                        rightAround = node.right.maxAround
                        if (not node.left):
                            node.maxUp = max(node.val, rightUp + node.val)
                            node.maxAround = max(rightAround, node.maxUp)
                    if (node.left and node.right):
                        node.maxUp = max(node.val, leftUp +
                                         node.val, rightUp + node.val)
                        node.maxAround = max(
                            leftAround, rightAround, node.maxUp, leftUp + rightUp + node.val)
                else:
                    stack.append(node)
                    if (node.left):
                        stack.append(node.left)
                    if (node.right):
                        stack.append(node.right)
        return root.maxAround

root = TreeNode(1)
left = TreeNode(2)
right = TreeNode(3)
root.left = left
root.right = right
son = TreeNode(4)
left.left = son

print(Solution().maxPathSum(root))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# File              : referential.py
# Author            : Chi Han, Jiayuan Mao
# Email             : haanchi@gmail.com, maojiayuan@gmail.com
# Date              : 23.10.2019
# Last Modified Date: 23.10.2019
# Last Modified By  : Chi Han, Jiayuan Mao
#
# This file is part of the VCML codebase
# Distributed under MIT license

import six
import sys
import torch
import numpy as np

from IPython.core import ultratb

import jacinle.random as random
from jacinle.utils.meter import GroupMeters

sys.excepthook = ultratb.FormattedTB(
    mode='Plain', color_scheme='Linux', call_pdb=True)

class Definition(object):
    annotation_attribute_names = ['color', 'material', 'shape', 'size']
    annotation_relation_names = ['behind', 'front', 'left', 'right']
    concepts = {
        'color': ['gray', 'red', 'blue', 'green',
                  'brown', 'purple', 'cyan', 'yellow'],
        'material': ['rubber', 'metal'],
        'shape': ['cube', 'sphere', 'cylinder'],
        'size': ['small', 'large']
    }
    concept2attribute = {
        v: k for k, vs in concepts.items() for v in vs
    }
    relational_concepts = {
        'spatial_relation': ['left', 'right', 'front', 'behind']
    }

    synonyms = {
        "thing": ["thing", "object"],
        "sphere": ["sphere", "ball"],
        "cube": ["cube", "block"],
        "cylinder": ["cylinder"],
        "large": ["large", "big"],
        "small": ["small", "tiny"],
        "metal": ["metallic", "metal", "shiny"],
        "rubber": ["rubber", "matte"],
    }

    word2lemma = {
        v: k for k, vs in synonyms.items() for v in vs
    }

def_ = Definition()

def get_desc(obj):
    names = [obj[k] for k in def_.annotation_attribute_names]
    for i, n in enumerate(names):
        if n in def_.synonyms:
            names[i] = random.choice_list(def_.synonyms[n])
    return names

def run_desc_obj(obj, desc):
    for d in desc:
        dd = def_.word2lemma.get(d, d)
        if dd != obj[def_.concept2attribute[dd]]:
            return False
    return True

def run_desc_pred(all_preds, desc):
    s = 10000
    for d in desc:
        s = np.fmin(s, all_preds[d])
    return s

def test(index, all_objs, all_preds, meter):
    obj = all_objs[index]
    nr_descriptors = random.randint(1, 3)
    desc = random.choice_list(get_desc(obj), size=nr_descriptors)
    if isinstance(desc, six.string_types):
        desc = [desc]

    filtered_objs = np.array([
        i for i, o in enumerate(all_objs)
        if not run_desc_obj(o, desc)], dtype=int)
    all_scores = run_desc_pred(all_preds, desc)
    rank = (all_scores[filtered_objs] > all_scores[index]).sum()

    meter.update('r@01', rank <= 1)
    meter.update('r@02', rank <= 2)
    meter.update('r@03', rank <= 3)
    meter.update('r@04', rank <= 4)
    meter.update('r@05', rank <= 5)

def get_preds(model, ref_dataset, logger):
    device = model.device
    args = model.args
    concepts = model.tools.concepts

    preds = {}

    def add_preds(data_batch):
        batch = {
            'batch_size': len(data_batch),
            'image': torch.Tensor(
                [data['image'] for data in data_batch]
            ).to(device),
            'objects': torch.Tensor(
                np.concatenate([data['objects'] for data in data_batch])
            ).to(device),
            'object_length': torch.LongTensor(
                [data['object_length'] for data in data_batch]),
            'image_id': [data['image_id'] for data in data_batch],
        }
        features = model.process_objects(batch)
        concept_embedding = model.embedding.get_all_concept_embeddings()
        for image_id, one_feature in zip(batch['image_id'], features):
            logits = model.embedding.logit_fn(
                one_feature, concept_embedding
            ).transpose(0, 1).detach().cpu().numpy()
            preds[image_id] = dict(zip(
                concepts, logits
            ))

    with torch.no_grad():
        pbar = logger.tqdm(ref_dataset.indexes)
        data_batch = []

        for image_index in pbar:
            data_batch.append(ref_dataset[image_index])
            if len(data_batch) == args.batch_size:
                add_preds(data_batch)
                data_batch.clear()
        if len(data_batch) != 0:
            add_preds(data_batch)
        data_batch.clear()

    return preds

def ref_epoch(coach, prepare_fn, recording, ref_dataset):
    prepare_fn()
    model = coach.model
    recording.reset()

    preds = get_preds(model, ref_dataset, coach.logger)

    meter = GroupMeters()

    for image_id, pred in coach.logger.tqdm(preds.items(), mininterval=0.5):
        scene = ref_dataset.sceneGraphs[image_id]
        for j in range(len(scene['objects'])):
            test(j, list(scene['objects'].values()), pred, meter)

    recording.record(meter.avg)