# -*- coding: utf-8 -*-
"""Provides skosify as command line client."""

from __future__ import print_function

from .skosify import Skosify, DEFAULT_NAMESPACES, DEFAULT_OPTIONS

import optparse
import logging
import sys

from rdflib import URIRef, Namespace, RDF, RDFS

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

def get_option_parser(defaults):
    """Create and return an OptionParser with the given defaults."""
    # based on recipe from:
    # http://stackoverflow.com/questions/1880404/using-a-file-to-store-optparse-arguments

    # process command line parameters
    # e.g. skosify yso.owl -o yso-skos.rdf
    usage = "Usage: %prog [options] voc1 [voc2 ...]"
    parser = optparse.OptionParser(usage=usage)
    parser.set_defaults(**defaults)
    parser.add_option('-c', '--config', type='string',
                      help='Read default options '
                           'and transformation definitions '
                           'from the given configuration file.')
    parser.add_option('-o', '--output', type='string',
                      help='Output file name. Default is "-" (stdout).')
    parser.add_option('-D', '--debug', action="store_true",
                      help='Show debug output.')
    parser.add_option('-d', '--no-debug', dest="debug",
                      action="store_false", help='Hide debug output.')
    parser.add_option('-O', '--log', type='string',
                      help='Log file name. Default is to use standard error.')

    group = optparse.OptionGroup(parser, "Input and Output Options")
    group.add_option('-f', '--from-format', type='string',
                     help='Input format. '
                           'Default is to detect format '
                           'based on file extension. '
                           'Possible values: xml, n3, turtle, nt...')
    group.add_option('-F', '--to-format', type='string',
                     help='Output format. '
                           'Default is to detect format '
                           'based on file extension. '
                           'Possible values: xml, n3, turtle, nt...')
    group.add_option('--update-query', type='string',
                     help='SPARQL update query. '
                     'This query is executed against the input '
                     'data before processing it. '
                     'The value can be either the actual query, '
                     'or "@filename".')
    group.add_option('--construct-query', type='string',
                     help='SPARQL CONSTRUCT query. '
                     'This query is executed against the input '
                     'data and the result graph is used as the '
                     'actual input. '
                     'The value can be either the actual query, '
                     'or "@filename".')
    group.add_option('-I', '--infer', action="store_true",
                     help='Perform RDFS subclass/subproperty inference '
                          'before transforming input.')
    group.add_option('-i', '--no-infer', dest="infer", action="store_false",
                     help="Don't perform RDFS subclass/subproperty inference "
                          "before transforming input.")
    parser.add_option_group(group)

    group = optparse.OptionGroup(
        parser, "Concept Scheme and Labelling Options")
    group.add_option('-s', '--namespace', type='string',
                     help='Namespace of vocabulary '
                          '(usually optional; used to create a ConceptScheme)')
    group.add_option('-L', '--label', type='string',
                     help='Label/title for the vocabulary '
                          '(usually optional; used to label a ConceptScheme)')
    group.add_option('-l', '--default-language', type='string',
                     help='Language tag to set for labels '
                          'with no defined language.')
    group.add_option('-p', '--preflabel-policy', type='string',
                     help='Policy for handling multiple prefLabels '
                          'with the same language tag. '
                          'Possible values: shortest, longest, all.')
    group.add_option('--set-modified', dest="set_modified",
                     action="store_true",
                     help='Set modification date on the ConceptScheme')
    group.add_option('--no-set-modified', dest="set_modified",
                     action="store_false",
                     help="Don't set modification date on the ConceptScheme")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser, "Vocabulary Structure Options")
    group.add_option('-E', '--mark-top-concepts', action="store_true",
                     help='Mark top-level concepts in the hierarchy '
                          'as top concepts (entry points).')
    group.add_option('-e', '--no-mark-top-concepts',
                     dest="mark_top_concepts", action="store_false",
                     help="Don't mark top-level concepts in the hierarchy "
                          "as top concepts.")
    group.add_option('-N', '--narrower', action="store_true",
                     help='Include narrower/narrowerGeneric/narrowerPartitive '
                          'relationships in the output vocabulary.')
    group.add_option('-n', '--no-narrower',
                     dest="narrower", action="store_false",
                     help="Don't include "
                          "narrower/narrowerGeneric/narrowerPartitive "
                          "relationships in the output vocabulary.")
    group.add_option('-T', '--transitive', action="store_true",
                     help='Include transitive hierarchy relationships '
                          'in the output vocabulary.')
    group.add_option('-t', '--no-transitive',
                     dest="transitive", action="store_false",
                     help="Don't include transitive hierarchy relationships "
                          "in the output vocabulary.")
    group.add_option('-M', '--enrich-mappings', action="store_true",
                     help='Perform SKOS enrichments on mapping relationships.')
    group.add_option('-m', '--no-enrich-mappings', dest="enrich_mappings",
                     action="store_false",
                     help="Don't perform SKOS enrichments "
                          "on mapping relationships.")
    group.add_option('-A', '--aggregates', action="store_true",
                     help='Keep AggregateConcepts completely '
                          'in the output vocabulary.')
    group.add_option('-a', '--no-aggregates',
                     dest="aggregates", action="store_false",
                     help='Remove AggregateConcepts completely '
                          'from the output vocabulary.')
    group.add_option('-R', '--keep-related', action="store_true",
                     help="Keep skos:related relationships "
                          "within the same hierarchy.")
    group.add_option('-r', '--no-keep-related',
                     dest="keep_related", action="store_false",
                     help="Remove skos:related relationships "
                          "within the same hierarchy.")
    group.add_option('-B', '--break-cycles', action="store_true",
                     help="Break any cycles in the skos:broader hierarchy.")
    group.add_option('-b', '--no-break-cycles', dest="break_cycles",
                     action="store_false",
                     help="Don't break cycles in the skos:broader hierarchy.")
    group.add_option('--eliminate-redundancy', action="store_true",
                     help="Eliminate hierarchical redundancy in the "
                          "skos:broader hierarchy.")
    group.add_option('--no-eliminate-redundancy', dest="eliminate_redundancy",
                     action="store_false",
                     help="Don't eliminate hierarchical redundancy in the "
                          "skos:broader hierarchy.")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser, "Cleanup Options")
    group.add_option('--cleanup-classes', action="store_true",
                     help="Remove definitions of classes with no instances.")
    group.add_option('--no-cleanup-classes', dest='cleanup_classes',
                     action="store_false",
                     help="Don't remove definitions "
                          "of classes with no instances.")
    group.add_option('--cleanup-properties', action="store_true",
                     help="Remove definitions of properties "
                          "which have not been used.")
    group.add_option('--no-cleanup-properties', action="store_false",
                     dest='cleanup_properties',
                     help="Don't remove definitions of properties "
                          "which have not been used.")
    group.add_option('--cleanup-unreachable', action="store_true",
                     help="Remove triples which can not be reached "
                          "by a traversal from the main vocabulary graph.")
    group.add_option('--no-cleanup-unreachable', action="store_false",
                     dest='cleanup_unreachable',
                     help="Don't remove triples which can not be reached "
                          "by a traversal from the main vocabulary graph.")
    parser.add_option_group(group)

    return parser

def expand_curielike(namespaces, curie):
    """Expand a CURIE (or a CURIE-like string with a period instead of colon
    as separator) into URIRef. If the provided curie is not a CURIE, return it
    unchanged."""

    if curie == '':
        return None
    if sys.version < '3':  # Python 2 ConfigParser reads raw byte strings
        curie = curie.decode('UTF-8')  # ...make those into Unicode objects

    if curie.startswith('[') and curie.endswith(']'):
        # decode SafeCURIE
        curie = curie[1:-1]

    if ':' in curie:
        ns, localpart = curie.split(':', 1)
    elif '.' in curie:
        ns, localpart = curie.split('.', 1)
    else:
        return curie

    if ns in namespaces:
        return URIRef(namespaces[ns].term(localpart))
    else:
        logging.warning("Unknown namespace prefix %s", ns)
        return URIRef(curie)

def expand_mapping_target(namespaces, val):
    """Expand a mapping target, expressed as a comma-separated list of
    CURIE-like strings potentially prefixed with ^ to express inverse
    properties, into a list of (uri, inverse) tuples, where uri is a URIRef
    and inverse is a boolean."""

    vals = [v.strip() for v in val.split(',')]
    ret = []
    for v in vals:
        inverse = False
        if v.startswith('^'):
            inverse = True
            v = v[1:]
        ret.append((expand_curielike(namespaces, v), inverse))
    return ret

def write_rdf(rdf, filename, fmt):
    if filename == '-':
        out = sys.stdout
    else:
        out = open(filename, 'wb')

    if not fmt:
        # determine output format
        fmt = 'xml'  # default
        if filename.endswith('n3'):
            fmt = 'n3'
        if filename.endswith('nt'):
            fmt = 'nt'
        if filename.endswith('ttl'):
            fmt = 'turtle'

    logging.debug("Writing output file %s (format: %s)", filename, fmt)
    rdf.serialize(destination=out, format=fmt)

def main():
    """Read command line parameters and make a transform based on them."""

    skosify = Skosify()

    namespaces = DEFAULT_NAMESPACES
    typemap = {}
    literalmap = {}
    relationmap = {}

    defaults = DEFAULT_OPTIONS
    defaults['to_format'] = None
    defaults['output'] = '-'
    defaults['log'] = None
    defaults['debug'] = False

    options, remainingArgs = get_option_parser(defaults).parse_args()

    # configure logging
    logformat = '%(levelname)s: %(message)s'
    loglevel = logging.INFO
    if options.debug:
        loglevel = logging.DEBUG
    if options.log:
        logging.basicConfig(filename=options.log,
                            format=logformat, level=loglevel)
    else:  # logging messages go into stderr by default
        logging.basicConfig(format=logformat, level=loglevel)

    # read config file
    if options.config is not None:
        # read the supplied configuration file
        cfgparser = configparser.SafeConfigParser()
        # force case-sensitive handling of option names
        cfgparser.optionxform = str
        cfgparser.read(options.config)

        # parse namespaces from configuration file
        for prefix, uri in cfgparser.items('namespaces'):
            namespaces[prefix] = URIRef(uri)

        # parse types from configuration file
        for key, val in cfgparser.items('types'):
            typemap[expand_curielike(namespaces, key)] = \
                expand_mapping_target(namespaces, val)

        # parse literals from configuration file
        for key, val in cfgparser.items('literals'):
            literalmap[expand_curielike(namespaces, key)] = \
                expand_mapping_target(namespaces, val)

        # parse relations from configuration file
        for key, val in cfgparser.items('relations'):
            relationmap[expand_curielike(namespaces, key)] = \
                expand_mapping_target(namespaces, val)

        # parse options from configuration file
        for opt, val in cfgparser.items('options'):
            if opt not in defaults:
                logging.warning(
                    'Unknown option in configuration file: %s (ignored)', opt)
                continue
            if defaults[opt] in (True, False):  # is a Boolean option
                defaults[opt] = cfgparser.getboolean('options', opt)
            else:
                defaults[opt] = val

        # re-initialize and re-run OptionParser using defaults read from
        # configuration file
        options, remainingArgs = get_option_parser(defaults).parse_args()

    if remainingArgs:
        inputfiles = remainingArgs
    else:
        inputfiles = ['-']

    voc = skosify.skosify(inputfiles, namespaces, typemap,
                          literalmap, relationmap, options)

    write_rdf(voc, options.output, options.to_format)

if __name__ == '__main__':
    main()


import pickle
import numpy as np
import matplotlib as plt
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
from time import time
import itertools
from math import log10

np.set_printoptions(threshold=np.nan)
np.random.seed(400)

def plot_acc(acc, acc_val):
    plt.plot(acc)
    plt.plot(acc_val)
    plt.show()

def plot_cost(J_train, J_val, etas, lambdas):
    ###### PLOT COST ######
    train_labels = ['train_cost, (eta,lambdas): ('+str(x[0]) + ',' + str(x[1]) + ')' for x in itertools.product(etas,lambdas)]
    val_labels = ['val_cost:, (eta,lamdbas): ('+str(x[0]) + ',' + str(x[1]) + ')'  for x in list(itertools.product(etas,lambdas))]
    for i in range(J_train.shape[0]):
        plt.plot(J_train[i,:], label=train_labels[i])
        plt.plot(J_val[i,:], label=val_labels[i])
    plt.legend()
    plt.show()

class Params():
    def __init__(self, batch_size, n_epochs, _lambda, eta):
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.eta = eta
        self._lambda = _lambda

class NeuronLayer():
    def __init__(self, n_in, n_out, type, init):
        if init == 'random':
            self.W = np.random.normal(0, 0.001, (n_in,n_out))
        elif init == 'He':
            self.W = np.random.normal(0, np.sqrt((2/n_out)), (n_in,n_out))
        self.b = np.zeros((n_in,1))
        self.grad_W = np.zeros((n_in,n_out))
        self.grad_b = np.zeros((n_in,1))
        self.type = type
        self.error = None
        self.velocity_W = 0
        self.velocity_b = 0

    def _feedforward(self,input):
        return self.W.dot(input) + self.b

    def _activation(self, ain):
        if self.type=='relu':
            return self.__relu(ain)
        if self.type=='softmax':
            return self.__softmax(ain)

    def __relu(self,ain):
        return np.maximum(0, ain)

    def __softmax(self, ain):
#         print(np.sum(np.exp(ain),axis=0))
        return np.exp(ain) / np.sum(np.exp(ain),axis=0)

    def estimate_params(self,ain):
        # OUTPUT: vectors of shape: (ain.shape[0],) (ain_shape[0],)
        return np.array(ain).mean(1), np.array(ain).var(1)

    def batchNorm(self, ain, mu, var, eps=1e-15):
        var_diag_scaled = np.diag(np.power((var+eps),-0.5))
#         print(var_diag_scaled.shape)

        ain_centered = ain - mu.reshape(-1,1)

        normed = var_diag_scaled.dot(ain_centered)
        return normed

class NeuralNetwork():
    def __init__(self):
        self.layers = []

    def add(self,layer):
        self.layers.append(layer)

    def computeAccuracy(self, X, Y):
        # finds the max probablity in each column and compares to the target labels.
        self.feedforward(X)
        pred = np.argmax(self.x[-1],axis=0)
        return np.sum(np.where(Y==pred,1,0))/X.shape[1]*100

    def compareGradient(self, X, Y, h=1e-5): # WEIGHT GRADIENT KLARAR 1e-3. men FILTER GRADIENT GÖR INTE DET.
        """
        http://peterroelants.github.io/posts/neural_network_implementation_part04/#Gradient-checking
        """
        # SAVE ALL DEVIATING BIAS GRADIENTS
        deviating_grads = []
        # Combine all parameter matrices in a list
        old_b = self.layers[-2].b.copy()
        # Combine all parameter gradients in a list
        grad_params = [self.layers[-2].grad_b.copy()]
        print(""" BIAS GRADIENT """)
        self.layers[-1].num_grad_b = np.zeros(self.layers[-2].grad_b.shape)
        for row in range(old_b.shape[0]):

                p_matrix_min = old_b.copy()
                p_matrix_min[row,0] -= h
                self.layers[-2].b = p_matrix_min
                c1 = self.computeCost(X, Y, _lambda=0)

                p_matrix_plus = old_b.copy()
                p_matrix_plus[row,0] += h
#                 print(self.layers[1].b - p_matrix_plus)
                self.layers[-2].b = p_matrix_plus
                c2 = self.computeCost(X, Y, _lambda = 0)

                grad_num = (c2 - c1) / (2 * h)
                print(c1, c2)
#                 print(grad_num, grad_params[0][row,0])

                # RESTORE OLD VALUES
#                 self.layers[1].b = old_b.copy()
#                 self.layers[1].grad_b = grad_params[0].copy()

                # Raise error if the numerical grade is not close to the backprop gradient
                if not np.isclose(grad_num, grad_params[0][row,0], atol=h):
                    print('Numerical gradient of {:.6f} is not close to {:.6f}!'.format(float(grad_num), float(grad_params[0][row,0])))
                    deviating_grads.append({'num':grad_num, 'analytical': grad_params[0][row,0], 'layer':'b'})
        print("num gradient errors in bias = {}".format(len(deviating_grads)/(row+1)))
#
        # SAVE ALL DEVIATING WEIGHT GRADIENTS
        deviating_grads = []
        # Combine all parameter matrices in a list
        old_W = self.layers[-2].W.copy()
        # Combine all parameter gradients in a list
        grad_params = [self.layers[-2].grad_W.copy()]
        print(""" WEIGHT GRADIENT """)
        self.layers[-2].num_grad_W = np.zeros(self.layers[-2].grad_W.shape)
        for row in range(old_W.shape[0]):
            for col in range(old_W.shape[1]):

                p_matrix_min = old_W.copy()
                p_matrix_min[row,col] -= h
                self.layers[-2].W = p_matrix_min
                c1 = self.computeCost(X, Y, 0)

                p_matrix_plus = old_W.copy()
                p_matrix_plus[row,col] += h
                self.layers[-2].W = p_matrix_plus
                c2 = self.computeCost(X, Y, 0)

                grad_num = (c2 - c1) / (2 * h)
#                 print(grad_num, grad_params[0][row,col])

                # RESTORE OLD VALUES
#                 self.layers[1].W = old_W.copy()
#                 self.layers[1].grad_W = grad_params[0].copy()

                # Raise error if the numerical grade is not close to the backprop gradient
                if not np.isclose(grad_num, grad_params[0][row,col], atol=h):
                    print('Numerical gradient of {:.6f} is not close to {:.6f}!'.format(float(grad_num), float(grad_params[0][row,col])))
                    deviating_grads.append({'num':grad_num, 'analytical': grad_params[0][row,col], 'layer':'W'})
            print("num gradient errors in row {}/{} = {}".format(row,old_W.shape[0], len(deviating_grads)/(row+1)))
        print('total errors is {}'.format(len(deviating_grads)))
#         print('No gradient errors found')

    def feedforward(self,input, test_time = None):
        self.eps = 1e-15
#         self.s.append(input) # add input to use in last backward pass
        self.s = [None] * len(self.layers)
        self.x = [None] * len(self.layers)
        self.batch_norms = [None] * len(self.layers)
#         self.input_mu, self.input_var = self.layers[-1].estimate_params(input)
#         normalized = self.layers[-1].batchNorm(input,self.input_mu,self.input_var)
#         self.input_norm = normalized
        if test_time == 'test_time':
#             print('test_time')
            self.mu = self.mu_avg
            self.var = self.var_avg
        else:
            self.mu = [None] * (len(self.layers)-1)
            self.var = [None] * (len(self.layers)-1)
        for i, layer in enumerate(self.layers):
            ain = layer._feedforward(input)
            self.s[i] = ain
            if i < len(self.layers)-1:
                if not test_time:
                    self.mu[i], self.var[i] = layer.estimate_params(ain)
                norm_out = layer.batchNorm(ain,self.mu[i],self.var[i])
                self.batch_norms[i] = norm_out
                aout = layer._activation(norm_out)
            else:
                # DO NOT NORMALIZE LAST LAYER OUTPUT
#                 print('hej')
                aout = layer._activation(ain)
            self.x[i] = aout
            input = aout

    def batchNormBackPass(self, G, s, mu, var):
#         print(G.shape,s.shape,mu.shape,var.shape)
        dj_dvar = np.zeros((G.shape[0]))
        dj_dmu = np.zeros((G.shape[0]))
        n = s.shape[1]
        for i in range(n):
            dj_dvar += G[:,i] * (1/np.cbrt(var+self.eps)) * (s[:,i] - mu)
            dj_dmu += G[:,i] * (1/np.sqrt(var+self.eps))
#         dj_dvar = np.sum(G * (1/np.cbrt(var+self.eps)), axis = 1, keepdims = True) * np.sum(s - mu, axis = 1, keepdims = True)
#         dj_dmu = np.sum(G * (1/np.sqrt(var+self.eps)), axis = 1)

        dj_dvar *= (-1/2)
        dj_dmu *= (-1)

        for i in range(n):
            a = G[:,i].dot(1/np.sqrt(var))
            b = (2/n) * dj_dvar * (s[:,i] - mu)
            c = (1/n) * dj_dmu
            G[:,i] = a + b + c
        return G

    def batchGradient(self,X,Y,GDparams):
        for layer in self.layers:
            layer.grad_W = np.zeros(layer.grad_W.shape)
            layer.grad_b = np.zeros(layer.grad_b.shape)
        n = X.shape[1]
        # ERROR LAST FIRST LAYER
        G = np.zeros(self.x[-2].shape)
        for i in range(n):
            g = -(Y[:,i] - self.x[-1][:,i]).reshape(1,-1) # 1x10
            xi = self.x[-2][:,i].reshape(-1,1) # 50x1
            s_hat = self.batch_norms[-2][:,i]
            # SET GRADIENTS
            self.layers[-1].grad_W += (np.dot(g.T, xi.T)/n) + 2*GDparams._lambda*self.layers[-1].W # 10x1 dot 1x50
            self.layers[-1].grad_b += (g.T/n)

            g = np.dot(g,self.layers[-1].W)
#             print(g.shape)
#             print(np.diag(s_hat).shape)
            G[:,i] = np.dot(g,np.diag(np.where(s_hat > 0, 1, 0)))
            # SCALE GRADIENTS
#       # KEEP ON BACKPROPAGATING THE ERROR THROUGH THE NETWORK
        for i in range(len(self.layers)-2,-1,-1):
            if i > 0:
#                 print('batch Norm backpass')
                G = self.batchNormBackPass(G, self.s[i], self.mu[i], self.var[i])
                # ISNERT BACKPROP HERE
#                 G = np.zeros(self.x[i-1].shape)
#                 print(self.layers[i].type)
                for j in range(n):
                    g = G[:,j].reshape(1,-1)
                    xi = self.x[i-1][:,j].reshape(-1,1) # 50x1
                    s_hat = self.batch_norms[i-1][:,j]
                    # SET GRADIENTS
                    self.layers[i].grad_W += (np.dot(g.T, xi.T)/n) + 2*GDparams._lambda*self.layers[i].W # 10x1 dot 1x50
                    self.layers[i].grad_b += (g.T/n)

                    g = np.dot(g,self.layers[i].W)
        #             print(g.shape)
        #             print(np.diag(s_hat).shape)
                    G[:,j] = np.dot(g,np.diag(np.where(s_hat > 0, 1, 0)))

            else:
                for j in range(n):
                    g = G[:,j].reshape(1,-1)
                    xj = X[:,j].reshape(-1,1) # 50x1
                    # SET GRADIENTS
                    self.layers[i].grad_W += (np.dot(g.T, xj.T)/n) #+ 2*GDparams._lambda*self.layers[j].W
                    self.layers[i].grad_b += (g.T/n)
#
#     def computeGradient(self, X, Y, GDparams):
#         for layer in self.layers:
#             layer.grad_W = 0
#             layer.grad_b = 0
#         self.error = -(Y -  self.activation_out[-1])
#         for i in range(len(self.layers)-1,-1,-1):
#             if i > 0:
#                 self.layers[i].grad_W = np.dot(self.error, self.activation_out[i-1].T)
#                 self.layers[i].grad_b = self.error.sum(axis=1).reshape(-1,1)
#                 self.error = np.dot(self.error.T, self.layers[i].W)
#                 # derivative
#                 self.error = np.where(self.activation_in[i-1] > 0, self.error.T, 0)
#             else: # LAST STEP. MULTIPLY BY INPUT
#                 self.layers[i].grad_W = np.dot(self.error, X.T)
#                 self.layers[i].grad_b = self.error.sum(axis=1).reshape(-1,1)
#
#             # NORMALIZE INPUT WEIGHTS AND BIAS
#             self.layers[i].grad_W *= (1/X.shape[1])
#             self.layers[i].grad_W +=  2 * GDparams._lambda * self.layers[i].W
#             self.layers[i].grad_b *= (1/X.shape[1])

    def computeCost(self, X, Y, _lambda, test_time = None):
        D = X.shape[1]
        if test_time == 'test_time':
            self.feedforward(X, 'test_time')
        else:
            self.feedforward(X)
        J = (1/D) * np.sum(self.l_cross(Y))  + _lambda * sum([np.sum(layer.W**2) for layer in self.layers])
        return J

    def l_cross(self, Y):
        idx = np.argmax(self.x[-1] * Y,axis = 0)
        cost_vec = np.zeros((1,idx.shape[0]))
        for i, p in enumerate(idx):
            cost_vec[0,i] = self.x[-1][p,i]
        return -np.log(cost_vec)

    def getMovingAvgs(self,j):
#         print(self.mu)
#         print(self.var)
        alpha = 0.99
        if j == 1:
            self.mu_avg = self.mu
            self.var_avg = self.var
        else:
            self.mu_avg = np.multiply(alpha, self.mu_avg) + np.multiply((1-alpha), self.mu)
            self.var_avg = np.multiply(alpha, self.var_avg) + np.multiply((1-alpha), self.var)

    def miniBatchGD(self, X, Y, GDparams, momentum):
        gamma = 0.9
        for j in range(1,int(X.shape[1]/GDparams.batch_size)+1):
            j_start = int((j-1)*GDparams.batch_size)
            j_end = int(j*GDparams.batch_size)
            Xs = X[:, j_start:j_end]
            Ys = Y[:, j_start:j_end]
            self.feedforward(Xs)
#             self.computeGradient(Xs, Ys, GDparams)
            self.batchGradient(Xs, Ys, GDparams)
#             self.compareGradient(Xs, Ys)
            # KEEP TRACK OF MOOVING AVERAGES OF MU AND VARIANCE
            self.getMovingAvgs(j)

            # UPDATE WEIGHTS
            if momentum:
                # USE MOMENTUM
                for layer in self.layers:
                    layer.velocity_W = gamma * layer.velocity_W + GDparams.eta * layer.grad_W
                    layer.velocity_b = gamma * layer.velocity_b + GDparams.eta * layer.grad_b
                    layer.W = layer.W - layer.velocity_W
                    layer.b = layer.b - layer.velocity_b
            else:
            # standard UPDATE WEIGHTS AND BIAS
                for layer in self.layers:
                    layer.W = layer.W - (GDparams.eta * layer.grad_W)
                    layer.b = layer.b - (GDparams.eta * layer.grad_b)

    def train(self, X_train, X_val, y_oh, Y_train, y_oh_val, Y_val, GDparams,init, momentum, check_gradients):
        J_train = np.zeros((GDparams.n_epochs,))
        J_val = np.zeros((GDparams.n_epochs,))
        acc = np.zeros(GDparams.n_epochs,)
        acc_val = np.zeros(GDparams.n_epochs,)
        best_acc = 0
        print('Start training... eta {}, lambda {}, init: {}'.format(GDparams.eta,GDparams._lambda,init))
        for epoch in range(GDparams.n_epochs):
            self.miniBatchGD(X_train, y_oh, GDparams, momentum)
            J_train[epoch] = self.computeCost(X_train, y_oh, GDparams._lambda, 'test_time')
            acc[epoch] = self.computeAccuracy(X_train,Y_train)

            J_val[epoch] = self.computeCost(X_val, y_oh_val, GDparams._lambda, 'test_time')
            acc_val[epoch] = self.computeAccuracy(X_val,Y_val)
            print('Epoch: ' + str(epoch) + '/' + str(GDparams.n_epochs) + ' Loss: ' + str(J_train[epoch]) + ' Acc: ' + str(acc[epoch]) + \
                                                                            ' Val-Loss: ' + str(J_val[epoch]) + ' Val-Acc: ' + str(acc_val[epoch]))
#             GDparams.eta = GDparams.eta * 0.95
            if check_gradients and epoch == 3:
                self.feedforward(X_train)
                self.batchGradient(X_train, y_oh, GDparams)
#                 self.computeGradient(X_train, y_oh, GDparams)
                self.compareGradient(X_train, y_oh)

        return J_train, J_val, acc, acc_val

def main():
    num_classes = 10
    num_hidden = 50
    features = 3072
    samples = 10000
    epochs = 10
    all_data = False
    momentum = True
    check_gradients = True

    import get_data as preprocess
    X_train,X_val,X_test,y_oh,y_oh_val,y_oh_test,Y_train,Y_val,Y_test = preprocess.get_data(all_data,features,samples,num_classes)

    ############# HYPER PARAMS ##############################
    etas = [0.02] # 0.02
    lambdas = [0.0] #[0.002]
    ##############################################################

    # SAVE ALL COSTS
    J_train = np.zeros((len(etas)*len(lambdas),epochs)) # cost vector for plotting
    J_val = np.zeros((len(etas)*len(lambdas),epochs)) # cost vector for plotting
    best_param_dict = {}

    init_list = ['random']

    for init in init_list:
        for i, hyperparam in enumerate(list(itertools.product(etas,lambdas))): # ~0.02 decent eta
            GDparams = Params(batch_size=100, n_epochs=epochs, eta=hyperparam[0], _lambda=hyperparam[1])
            NN = NeuralNetwork()
            layer0 = NeuronLayer(num_hidden,features,'relu', init)
            NN.add(layer0)
            layer1 = NeuronLayer(50,50,'relu', init)
            NN.add(layer1)
#             layer2 = NeuronLayer(30,50,'relu', init)
#             NN.add(layer2)
            layer3 = NeuronLayer(10,50,'softmax', init)
            NN.add(layer3)
            J_train[i], J_val[i], acc_vec, acc_val_vec = NN.train(X_train, X_val, y_oh, Y_train, y_oh_val, Y_val, GDparams,init,
                                                                                momentum,
                                                                                check_gradients)
            test_acc = NN.computeAccuracy(X_test,Y_test)
            print('test Acc: ' + str(test_acc) + ' eta: ' + str(hyperparam[0]) + ' lambda: ' + str(hyperparam[1]))
#         plt.plot(J_train, label = init)
#     plt.legend()
#     plt.show()
#         plot_cost(J_train, J_val, etas, lambdas)
#         plot_acc(acc_vec,acc_val_vec)

main()


#
# @lc app=leetcode.cn id=189 lang=python3
#
# [189] 旋转数组
#

# @lc code=start
class Solution:
    def rotate(self, nums: List[int], k: int) -> None:
        """
        Do not return anything, modify nums in-place instead.
        """
        # 方法一：观察结果，充分利用Python的切片功能
        # k = k % len(nums)
        # nums[:] = nums[-k:] + nums[:-k]

        # k = k % len(nums)
        # if not k:
        #     return
        # nums[:k], nums[k:] = nums[-k:], nums[:-k]

        # 递归的这种方法，占用空间最小
        # 三次反转
        if k is None or k <= 0:
            return
        k, end = k % len(nums), len(nums) - 1

        def reverse(nums, start, end):
            while start < end:
                nums[start], nums[end] = nums[end], nums[start]
                start += 1
                end -= 1
        # 第一次整体反转
        reverse(nums, 0, end)
        # 第二次反转前k个数
        reverse(nums, 0, k-1)
        # 第三次反转后面的部分
        reverse(nums,k, end)

# @lc code=end


from requester import Requester
from common import config_reader, config_intervaltime
import post
import time
import rediscache
import logging

class TiebaCrawler(Requester):
    """ Post crawler  , gather information of posts in given bar
        can't get image submmited in post
    """

    def __init__(self, tieba_name="steam", cookie=None):
        """Initialize Crawler
        :param tieba_name:
        :param cookie:
        """
        Requester.__init__(self, tieba_name, cookie)

        self.__available_check()

    def __available_check(self):
        """
        Checking cookie available , tieba existence , manage rights validity
        :return boolean:
        """
        logging.info("Doing available_check")

        response = self.get_content(self.tieba_base)

        if "参与本吧讨论请先" in response:
            logging.warning("Doing available_check : FAILED")
            raise Exception("User Cookie not available")

        logging.info("Checked")

        logging.info("Checking tieba existence......")
        if "尚未建立" in response:
            logging.warning("Checking tieba existence : FAILED")
            raise Exception("Tieba doesn't exist")

        logging.info("Checked")

        logging.info("All check done")
        return True

    def get_posts(self):
        """Get all posts on first page of tieba , and generate a post objects list
        :return list of Post object:
        """
        soup = self.get_content(self.tieba_base)

        post_a = self.__get_posts_a(soup)

        url_list = [self.url_base + tag.get('href') for tag in post_a]

        post_dict = self.__get_content_list(url_list)
        post_list = [post.Post(url, soup) for url, soup in post_dict.items()]

        return post_list

    @rediscache.postcache
    def __get_content_list(self, url_list):
        """Get post content with given url list
        :param url_list:
        :return dict:
        """
        content_list = {}

        for url in url_list:
            content = self.get_content(url)
            if content:
                content_list[url] = content

            time.sleep(config_intervaltime())

        return content_list

    def __get_posts_a(self, soup):
        """Get all post url from the soup of first page of tieba's first page
        :param soup:
        :return list of posts' url:
        """
        posts_list = soup.findAll('div', {'class': 'i'})
        posts_list = [tag.find('a') for tag in posts_list if not tag.find('span', {'class': 'light'})]
        return posts_list

if __name__ == "__main__":
    cookie, _ = config_reader()
    tieba_worker = TiebaCrawler(cookie=cookie, tieba_name='dota2提问')
    posts = tieba_worker.get_posts()
    print(len(list(map(str, posts))))
    posts = tieba_worker.get_posts()
    print(len(list(map(str, posts))))
    # print(list(map(str, posts[0].reply_list)))


import csv

all_output_csv_path = 'C:/Users/TIM58/Downloads/hurricane-need-vis-master_final/hurricane-need-vis-master/data/All_Dataset.csv'
days = ["0823", "0824", "0825", "0826", "0827", "0828", "0829", "0830", "0831", "0901"]

with open(all_output_csv_path, 'r', encoding="utf-8_sig") as csv_reader:
    rows = csv.reader(csv_reader)
    my_dict = {}
    dist = locals()
    for day in days:
        my_dict[str(day)] = {}
        for hour in range(24):
            if hour < 10:
                hour = '0' + str(hour)
            my_dict[str(day)][str(hour)] = {}
            for dist_num in range(1, 89):
                my_dict[str(day)][str(hour)][str(dist_num)] = {}
                for i in range(2):
                    my_dict[str(day)][str(hour)][str(dist_num)][str(i)] = 0
    # print(my_dict)
    # num = my_dict['0823']['00']['1']['1']
    # num += 1
    # my_dict['0823']['00']['1']['1'] = num
    # print(my_dict['0823']['00']['1']['1'])
    for row in rows:
        userId = row[0]
        tweet = row[1]
        dist_num = row[2]
        region = row[3]
        timestamp = row[4]
        day = row[5]
        hour = row[6]
        needs = row[7]
        count = row[8]
        needs_sum = row[9]
        if len(needs) > 0:
            num1 = my_dict[day][hour][dist_num]['1']
            num1 += 1
            my_dict[day][hour][dist_num]['1'] = num1
        else:
            num0 = my_dict[day][hour][dist_num]['0']
            num0 += 1
            my_dict[day][hour][dist_num]['0'] = num0
    for i in range(24):
        if i < 10:
            i = '0' + str(i)
        else:
            i = str(i)
        print(my_dict['0823'][i]['1'])

csv_reader.close()


import logging

from random import randint

from flask import Flask, render_template

from flask_ask import Ask, statement, question, session
import requests

app = Flask(__name__)

ask = Ask(app, "/")

logging.getLogger("flask_ask").setLevel(logging.DEBUG)

@ask.launch

def new_game():

    welcome_msg = render_template('welcome')

    return question(welcome_msg).reprompt(render_template('reprom'))

@ask.intent("SalesIntent",convert={'InvOne': str, 'InvTwo': str, 'csname':str},default={'InvOne': 'None','InvTwo': 'None','csname': 'None'})

def Sales(InvOne, InvTwo,csname):
    print(csname)
    final_sentence = ""
    if InvOne != 'None' and InvTwo != 'None' and csname == 'None':
        url = "http://122.179.136.81:1531/henkel/sale"
        params = {"SalOne": InvOne, "SalTwo": InvTwo}
        data = requests.get(url=url, params=params)
        data = data.json()
        final_sentence = data["Response"]
    elif InvOne == 'None' and InvTwo == 'None' and csname != 'None':
        url = "http://122.179.136.81:1531/henkel/customer"
        params = {"CusId": str(csname)}
        data = requests.get(url=url, params=params)
        data = data.json()
        final_sentence = data["Response"]
    else:
        final_sentence = "Kindly provide customer name"
    return question(final_sentence).reprompt(render_template('reprom'))

@ask.intent("InventoryIntent", convert={'InvOne': str, 'InvTwo': str})

def inventory(InvOne, InvTwo):
    final_sentence = ""
    url = "http://122.179.136.81:1531/henkel/inventory"
    params = {"InvOne":InvOne,"InvTwo":InvTwo}
    data = requests.get(url=url,params=params)
    data = data.json()
    final_sentence = data["Response"]
    return question(final_sentence).reprompt(render_template('reprom'))

@ask.intent("OpenOrdersIntent", convert={'InvOne': str, 'InvTwo': str,'InvThree':str},default={'InvThree': 'None'})

def order(InvOne, InvTwo,InvThree):
    final_sentence = ""
    url = "http://122.179.136.81:1531/henkel/order"
    params = {"OrdOne":InvOne,"OrdTwo":InvTwo,"OrdThree":InvThree}
    data = requests.get(url=url,params=params)
    data = data.json()
    final_sentence = data["Response"]
    return question(final_sentence).reprompt(render_template('reprom'))

@ask.intent("AMAZON.StopIntent")

def stop():
    final_sentence = "Alright then, Thank you for using insight bot. Bye."
    return statement(final_sentence)

@ask.intent("AMAZON.YesIntent")

def yes():
    final_sentence = "Okay. what else you want to know ?"
    return question(final_sentence).reprompt(render_template('reprom'))

@ask.intent("CustomerIntent",convert={'number': str},default={'number': 'None'})

def customer_val(csname):
    print(csname)
    final_sentence = ""
    url = "http://122.179.136.81:1531/henkel/customer"
    params = {"CusId": csname}
    data = requests.get(url=url, params=params)
    data = data.json()
    final_sentence = data["Response"]
    return question(final_sentence)

@ask.intent("AMAZON.NoIntent")

def no():
    final_sentence = "Alright then, Thank you for using insight bot. Bye."
    return question(final_sentence)

# @ask.intent("AMAZON.NoIntent")
#
# def no():
#     final_sentence = "Alright then, Thank you for using henkel bot. Bye Bye"
#     return statement(final_sentence)

if __name__ == '__main__':

    app.run(debug=True)


#!/usr/bin/env python
# encoding: utf-8
# Philipp Bender, 2012
# Matt Clarkson, 2012

import re
from waflib.Task import Task
from waflib.TaskGen import extension
from waflib import Errors

"""
A simple tool to integrate protocol buffers into your build system.

Example for C++:

    def configure(conf):
        conf.load('compiler_cxx cxx protoc')

    def build(bld):
        bld(
                features = 'cxx cxxprogram'
                source   = 'main.cpp file1.proto proto/file2.proto',
                includes = '. proto',
                target   = 'executable')

Example for Python:

    def configure(conf):
        conf.load('python protoc')

    def build(bld):
        bld(
                features = 'py'
                source   = 'main.py file1.proto proto/file2.proto',
                protoc_includes  = 'proto')

Example for both Python and C++ at same time:

    def configure(conf):
        conf.load('cxx python protoc')

    def build(bld):
        bld(
                features = 'cxx py'
                source   = 'file1.proto proto/file2.proto',
                protoc_includes  = 'proto')	# or includes

Notes when using this tool:

- protoc command line parsing is tricky.

  The generated files can be put in subfolders which depend on
  the order of the include paths.

  Try to be simple when creating task generators
  containing protoc stuff.

"""

class protoc(Task):
	run_str = '${PROTOC} ${PROTOC_FL:PROTOC_FLAGS} ${PROTOC_ST:INCPATHS} ${PROTOC_ST:PROTOC_INCPATHS} ${SRC[0].bldpath()}'
	color   = 'BLUE'
	ext_out = ['.h', 'pb.cc', '.py']
	def scan(self):
		"""
		Scan .proto dependencies
		"""
		node = self.inputs[0]

		nodes = []
		names = []
		seen = []
		search_nodes = []

		if not node:
			return (nodes, names)

		if 'cxx' in self.generator.features:
			search_nodes = self.generator.includes_nodes

		if 'py' in self.generator.features:
			for incpath in getattr(self.generator, 'protoc_includes', []):
				search_nodes.append(self.generator.bld.path.find_node(incpath))

		def parse_node(node):
			if node in seen:
				return
			seen.append(node)
			code = node.read().splitlines()
			for line in code:
				m = re.search(r'^import\s+"(.*)";.*(//)?.*', line)
				if m:
					dep = m.groups()[0]
					for incnode in search_nodes:
						found = incnode.find_resource(dep)
						if found:
							nodes.append(found)
							parse_node(found)
						else:
							names.append(dep)

		parse_node(node)
		return (nodes, names)

@extension('.proto')
def process_protoc(self, node):
	incdirs = []
	out_nodes = []
	protoc_flags = []

	# ensure PROTOC_FLAGS is a list; a copy is used below anyway
	self.env.PROTOC_FLAGS = self.to_list(self.env.PROTOC_FLAGS)

	if 'cxx' in self.features:
		cpp_node = node.change_ext('.pb.cc')
		hpp_node = node.change_ext('.pb.h')
		self.source.append(cpp_node)
		out_nodes.append(cpp_node)
		out_nodes.append(hpp_node)
		protoc_flags.append('--cpp_out=%s' % node.parent.get_bld().bldpath())

	if 'py' in self.features:
		py_node = node.change_ext('_pb2.py')
		self.source.append(py_node)
		out_nodes.append(py_node)
		protoc_flags.append('--python_out=%s' % node.parent.get_bld().bldpath())

	if not out_nodes:
		raise Errors.WafError('Feature %r not supported by protoc extra' % self.features)

	tsk = self.create_task('protoc', node, out_nodes)
	tsk.env.append_value('PROTOC_FLAGS', protoc_flags)

	# Instruct protoc where to search for .proto included files.
	# For C++ standard include files dirs are used,
	# but this doesn't apply to Python for example
	for incpath in getattr(self, 'protoc_includes', []):
		incdirs.append(self.bld.path.find_node(incpath).bldpath())
	tsk.env.PROTOC_INCPATHS = incdirs

	use = getattr(self, 'use', '')
	if not 'PROTOBUF' in use:
		self.use = self.to_list(use) + ['PROTOBUF']

def configure(conf):
	conf.check_cfg(package='protobuf', uselib_store='PROTOBUF', args=['--cflags', '--libs'])
	conf.find_program('protoc', var='PROTOC')
	conf.env.PROTOC_ST = '-I%s'
	conf.env.PROTOC_FL = '%s'


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /usr/local/lib/python3.6/dist-packages/pyxrd/mixture/models/insitu_mixture.py
# Compiled at: 2020-03-07 03:51:50
# Size of source mod 2**32: 9931 bytes
raise NotImplemented('This module is not yet implemented')
import sys, imp
from contextlib import contextmanager
import numpy as np
from pyxrd.generic.io.custom_io import storables
from pyxrd.mixture.models.mixture import Mixture
from copy import deepcopy

@storables.register()
class InSituMixture(Mixture):
    __doc__ = '\n        Advanced model for optimizing and refining of calculated in-situ XRD data.\n        Is a sub-class of the basic Mixture class.\n        \n        Each phase is linked with an additional Behaviour instance - these are created at the\n        project level.\n    '

    class Meta(Mixture.Meta):
        store_id = 'InSituMixture'

    _data_object = None

    @property
    def data_object(self):
        self._data_object = Mixture.data_object.fget(self)
        return self._data_object

    def get_phase_data_object(self, specimen_index, z_index, phase_index):
        behav = self.behaviour_matrix[(specimen_index, ...)].flatten()[phase_index] if self.behaviour_matrix is not None else None
        phase = self.phase_matrix[(specimen_index, ...)].flatten()[phase_index] if self.phase_matrix is not None else None
        if behav is not None:
            if phase is not None:
                behav.apply(phase, self.specimens[specimen_index].get_z_list()[z_index])
        if phase is not None:
            return deepcopy(phase.data_object)

    behaviour_matrix = None

    def __init__(self, *args, **kwargs):
        my_kwargs = (self.pop_kwargs)(kwargs, 'behaviour_uuids', *[prop.label for prop in InSituMixture.Meta.get_local_persistent_properties()])
        (super(InSituMixture, self).__init__)(*args, **kwargs)
        kwargs = my_kwargs
        with self.data_changed.hold():
            behaviour_uuids = self.get_kwarg(kwargs, None, 'behaviour_uuids')
            if behaviour_uuids is not None:
                self.behaviour_matrix = np.array([[type(type(self)).object_pool.get_object(uuid) if uuid else None for uuid in row] for row in behaviour_uuids], dtype=(np.object_))
            else:
                self.behaviour_matrix = np.empty(shape=(0, 0), dtype=(np.object_))
            print(behaviour_uuids, self.behaviour_matrix)

    def json_properties(self):
        retval = super(InSituMixture, self).json_properties()
        retval['behaviour_uuids'] = [[item.uuid if item else None for item in row] for row in map(list, self.behaviour_matrix)]
        return retval

    @staticmethod
    def from_json(**kwargs):
        if 'refinables' in kwargs:
            del kwargs['refinables']
        return InSituMixture(**kwargs)

    def get_behaviour(self, specimen_slot, phase_slot):
        """Returns the behaviour at the given slot positions or None if not set"""
        return self.behaviour_matrix[(specimen_slot, phase_slot)]

    def set_behaviour(self, specimen_slot, phase_slot, behaviour):
        """Sets the behaviour at the given slot positions"""
        if self.parent is not None:
            with self.needs_update.hold_and_emit():
                with self.data_changed.hold():
                    if behaviour is None:
                        raise RuntimeError('Behaviour can not be set to None, use unset_behaviour to clear!')
                    self.behaviour_matrix[(specimen_slot, phase_slot)] = behaviour
                    self.refinement.update_refinement_treestore()

    def unset_behaviour(self, behaviour):
        """ Clears a behaviour slot in the phase matrix """
        with self.needs_update.hold_and_emit():
            with self.data_changed.hold():
                shape = self.behaviour_matrix.shape
                for i in range(shape[0]):
                    for j in range(shape[1]):
                        if self.behaviour_matrix[(i, j)] == behaviour:
                            self.behaviour_matrix[(i, j)] = None

                self.refinement.update_refinement_treestore()

    @contextmanager
    def _relieve_and_observe_behaviours(self):
        self._relieve_behaviours()
        yield
        self._observe_behaviours()

    def _observe_behaviours(self):
        """ Starts observing behaviour in the behaviour matrix"""
        for behaviour in self.behaviour_matrix.flat:
            if behaviour is not None:
                self.observe_model(behaviour)

    def _relieve_behaviours(self):
        """ Relieves behaviour observer calls """
        for behaviour in self.behaviour_matrix.flat:
            if behaviour is not None:
                self.relieve_model(behaviour)

    def add_phase_slot(self, phase_name, fraction):
        with self.needs_update.hold_and_emit():
            with self.data_changed.hold():
                n, m = self.behaviour_matrix.shape if self.behaviour_matrix.ndim == 2 else (0,
                                                                                0)
                if self.behaviour_matrix.size == 0:
                    self.behaviour_matrix = np.resize(self.behaviour_matrix.copy(), (n, m + 1))
                    self.behaviour_matrix[:] = None
                else:
                    self.behaviour_matrix = np.concatenate([self.behaviour_matrix.copy(), [[None]] * n], axis=1)
                    self.behaviour_matrix[:, m] = None
                return super(InSituMixture, self).add_phase_slot(phase_name, fraction)

    def del_phase_slot(self, phase_slot):
        with self.needs_update.hold_and_emit():
            with self.data_changed.hold():
                with self._relieve_and_observe_behaviours():
                    self.behaviour_matrix = np.delete((self.behaviour_matrix), phase_slot, axis=1)
                return super(InSituMixture, self).del_phase_slot(phase_slot)

    def add_specimen_slot(self, specimen, scale, bgs):
        with self.needs_update.hold_and_emit():
            with self.data_changed.hold():
                n, m = self.behaviour_matrix.shape if self.behaviour_matrix.ndim == 2 else (0,
                                                                                0)
                if self.behaviour_matrix.size == 0:
                    self.behaviour_matrix = np.resize(self.behaviour_matrix.copy(), (n + 1, m))
                    self.behaviour_matrix[:] = None
                else:
                    self.behaviour_matrix = np.concatenate([self.behaviour_matrix.copy(), [[None] * m]], axis=0)
                    self.behaviour_matrix[n, :] = None
                return super(InSituMixture, self).add_specimen_slot(specimen, scale, bgs)

    def del_specimen_slot(self, specimen_slot):
        with self.needs_update.hold_and_emit():
            with self.data_changed.hold():
                self.behaviour_matrix = np.delete((self.behaviour_matrix), specimen_slot, axis=0)
                return super(InSituMixture, self).del_specimen_slot(specimen_slot)

    def set_data_object(self, mixture, calculate=False):
        """
            Sets the fractions, scales and bgshifts of this mixture.
        """
        if mixture is not None:
            with self.needs_update.ignore():
                with self.data_changed.hold_and_emit():
                    self.fractions[:] = list(mixture.fractions)
                    self.scales[:] = list(mixture.scales)
                    self.bgshifts[:] = list(mixture.bgshifts)
                    if calculate:
                        mixture = self.optimizer.calculate(mixture)
                    for i, (specimen_data, specimen) in enumerate(zip(mixture.specimens, self.specimens)):
                        if specimen is not None:
                            with specimen.data_changed.ignore():
                                specimen.update_pattern(specimen_data.total_intensity, specimen_data.phase_intensities * self.fractions[:, np.newaxis] * self.scales[i], self.phase_matrix[i, :])

    def load_behaviour_from_module(self, path):
        """Loads the behaviour class from the given filepath"""
        mod = imp.load_source('Behaviour', path)
        del sys.modules['Behaviour']
        sys.modules[mod.BEHAVIOUR_CLASS] = mod
        klass = mod[mod.BEHAVIOUR_CLASS]
        behaviour = klass()
        behaviour._InSituMixture__path = path
        return behaviour


import json
import re
import pandas as pd
import csv
import os

seperator = '|'
#file_cnt = 561
pkt_dir = "./pcap_data/"

#read files from dir
def iter_dir():
    path = ""

def find_mac_obj(json_root_arr, mac):
    for json_pkt_obj in json_root_arr:
        if(json_pkt_obj['mac'] == mac):
            return json_pkt_obj
    #create mac_obj and add to json_root_arr
    json_pkt_obj = {}
    json_pkt_obj['mac'] = mac
    json_root_arr.append(json_pkt_obj)
    return json_pkt_obj

def extract_ptcl(layer_obj):
    frame_obj = layer_obj['frame']
    ptcl_str = frame_obj['frame.protocols']
    ptcl = ptcl_str.split(':')[-1]
    if ('data' == ptcl) and ('udp' in layer_obj):
        dst_port = layer_obj['udp']['udp.dstport']
        ptcl = 'udp_' + dst_port
    elif ('data' == ptcl):
        ptcl = ptcl_str.split(':')[-2]
    return ptcl

def gene_ptcl_obj(json_pkt_obj, ptcl):
    if 'protocol' in json_pkt_obj:
        ptcl_str = json_pkt_obj['protocol']
        json_pkt_obj['protocol'] = ptcl_str + ',' + ptcl
    else:
        json_pkt_obj['protocol'] = ptcl

#2 layer:     "ssdp":{...}
def add_ptcl_obj(json_pkt_obj, ptcl):
    if ptcl in json_pkt_obj:
        return
    else:
        ptcl_obj = {}
        json_pkt_obj[ptcl] = ptcl_obj

#suit for llmnr/nbns/smb
def add_ptcl_val(json_pkt_obj, ptcl, val):
    if ptcl in json_pkt_obj:
        ptcl_str = json_pkt_obj[ptcl]
        if(-1 == ptcl_str.find(val)):
            json_pkt_obj[ptcl] = ptcl_str + '_nextpacket_' + val
    else:
        json_pkt_obj[ptcl] = val

def add_mdns_rrs(layer_obj, key):
    val = ''
    if type(layer_obj['mdns'][key]) == str:
        if ('' != layer_obj['mdns'][key]):
            val = layer_obj['mdns'][key]
        return val
    keys = layer_obj['mdns'][key].keys()
    model = ''
    for rr_key in keys:
        if not val:
            val = rr_key.replace(':', seperator).replace('.', seperator).replace(',', seperator)
        else:
            val = val + '_nextrr_' + rr_key.replace(':', seperator).replace('.', seperator).replace(',', seperator)

        if 'dns.resp.type' in layer_obj['mdns'][key][rr_key]:
            if '16' == layer_obj['mdns'][key][rr_key]['dns.resp.type']:
                txt_list = []
                for rr_val_key in layer_obj['mdns'][key][rr_key].keys():
                    txt_list.append(rr_val_key)
                txt_list.sort()
                for txt_key in txt_list:
                    if ('dns.txt' in txt_key) and ('dns.txt.length' not in txt_key):
                        #print(layer_obj['mdns'][key][rr_key][txt_key])
                        val = val + seperator + layer_obj['mdns'][key][rr_key][txt_key]
                        if 'model=' in layer_obj['mdns'][key][rr_key][txt_key]:
                            model = layer_obj['mdns'][key][rr_key][txt_key].split('=')[1]
    if model:
        val = val + '&&' + model
    return val

def add_list_val(json_obj, key, val):
    if key in json_obj:
        key_str = json_obj[key]
        ptn1 = "_" + val + "_"
        ptn2 = "_" + val + "$"
        ptn3 = "^" + val + "_"
        ptn4 = "^" + val + "$"
        if (not re.search(ptn1, key_str)) and (not re.search(ptn2, key_str)) and (not re.search(ptn3, key_str)) and (not re.search(ptn4, key_str)):
            json_obj[key] = key_str + '_nextpacket_' + val
    else:
        json_obj[key] = val

#for dhcpv6
def req_list2comma_list(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums)/2)):
        temp = nums[i*2] + nums[i*2+1]
        temp_num = int(temp, 16)
        if not new_str:
            new_str = str(temp_num)
        else:
            new_str = new_str + ',' + str(temp_num)
    return new_str

#for dhcp
def dhcp_req_list2comma_list(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums))):
        temp = nums[i]
        temp_num = int(temp, 16)
        if not new_str:
            new_str = str(temp_num)
        else:
            new_str = new_str + ',' + str(temp_num)
    return new_str

#for udp
def hex_str2char_str(raw_list):
    new_str = ''
    nums = raw_list.split(':')
    for i in range(int(len(nums))):
        temp = nums[i]
        temp_num = int(temp, 16)
        temp_chr = chr(temp_num)
        if not new_str:
            new_str = temp_chr
        else:
            new_str = new_str + temp_chr
    return new_str

def extract_mdns_dot1(raw_str):
    new_str = ''
    packet_list = raw_str.split('_nextpacket_')
    for pkt in packet_list:
        rr_list = pkt.split('_nextrr_')
        for rr in rr_list:
            if not new_str:
                new_str = rr.split(seperator)[0]
            elif (rr.split(seperator)[0] not in new_str):
                new_str = new_str + ',' + rr.split(seperator)[0]
    return new_str

def mac_gene_feature(json_mac_obj, resolved_mac, model):
    if 'feature_label' not in json_mac_obj:
        json_mac_obj['feature_label'] = {}
    json_feature = json_mac_obj['feature_label']
    json_feature['vendor'] = resolved_mac.split('_')[0]
    if 'dhcp' in json_mac_obj:
        if 'request' in json_mac_obj['dhcp']:
            if '12' in json_mac_obj['dhcp']['request']:
                json_feature['dhcp_hostname'] = json_mac_obj['dhcp']['request']['12']
            if '60' in json_mac_obj['dhcp']['request']:
                json_feature['dhcp_vendorclass'] = json_mac_obj['dhcp']['request']['60']
            if '77' in json_mac_obj['dhcp']['request']:
                mod_name = re.findall(r".*Mod=(.+?)\n.*", json_mac_obj['dhcp']['request']['77'])
                if mod_name:
                    json_feature['dhcp_userclass'] = mod_name[0]
        if 'discover' in json_mac_obj['dhcp']:
            if '12' in json_mac_obj['dhcp']['discover']:
                json_feature['dhcp_hostname'] = json_mac_obj['dhcp']['discover']['12']
            if '60' in json_mac_obj['dhcp']['discover']:
                json_feature['dhcp_vendorclass'] = json_mac_obj['dhcp']['discover']['60']
            if '77' in json_mac_obj['dhcp']['discover']:
                mod_name = re.findall(r".*Mod=(.+?)\n.*", json_mac_obj['dhcp']['discover']['77'])
                if mod_name:
                    json_feature['dhcp_userclass'] = mod_name[0]
        if ('ack' in json_mac_obj['dhcp']) or ('offer' in json_mac_obj['dhcp']):
            json_feature['dhcp_router'] = 'router'
    if 'dhcpv6' in json_mac_obj:
        if '39' in json_mac_obj['dhcpv6']:
            json_feature['dhcpv6'] = json_mac_obj['dhcpv6']['39']
    if 'mdns' in json_mac_obj:
        if 'response' in json_mac_obj['mdns']:
            if 'answer' in json_mac_obj['mdns']['response']:
                raw_q_str = json_mac_obj['mdns']['response']['answer'] #query:query, response:answer
                q_name = extract_mdns_dot1(raw_q_str)
                json_feature['mdns_name'] = q_name
        if model:
            json_feature['mdns_model'] = model
    if 'llmnr' in json_mac_obj:
        json_feature['llmnr'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['llmnr'])
    if 'browser' in json_mac_obj:
        json_feature['browser'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['browser'])
    if 'nbns' in json_mac_obj:
        json_feature['nbns'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['nbns'])
    if 'ssdp' in json_mac_obj:
        if 'search' in json_mac_obj['ssdp']:
            if 'user-agent' in json_mac_obj['ssdp']['search']:
                json_feature['ssdp'] = {}
                json_feature['ssdp']['user-agent'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['search']['user-agent'])
            #used for sony tv
            if 'X-AV-Client-Info' in json_mac_obj['ssdp']['search']:
                cn = re.findall(r".*cn=\"(.+?)\";.*", json_mac_obj['ssdp']['search']['X-AV-Client-Info'])[0]
                mn = re.findall(r".*mn=\"(.+?)\";.*", json_mac_obj['ssdp']['search']['X-AV-Client-Info'])[0]
                if cn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['clientcn'] = cn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['clientcn'] = cn
                if mn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['clientmn'] = mn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['clientmn'] = mn
        if 'notify' in json_mac_obj['ssdp']:
            if 'server' in json_mac_obj['ssdp']['notify']:
                if 'ssdp' not in json_feature:
                    json_feature['ssdp'] = {}
                    json_feature['ssdp']['server'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['notify']['server'])
                else:
                    json_ssdp_feature = json_feature['ssdp']
                    json_ssdp_feature['server'] = re.sub(r'_nextpacket_|_nextrr_', ',', json_mac_obj['ssdp']['notify']['server'])
            #used for sony tv
            if 'X-AV-Server-Info' in json_mac_obj['ssdp']['notify']:
                cn = re.findall(r".*cn=\"(.+?)\";.*", json_mac_obj['ssdp']['notify']['X-AV-Server-Info'])[0]
                mn = re.findall(r".*mn=\"(.+?)\";.*", json_mac_obj['ssdp']['notify']['X-AV-Server-Info'])[0]
                if cn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['servercn'] = cn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['servercn'] = cn
                if mn:
                    if 'ssdp' not in json_feature:
                        json_feature['ssdp'] = {}
                        json_feature['ssdp']['servermn'] = mn
                    else:
                        json_ssdp_feature = json_feature['ssdp']
                        json_ssdp_feature['servermn'] = mn
    if 'udp' in json_mac_obj:
        json_feature['udp'] = json_mac_obj['udp']

def mac_gene_priority_feature(json_root_arr, json_feature_arr):
    for mac_obj in json_root_arr:
        mac_feature_obj = find_mac_obj(json_feature_arr, mac_obj['mac'])
        #mac prefix
        mac_addr_obj['vendor'] = mac_obj['feature_label']['vendor']
        #dhcp
        if 'dhcp_hostname' in mac_obj['feature_label']:
            dhcp_hostname_obj['dhcp_hostname'] = mac_obj['feature_label']['dhcp_hostname']
        if 'dhcp_userclass' in mac_obj['feature_label']:
            dhcp_userclass_obj['dhcp_userclass'] = mac_obj['feature_label']['dhcp_userclass']
        if 'dhcp_vendorclass' in mac_obj['feature_label']:
            dhcp_vendorclass_obj['dhcp_vendorclass'] = mac_obj['feature_label']['dhcp_vendorclass']
        #dhcpv6
        if 'dhcpv6' in mac_obj['feature_label']:
            dhcpv6_obj['dhcpv6'] = mac_obj['feature_label']['dhcpv6']
        #mdns
        if 'mdns_name' in mac_obj['feature_label']:
            mdns_name_obj['mdns_name'] = mac_obj['feature_label']['mdns_name']
        if 'mdns_model' in mac_obj['feature_label']:
            mdns_model_obj['mdns_model'] = mac_obj['feature_label']['mdns_model']
        #ssdp

#read csv to dict
def csv2dict(in_file,key,value):
    new_dict = {}
    with open(in_file, 'r') as f:
        reader = csv.reader(f, delimiter=',')
        fieldnames = next(reader)
        reader = csv.DictReader(f, fieldnames=fieldnames, delimiter=',')
        for row in reader:
            new_dict[row[key]] = row[value]
    return new_dict

for root, dirs, files in os.walk(pkt_dir, False):
    if files:
        for line in files:
            if '.json' not in line:
                #file_cnt = file_cnt + 1
                file_cnt = line.split('-')[0]
		        #if (int(file_cnt) <= 34):
		        #    continue
                if (int(file_cnt) <= 666):
                    continue
                filepath = os.path.join(root, line)
                file_name = filepath

                csv_file = './support/knowledge.csv'
                output_json_name = 'train_data' + str(file_cnt) + '.json'
                output_csv_name = 'result' + str(file_cnt) +'.csv'

                #create json_root_arr
                json_root_arr = []
                json_feature_arr = []

                bootp_option_type_cnt = 0
                bootp_option_type_tree_cnt = 0
                bootp_option_request_list_item_cnt = 0
                http_request_line_cnt = 0
                dhcpv6_option_type_cnt = 0
                dns_txt_cnt = 0
                http_cnt = 0

                #remove key duplicate
                wf = open(file_name+'.json', 'w+', encoding='utf-8')
                with open(file_name,'r') as raw_f:
                    for line in raw_f.readlines():
                        if 'ip6.arpa' in line:
                            line = re.sub(r'(?<=[\"\s])[0-9A-F\.]{64}(?=ip6\.arpa)', 'ipv6,', line)
                        line = re.sub(r'((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)', 'ipv4', line)
                        if ('dhcp' not in line) and ('bootp' not in line) and ('data.data' not in line):
                            line = re.sub(r'(?<![:])([a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7})(?!:)', 'ipv6', line)
                        if 'bootp.option.type\"' in line:
                            bootp_option_type_cnt += 1
                            new_str = str(bootp_option_type_cnt) + ',bootp.option.type'
                            new_line = line.replace('bootp.option.type', new_str)
                        elif 'bootp.option.type_tree' in line:
                            bootp_option_type_tree_cnt += 1
                            new_str = str(bootp_option_type_tree_cnt) + ',bootp.option.type_tree'
                            new_line = line.replace('bootp.option.type_tree', new_str)
                        elif 'bootp.option.request_list_item' in line:
                            bootp_option_request_list_item_cnt += 1
                            new_str = 'bootp.option.request_list_item,' + str(bootp_option_request_list_item_cnt)
                            new_line = line.replace('bootp.option.request_list_item', new_str)
                        #ssdp
                        elif 'http.request.line' in line:
                            http_request_line_cnt += 1
                            new_str = 'http.request.line' + str(http_request_line_cnt)
                            new_line = line.replace('http.request.line', new_str)
                        elif 'dhcpv6.option.type' in line:
                            dhcpv6_option_type_cnt += 1
                            new_str = str(dhcpv6_option_type_cnt) + ',dhcpv6.option.type'
                            new_line = line.replace('dhcpv6.option.type', new_str)
                        elif 'dns.txt' in line:
                            dns_txt_cnt += 1
                            new_str = str(dns_txt_cnt) + ',dns.txt'
                            new_line = line.replace('dns.txt', new_str)
                        #mdns
                        elif ('http' in line) and ('class' not in line):
                            http_cnt += 1
                            new_str = str(http_cnt) + ',http'
                            new_line = re.sub(r'http(?!:)', new_str, line)
                        else:
                            new_line = line
                        wf.write(new_line)
                wf.close()

                with open(file_name+'.json', 'r') as load_f:
                    pkt_arr = json.load(load_f)

                for pkt_obj in pkt_arr:
                    model = ''

                    src_obj = pkt_obj['_source']
                    layer_obj = src_obj['layers']

                    #src mac:
                    eth_obj = layer_obj['eth']
                    mac = eth_obj['eth.src']
                    json_mac_obj = find_mac_obj(json_root_arr, mac)

                    #resolved src mac:
                    mac_tree_obj = eth_obj['eth.src_tree']
                    resolved_mac = mac_tree_obj['eth.src_resolved']

                    #protocol:
                    ptcl = extract_ptcl(layer_obj)
                    gene_ptcl_obj(json_mac_obj, ptcl)

                    if('bootp' == ptcl):
                        bootp_keys = layer_obj['bootp'].keys()
                        add_ptcl_obj(json_mac_obj, 'dhcp')
                        json_dhcp_obj = json_mac_obj['dhcp']
                        json_dhcp_type_obj = {}

                        flag_cnt = 0
                        while(flag_cnt < bootp_option_type_cnt):
                            flag_cnt += 1
                            option_key = str(flag_cnt) + ',' + 'bootp.option.type'
                            val_key = str(flag_cnt) + ',' + 'bootp.option.type_tree'
                            add_key = ''
                            add_val = ''
                            for key in bootp_keys:
                                if(option_key == key):
                                    add_key = layer_obj['bootp'][key]
                            if('53' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        add_val = layer_obj['bootp'][key]['bootp.option.dhcp']

                            #add to json
                            if('53' == add_key) and ('1' == add_val):
                                dhcp_type = 'discover'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('3' == add_val):
                                dhcp_type = 'request'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('6' == add_val):
                                dhcp_type = 'nak'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('2' == add_val):
                                dhcp_type = 'offer'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('5' == add_val):
                                dhcp_type = 'ack'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]
                            elif('53' == add_key) and ('8' == add_val):
                                dhcp_type = 'inform'
                                add_ptcl_obj(json_dhcp_obj, dhcp_type)
                                json_dhcp_type_obj = json_dhcp_obj[dhcp_type]

                        option_cnt = 0
                        series_str = ''
                        while(option_cnt < bootp_option_type_cnt):
                            option_cnt += 1
                            option_key = str(option_cnt) + ',' + 'bootp.option.type'
                            val_key = str(option_cnt) + ',' + 'bootp.option.type_tree'
                            add_key = ''
                            add_val = ''
                            for key in bootp_keys:
                                if (option_key == key):
                                    add_key = layer_obj['bootp'][key]
                                    if not series_str:
                                        series_str = add_key
                                    else:
                                        series_str = series_str + ',' + add_key
                            if('55' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        add_val = layer_obj['bootp'][key]['bootp.option.value']
                            elif('81' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key) and ('bootp.fqdn.name' in layer_obj['bootp'][key]):
                                        add_val = layer_obj['bootp'][key]['bootp.fqdn.name']
                            elif('77' == add_key):
                                for key in bootp_keys:
                                    if(val_key == key):
                                        raw_str = layer_obj['bootp'][key]['bootp.option.user_class_tree']['bootp.option.user_class.data']
                                        add_val = hex_str2char_str(raw_str)
                            elif('0' != add_key):
                                for key in bootp_keys:
                                    if (val_key == key) and (len(layer_obj['bootp'][key]) > 1):
                                        tree_keys = layer_obj['bootp'][key].keys()
                                        for tree_key in tree_keys:
                                            if('bootp.option.length' != tree_key) and ('bootp.option.value' != tree_key) and ('bootp.hw.type' != tree_key):
                                                add_val = layer_obj['bootp'][key][tree_key]
                                                if mac == add_val:
                                                    add_val = resolved_mac.split('_')[0]

                            if('55' == add_key):
                                add_val_str = dhcp_req_list2comma_list(add_val)
                                add_list_val(json_dhcp_type_obj, add_key, add_val_str)
                            elif('0' != add_key) and ('' != add_key):
                                add_ptcl_val(json_dhcp_type_obj, add_key, add_val.replace('.', ','))
                        add_ptcl_val(json_dhcp_type_obj, 'series', series_str)

                    elif('dhcpv6' == ptcl):
                        dhcpv6_keys = layer_obj['dhcpv6'].keys()
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_dhcpv6_obj = json_mac_obj[ptcl]
                        series_str = ''
                        opt_cnt = 0
                        while(opt_cnt < dhcpv6_option_type_cnt):
                            opt_cnt += 1
                            add_key = ''
                            add_val = ''
                            type_key = str(opt_cnt) + ',' + 'dhcpv6.option.type'
                            for key in dhcpv6_keys:
                                if ('dhcpv6.msgtype' != key and 'dhcpv6.xid' != key):
                                    type_trees = layer_obj['dhcpv6'][key].keys()
                                    for type_tree in type_trees:
                                        if (type_key == type_tree):
                                            add_key = layer_obj['dhcpv6'][key][type_tree]
                                            if not series_str:
                                                series_str = add_key
                                            else:
                                                series_str = series_str + ',' + add_key

                                            if('39' == add_key):
                                                add_ptcl_val(json_dhcpv6_obj, add_key, layer_obj['dhcpv6'][key]['dhcpv6.client_fqdn'].replace('.', ','))
                                            elif('6' == add_key):
                                                req_list = layer_obj['dhcpv6'][key]['dhcpv6.option.value']
                                                req_str = req_list2comma_list(req_list)
                                                add_list_val(json_dhcpv6_obj, add_key, req_str)
                                            else:
                                                add_ptcl_val(json_dhcpv6_obj, add_key, 'null')
                        add_ptcl_val(json_dhcpv6_obj, 'series', series_str)

                    if('mdns' == ptcl):
                        mdns_keys = layer_obj['mdns'].keys()
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_mdns_obj = json_mac_obj[ptcl]
                        json_mdns_type_obj = {}
                        for key in mdns_keys:
                            if('dns.flags' == key):
                                if('0x00000000' == layer_obj['mdns'][key]):
                                    mdns_type = 'query'
                                    add_ptcl_obj(json_mdns_obj, mdns_type)
                                    json_mdns_type_obj = json_mdns_obj[mdns_type]
                                elif('0x00008400' == layer_obj['mdns'][key]):
                                    mdns_type = 'response'
                                    add_ptcl_obj(json_mdns_obj, mdns_type)
                                    json_mdns_type_obj = json_mdns_obj[mdns_type]
                        for key in mdns_keys:
                            if('Queries' == key):
                                q_val = ''
                                query_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in query_val:
                                    q_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    q_val = query_val
                                add_ptcl_val(json_mdns_type_obj, 'query', q_val)
                            elif('Answers' == key):
                                a_val = ''
                                answer_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in answer_val:
                                    a_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    a_val = answer_val
                                add_ptcl_val(json_mdns_type_obj, 'answer', a_val)
                            elif('Additional records' == key):
                                add_val = ''
                                addition_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in addition_val:
                                    add_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    add_val = addition_val
                                add_ptcl_val(json_mdns_type_obj, 'additional', add_val)
                            elif('Authoritative nameservers' == key):
                                auth_val = ''
                                authentity_val = add_mdns_rrs(layer_obj, key)
                                if '&&' in authentity_val:
                                    auth_val = add_mdns_rrs(layer_obj, key).split('&&')[0]
                                    model = add_mdns_rrs(layer_obj, key).split('&&')[1]
                                else:
                                    auth_val = authentity_val
                                add_ptcl_val(json_mdns_type_obj, 'authoritative', auth_val)
                    elif('llmnr' == ptcl):
                        if isinstance(layer_obj['llmnr']['Queries'], str):
                            print(mac)
                        else:
                            llmnr_keys = layer_obj['llmnr']['Queries'].keys()
                            for key in llmnr_keys: #only one key
                                llmnr_qname = key.split(':')[0]
                            add_ptcl_val(json_mac_obj, ptcl, llmnr_qname)
                    elif('browser' == ptcl):
                        smb_srcname = layer_obj['nbdgm']['nbdgm.source_name']  #only one sourcename
                        add_ptcl_val(json_mac_obj, ptcl, smb_srcname)
                    elif('nbns' == ptcl):
                        if 'Queries' in layer_obj['nbns']:
                            if isinstance(layer_obj['nbns']['Queries'], str):
                                print(layer_obj['nbns']['Queries'])
                            else:
                                nbns_keys = layer_obj['nbns']['Queries'].keys()
                                for key in nbns_keys:
                                    nbns_qname = key.split(':')[0]
                                add_ptcl_val(json_mac_obj, ptcl, nbns_qname)
                    elif('ssdp' == ptcl):
                        ssdp_keys = layer_obj['ssdp'].keys() #key is m-search or notify
                        add_ptcl_obj(json_mac_obj, ptcl)
                        json_ssdp_obj = json_mac_obj[ptcl]
                        json_ssdp_type_obj = {}
                        request_line_list = []
                        ssdp_type = ''
                        for key in ssdp_keys:
                            if('M-SEARCH' == key.split(' ')[0]):
                                ssdp_type = 'search'
                                add_ptcl_obj(json_ssdp_obj, ssdp_type)
                                json_ssdp_type_obj = json_ssdp_obj[ssdp_type]
                            elif('NOTIFY' == key.split(' ')[0]):
                                ssdp_type = 'notify'
                                add_ptcl_obj(json_ssdp_obj, ssdp_type)
                                json_ssdp_type_obj = json_ssdp_obj[ssdp_type]
                        for key in ssdp_keys:
                            if ('search' == ssdp_type) and ('http.request.line' in key):
                                request_line_list.append(key)
                            elif ('notify' == ssdp_type) and ('http' in key):
                                request_line_list.append(key)
                        request_line_list.sort()
                        series_str = ''
                        add_key = ''
                        add_val = ''
                        for item in request_line_list:
                            if 'notify' == ssdp_type:
                                if 'http.host' in item:
                                    add_key = 'host'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.cache_control' in item:
                                    add_key = 'cache_control'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.location' in item:
                                    add_key = 'location'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.server' in item:
                                    add_key = 'server'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.request.full_uri' in item:
                                    add_key = 'full_uri'
                                    add_val = layer_obj['ssdp'][item]
                                elif 'http.unknown_header' in item:
                                    add_key = layer_obj['ssdp'][item].split(':')[0]
                                    if len(layer_obj['ssdp'][item].split(':')) > 1:
                                        add_val = layer_obj['ssdp'][item].split(':')[1]
                                    else:
                                        add_val = 'nak'
                                else:
                                    add_key = ''
                            elif 'search' == ssdp_type:
                                add_key = layer_obj['ssdp'][item].split(':')[0]
                                add_val = layer_obj['ssdp'][item].split(':')[1]

                            if add_key:
                                if (not series_str):
                                    series_str = add_key
                                else:
                                    series_str = series_str + ',' + add_key

                            if('user-agent' == add_key):
                                add_val = add_val.replace(' ', ',')
                            if add_key:
                                add_ptcl_val(json_ssdp_type_obj, add_key, add_val.replace(':', ','))
                        add_ptcl_val(json_ssdp_type_obj, 'series', series_str)
                    elif 'udp' in ptcl:
                        if 'data' in layer_obj:
                            raw_data_str = layer_obj['data']['data.data']
                            data_str = hex_str2char_str(raw_data_str)
                            device_name = re.findall(r".*DEVICE_NAME=(.+?)\n.*", data_str)
                            if device_name:
                                add_ptcl_val(json_mac_obj, 'udp', device_name[0])
                    mac_gene_feature(json_mac_obj, resolved_mac, model)
                #generate priority feature
                #mac_gene_priority_feature(json_root_arr, json_feature_arr)

                with open(output_json_name, 'w') as outfile:
                    json.dump(json_root_arr, outfile, indent=4)

                #read in knowledge csv
                vendor_know_dict = csv2dict(csv_file, 'vendor_key', 'vendor')
                type_know_dict = csv2dict(csv_file, 'type_key', 'type')
                model_know_dict = csv2dict(csv_file, 'model_key', 'model')

                #add prefix
                vendor_know_arr = []
                type_know_arr = []
                model_know_arr = []
                for vendor_key in vendor_know_dict.keys():
                    vendor_know_arr.append(vendor_key)
                    vendor_know_arr.sort(reverse = True)
                for type_key in type_know_dict.keys():
                    type_know_arr.append(type_key)
                    type_know_arr.sort(reverse = True)
                for model_key in model_know_dict.keys():
                    model_know_arr.append(model_key)
                    model_know_arr.sort(reverse = True)

                #generate csv file
                #mac|feature
                mac_list = []
                feature_list = []
                mac_prefix_list = []
                vendor_list = []
                type_list = []
                model_list = []
                for mac_obj in json_root_arr:
                    mac_list.append(mac_obj['mac'])

                    #mac_prefix
                    mac_arr = mac_obj['mac'].split(':')
                    mac_prefix = mac_arr[0]
                    mac_prefix = mac_prefix + mac_arr[1]
                    mac_prefix = mac_prefix + mac_arr[2]

                    feature_str = mac_prefix + ','
                    for key in mac_obj['feature_label'].keys():
                        if 'ssdp' == key:
                            for type_key in mac_obj['feature_label']['ssdp'].keys():
                                if not feature_str:
                                    feature_str = mac_obj['feature_label']['ssdp'][type_key].replace('\r\n', '')
                                else:
                                    feature_str = feature_str + ',' + mac_obj['feature_label']['ssdp'][type_key].replace('\r\n', '')
                        else:
                            if not feature_str:
                                feature_str = mac_obj['feature_label'][key].replace('\r\n', '')
                            else:
                                feature_str = feature_str + ',' + mac_obj['feature_label'][key].replace('\r\n', '')
                    feature_list.append(feature_str)

                    #mac_prefix label:
                    mac_prefix_list.append(mac_prefix)

                    #vendor label:
                    vendor = ''
                    for vendor_key in vendor_know_arr:
                        if vendor_key.lower() in feature_str.lower():
                            vendor = vendor_know_dict[vendor_key]
                            break
                    vendor_list.append(vendor)
                    #type label:
                    dev_type = ''
                    for type_key in type_know_arr:
                        if type_key.lower() in feature_str.lower():
                            dev_type = type_know_dict[type_key]
                            break
                        if 'router' in feature_str.lower():
                            #dev_type = vendor + '_router'
                            print(str(file_cnt) + ' : router')
                            break
                    type_list.append(dev_type)
                    #model label:
                    model = ''
                    for model_key in model_know_arr:
                        if model_key.lower() in feature_str.lower():
                            model = model_know_dict[model_key]
                            break
                    model_list.append(model)

                dataframe = pd.DataFrame({'mac':mac_list, 'feature':feature_list, 'mac_prefix':mac_prefix_list, 'vendor':vendor_list, 'type':type_list, 'model':model_list}, columns=["mac", "feature", "mac_prefix", "vendor", "type", "model"])
                dataframe.to_csv(output_csv_name, sep=',')
                #print(json_root_arr)


import pymongo
from luigi.mongo import MongoTarget
import datetime
import unittest

host = 'localhost'
port = 27017
database = 'testdatabase'
collection = 'testcollection'

class MongoTargetTest(unittest.TestCase):
    """ testing the mongo-target extension """
    def setUp(self):
        self.target = MongoTarget(database, collection)
        self.doc = { "timestamp":datetime.datetime.now() }

    def test_insert(self):
        self.target.insert(self.doc)

    def test_exists(self):
        self.target.exists()


DP = [1]
for i in range(1, 13):
    DP.append((4*i-2)*DP[i-1]//(i+1))

import sys
frst = 1
for line in sys.stdin:
    l = line.strip()
    if len(l) > 0:
        if not frst:
            print()
        else:
            frst = 0
        print(DP[int(line)])


import requests
from PIL import Image
from io import BytesIO
import re
from immkdir import mkdir
import xiangceList

url = ''
host = 'https://www.nvshens.com/'
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}

def getpage(url):  # 获取网页源码
    r = requests.get(url, headers=headers)
    r.encoding = "utf-8"
    page_code = r.text
    return page_code

def get_page_name_num() -> object:  # 获取相册页数、名字、相片数量
    page_code = getpage(url)
    reg = r"(.*?)"
    reg = re.compile(reg, re.S)
    num = re.findall(reg, page_code)
    num = num[0]
    num = int(re.findall("\d+", num)[0])
    if num % 3 == 0:
        page = int(num/3)
    else:
        page = int(num/3+1)
    reg = r"

(.*?)
"
    reg = re.compile(reg, re.S)
    name = re.findall(reg, page_code)
    name = name[0]
    if name == "":
        name = "无名"
    mkdir(name)
    print('{}的相册，有{}页，共有{}张相片。'.format(name,page,num))
    return name, num, page

def get_page_link():  # 页面地址拼接并返回拼接后的列表
    page_links = []
    page_num = get_page_name_num()
    page_num = page_num[2]
    for i in range(1, page_num+1):
        page_links.append('{}{}.html'.format(url, i))
    return page_links

def get_img_link():
    page_links = get_page_link()
    page_code = getpage(page_links[0])
    reg = r""
    reg = re.compile(reg, re.S)
    urlname = re.findall(reg, page_code)
    return urlname

def get_Img(imgLink,imgName):
    img = requests.get(imgLink, headers=headers)
    image = Image.open(BytesIO(img.content))
    image.save('D:/1/{}/{}.jpg'.format(name,imgName))
xiangceList.url1 = 'https://www.nvshens.com/gallery/qizhi/'
for j in xiangceList.get_typePageLink(3):
    url = j
    for i in get_page_link():
        url = i
        temp = get_page_name_num()
        name = temp[0]
        for j in get_img_link():
            imgUrl =j[0]
            a =imgUrl.replace('/s/','/')
            get_Img(a, j[1])
            print("正在保存‘{}’".format(j[1]))
    print('下载完毕！')