import numpy as np
import tensorflow as tf
import sys
import math
import matplotlib.pyplot as plt

import amplitf.interface as atfi
import tfa.rootio as atfr
import tfa.plotting as atfp

def create_weights_biases(n_input, layers, sigma=1.0, n_output=1):
    """
    Create arrays of weights and vectors of biases for the multilayer perceptron
    if a given configuration (with a single output neuron).
      n_input : number of input neurons
      layers  : list of numbers of neurons in the hidden layers
    output :
      weights, biases
    """
    n_hidden = [n_input] + layers
    weights = []
    biases = []
    for i in range(len(n_hidden) - 1):
        weights += [
            tf.Variable(
                sigma * np.random.normal(size=[n_hidden[i], n_hidden[i + 1]]),
                dtype=atfi.fptype(),
            )
        ]
        biases += [
            tf.Variable(
                sigma * np.random.normal(size=[n_hidden[i + 1]]), dtype=atfi.fptype()
            )
        ]
    weights += [
        tf.Variable(
            sigma * np.random.normal(size=[n_hidden[-1], n_output]), dtype=atfi.fptype()
        )
    ]
    biases += [
        tf.Variable(sigma * np.random.normal(size=[n_output]), dtype=atfi.fptype())
    ]
    return (weights, biases)

def init_weights_biases(init):
    """
    Initialise variable weights and biases from numpy array
    """
    init_weights = init[0]
    init_biases = init[1]
    weights = []
    biases = []
    for i in range(len(init_weights) - 1):
        weights += [tf.Variable(init_weights[i], dtype=atfi.fptype())]
        biases += [tf.Variable(init_biases[i], dtype=atfi.fptype())]
    weights += [tf.Variable(init_weights[-1], dtype=atfi.fptype())]
    biases += [tf.Variable(init_biases[-1], dtype=atfi.fptype())]
    return (weights, biases)

def init_fixed_weights_biases(init):
    """
    Initialise constant weights and biases from numpy array
    """
    init_weights = init[0]
    init_biases = init[1]
    weights = []
    biases = []
    for i in range(len(init_weights) - 1):
        weights += [tf.constant(init_weights[i], dtype=atfi.fptype())]
        biases += [tf.constant(init_biases[i], dtype=atfi.fptype())]
    weights += [tf.constant(init_weights[-1], dtype=atfi.fptype())]
    biases += [tf.constant(init_biases[-1], dtype=atfi.fptype())]
    return (weights, biases)

def normalise(array, ranges):
    v = [
        (array[:, i] - atfi.const(ranges[i][0])) / (atfi.const(ranges[i][1]) - atfi.const(ranges[i][0]))
        for i in range(len(ranges))
    ]
    return tf.stack(v, axis=1)

@tf.function
def multilayer_perceptron(x, ranges, weights, biases, multiple=False):
    """
    Multilayer perceptron with fully connected layers defined by matrices of weights and biases.
    Use sigmoid function as activation.
    """
    layer = normalise(x, ranges)
    for i in range(len(weights)):
        layer = tf.nn.sigmoid(tf.add(tf.matmul(layer, weights[i]), biases[i]))
    if multiple:
        return layer
    else:
        return layer[:, 0]

def l2_regularisation(weights):
    """
    L2 regularisation term for a list of weight matrices
    """
    penalty = 0.0
    for w in weights:
        penalty += tf.reduce_sum(tf.square(w))
    return penalty

def estimate_density(
    phsp,
    data,
    ranges,
    labels,
    weight=None,
    transform = None,
    transform_ranges = None,
    learning_rate=0.001,
    training_epochs=100000,
    norm_size=1000000,
    print_step=50,
    display_step=500,
    weight_penalty=1.,
    n_hidden=[32, 8],
    initfile="init.npy",
    outfile="train",
    seed=1,
    fig = None,
    axes = None,
    units = None,
    model = None,
    regularisation = None,
):

    n_input = len(ranges)

    bins = n_input*[ 50 ]

    atfi.set_seed(seed)

    if model is None :
        try:
            init_w = np.load(initfile, allow_pickle = True)
        except:
            init_w = None

        if isinstance(init_w, np.ndarray):
            print("Loading saved weights")
            (weights, biases) = init_weights_biases(init_w[2:])
        else:
            print("Creating random weights")
            (weights, biases) = create_weights_biases(n_input, n_hidden)

        if not transform_ranges : transform_ranges = ranges

        def fitmodel(x):
            if transform : x2 = transform(x)
            else : x2 = x
            # to make sure PDF is always strictly positive
            return multilayer_perceptron(x2, transform_ranges, weights, biases) + 1e-20

        parameters = [weights, biases]
    else :
        fitmodel, parameters = model

    @tf.function
    def unbinned_nll(pdf, integral) :
        return -tf.reduce_sum(atfi.log(pdf/integral))

    @tf.function
    def unbinned_weighted_nll(pdf, integral, w) :
        return -tf.reduce_sum(atfi.log(pdf/integral)*w)

    @tf.function
    def integral(pdf) :
        return tf.reduce_mean(pdf)

    opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)

    data_inside = phsp.inside(data)
    data_sample = atfi.cast_real(data[data_inside])
    if weight is not None :
        weight_sample = weight[data_inside]
    else :
        weight_sample = None

    if isinstance(norm_size, int) :
        norm_sample = phsp.uniform_sample(norm_size)
    else :
        norm_sample = norm_size
    print("Normalisation sample size = ", len(norm_sample))
    print(norm_sample)
    print("Data sample size = ", len(data_sample))
    print(data_sample)

    if weight_sample is None :
        @tf.function
        def raw_nll() :
            return unbinned_nll(fitmodel(data_sample), integral(fitmodel(norm_sample)))
    else :
        @tf.function
        def raw_nll() :
            return unbinned_weighted_nll(fitmodel(data_sample), integral(fitmodel(norm_sample)), weight_sample)

    if model is None :
        # Define loss and optimizer
        if regularisation is None :
            @tf.function
            def nll() :
                return raw_nll() + l2_regularisation(weights)*weight_penalty
        else :
            @tf.function
            def nll() :
                return raw_nll() + regularisation(weights)
    else :
        @tf.function
        def nll() :
            return raw_nll()

    # Training cycle
    best_cost = 1e10

    display = atfp.MultidimDisplay(
        data_sample,
        norm_sample,
        bins,
        ranges,
        labels,
        fig,
        axes,
        units = units,
        dataweights = weight_sample
    )
    plt.ion()
    plt.show()
    plt.pause(0.1)

    for epoch in range(training_epochs):

        if epoch % display_step == 0 and fig :
            w = fitmodel(norm_sample)
            display.draw(w)
            plt.tight_layout(pad=0., w_pad=0., h_pad=0.)
            plt.draw()
            plt.pause(0.1)
            plt.savefig(outfile + ".pdf")

        if epoch % print_step == 0 :
            c = nll()
            s = "Epoch %d, cost %.9f" % (epoch+1, c)
            print(s)
            if c < best_cost:
                best_cost = c
                w = fitmodel(norm_sample)
                scale = 1./np.mean(w)
                if model is None :
                    weights = [w.numpy() for w in parameters[0]]
                    biases =  [b.numpy() for b in parameters[1]]
                    np.save(outfile, [ scale, transform_ranges, weights, biases] )
                else :
                    np.save(outfile, [ scale, transform_ranges] + [p.numpy() for p in parameters() ] )
                f = open(outfile + ".txt", "w")
                f.write(s + "\n")
                f.close()

        opt.minimize(nll, parameters)

    print("Optimization Finished!")

def estimate_density_old(
    phsp,
    data,
    ranges,
    labels,
    weight=None,
    transform=None,
    transform_ranges=None,
    learning_rate=0.001,
    training_epochs=100000,
    norm_size=1000000,
    print_step=50,
    display_step=500,
    weight_penalty=1.0,
    n_hidden=[32, 8],
    initfile="init.npy",
    outfile="train",
    seed=1,
    fig=None,
    axes=None,
):

    tf.compat.v1.disable_eager_execution()

    n_input = len(ranges)

    bins = n_input * [50]

    #tf.compat.v1.set_random_seed(seed)
    #np.random.seed(seed + 12345)

    try:
        init_w = np.load(initfile, allow_pickle=True)
    except:
        init_w = None

    if isinstance(init_w, np.ndarray):
        print("Loading saved weights")
        (weights, biases) = init_weights_biases(init_w)
    else:
        print("Creating random weights")
        (weights, biases) = create_weights_biases(n_input, n_hidden)

    data_ph = tf.compat.v1.placeholder(atfi.fptype(), shape=(None, None), name="data")
    norm_ph = tf.compat.v1.placeholder(atfi.fptype(), shape=(None, None), name="norm")

    if not transform_ranges:
        transform_ranges = ranges

    def model(x):
        if transform:
            x2 = transform(x)
        else:
            x2 = x
        # to make sure PDF is always strictly positive
        return multilayer_perceptron(x2, transform_ranges, weights, biases) + 1e-20

    data_model = model(data_ph)
    norm_model = model(norm_ph)

    def unbinned_nll(pdf, integral):
        return -tf.reduce_sum(atfi.log(pdf / integral))

    def integral(pdf):
        return tf.reduce_mean(pdf)

    # Define loss and optimizer
    nll = (
        unbinned_nll(data_model, integral(norm_model))
        + l2_regularisation(weights) * weight_penalty
    )

    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)
    train_op = optimizer.minimize(nll)

    # Initializing the variables
    init = tf.compat.v1.global_variables_initializer()

    with tf.compat.v1.Session() as sess:
        sess.run(init)

        data_sample = sess.run(phsp.filter(data_ph), feed_dict={data_ph: data})

        norm_sample = sess.run(phsp.uniform_sample(norm_size))
        print("Normalisation sample size = ", len(norm_sample))
        print(norm_sample[:,0])
        print(norm_sample[:,1])
        print(norm_sample[:,2])
        print(norm_sample[:,3])
        print("Data sample size = ", len(data_sample))
        print(data_sample)

        # Training cycle
        best_cost = 1e10

        display = atfp.MultidimDisplay(
            data_sample, norm_sample, bins, ranges, labels, fig, axes
        )
        plt.ion()
        plt.show()
        plt.pause(0.1)

        for epoch in range(training_epochs):

            _, c = sess.run(
                [train_op, nll], feed_dict={data_ph: data_sample, norm_ph: norm_sample}
            )

            if epoch % display_step == 0 and fig:
                w = sess.run(norm_model, feed_dict={norm_ph: norm_sample})
                display.draw(w)
                plt.tight_layout(pad=1.0, w_pad=1.0, h_pad=1.0)
                plt.draw()
                plt.pause(0.1)
                plt.savefig(outfile + ".pdf")

            if epoch % print_step == 0:
                s = "Epoch %d, cost %.9f" % (epoch + 1, c)
                print(s)
                if c < best_cost:
                    best_cost = c
                    w = sess.run(norm_model, feed_dict={norm_ph: norm_sample})
                    scale = 1.0 / np.mean(w)
                    np.save(
                        outfile, [scale, transform_ranges] + sess.run([weights, biases])
                    )
                    f = open(outfile + ".txt", "w")
                    f.write(s + "\n")
                    f.close()

        print("Optimization Finished!")


#shoaling
import numpy as np
from numpy.lib import scimath as SM
from scipy.optimize import fsolve
import matplotlib.pyplot as plt
from scipy.special import gamma
from scipy.interpolate import interp1d
from scipy.integrate import dblquad, quad, simps
from scipy import integrate
from matplotlib import cm
from help_tools import plotting_interface

class Pol2Cart:

    def __init__(self, r, theta, x, y):
        '''
        Based on uniform axis defined in x and y the
        '''
        self.x_mesh, self.y_mesh = np.meshgrid(x, y, indexing='ij')
        self.Nx, self.Ny = self.x_mesh.shape
        self.r_mesh, self.theta_mesh = np.meshgrid(r, theta, indexing='ij')
        x_pol = (self.r_mesh*np.cos(self.theta_mesh)).flatten()
        y_pol = (self.r_mesh*np.sin(self.theta_mesh)).flatten()
        if np.max(x_pol)>np.max(x):
            print('Out of range along positive x axis')
        if np.min(x_pol)<np.min(x):
            print('Out of range along negative x axis')
        self.x_cart_indices = np.zeros(len(x_pol), dtype=int)
        self.y_cart_indices = np.zeros(len(y_pol), dtype=int)
        if np.max(y_pol)>np.max(y):
            print('Out of range along positive y axis')
        if np.min(y_pol)<np.min(y):
            print('Out of range along negative y axis')
        for i in range(0, len(x_pol)):
            self.x_cart_indices[i] = np.argmin(np.abs(x_pol[i] - x))
            self.y_cart_indices[i] = np.argmin(np.abs(y_pol[i] - y))

    def transform(self, A):
        if A.shape != self.r_mesh.shape:
            print('input format does not have the right shape')
        else:
            A_out = np.zeros((self.Nx, self.Ny), dtype=A.dtype)
            A_flat = A.flatten()
            for i in range(0, A_flat.size):
                mapped_i = self.x_cart_indices[i]
                mapped_j = self.y_cart_indices[i]
                A_out[mapped_i, mapped_j] = A_flat[i]
        return A_out

class Bathymetry:
    def __init__(self, x, y):
        dy = y[1] - y[0]
        y1 = np.arange(400, 500 + dy, dy)
        y2 = np.arange(550, 700 + dy, dy)
        y3 = np.arange(750, 1100 + dy, dy)
        y_u = np.block([y1, y2, y3])
        self.x = x
        self.y = y
        self.Nx = len(self.x)
        self.Ny = len(self.y)
        bathy1 = -2 * (y_u<=500)
        bathy2 = (-0.1*y_u+50)*(np.logical_and(y_u>=550, y_u<= 700))
        bathy3 = -25*(y_u>=750)
        bathy = bathy1 + bathy2 + bathy3
        h_func = interp1d(y_u, bathy, kind='cubic')
        self.h = h_func(y)
        self.H = -np.outer(np.ones(len(x)), self.h)

    def plot1d(self):
        plt.figure()
        plt.plot(self.y, self.h, 'x')

    def plot2d(self):
        X, Y= np.meshgrid(self.x, self.y, indexing='ij')
        fig = plt.figure(figsize=(14,8))
        ax = fig.add_subplot(111, projection='3d')
        surf = ax.plot_surface(X, Y, self.H, linewidth=0, antialiased=False)
        fig.colorbar(surf, shrink=0.5, aspect=5)

    def calc_wavenumber(self, f_r):
        N_f = len(f_r)
        k_out = np.zeros((N_f, len(self.x), len(self.y)))
        for i in range(N_f):
            w = 2*np.pi*f_r[i,0]
            ki = w**2/(9.81)
            wt = np.sqrt(9.81*ki*np.tanh(ki*(-self.h)))
            eps = 10**(-6)
            N_max = 100
            count=0
            while np.max(np.abs(w - wt))>eps and count<N_max:
                latter = 9.81*np.tanh(ki*(-self.h))
                ki = w**2/(latter)
                wt = np.sqrt(latter)
                count = count + 1
            k_out[i,:,:] = np.outer(np.ones(self.Nx), ki)
        return k_out

class DirectionalSpectrum:
    def __init__(self, Tp, theta_p, gam, c, F):
        self.Tp = Tp
        self.fp = 1./Tp
        self.theta_p = theta_p
        self.gam = gam
        g = 9.81
        U = lambda UU: 3.5*(g/UU)*(g/UU**2*F)**(-0.33)-self.fp
        self.U10 = fsolve(U, 10,  xtol=1e-04)[0]
        self.xxn = g/self.U10**2*F
        self.S = lambda f:(0.076*self.xxn**(-0.22)*g**2/(2*np.pi)**4*(f)**(-5)*np.exp(-5/4*(self.fp/f)**4)
         *gam**np.exp(-((f-self.fp)**2)/(2*(self.fp*(0.07*(1/2 + 1/2*np.sign(self.fp - f))
        +0.09*(1/2 -1/2*np.sign(self.fp - f))))**2)))

        self.s = lambda f:((c*(2*np.pi*self.fp*self.U10/g)**(-2.5)*(f/self.fp)**5)*(1/2 + 1/2*np.sign(self.fp - f)) +
    (c*(2*np.pi*self.fp*self.U10/g)**(-2.5)*(f/self.fp)**(-2.5))*(1/2 -1/2*np.sign(self.fp - f)))
        self.D = lambda f, theta: (2**(2*self.s(f)-1)/np.pi*(gamma(self.s(f)+1))**2./gamma(2*self.s(f)+1)*(np.abs(np.cos((theta-theta_p)/2)))**(2*self.s(f)))
        self.Sdir=lambda f, theta: (self.S(f)*self.D(f,theta))

    def plot_s(self, f_min, f_max, N=200):
        f = np.linspace(f_min, f_max, N)
        plt.figure()
        plt.plot(f, self.s(f))
        plt.xlabel(r'$f~[Hz]$')
        plt.ylabel(r'$\mathrm{s}(f)$')

    def plot_S(self, f_min, f_max, N=200):
        f = np.linspace(f_min, f_max, N)
        plt.figure()
        plt.plot(f, self.S(f))
        plt.xlabel(r'$f~[Hz]$')
        plt.ylabel(r'$\mathrm{S}(f)$')

    def plot_D(self, f_min, f_max, theta_min, theta_max, N_f=200, N_theta=100):
        f_1d = np.linspace(f_min, f_max, N_f)
        theta_1d = np.linspace(theta_min, theta_max, N_theta)
        f, theta = np.meshgrid(f_1d, theta_1d, indexing='ij')
        plt.figure()
        plt.contour(f, theta, self.D(f, theta), 10)
        plt.xlabel(r'$f~[Hz]$')
        plt.ylabel(r'$\theta$')

    def seed_f(self, f_min, f_max, N_f, plot_it=False):
        #'''
        f = []
        while len(f)<N_f:
            fi = f_min + (f_max - f_min) * np.random.uniform()
            eta = self.S(self.fp) * np.random.uniform() + 1
            if np.sqrt(eta) < np.sqrt(self.S(fi)) + 1:
                f.append(fi)
        #'''

        f = np.sort(f)
        if plot_it:
            plt.figure()
            plt.plot(f, self.S(f), 'x')
            plt.xlabel(r'$f~[Hz]$')
            plt.ylabel(r'$\mathrm{S}(f)$')
            plt.show()
        return f

    def seed_f_theta(self, f_min, f_max, theta_min, theta_max, N_f, N_theta, plot_it=True):
        f = self.seed_f(f_min, f_max, N_f)
        print('f seeded')
        Theta = np.zeros((N_f, N_theta))
        ff = np.zeros((N_f, N_theta))
        for i in range(0, N_f):
            theta = []
            #'''
            while len(theta)<N_theta:
                thetai = theta_min + (theta_max - theta_min) * np.random.uniform()
                eta = self.D(self.fp, self.theta_p) * np.random.uniform()
                if eta < self.D(f[i], thetai):
                    theta.append(thetai)
            #'''
            #theta = theta_min + (theta_max - theta_min) * np.random.uniform(size=N_theta)
            Theta[i,:] = np.sort(theta)
            ff[i,:] = f[i]*np.ones(N_theta)

        if plot_it:
            fig = plt.figure(figsize=(14,8))
            ax = fig.add_subplot(111, projection='3d')
            xxx = np.arange(0, N_f)
            yyy = np.arange(0, N_theta)
            bla, blu = np.meshgrid(xxx, yyy, indexing='ij')
            surf = ax.plot_surface(bla, blu, Theta, cmap=cm.coolwarm, linewidth=0, antialiased=True)
        return ff, Theta

    def define_realization(self, f_min, f_max, theta_min, theta_max, N_f, N_theta, plot_it=True):
        f_r, Theta_r = self.seed_f_theta(f_min, f_max, theta_min, theta_max, N_f, N_theta)

        a = np.zeros((N_f, N_theta))
        '''
        for i in range(1, N_f-1):
            for j in range(1, N_theta-1):
                #a[i,j] = np.sqrt(2*integrate.nquad(self.Sdir, [[(f_r[i,j]+f_r[i-1,j])/2,(f_r[i,j]+f_r[i+1,j])/2],[(Theta_r[i,j]+Theta_r[i,j-1])/2,(Theta_r[i,j]+Theta_r[i,j+1])/2]]))[0]
                a[i,j] = np.sqrt(2*self.Sdir(f_r[i,j], Theta_r[i,j]) * ((f_r[i,j])**2-(f_r[i-1,j])**2)*(Theta_r[i,j]-Theta_r[i,j-1])/3)

        a[:,0] = a[:,1]
        a[0,:] = a[1,:]
        a[-1,:] = a[-2,:]
        a[:,-1] = a[:,-2]
        '''
        df = np.gradient(f_r, axis=0)
        dTheta = np.gradient(Theta_r, axis=1)
        dA = df*dTheta
        a = np.sqrt(2*self.Sdir(f_r, Theta_r) *dA)
        if plot_it:
            fig = plt.figure(figsize=(14,8))
            ax = fig.add_subplot(111, projection='3d')
            xxx = np.arange(0, N_f)
            yyy = np.arange(0, N_theta)
            bla, blu = np.meshgrid(xxx, yyy, indexing='ij')
            surf = ax.plot_surface(bla, blu, a, cmap=cm.coolwarm, linewidth=0, antialiased=True)
        return f_r, Theta_r, a

class SpectralRealization:

    def __init__(self, DirSpec, f_min, f_max, theta_min, theta_max, N_f, N_theta):
        self.DirSpec = DirSpec
        self.f_min = f_min
        self.f_max = f_max
        self.theta_min = theta_min
        self.theta_max = theta_max
        self.N_f = N_f
        self.N_theta = N_theta
        self.f_r, self.Theta_r, self.a = DirSpec.define_realization(f_min, f_max, theta_min, theta_max, N_f, N_theta)

    def calc_wavenumber(self, Nx, Ny, bathy=None, h=1000, plot_it=False):
        '''
        function to calculate the wave number for f_r
        Parameters:
        -----------
            bathy       : optional Bathymetry object
                          default: None; if provided defines the bathymetry for the calculation
            h           : float
                          default: 1000 meters; is used when bathymetry not present; defines a constant waterdepth
        '''
        if bathy==None:
            k_loc_f = fsolve((lambda k: ((9.81*k*np.tanh(k*h)) - (2*np.pi*self.f_r[:,0])**2)), 0.01*np.ones(self.N_f))
            k_loc = np.outer(k_loc_f, np.ones(Nx*Ny)).reshape((self.N_f, Nx, Ny))
        else:
            k_loc = bathy.calc_wavenumber(self.f_r)

        if plot_it:
            plt.figure()
            plt.plot(k_loc[:,0,0], self.f_r[:,0])
            if bathy!=None:
                for i in range(1,bathy.Ny):
                    plt.plot(k_loc[i,:,0], self.f_r[:,0])
        return k_loc

    def invert(self, bathy, ti, x, y, h=1000, plot_it=False):
        # es sollte hier 2 Methoden geben: das argument des cos zu kartesischen konvertieren
        # man könnte auch einfach auf dem kart. System arbeiten (uniform) und dann nur die richtigen as dran multiplizieren. Dann bräuchte man vorher sicher auch die integrale nicht?
        # natürlich könnte man auch einfach zu Polarkoordinaten transferieren
        #man braucht auch noch ein omega-grid... wie kann man dann da die echten omega dranmachen?
        #eta = np.dot(A_cart, np.outer(np.cos(w*t[i]-kx_mesh*)))
        # TODO different option for bathymetry or const depth
        Nx = len(x)
        Ny = len(y)
        k = self.calc_wavenumber(Nx, Ny, bathy)
        print('wavenumber calculated')
        w = 2*np.pi*self.f_r

        X, Y = np.meshgrid(x, y, indexing='ij')
        theta = self.Theta_r
        dy = np.gradient(y)

        if not bathy is  None:
            H = bathy.H
        else:
            H = h

        eta = np.zeros((Nx, Ny))
        #phase = np.random.uniform(size=self.N_f)*np.pi*2
        print('before loop')
        for i in range(0, self.N_f):
            k2H_by_sinh_2kH = np.where(k[i,:,:]*H < 50,  2*k[i,:,:]*H / np.sinh(2*k[i,:,:]*H), 0)
            for j in range(0, self.N_theta):
                thetaxy = np.abs(np.arcsin(np.sin(theta[i,j])*k[i,-1,-1]/k[i,:,:]))
                ky = k[i,:,:] * np.cos(thetaxy)
                ksh = np.cumsum(ky[0,:])*dy
                Cgx =  w[i,j]/(2*k[i,:,:] * (1+k2H_by_sinh_2kH))*(np.cos(thetaxy))
                Cg0x = w[i,j]/(2*k[i,-1,-1] * (1+k2H_by_sinh_2kH[-1,-1]))*(np.cos(theta[i,j]))
                phase = np.random.uniform()*np.pi*2
                eta += self.a[i,j]* np.abs(SM.sqrt(Cg0x/Cgx))*np.cos(phase+w[i, j]*ti-k[i,-1,-1]*np.sin(theta[i,j])*X
                +np.outer(np.ones(Nx), ksh))
        if plot_it:
            plotting_interface.plot_3d_as_2d(x, y, eta)
            plt.colorbar()
            plt.xlabel('$x~[\mathrm{m}]$')
            plt.ylabel('$y~[\mathrm{m}]$')
            plt.figure()
            plt.plot(x, eta[:,150])
            plt.xlabel('$x~[\mathrm{m}]$')
            plt.ylabel('$\eta~[\mathrm{m}]$')
            plt.figure()
            plt.plot(y, eta[400,:])
            plt.xlabel('$y~[\mathrm{m}]$')
            plt.ylabel('$\eta~[\mathrm{m}]$')
        return eta

if __name__=='__main__':

    dx = 2
    dy = dx
    x = np.arange(-500, 500 + dx, dx)
    y = np.arange(400, 1100 + dy, dy)
    g = 9.81
    Tp = 10
    fp = 1./Tp
    theta_p = -5*np.pi/180
    gam = 3.3
    N_f = 100
    f_min = 0.001
    f_max = 0.4
    N_theta = 40
    theta_min = -np.pi
    theta_max = np.pi
    c = 50
    F = 300000

    DirSpec = DirectionalSpectrum(Tp, theta_p, gam, c, F)
    #DirSpec.plot_S(f_min, f_max)
    #DirSpec.plot_s(f_min, f_max)
    #DirSpec.plot_D(f_min, f_max, theta_min, theta_max)
    realization = SpectralRealization(DirSpec, f_min, f_max, theta_min, theta_max, N_f, N_theta)
    print('Directional Spectrum defined')
    b = Bathymetry(x, y)
    print('Bathymetry defined')
    #b.plot1d()
    #b.plot2d()
    eta = realization.invert(b, 0, x, y, plot_it=True)

    plt.show()


#!/usr/bin/env python3
"""
Created on Wed Dec 25 2020

@author: beizishaozi
"""

import logging
import sys
import shutil
import re

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

import os
from libs.modules.BaseModule import BaseModule

logging.basicConfig(stream=sys.stdout, format="%(levelname)s: %(message)s", level=logging.INFO)
log = logging.getLogger(__name__)

class AppsGeyser(BaseModule):
    def doSigCheck(self):
        if self.host_os == "android":
            return self._find_main_activity('com.appsgeyser.multiTabApp.MainNavigationActivity')
        elif self.host_os == "ios":
            log.error("not support yet.")
            return False

        return False

    def doExtract(self, working_folder):
        extract_folder = self._format_working_folder(working_folder)
        if os.access(extract_folder, os.R_OK):
            shutil.rmtree(extract_folder)
        os.makedirs(extract_folder, exist_ok = True)
        tmp_folder = os.path.join(os.getcwd(), extract_folder, "tmp")
        os.makedirs(tmp_folder, exist_ok = True)
        self._apktool(tmp_folder)

        launch_path = ""
        for dirpath, dirnames, ifilenames in os.walk(tmp_folder):
            #audience_network.dex文件是框架自带的，经过apktool反编译之后在assets下会有一个文件夹，这部分信息没有必要提取
            if dirpath.find("audience_network") != -1:
                continue
            if dirpath.find("assets") != -1:   # store web resource
                for fs in ifilenames:
                    #这三个文件是框架自带的，没有必要提取
                    if fs == "audience_network.dex" or fs == "splash_screen.png" or fs == "user_custom_script.js":
                        continue
                    f = os.path.join(dirpath, fs)
                    matchObj = re.match(r'(.*)assets/(.*)', f, re.S)
                    newRP = matchObj.group(2)

                    tf = os.path.join(extract_folder, newRP)
                    if not os.access(os.path.dirname(tf), os.R_OK):
                        os.makedirs(os.path.dirname(tf))
                    with open(tf, "wb") as fwh:  #output the plain
                        # ugly coding
                        fp = open(os.path.join(dirpath, fs), "rb")
                        c = fp.read()
                        fp.close()
                        fwh.write(c)
                    fwh.close()
            elif dirpath.endswith("res/raw") != -1:
                for fs in ifilenames:
                    if fs != "configuration.xml":  #在configuration.xml里配置了起始页链接
                        continue
                    t = ET.ElementTree(file=os.path.join(dirpath, fs))
                    for elem in t.iter(tag='fullScreenMode'):
                        for subelem in elem.iter(tag='content'):
                            for linkelem in subelem.iter(tag='link'):
                                launch_path = linkelem.text

        self._dump_info(extract_folder, launch_path)
        # clean env
        shutil.rmtree(tmp_folder)
        return extract_folder, launch_path

def main():
    f = "./test_case/AppsGeyser/examplehtmlzip.apk"    #后续会将当前脚本路径与之相拼接，得到最终detect_file路径
    appsgeyser = AppsGeyser(f, "android")
    if appsgeyser.doSigCheck():
        logging.info("Andromo signature Match")

        extract_folder, launch_path = appsgeyser.doExtract("working_folder")
        log.info("{} is extracted to {}, the start page is {}".format(f, extract_folder, launch_path))

    return

if __name__ == "__main__":
    sys.exit(main())


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

def solution(answers):
    order = []
    points = [0, 0, 0]
    patterns = [
        [1, 2, 3, 4, 5],
        [2, 1, 2, 3, 2, 4, 2, 5],
        [3, 3, 1, 1, 2, 2, 4, 4, 5, 5]
    ]

    for index, answer in enumerate(answers):
        if answer == patterns[0][index % 5]: points[0] += 1
        if answer == patterns[1][index % 8]: points[1] += 1
        if answer == patterns[2][index % 10]: points[2] += 1

    for index, point in enumerate(points, 1):
        if point == max(points):
            order.append(index)

    return order


import jsonlines
import os

# data_path = "D:\IUSTIN\DoinaData\data_splits\humanitarian_orig"
# in_file1 = "train_bt_en-de-en.jsonl"
# in_file2 = "train_bt_en-de-fr-en.jsonl"
# out_file = "train_bt.jsonl"

data_path = r"D:/IUSTIN/DoinaData/data_splits/unlabeled"
in_file1 = "unlabeled_bt_en-de-en.jsonl"
in_file2 = "unlabeled_bt_en-de-fr-en.jsonl"
out_file = "unlabeled_bt.jsonl"

def main():

    with jsonlines.open(os.path.join(data_path, in_file1)) as f1, \
        jsonlines.open(os.path.join(data_path, in_file2)) as f2, \
        jsonlines.open(os.path.join(data_path, out_file), 'w') as f_out:
            for obj1, obj2 in zip(f1, f2):
                assert obj1['id'] == obj2['id']
                obj = {**obj1, **obj2}
                f_out.write(obj)

if __name__ == "__main__":
    main()


import torch
import torch.nn as nn
import torch.optim as optim
from src.utils.images import load_traning_images
from torch import device, cuda
from src.models.gender_classifier import GenderClassifier
from src.core.config import settings

def train_model():
    device('gpu') if cuda.is_available() else device('cpu')
    images, labels = load_traning_images()
    dataset = torch.utils.data.TensorDataset(images, labels)
    data_loader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)
    model = GenderClassifier()
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=0.001)
    num_epochs = settings.NUM_EPOCH
    print(type(num_epochs))
    for epoch in range(num_epochs):
        running_loss = 0.0
        for inputs, labels in data_loader:
            optimizer.zero_grad()
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            running_loss += loss.item() * inputs.size(0)
        epoch_loss = running_loss / len(dataset)
        print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, epoch_loss))
    return model


import numpy as np
import pandas as pd
import ccm
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import LassoCV
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import mutual_info_classif
from sklearn.feature_selection import SelectFromModel
def mutual_mutual(data,label,k=200):
    model_mutual= SelectKBest(mutual_info_classif, k=k)
    new_data=model_mutual.fit_transform(data, label)
    mask=model_mutual.get_support(indices=True)
    return new_data,mask
def elasticNet(data,label,alpha =np.array([0.01]),l1_ratio=0.1):
    enet=ElasticNet(alpha=alpha, l1_ratio=l1_ratio)
    enet.fit(data,label)
    mask_ = enet.coef_
    mask=np.nonzero(mask_)
    new_data = data[:,mask]
    return new_data,mask
def lassodimension(data,label,alpha=np.array([0.01,0.02,0.03])):
    lassocv=LassoCV(cv=5, alphas=alpha).fit(data, label)
    x_lasso = lassocv.fit(data,label)
    mask = np.nonzero(x_lasso.coef_)
    new_data = data[:,mask]
    return new_data,mask
def ET(data,label):
    clf=ExtraTreesClassifier(n_estimators=200)
    clf=clf.fit(data,label)
    model=SelectFromModel(clf, prefit=True)
    X_new=model.transform(data)
    return X_new
def CCM_feature(X_res,y_res):
    epsilon = 0.1; num_features = 200; type_Y = 'binary'
    rank = ccm.ccm(X_res, y_res, num_features, type_Y, epsilon, iterations = 50, verbose = True)
    selected_feats = np.argsort(rank)[:200]
    X=X_res[:,selected_feats]
    return X


import redis

from flask import Flask, session, request, abort, jsonify

app = Flask(__name__)

app.config.from_object('config')

@app.route('/<key>')
def index(key):
    session.permament = True
    if 'value' not in request.args:
        value = app.config['CACHE'].get(key)
        if not value:
            abort(404)
        return jsonify({'value': value })

    app.config['CACHE'].set(key, request.args.get('value'))
    return jsonify({'success': True})


__author__ = 'Andrew J. Lambert, andy@andyroid.co.uk'

from pybrain.structure.modules.module import Module
from pybrain.structure.networks.recurrent import RecurrentNetwork
from pygfnn.structure.modules.gfnn import GFNNLayer
from pygfnn.structure.connections.gfnn import GFNNIntConnection

import numpy as np

class GFNN(RecurrentNetwork):

    fs = None
    dt = None

    def __init__(self, name=None, fs=40., *args, **kwargs):
        kwargs['name'] = name
        RecurrentNetwork.__init__(self, *args, **kwargs)
        self.setArgs(fs=np.float32(fs), dt=np.float32(1./fs))

    def addModule(self, m):
        super(RecurrentNetwork, self).addModule(m)
        if isinstance(m, GFNNLayer):
            m.fs = self.fs
            m.dt = self.dt

    def reset(self, randomizeOscs=True, randomizeConnPhase=True):
        """Reset all component modules and the network."""
        Module.reset(self)
        for m in self.modules:
            if isinstance(m, GFNNLayer):
                m.reset(randomizeOscs)
            else:
                m.reset()
        for c in self.recurrentConns:
            if isinstance(c, GFNNIntConnection):
                c.reset(randomizeConnPhase)


import pickle
import socket
from datetime import timedelta, datetime
from dnslib import DNSError, DNSRecord

class Packet:
    def __init__(self, rr, create_time):
        self.resource_record = rr  # ресурсная запись
        self.create_time = create_time  # время создания записи

#  загркзка кэша из файла
def load_history():
    try:
        with open('data.pickle', 'rb') as f:
            database = pickle.load(f)
        print('history loaded')
    except:
        print('history not exist')
        return {}
    return database

#  дампинг кеша в файл
def save_history(data):
    try:
        with open('data.pickle', 'wb') as f:
            pickle.dump(data, f)
        print('dumping successful')
    except:
        print('dumping error')

#  проверка записи, истёк ttl или нет
def is_expired(packet):
    return datetime.now() - packet.create_time > timedelta(seconds=packet.resource_record.ttl)

#  очистка просроченного кеша
def clear_outdated_cash():
    cache_delta = 0
    for key, value in database.items():
        old_length = len(value)
        database[key] = set(packet for packet in value if not is_expired(packet))
        cache_delta += old_length - len(database[key])
    if cache_delta > 0:
        print(str(datetime.now()) + " - cleared " + str(cache_delta) + " resource records")

#  добавление/обновление записи в кеше
def add_record(rr, date_time):
    k = (str(rr.rname).lower(), rr.rtype)
    if k in database:
        database[k].add(Packet(rr, date_time))
    else:
        database[k] = {Packet(rr, date_time)}

#  извлечение ресурсных записей с ответом, записей с указанием на уполномоченный сервер, записей с дополнительной информацией
def add_records(dns_record):
    for r in dns_record.rr + dns_record.auth + dns_record.ar:
        date_time = datetime.now()
        add_record(r, date_time)
        print(str(date_time) + " - DNS record added.")

#  получение ответа из кеша, если он там есть
def get_response(dns_record):
    key = (str(dns_record.q.qname).lower(), dns_record.q.qtype)
    if key in database and database[key]:
        reply = dns_record.reply()
        reply.rr = [p.resource_record for p in database[key]]
        return reply

#  отправить результат юзеру
def send_response(response, addr):
    sock.connect(addr)
    sock.sendall(response)
    sock.close()

#  основной цикл
def work_loop():
    global sock

    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.bind(("", 40000))

    try:
        while True:
            data, addr = sock.recvfrom(2048)

            if database:
                clear_outdated_cash()

            try:
                dns_record = DNSRecord.parse(data)
            except DNSError:
                print('getting incorrect query, packet ignored')
                continue

            add_records(dns_record)
            if not dns_record.header.qr:
                response = get_response(dns_record)
                try:
                    if response:
                        send_response(response.pack(), addr)
                    else:
                        resp = dns_record.send("dns.yandex.ru")
                        add_records(DNSRecord.parse(resp))
                        send_response(resp, addr)
                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    sock.bind(("", 40000))
                    print(str(datetime.now()) + " - response send")
                except (OSError, DNSError):
                    print(str(datetime.now()) + " - transmission error")
    except:
        print('server error')

def main():
    print('server started')

    global database

    database = load_history()

    try:
        work_loop()
    finally:
        if database:
            save_history(database)
        print('server stop')

if __name__ == '__main__':
    main()


# Edit Distance

class Solution(object):

    def getMinEdits(self,w1, w2, word1, word2,cache):
        if (w1,w2) in cache:
            return cache[(w1,w2)]
        if w2 >= len(word2) and w1 < len(word1):
             return len(word1) - w1
        if w1 >= len(word1)    and w2 < len(word2):
            return len(word2) - w2

        min_edits = 0
        while w1 < len(word1) and w2 < len(word2) and word1[w1] == word2[w2]:
            w1 += 1
            w2 += 1

        if w2 < len(word2) or w1 < len(word1):
            min_edits += 1 + min(
                self.getMinEdits(w1+1, w2+1, word1, word2,cache), # replace chr in word1
                self.getMinEdits(w1, w2+1, word1, word2,cache), # insert new chr in word1
                self.getMinEdits(w1+1,w2,word1,word2,cache)# delete chr from word1
            )
        cache[(w1,w2)] = min_edits
        return min_edits

    def minDistance(self, word1, word2):
        """
        :type word1: str
        :type word2: str
        :rtype: int
        """
        cache = {}
        return self.getMinEdits(0,0,word1,word2,cache)


import sys
from datetime import datetime

import tcpparser.environment as environment
import tcpparser.utils as utils

def is_blocked(ip: str) -> bool:
    """ Check if ip is already blocked """
    cmd = "{} {} {} {} {}".format("iptables", "-C INPUT", "-s", ip, "-j DROP")
    return environment.execute(cmd)

def block_ip(ip: str) -> None:
    """
    Blocks source ip
    e.g: iptables -I INPUT -s 12.34.56.78/32 -j DROP
    """
    # SHORTCUT: Ideally should check if the ip is not one of the ips on any
    # physic or virtual network device before blocking.
    cmd = "{} {} {} {}/32 {}".format("iptables", "-I INPUT", "-s", ip, "-j DROP")
    exe = cmd.split(' ')[0]

    if environment.exe_exists(exe):

        if not is_blocked(ip):
            return environment.execute(cmd)

    else:
        print(f"[ERROR]: Unable to execute: {exe}", file=sys.stderr)
        print(f"Please check if {exe} is installed and is in the right path")

def update_connections(cnx: dict, data: list, popfirst: bool) -> None:
    """ Update a dictionary (cnx) with the connections from the last minute """

    # dict key
    epoch = int(datetime.now().timestamp() * 1000)

    # After 1 minute, start removing the first entry
    # cnx is an ordered dictionary
    # FIFO
    if popfirst:
        cnx.pop(next(iter(cnx)))

    # Uses set to avoid duplication of values
    group_conn = set()

    for ldata in data:

        if ldata["DIRECTION"] == "->":
            conn = tuple((ldata["PEER"].split(':')[0], ldata["LOCAL"]))
            group_conn.add(conn)

    cnx[epoch] = group_conn

def check_portscan(cnx: dict, blocked_ips: list) -> None:
    """ Checks established connections from a source ip """

    peers = utils.organize_peer_connections(cnx)

    # Checks if a single peer address has more than 3 connection in the previous minute
    # and shows "Port scan detected"
    for host_ip, sockets in peers.items():
        fail = False

        # Check if IP wasn't already blocked and has more than 3 ports
        # open on host running tcpparser
        if len(sockets) > 3 and host_ip not in blocked_ips:
            blocked_ips.append(host_ip)
            now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            ports = []
            reason = "Port scan detected"

            for netaddr in sockets:
                ports.append(str(netaddr.split(':')[1]))

            local = str(netaddr.split(':')[0])
            print("{:>19}: {:>18}: {:>21} {} {:<21}".format(
                now, reason, host_ip, "->", local + " on ports " + ",".join(ports)
            ))

            # Blocks the source ip only once
            if not fail:
                fail = block_ip(host_ip)


PAGE_TOP = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>'''
PAGE_MIDDLE = '''</title>

'''
PAGE_BOTTOM = '''
'''

CONTENT_HTML = "text/html; charset=utf-8"

def html_page_error_custom(msg):
    return PAGE_TOP + msg + PAGE_MIDDLE + "
" + msg + "

" + PAGE_BOTTOM

PAGE_ERROR_400 = html_page_error_custom("400 - Bad request")
PAGE_ERROR_403 = html_page_error_custom("403 - Forbidden")
PAGE_ERROR_404 = html_page_error_custom("404 - Not found")
PAGE_ERROR_500 = html_page_error_custom("500 - Internal server error")

FORM_TITLE = u"%s website url for group %i"

PAGE_FORM_1 = u'''<form name="sendurl" action="/submit" method="POST">'''

PAGE_FORM_2 = u'''<fieldset style="margin: auto;">
	<legend>
%s
</legend>


import requests
import json

def send_fcm_token_for_device(serverToken, deviceToken, order_id, status):

  headers = {
        'Content-Type': 'application/json',
        'Authorization': 'key=' + serverToken,
      }

  body = {
          'notification': {'title': 'Your order #' + order_id +' status : '+status,
                            'body': 'Thank you for your purchase'
                            },
          'to':
              '/topics/all', ##for all /topics/all , for individual deviceToken is passed
          'priority': 'high',
        #   'data': dataPayLoad,
        }
  response = requests.post("https://fcm.googleapis.com/fcm/send",headers = headers, data=json.dumps(body))
  print(response.status_code)

  print(response.json())


from itertools import chain

class GameState:
    def __init__(self, data):
        self.grid = data

    @classmethod
    def from_array(cls, grid):
        return cls(grid)

    @classmethod
    def empty(cls):
        grid = [[0, 0, 0, 0],
                [0, 0, 0, 0],
                [0, 0, 0, 0],
                [0, 0, 0, 0]]
        return cls(grid)

    def validate_sudoku_solution(self):

        def validate_rows(grid):
            for i in range(len(grid)):
                if sum(grid[i]) != sum(set(grid[i])):
                    return False
            return True

        def validate_columns(grid):
            t_grid = list(zip(*grid))
            return validate_rows(t_grid)

        def validate_squares(grid):
            for i in range(0, 4, 2):
                for j in range(0, 4, 2):
                    square = list(chain(*(row[j:j + 2] for row in grid[i:i + 2])))
                    if sum(square) != sum(set(square)):
                        return False
            return True

        rows_status = validate_rows(self.grid)
        columns_status = validate_columns(self.grid)
        squares_status = validate_squares(self.grid)

        valid = rows_status and columns_status and squares_status

        if valid:
            return 'Solution valid!'

        message = 'Failed: '
        if not rows_status:
            message += 'rows '
        if not columns_status:
            message += 'columns '
        if not squares_status:
            message += 'squares '

        return message


import json
import logging
import psycopg2
import redis
import requests
import time
from contextlib import contextmanager

LOG = logging.getLogger()
APP_PORT = 8080
REDIS_PORT = 8081
PSQL_PORT = 8082

class DBConnector():
    def __init__(self, host='localhost', port=PSQL_PORT, db='postgres', user='postgres'):
        self.connect = psycopg2.connect(
            host=host,
            port=port,
            dbname=db,
            user=user
        )

    def fetch_all_votes(self):
        """
        Fetches all rows from the votes table and returns a tuple
        of tuples in the format ((id1, vote1), (id2, vote2), ...)
        """
        with self.connect.cursor() as curs:
            curs.execute(
                """
                SELECT * FROM votes;
                """
            )
            return curs.fetchall()

    def fetch_vote_by_id(self, voter_id):
        """
        Fetches the vote value of the specified `voter_id`.  Raises
        exception on multiple matches.  Returns None for non-match.
        """
        with self.connect.cursor() as curs:
            curs.execute(
                """
                SELECT vote from votes where id = %(id)s;
                """,
                {'id': voter_id}
            )
            results = curs.fetchall()

            if len(results) == 0:
                return None
            assert len(results) == 1, "Votes for id {} not equal to 1: {}".format(voter_id, results)
            return results[0][0]

    @contextmanager
    def expect_vote_updated(self, voter_id, timeout=60):
        """
        Context manager that waits 60 seconds for the db to update
        before return.
        """
        old = self.fetch_vote_by_id(voter_id)

        try:
            yield self
        finally:
            timeout = time.time() + 60
            while time.time() < timeout:
                new = self.fetch_vote_by_id(voter_id)
                if old != new:
                    return
                LOG.warning("Expected vote value not updated")
                time.sleep(1)

class RedisConnector():
    def __init__(self, host='localhost', port=REDIS_PORT):
        self.client = redis.Redis(
            host=host,
            port=port
        )

    def push_vote(self, vote, voter_id):
        """
        Pushes a vote in the back of the queue
        """
        ret = self.client.rpush(
            'votes',
            json.dumps({
                "vote": vote,
                "voter_id": voter_id
            })
        )
        assert ret == 1

    def insert_vote(self, vote, voter_id):
        """
        Inserts a vote in the front of the queue
        """
        ret = self.client.lpush(
            'votes',
            json.dumps({
                "vote": vote,
                "voter_id": voter_id
            })
        )
        assert ret == 1

class APIConnector():
    def __init__(self, host='localhost', port=APP_PORT):
        self.host = host
        self.port = port
        self.session = requests.Session()

    @property
    def voter_id(self):
        """
        Returns the voter_id from the session cookie.  Opens a
        new session if not yet opened
        """
        if 'voter_id' not in self.session.cookies:
            LOG.warning("No voter_id cookie")
            self._init_cookies()
        return self.session.cookies.get('voter_id')

    @property
    def url(self):
        """
        Returns the formated url
        """
        return "http://{}:{}/".format(self.host, self.port)

    def _init_cookies(self):
        """
        GET the web service for initializing the cookie
        """
        LOG.debug("Getting voter_id cookie")
        self.session.get(self.url)

    def _post(self, data, *args, **kwargs):
        """
        Wrapper session.post.  Raises exception on HTTP
        error codes.
        """
        resp = self.session.post(
            self.url,
            data,
            *args,
            **kwargs
        )
        resp.raise_for_status()
        return resp

    def post_vote(self, vote, voter_id=None):
        """
        POST a vote to the web service
        """
        if voter_id:
            return self._post(
                data={'vote': vote},
                cookies={'voter_id', voter_id}
            )
        return self._post(
            data={'vote': vote}
        )


from typing import List
from os.path import isfile
from collections import OrderedDict
import logging

from scapy.layers.dot11 import RadioTap, Dot11, Dot11FCS
from scapy.packet import Packet, Raw
from scapy.utils import rdpcap
import pcapy

from netzob.Common.NetzobException import NetzobImportException
from netzob.Common.Utils.SortedTypedList import SortedTypedList
from netzob.Import.PCAPImporter.PCAPImporter import PCAPImporter
from netzob.Model.Vocabulary.Messages.AbstractMessage import AbstractMessage
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
from netzob.Model.Vocabulary.Messages.L2NetworkMessage import L2NetworkMessage
from netzob.Model.Vocabulary.Messages.L4NetworkMessage import L4NetworkMessage

from nemere.validation.messageParser import ParsingConstants

class BaseLoader(object):
    """
    Wrapper for loading messages from memory as specimens.
    Prepares and represents messages to support further analysis.
    Especially useful for on-the-fly creation of test cases.
    """

    def __init__(self, l5msgs, l1msgs=None, baselayer=None):
        """
        Load messages from memory. Base class for other loaders, e. g. loading from PCAP file.

        :param l5msgs:
        :param l1msgs:
        :param baselayer: Optionally, set the base layer explicitly.
        """
        if not hasattr(self, 'pcapFileName'):
            self.pcapFileName = 'from-memory'
        self.messagePool = OrderedDict()  # type: OrderedDict[AbstractMessage, RawMessage]
        """maps the message representations for Netzob and tshark
        dict of { application layer of messages L5Messages : RawMessage }"""
        if not l1msgs:
            l1msgs = l5msgs
        for k, m in zip(l5msgs, l1msgs):
            self.messagePool[k] = m
            # TODO replace the above quickfix not to read the file a second time (should we?)
            # probably we could use msgs = ParsedMessage.parseMultiple(l1msgs); for m in msgs:
            # ... append(RawMessage(m.protocolbytes))
        self._baselayer = baselayer

    def getBaseLayerOfPCAP(self):
        """
        see ParsingConstants.LINKTYPES

        :return: Determine lowest encapsulating layer of PCAP.
        """
        if self._baselayer is not None:
            return self._baselayer

        try:
            # looking at just one message should reveal lowest encapsulating layer of the whole PCAP
            al5msg = next(iter(self.messagePool.keys()))
        except StopIteration:
            raise ValueError('No message could be imported. See previous errors for more details.')
        if isinstance(al5msg, L2NetworkMessage):
            if al5msg.l2Protocol == 'Ethernet':
                return ParsingConstants.LINKTYPES['ETHERNET']
            # for some reason, its not possible to access the class variable "name", so instantiate a dummy object
            elif al5msg.l2Protocol == Dot11().name or al5msg.l2Protocol == Dot11FCS().name:
                return ParsingConstants.LINKTYPES['IEEE802_11']  # 802.11
            elif al5msg.l2Protocol == 'None':  # no ethernet
                if al5msg.l3Protocol == 'IP':
                    return ParsingConstants.LINKTYPES['RAW_IP']  # IP
                else:
                    raise NotImplementedError("Linktype on layer 3 unknown. Protocol is {}".format(al5msg.l3Protocol))
            else:
                raise NotImplementedError("Linktype on layer 2 unknown. Protocol is {}".format(al5msg.l2Protocol))
        else:
            return ParsingConstants.LINKTYPES['undecoded']  # non-decoded raw trace without link type information

    def __repr__(self):
        return type(self).__name__ + ": " + self.pcapFileName + f" on layer {self.getBaseLayerOfPCAP()}"

    @property
    def maximumMessageLength(self):
        """
        :return: The maximum message length in bytes of the relevant network layer without its encapsulation for
            the messages in this specimen's pool.
        """
        return max(len(line.data) for line in self.messagePool.keys())

    @property
    def cumulatedMessageLength(self):
        """
        :return: The sum of all message lengths in bytes of the relevant network layer without its encapsulation for
            the messages in this specimen's pool. I. e., the cumulated size of all payload in the trace.
        """
        return sum(len(line.data) for line in self.messagePool.keys())

class SpecimenLoader(BaseLoader):
    """
    Wrapper for loading messages from a PCAP as specimens.
    Prepares and represents messages to support further analysis.
    """

    def __init__(self, pcap: str, layer:int=-1, relativeToIP:bool=False):
        """
        Load the messages from the PCAP file of the given name.

        >>> from nemere.utils.loader import SpecimenLoader
        >>> sl = SpecimenLoader('../input/hide/random-100-continuous.pcap', layer=0, relativeToIP=True)
        >>> firstmessage = list(sl.messagePool.items())[0]
        >>> print(firstmessage[0].data.hex())  # the whole message
        450000780001000040007c837f0000017f000001da362a9b658bcc70d1a8323979e8fc0affda6f9fca9de16a3b051303601280f2c249abdd05f85ce98aebaf67626c07b4698a3c87abb95af87abf108735bac6ffd8823e80ac498622e7347852e0f2e7a8759911782b00a74b5744109b9fe59f5d311252a1
        >>> print(firstmessage[1].data.hex())  # only the payload of the target layer
        ffffffffffff0000000000000800450000780001000040007c837f0000017f000001da362a9b658bcc70d1a8323979e8fc0affda6f9fca9de16a3b051303601280f2c249abdd05f85ce98aebaf67626c07b4698a3c87abb95af87abf108735bac6ffd8823e80ac498622e7347852e0f2e7a8759911782b00a74b5744109b9fe59f5d311252a1

        :param pcap: PCAP input file name.
        :param layer: The protocol layer to extract. If not set or negative, use the top layer.
        :param relativeToIP: If True, extract the given layer relative to the IP layer.

        """
        if not isfile(pcap):
            raise FileNotFoundError('File not found:', pcap)
        self.pcapFileName = pcap
        self.layer = layer
        self.relativeToIP = relativeToIP
        absLayer = 2 + layer if relativeToIP else layer

        # prevent Netzob from producing debug output in certain cases.
        logging.getLogger().setLevel(30)

        try:
            if layer < 0:
                # read messages at layer 5 for the Netzob inference
                l5msgs = PCAPImporter.readFile(pcap, importLayer=5).values()  # type: List[L4NetworkMessage]
            else:
                # read messages at given layer for the Netzob inference
                l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values()  # type: List[AbstractMessage]
            # read messages as raw for tshark input
            l1msgs = PCAPImporter.readFile(pcap, importLayer=1).values()  # type: List[RawMessage]
        except (NetzobImportException, pcapy.PcapError):
            importer = ScaPyCAPimporter(self.pcapFileName, absLayer)
            l5msgs = importer.messages
            l1msgs = importer.rawMessages
        super().__init__(l5msgs, l1msgs)

    # The int value of some pcapy datalink denotations is different from the tcpdump ones: https://www.tcpdump.org/linktypes.html
    # http://vpnb.leipzig.freifunk.net:8004/srv2/lede/lede-20171116/build_dir/target-mips_24kc_musl/python-pcapy-0.11.1/pcapy.html#idp8777598240
    pcapyDatalinkTranslation = {
        pcapy.DLT_RAW: ParsingConstants.LINKTYPES['RAW_IP']
    }
    """Translates pcapy linktype values to tcpdump ones."""

    def getBaseLayerOfPCAP(self):
        pcap = pcapy.open_offline(self.pcapFileName)
        dl = pcap.datalink()
        # Translates pcapy linktype values to tcpdump ones if in dict, otherwise the value is used unchanged
        return dl if dl not in SpecimenLoader.pcapyDatalinkTranslation else SpecimenLoader.pcapyDatalinkTranslation[dl]

class ScaPyCAPimporter(object):
    def __init__(self, pcapfilename, importLayer=5):
        # l5msgs = PCAPImporter.readFile(pcap, importLayer=absLayer).values()  # type: List[AbstractMessage]
        self.importLayer = importLayer
        self.packets = rdpcap(pcapfilename)
        self._messages = SortedTypedList(AbstractMessage)
        self._rawmessages = SortedTypedList(AbstractMessage)

        for pkt in self.packets:  # type: Packet
            self.packetHandler(pkt)

    @property
    def messages(self):
        return self._messages.values()

    @property
    def rawMessages(self):
        return self._rawmessages.values()

    def packetHandler(self, packet: Packet):
        epoch = packet.time
        l1Payload = bytes(packet)
        if len(l1Payload) == 0:
            return
        # Build the RawMessage
        rawMessage = RawMessage(l1Payload, epoch, source=None, destination=None)

        if isinstance(packet, RadioTap):
            # lift layer to Dot11 if there is a RadioTap dummy frame
            packet = packet.payload
        if self.importLayer == 2:
            (l2Proto, l2SrcAddr, l2DstAddr, l2Payload) = self.__decodeLayer2(packet)
            if len(l2Payload) == 0:
                return
            # Build the L2NetworkMessage
            l2Message = L2NetworkMessage(l2Payload, epoch, l2Proto, l2SrcAddr, l2DstAddr)
            self._messages.add(l2Message)
            self._rawmessages.add(rawMessage)
        else:
            # Use Netzob's PCAPImporter if layer 2 is not WLAN
            raise NetzobImportException("PCAP", "Unsupported import layer. Currently only handles layer 2.",
                                        PCAPImporter.INVALID_LAYER2)

    def __decodeLayer2(self, packet: Packet):
        """Internal method that parses the specified header and extracts
        layer2 related proprieties."""
        l2Proto = packet.name
        if isinstance(packet, Raw):
            print("Ignoring undecoded packet with values:", bytes(packet).hex())
            return l2Proto, None, None, ""
        if isinstance(packet, Dot11):
            l2DstAddr = packet.fields['addr1']  # receiver address, alt: packet.fields['addr3'] destination address
            l2SrcAddr = packet.fields['addr2']  # transmitter address, alt: packet.fields['addr4'] source address
        else:
            raise NetzobImportException("NEMERE_PCAP", "Unsupported layer 2 protocol " + l2Proto,
                                        PCAPImporter.INVALID_LAYER2)
        l2Payload = bytes(packet.payload)
        return l2Proto, l2SrcAddr, l2DstAddr, l2Payload