import matplotlib.pyplot as plt
import numpy as np
import math
import os

class Surfaces_math_functions:

    def __init__(self, X_MIN, X_MAX, Y_MIN, Y_MAX, COFF, OFFSET, equation_name):
        self.X_MIN = X_MIN
        self.X_MAX = X_MAX
        self.Y_MIN = Y_MIN
        self.Y_MAX = Y_MAX
        self.COFF = COFF
        self.OFFSET = OFFSET
        self.equation_name = equation_name
        self.equation = self.__get_equation(equation_name)

    # ---- Private methods ----

    # Private getters
    def __get_equation(self, string):
        string = string.lower()
        equation_table = {
            'paraboloide hiperbolico' or 'silla de montura':
                [
                    lambda x, y: ((y**2)/5 - (x**2)/3) * self.COFF,
                    "y^2/5 - x^2/3"
                ],
            'seno' or 'sin':
                [
                    lambda x, y: (np.sin(x+y)) * self.COFF,
                    "sin(x+y)"
                ],
            'coseno' or 'cos':
                [
                    lambda x, y: (np.cos(x+y)) * self.COFF,
                    "cos(x+y)"
                ],
            'tangente' or 'tan':
                [
                    lambda x, y: (np.tan(x+y)) * self.COFF,
                    "tan(x+y)"
                ],
            'exponencial' or 'exp':
                [
                    lambda x, y: (np.exp(x+y)) * self.COFF,
                    "e^(x+y)"
                ],
            'logaritmo' or 'log':
                [
                    lambda x, y: (np.log(x**2+y**2+1)) * self.COFF,
                    "log(x^2+y^2+1)"
                ],
            'raiz cuadrada' or 'sqrt':
                [
                    lambda x, y: (np.sqrt(x**2+y**2)) * self.COFF,
                    "sqrt(x^2+y^2)"
                ],
            'seno hiperbolico' or 'sinh':
                [
                    lambda x, y: (np.sinh(x+y)) * self.COFF,
                    "sinh(x+y)"
                ],
            'coseno hiperbolico' or 'coseno hiperbolico' or 'cosh':
                [
                    lambda x, y: (np.cosh(x+y)) * self.COFF,
                    "cosh(x+y)"
                ],
            'superficie 1' or 's1':
                [
                    lambda x, y: ((1-x/2+x**5+y**3) *
                                  np.exp(-x**2-y**2)) * self.COFF,
                    "(1-x/2+x^5+y^3) * e^(-x^2-y^2)"
                ],
            'superficie 2' or 's2':
                [
                    lambda x, y: ((x**2+y**2)*np.exp(-x**2-y**2)) * self.COFF,
                    "(x^2+y^2) * e^(-x^2-y^2)"
                ],
            'superficie 3' or 's3':
                [
                    lambda x, y: ((x**2+y**2)/(x**2+y**4)) * self.COFF,
                    "(x^2+y^2)/(x^2+y^4)"
                ],
            'superficie 4' or 's4':
                [
                    lambda x, y: ((x**2*y)/(x**2+y**4)) * self.COFF,
                    "(x^2*y)/(x^2+y^4)"
                ],
            'superficie 5' or 's5':
                [
                    lambda x, y: ((x**2*y**2)/(x**2+y**4)) * self.COFF,
                    "(x^2*y^2)/(x^2+y^4)"
                ],
        }

        return equation_table[string]

    def __get_surface_image(self):
        x = np.arange(self.X_MIN, self.X_MAX, 0.6)
        y = np.arange(self.Y_MIN, self.Y_MAX, 0.6)
        X, Y = np.meshgrid(x, y)
        Z = self.equation[0](X, Y)
        # reshaped_Z = np.reshape(Z, (len(x), len(y)))
        return X, Y, Z

    # ---- Public methods ----

    # Public getters

    def get_plot_data(self):
        X, Y, Z = self.__get_surface_image()
        return X, Y, Z

    def get_equation_function(self):
        return self.equation[0]

    def get_cartesian_equation_str(self):
        return self.equation[1]

class Fan_mapping:

    def __init__(self, QTY_FAN, V_FAN, I_FAN, surface_eq):
        self.QTY_FAN = QTY_FAN
        self.ROWS_FAN, self.COLS_FAN = self.__get_row_col()
        self.V_FAN = V_FAN
        self.I_FAN = I_FAN
        self.surface_eq = surface_eq

    # ---- Private methods ----

    # Create a matrix given the fan's quantity
    def __fan_to_grid(self):
        # 1.01 to avoid 0 (common discontinuity)
        _x = np.arange(-self.ROWS_FAN/2, self.ROWS_FAN/2, 1.01)
        # 1.01 to avoid 0 (common discontinuity)
        _y = np.arange(-self.COLS_FAN/2, self.COLS_FAN/2, 1.01)
        _xx, _yy = np.meshgrid(_x, _y)
        X, Y = _xx.ravel(), _yy.ravel()
        return X, Y

    # Private getters
    def __get_row_col(self):
        rows = math.ceil(math.sqrt(self.QTY_FAN))
        cols = math.ceil(self.QTY_FAN / rows)
        return rows, cols

    def __get_voltage_point(self, X, Y):

        voltage = self.surface_eq.get_equation_function()(X, Y)

        # Normalize values
        self.max_value = np.amax(voltage)
        self.min_value = np.amin(voltage)
        self.mean_value = np.mean(voltage)
        voltage = (voltage - self.min_value) / \
            (self.max_value - self.min_value)
        return voltage * self.V_FAN

    def __get_voltage_map(self):
        X, Y = self.__fan_to_grid()
        Z = self.__get_voltage_point(X, Y)
        self.max_value = round(np.max(Z), 2)
        self.min_value = round(np.min(Z), 2)
        self.mean_value = round(np.mean(Z), 2)
        self.power_consumption = round(np.sum(Z * self.I_FAN), 2)
        return X, Y, Z

    # ---- Public methods ----

    # Public getters

    def get_max_value(self):
        return self.max_value

    def get_min_value(self):
        return self.min_value

    def get_mean_value(self):
        return self.mean_value

    def get_power_consumption(self):
        return self.power_consumption

    def get_plot_data(self):
        X, Y, Z = self.__get_voltage_map()
        return X, Y, Z

    def get_row_col(self):
        return self.ROWS_FAN, self.COLS_FAN

def input_data():

    # TODO Check if the limits are correct

    qty_fan = int(input("¿Cuantos ventiladores hay disponibles? "))
    if qty_fan <= 1:
        raise ValueError("La cantidad de ventiladores debe ser mayor a 0")

    v_fan = float(input("¿Cuál es la tension nominal de los ventiladores? "))
    if v_fan <= 1:
        raise ValueError("La tension de los ventiladores debe ser mayor a 0")

    i_fan = float(input("¿Cuál es la corriente nominal de los ventiladores? "))
    if i_fan <= 0:
        raise ValueError("La corriente de los ventiladores debe ser mayor a 0")

    return qty_fan, v_fan, i_fan

def main():

    # ---- General parameters ----
    eq_available = ["paraboloide hiperbolico",
                    # "superficie 1",
                    # "superficie 2",
                    # "superficie 3",
                    # "superficie 4",
                    # "superficie 5",
                    # "seno",
                    # "coseno",
                    # "tangente",
                    # "exponencial",AAAz

                    # "logaritmo",
                    # "raiz cuadrada",
                    # "seno hiperbolico",
                    # "coseno hiperbolico"
                    ]

    X_MIN = -3
    X_MAX = 3
    Y_MIN = -3
    Y_MAX = 3
    COFF = 1
    OFFSET = 0

    # ---- Input data ----
    qty_fan, v_fan, i_fan = input_data()
    # qty_fan, v_fan, i_fan = 50, 12, 0.3

    # ---- Folder creation ----
    folder_name = "./" + str(qty_fan) + " ventiladores"
    if not os.path.exists(folder_name):

        os.makedirs(folder_name)

    # General parameters log
    with open(folder_name + "/parametros.txt", "w") as f:
        f.write("Cantidad de ventiladores: " + str(qty_fan) + "\n")
        f.write("Tension nominal de los ventiladores: " + str(v_fan) + "\n")
        f.write("Corriente nominal de los ventiladores: " + str(i_fan) + "\n")
        f.write("X_MIN: " + str(X_MIN) + "\n")
        f.write("X_MAX: " + str(X_MAX) + "\n")
        f.write("Y_MIN: " + str(Y_MIN) + "\n")
        f.write("Y_MAX: " + str(Y_MAX) + "\n")
        f.write("COFF: " + str(COFF) + "\n")
        f.write("OFFSET: " + str(OFFSET) + "\n")

    figs_pairs = []
    for i in range(len(eq_available)):
        figs_pairs.append(plt.figure(figsize=(8, 4)))
        figs_pairs[i].suptitle(eq_available[i], fontsize=12)
        figs_pairs[i].subplots_adjust(hspace=0.5)
        figs_pairs[i].subplots_adjust(wspace=0.5)
        figs_pairs[i].subplots_adjust(
            left=0.11, right=0.85, top=0.9, bottom=0.3)

    # Generate plot for each surface-fan_voltage_map pair
    for i, eq_str in enumerate(eq_available):

        # Instantiate surface equation
        surface_eq = Surfaces_math_functions(
            X_MIN, X_MAX, Y_MIN, Y_MAX, COFF, OFFSET, eq_str)

        # Instantiate fan_voltage_map
        fan_map = Fan_mapping(QTY_FAN=qty_fan, V_FAN=v_fan,
                              I_FAN=i_fan, surface_eq=surface_eq)

        # Surface plot data
        x_s, y_s, z_s = surface_eq.get_plot_data()
        # Voltage map data
        x_f, y_f, z_f = fan_map.get_plot_data()

        # Plot surface
        surface_plot = figs_pairs[i].add_subplot(1, 2, 1, projection='3d')
        surface_plot.plot_surface(
            x_s, y_s, z_s, rstride=1, cstride=1, cmap='inferno', edgecolor='none')
        surface_plot.set_title(
            "f(x,y)= " + surface_eq.get_cartesian_equation_str())
        surface_plot.set_xlabel("X")
        surface_plot.set_ylabel("Y")
        surface_plot.set_zlabel("Z")
        surface_plot.set_xlim3d(X_MIN, X_MAX)
        surface_plot.set_ylim3d(Y_MIN, Y_MAX)

        # Plot fan voltage map
        fan_plot = figs_pairs[i].add_subplot(1, 2, 2, projection='3d')
        fan_plot.bar3d(x_f, y_f, 0, 1, 1, z_f, shade=True, color='red')
        fan_plot.set_title("Mapa de ventiladores")
        fan_plot.set_xlabel("X")
        fan_plot.set_ylabel("Y")
        fan_plot.set_zlabel("Z")
        fan_plot.set_xlim3d(-fan_map.ROWS_FAN/2, fan_map.ROWS_FAN/2)
        fan_plot.set_ylim3d(-fan_map.COLS_FAN/2, fan_map.COLS_FAN/2)
        fan_plot.set_zlim3d(0, fan_map.V_FAN)

        #Labels (V_max, V_min, P_total)
        figs_pairs[i].text(0.3, 0.15, "V_max: " + str(fan_map.get_max_value()) +
                           " V", ha='center', va='center', transform=figs_pairs[i].transFigure)
        figs_pairs[i].text(0.5, 0.15, "V_min: " + str(fan_map.get_min_value()) +
                           " V", ha='center', va='center', transform=figs_pairs[i].transFigure)
        figs_pairs[i].text(0.7, 0.15, "V_mean: " + str(fan_map.get_mean_value()) +
                           " V", ha='center', va='center', transform=figs_pairs[i].transFigure)
        figs_pairs[i].text(0.5, 0.05, "P_total: " + str(fan_map.get_power_consumption()) +
                           " W", ha='center', va='center', transform=figs_pairs[i].transFigure)

        # Save figure
        figs_pairs[i].savefig(folder_name + "/" + eq_str + ".png")

        print("x_f: " + str(x_f))
        print("y_f: " + str(y_f))
        print("z_f: " + str(z_f))

        #Write in file the fan_voltage_map as lookup table (for the Arduino)
        with open(folder_name + "/" + eq_str + ".txt", "w") as f:
            rows,cols = fan_map.get_row_col()
            f.write("const static float fan_voltage_map[" + str(rows) + "][" + str(cols) + "] = {")
            # for i in range(rows):
            #     for j in range(cols):
            #         f.write(str(fan_map.get_value(i,j)) + ",")
            #     f.write("\n")
            f.write("};")

    plt.show()
    plt.close()

if __name__ == '__main__':
    main()


import random

import math
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
import sklearn.gaussian_process as gp
from mpl_toolkits.mplot3d import Axes3D

from tensorflow import keras
import tensorflow as tf

import tensorflow.keras.optimizers as tko
import tensorflow.keras.activations as tka
import tensorflow.keras.losses as tkl
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.models import Model

##########################################################################################################

def __build_activation_function(af):
    if af.lower() == 'none':
        return None
    exp_af = 'lambda _ : tka.' + af
    return eval(exp_af)(None)

def __build_optimizer(optimizer):
    opt_init = optimizer
    exp_po = 'lambda _ : tko.' + opt_init
    optimizer = eval(exp_po)(None)
    return optimizer

def __build_loss(loss):
    exp_loss = 'lambda _ : tkl.' + loss
    return eval(exp_loss)(None)

def __build_model (hidden_layers_layout, activation_functions):
    inputs = Input(shape=(2,))
    hidden = inputs
    for i in range(0, len(hidden_layers_layout)):
        hidden = Dense(hidden_layers_layout[i], \
                activation= __build_activation_function(activation_functions[i]))(hidden)
    outputs = Dense(1)(hidden)
    model = Model(inputs=inputs, outputs=outputs)

    return model

##########################################################################################################

def build_model_NN_2 ():

    hidden_layers_layout = [100, 100]
    activation_functions = ["relu", "relu"]
    inoptimizer = "SGD(decay=1e-6, momentum=0.9, nesterov=True)"
    loss = 'MeanSquaredError()'

    model = __build_model (hidden_layers_layout, activation_functions)
    optimizer = __build_optimizer(inoptimizer)
    model.compile(loss= __build_loss(loss), optimizer=optimizer)
    #model.summary()

    return model

##########################################################################################################

def build_model_NN_3 ():

    model = keras.Sequential()
    model.add(keras.layers.Dense(units = 2, activation = 'linear', input_shape=[2]))
    model.add(keras.layers.Dense(units = 100, activation = 'relu'))
    model.add(keras.layers.Dense(units = 100, activation = 'relu'))
    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')

    return model

##########################################################################################################

def build_model_NN_1 ():

    model = keras.Sequential()
    model.add(keras.layers.Dense(units = 2, activation = 'linear', input_shape=[2]))
    model.add(keras.layers.Dense(units = 32, activation = 'relu'))
    model.add(keras.layers.Dense(units = 64, activation = 'relu'))
    model.add(keras.layers.Dense(units = 128, activation = 'relu'))
    model.add(keras.layers.Dense(units = 32, activation = 'relu'))
    #model.add(keras.layers.Dense(units = 16, activation = 'relu'))
    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')
    #model.summary()

    return model

##########################################################################################################3

def build_model_GP_1 (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    kernel = gp.kernels.ConstantKernel(1.0, (1e-3, 1e3)) * gp.kernels.RBF([5,5], (1e-2, 1e2))
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################3

def build_model_GP_2 (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################3

def build_model_GP_2D (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################

def build_model_GP_3D (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

#####################################################################################################

def filterinitialset (filename, headername = "vibrational level v\Temperature(K)",  \
    factor = 1.0, normalize = False):

    dfin = pd.read_excel(filename)

    #print(dfin)

    dfdict = {}

    tempvalues = list(dfin.columns[1:])
    vibvalues = list(dfin[headername].values)

    min = float("inf")
    max = float("-inf")
    for c in dfin.columns:
        dfdict[c] = []
        if c == headername:
            dfdict[c] = list(dfin[c].values)
        else:
            for v in dfin[c].values:
                val = math.log10(factor*v)
                if val > max:
                    max = val
                if val < min:
                    min = val

    for c in dfin.columns:
        if c != headername:
            for v in dfin[c].values:
                val = math.log10(factor*v)
                valp = (val - min) / (max - min)
                if normalize:
                    dfdict[c].append(valp)
                else:
                    dfdict[c].append(val)

    df = pd.DataFrame.from_dict(dfdict)

    return df, vibvalues, tempvalues, min, max

##############################################################################

def plotfull3dcurve (df, vib_values, temp_values):

    y = []
    x = []
    for t in temp_values:
        for idx in range(len(vib_values)):
            x.append([float(t), float(vib_values[idx])])
            y.append(df[t].values[idx])

    X = np.array(x)
    Y = np.array(y)

    xdim = len(temp_values)
    ydim = len(vib_values)

    Xp = np.zeros((xdim, ydim), dtype=float)
    Yp = np.zeros((xdim, ydim), dtype=float)
    Zp = np.zeros((xdim, ydim), dtype=float)
    for xidx in range(xdim):
        t = temp_values[xidx]
        for yidx in range(ydim):
            v =  vib_values[yidx]
            Xp[xidx, yidx] = float(t)
            Yp[xidx, yidx] = float(v)
            Zp[xidx, yidx] = df[t].values[yidx]

    #fig = plt.figure(figsize=(10,8))
    fig = plt.figure(figsize=plt.figaspect(2.))
    plt.gcf().set_size_inches(40, 30)
    ax = fig.add_subplot(2,1,1, projection='3d')
    surf = ax.plot_surface(Xp, Yp, Zp, rstride=1, cstride=1, cmap='jet', linewidth=0, antialiased=False)
    plt.show()

##########################################################################################################

def get_train_and_test_rmt (temp_values, vib_values, df, \
    removetemps=[]):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    for tidx, t in enumerate(temp_values):
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        if t not in removetemps:
            tnorm = (t - mint)/(maxt - mint)

            for vidx, v in enumerate(vib_values):
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
        else:
            tnorm = (t - mint)/(maxt - mint)
            for vidx, v in enumerate(vib_values):
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##########################################################################################################

def get_train_and_test_rmv (temp_values, vib_values, df, \
    removevibs=[]):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    for tidx, t in enumerate(temp_values):
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)

        for vidx, v in enumerate(vib_values):
            if v not in removevibs:
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
            else:
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##########################################################################################################

def get_train_and_test_random (temp_values, vib_values, df, \
    perc):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    totnumber = 0;
    for t in temp_values:
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            totnumber += 1

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    random.seed(1)

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for vidx, v in enumerate(vib_values):
            rv = random.uniform(0.0, float(totnumber))
            #print(rv, perc*float(totnumber))
            if rv > (perc*float(totnumber)):
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
            else:
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##############################################################################

def get_train_full (temp_values, vib_values, df, vtopredict):

    maxt = max(temp_values)
    mint = min(temp_values)

    train_xy = []
    train_z = []

    maxz = float("-inf")
    minz = float("+inf")

    minv = min(vib_values)
    maxv = max(vib_values)

    totnumber = 0;
    for t in temp_values:
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            totnumber += 1

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for vidx, v in enumerate(vib_values):
            vnorm  = (v - minv)/(maxv - minv)
            train_xy.append([tnorm, vnorm])
            z = df[t].values[vidx]
            znorm = (z - minz)/(maxz - minz)
            train_z.append(znorm)

    test_xy = []
    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for v in vtopredict:
            vnorm  = (v - minv)/(maxv - minv)
            test_xy.append([tnorm, vnorm])

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)
    test_xy = np.asarray(test_xy)

    return train_xy, train_z, test_xy

##############################################################################

def plotfull3dcurve (columntorm, x, y):

    yv = []
    xv = []
    for i, v in enumerate(x):
        toappend = []
        for j in range(len(v)):
            if j != columntorm:
                toappend.append(v[j])
        xv.append(toappend)
        yv.append(y[i])

    X = np.array(xv)
    Y = np.array(yv)

    #for i in range(X.shape[0]):
    #    print("%4d %5d %10.5e"%(X[i,0], X[i,1], Y[i]))

    #print(X.shape)
    #print(Y.shape)

    x1set = sorted(list(set(X[:,0])))
    x2set = sorted(list(set(X[:,1])))

    x1dim = len(x1set)
    x2dim = len(x2set)

    Xp = np.zeros((x1dim, x2dim), dtype=float)
    Yp = np.zeros((x1dim, x2dim), dtype=float)
    Zp = np.zeros((x1dim, x2dim), dtype=float)
    for x1idx in range(x1dim):
        x1 = x1set[x1idx]
        for x2idx in range(x2dim):
            x2 =  x2set[x2idx]
            Xp[x1idx, x2idx] = float(x1)
            Yp[x1idx, x2idx] = float(x2)

            zval = None
            for i in range(X.shape[0]):
                if X[i,0] == x1 and X[i,1] == x2:
                    zval = Y[i]
                    break

            Zp[x1idx, x2idx] = zval

    #print(Xp.shape, Yp.shape, Zp.shape)

    #fig = plt.figure(figsize=(10,8))
    fig = plt.figure(figsize=plt.figaspect(2.))
    #plt.gcf().set_size_inches(40, 30)
    ax = fig.add_subplot(2, 1, 1, projection='3d')
    surf = ax.plot_surface(Xp, Yp, Zp, rstride=1, cstride=1, cmap='jet', linewidth=0, antialiased=True)
    plt.show()

#######################################################################

def plotfull3dscatter (columntorm, x, y):

    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')

    for i, v in enumerate(x):
        toappend = []
        for j in range(len(v)):
            if j != columntorm:
                toappend.append(v[j])
        xs = toappend[0]
        ys = toappend[1]
        zs = y[i]
        ax.scatter(xs, ys, zs, marker='o')

    plt.show()

#######################################################################

def test_train_split (column, valuestotest, x, y):

    xtest = []
    ytest = []
    xtrain = []
    ytrain = []

    for i, xv in enumerate(x[:,column]):
        if xv in valuestotest:
            xtest.append(x[i,:])
            ytest.append(y[i])
        else:
            xtrain.append(x[i,:])
            ytrain.append(y[i])

    #print(len(xtest), len(ytest))
    #print(len(xtrain), len(ytrain))

    return np.asarray(xtrain), np.asarray(xtest), \
        np.asarray(ytrain), np.asarray(ytest)

#######################################################################

def buildmodel(modelshape, inputshape=3):
    model = keras.Sequential()
    model.add(keras.layers.Input(shape=(inputshape)))

    for n in modelshape:
        model.add(keras.layers.Dense(units = n, activation = 'relu'))

    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')

    return model

#######################################################################


# -*- coding: utf-8 -*-
# @Author: Theo Lemaire
# @Email: theo.lemaire@epfl.ch
# @Date:   2017-02-15 15:59:37
# @Last Modified by:   Theo Lemaire
# @Last Modified time: 2020-02-03 21:21:58

''' Plot the effective variables as a function of charge density with color code. '''

import numpy as np
import logging
import matplotlib.pyplot as plt

from PySONIC.plt import plotEffectiveVariables
from PySONIC.utils import logger
from PySONIC.parsers import MechSimParser

# Set logging level
logger.setLevel(logging.INFO)

def main():

    parser = MechSimParser()
    parser.addNeuron()
    parser.addNColumns()
    parser.addNLevels()
    parser.defaults['neuron'] = 'RS'
    parser.defaults['radius'] = np.nan
    parser.defaults['freq'] = np.nan
    parser.defaults['amp'] = np.nan
    args = parser.parse()
    for k in ['charge', 'embedding', 'Cm0', 'Qm0', 'fs', 'mpi', 'pltscheme', 'plot']:
        del args[k]
    logger.setLevel(args['loglevel'])

    # Restrict radius, frequency and amplitude to single values
    for k in ['radius', 'freq', 'amp']:
        if len(args[k]) > 1:
            logger.error(f'multiple {k} values not allowed')
        val = args[k][0]
        if np.isnan(val):
            args[k] = None
        else:
            args[k] = val

    for pneuron in args['neuron']:
        plotEffectiveVariables(
            pneuron, a=args['radius'], f=args['freq'], A=args['amp'],
            zscale=args['cscale'], cmap=args['cmap'], ncolmax=args['ncol'], nlevels=args['nlevels'])

    plt.show()

if __name__ == '__main__':
    main()


import cv2
import os
import numpy as np
import matplotlib.pyplot as plt
import sqlite3

subjects = ["", "El gran santiago", ""]
labels_recorded = []
conn = sqlite3.connect("records.sqlite")
c = conn.cursor()

def recordDetection(label):
    if(label not in labels_recorded):
        labels_recorded.append(label)
        try:
            c.execute("INSERT INTO records (id,identifier, site, cam_code) VALUES (NULL, '" + label + "', 'Door office 1' , 'CodeCAM')")
            conn.commit()
            #conn.close()
        except sqlite3.IntegrityError:
            print('ERROR: ID already exists in PRIMARY KEY column {}'.format(id_column))

def draw_rectangle(img, rect):
    (x, y, w, h) = rect
    cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 2)

def draw_text(img, text, x, y):
    return cv2.putText(img, text, (x, y), cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 255, 0), 2)

def detect_face(img):
    gray = img
    #cascada entrenada para detectar rostros frontales.
    face_cascade_profile = cv2.CascadeClassifier('trainer-haar-files/haarcascade_profileface.xml')
    face_cascade_frontal = cv2.CascadeClassifier('trainer-haar-files/haarcascade_frontalface_alt.xml')

    faces_frontal = face_cascade_frontal.detectMultiScale(gray, scaleFactor=1.2, minNeighbors=5);
    detected = False;

    if (len(faces_frontal) > 0):
        return faces_frontal
        detected = True

    if (detected is not True):
        faces_profile_right = face_cascade_profile.detectMultiScale(gray, scaleFactor=1.2, minNeighbors=5);
        if (len(faces_profile_right) > 0):
            return faces_profile_right
            detected = True

    if (detected is False):
        return None

def predict(test_img):
    img = test_img.copy()
    faces = detect_face(img)

    if(faces is not None):
        for face in faces:
            rect = (x, y, w, h) = face
            label = face_recognizer.predict(img[y:y+w, x:x+h])
            #label_text = subjects[label[0]] + " " + str(label[1])
            label_text = subjects[label[0]]
            draw_rectangle(img, rect)
            draw_text(img, label_text, rect[0], rect[1]-5)
            recordDetection(label_text)

    return img

#stest_img1 = cv2.imread("test-data/test1.jpg")
#predicted_img1 = predict(test_img1)
#create a figure of 2 plots (one for each test image)

face_recognizer = cv2.face.LBPHFaceRecognizer_create()
face_recognizer.setThreshold(40)
face_recognizer.read("trainer.yml")

cap = cv2.VideoCapture(1)
w = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
print("Resolution : " + str(w) + "x" + str(h), 0, 0)

while(True):
    ret, frame = cap.read()
    test_img1 = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    predicted_img1 = predict(test_img1)

    cv2.imshow("frame", predicted_img1)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


#!/usr/bin/env python
# coding: utf-8

# # conterfactual expalantion for DRL baced-on policy distillation

# ### import packages

# In[1]:

import gym
from stable_baselines3 import DQN
from stable_baselines3.common.vec_env import VecVideoRecorder, DummyVecEnv
from stable_baselines3 import DQN
from stable_baselines3.common.vec_env import VecVideoRecorder, DummyVecEnv
from policy_distillation import main as policy_distillation
import random
from tkinter import Y
import numpy as np
import torch
import torch.nn as nn
import gym
import numpy as np
from IPython.display import clear_output
import matplotlib.pyplot as plt
# import matplotlib
# matplotlib.use("Qt5Agg")
# from matplotlib import pyplot as plt
import gym
import highway_env
from matplotlib import pyplot as plt

from scipy.signal import convolve, gaussian
from copy import deepcopy
import os
import io
import base64
import time
import glob
from IPython.display import HTML
import local_lib as myLib

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
from stable_baselines3 import SAC,A2C,DDPG, DQN

from itertools import count
from time import time, strftime, localtime
import gym
import scipy.optimize
from tensorboardX import SummaryWriter
from core.models import *
from core.agent_ray_pd import AgentCollection
from utils.utils import *
import numpy as np
import ray
import envs
from trpo import trpo
from student import Student
from teacher import Teacher
import os
import pickle
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("device: ",device)

# ### environment setup

# In[2]:
num_of_actions=5
num_Of_vehicles_Under_vision = 5
vehicle_attr = 4

low_dimention_env = gym.make("highway-fast-v0")
low_dimention_env.configure({
    "observation": {
        "type": "Kinematics",
        "vehicles_count": num_Of_vehicles_Under_vision,
        "features": ["x", "y", "vx", "vy"],
        "normalize": False,
        "absolute": True,
        # "order": "sorted"
    }
})
low_dimention_env.reset()

high_dimention_env = gym.make("highway-fast-v0")
high_dimention_env.configure({
    "observation": {
        "type": "GrayscaleObservation",
        "observation_shape": (128, 64),
        "stack_size": 4,
        "weights": [0.2989, 0.5870, 0.1140],  # weights for RGB conversion
        "scaling": 1.75,
    },
})
high_dimention_env.reset()

# ## teacher training phase

# In[3]:

# model = myLib.DQN_creation(policy='MlpPolicy', env=env, policy_kwargs=dict(net_arch=[256, 256]), learning_rate=5e-4,
#                 buffer_size=15000, learning_starts=200, batch_size=128, gamma=0.8, train_freq=1, gradient_steps=1,
#                 target_update_interval=50, exploration_fraction=0.7, verbose=1, tensorboard_log="highway_dqn/")
# model.learn(int(10e4))
fileName = "model"
# myLib.Save_DQN_model(model,fileName)
teacher_model = myLib.Load_DQN_model(fileName=fileName)

# ## student training phase

# In[5]:

import argparse

parser = argparse.ArgumentParser(description='Policy distillation')
# Network, env, MDP, seed
parser.add_argument('--hidden-size', type=int, default=256,
                    help='number of hidden units per layer')
parser.add_argument('--num-layers', type=int, default=2,
                    help='number of hidden layers')
parser.add_argument('--env-name', default="highway-fast-v0", metavar='G',
                    help='name of the environment to run')
parser.add_argument('--gamma', type=float, default=0.995, metavar='G',
                    help='discount factor (default: 0.995)')
parser.add_argument('--tau', type=float, default=0.97, metavar='G',
                    help='gae (default: 0.97)')
parser.add_argument('--seed', type=int, default=1, metavar='N',
                    help='random seed (default: 1)')
parser.add_argument('--load-models', default=True, action='store_true',
                    help='load_pretrained_models')

# Teacher policy training
parser.add_argument('--agent-count', type=int, default=10, metavar='N',
                    help='number of agents (default: 100)')
parser.add_argument('--num-teachers', type=int, default=1, metavar='N',
                    help='number of teacher policies (default: 1)')
parser.add_argument('--max-kl', type=float, default=1e-2, metavar='G',
                    help='max kl value (default: 1e-2)')
parser.add_argument('--cg-damping', type=float, default=1e-2, metavar='G',
                    help='damping for conjugate gradient (default: 1e-2)')
parser.add_argument('--cg-iter', type=int, default=10, metavar='G',
                    help='maximum iteration of conjugate gradient (default: 1e-1)')
parser.add_argument('--l2-reg', type=float, default=1e-3, metavar='G',
                    help='l2 regularization parameter for critics (default: 1e-3)')
parser.add_argument('--teacher-batch-size', type=int, default=1000, metavar='N',
                    help='per-iteration batch size for each agent (default: 1000)')
parser.add_argument('--sample-batch-size', type=int, default=10000, metavar='N',
                    help='expert batch size for each teacher (default: 10000)')
parser.add_argument('--render', action='store_true',
                    help='render the environment')
parser.add_argument('--log-interval', type=int, default=1, metavar='N',
                    help='interval between training status logs (default: 10)')
parser.add_argument('--device', type=str, default=device,
                    help='set the device (cpu or cuda)')
parser.add_argument('--num-workers', type=int, default=10,
                    help='number of workers for parallel computing')
parser.add_argument('--num-teacher-episodes', type=int, default=10, metavar='N',
                    help='num of teacher training episodes (default: 100)')

# Student policy training
parser.add_argument('--lr', type=float, default=1e-5, metavar='G',
                    help='adam learnig rate (default: 1e-3)')
parser.add_argument('--test-interval', type=int, default=10, metavar='N',
                    help='interval between training status logs (default: 10)')
parser.add_argument('--student-batch-size', type=int, default=1000, metavar='N',
                    help='per-iteration batch size for student (default: 1000)')
parser.add_argument('--sample-interval', type=int, default=10, metavar='N',
                    help='frequency to update expert data (default: 10)')
parser.add_argument('--testing-batch-size', type=int, default=10000, metavar='N',
                    help='batch size for testing student policy (default: 10000)')
parser.add_argument('--num-student-episodes', type=int, default=1000, metavar='N',
                    help='num of teacher training episodes (default: 1000)')
parser.add_argument('--loss-metric', type=str, default='kl',
                    help='metric to build student objective')
parser.add_argument('--algo', type=str, default='sgd',
                    help='update method')
parser.add_argument('--storm-interval', type=int, default=10, metavar='N',
                    help='frequency of storm (default: 10)')
parser.add_argument('--init-alpha', type=float, default=1.0, metavar='G',
                    help='storm init alpha (default: 1.0)')
args = parser.parse_args()

student = policy_distillation(args, low_dimention_env)

# ## potential Counterfactuals
    #################################for test only
env = gym.make("highway-fast-v0")
config = {
    "observation": {
        "type": "Kinematics",
        "vehicles_count": num_Of_vehicles_Under_vision,
        "features": ["x", "y", "vx", "vy"],
        "normalize": False,
        "absolute": True,
        # "order": "sorted"
    }
}
env.configure(config)

obs,done = env.reset(), False
for _ in range(20):
    while done == False:
        action = student.policy(torch.tensor(obs.reshape(obs.size), device=device)).loc.argmax().tolist()
        obs, reward, done, info = env.step(action)
        env.render()
    obs, done = env.reset(),False
env.close()
#################################end test
# ### second environment (lower dimensional input)

obs = env.reset()
env.render()

exp_is_valid = []
indx=np.array([1])
CFs, desireAction = local_lib.CF_find_2(env=env,student_model=student_model,obs=obs,device=device,indx=indx,
                      num_Of_vehicles_Under_vision=num_Of_vehicles_Under_vision,vehicle_attr=vehicle_attr)

# ## counterfactuals validation

# In[ ]:

image_env = DummyVecEnv([train_env])
video_length = 2 * image_env.envs[0].config["duration"]
image_env = VecVideoRecorder(image_env, "highway_cnn/videos/",
                       record_video_trigger=lambda x: x == 0, video_length=video_length,
                       name_prefix="dqn-agent")

for i, cf in enumerate(CFs):
    obs_image = image_env.reset()
    obs_kinematics = myLib.CF2Env3(obs, cf, indx, env)
    for index, vehicles in enumerate(image_env.venv.envs[0].env.road.vehicles[0:num_Of_vehicles_Under_vision]):
        vehicles.position=cf[index,0:2]
        # vehicles.velocity=CF[indices,2:4]
#applying the CFs to environments
    image_env.venv.envs[0].env.render()
    image_env.venv.envs[0].env.render()
    image_env.venv.envs[0].env.render()
    obs_image=image_env.venv.envs[0].env.observation_type.observe()
    ##prepare data
    plt.imshow(obs_image[3].T, cmap=plt.get_cmap('gray'))
    plt.show()
    action,_ = teacher_model.predict(obs_image)
    action_prim = student_model(torch.tensor(obs_kinematics.observation_type.observe()        .reshape(num_Of_vehicles_Under_vision*vehicle_attr), device=device)).argmax(dim=-1).item()
    # if action_prim==action_desire then the exp is valide
    if(action == action_prim == desireAction[i]):
        exp_is_valid.append(1)
    else:
        exp_is_valid.append(0)
    print(exp_is_valid)

# ## visulize the finded CFs

# In[ ]:


import torch
from code_slac.network.base import BaseNetwork
from code_slac.network.latent import ConstantGaussian

from diayn_seq_code_revised.networks.my_gaussian import MyGaussian as Gaussian

from self_supervised.utils.my_pytorch_util import tensor_equality

from latent_with_splitseqs.base.slac_latent_base import SlacLatentBase

# Adaption of the SLAC latent network architecture
class SlacLatentNetConditionedOnSingleSkill(SlacLatentBase):

    def __init__(self,
                 *args,
                 obs_dim,
                 skill_dim,
                 latent1_dim=32,
                 latent2_dim=256,
                 hidden_units=(256, 256),
                 leaky_slope=0.2,
                 dropout=0.,
                 **kwargs
                 ):
        super(SlacLatentNetConditionedOnSingleSkill, self).__init__(
            *args,
            latent1_dim=latent1_dim,
            latent2_dim=latent2_dim,
            **kwargs
        )
        # We use the observations as actions for this model
        # and the infered skill as observaton

        # p(z1(0)) = N(0, I)
        self.latent1_init_prior = ConstantGaussian(latent1_dim)
        # p(z2(0) | z1(0))
        self.latent2_init_prior = Gaussian(
            input_dim=latent1_dim,
            output_dim=latent2_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # p(z1(t+1) | z2(t), a(t))
        self.latent1_prior = Gaussian(
            input_dim=latent2_dim + obs_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # p(z2(t+1) | z1(t+1), z2(t), a(t))
        self.latent2_prior = Gaussian(
            input_dim=latent1_dim + latent2_dim + obs_dim,
            output_dim=latent2_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )

        # q(z1(0) | feat(0))
        self.latent1_init_posterior = Gaussian(
            input_dim=skill_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # q(z2(0) | z1(0)) = p(z2(0) | z1(0))
        self.latent2_init_posterior = self.latent2_init_prior
        # q(z1(t+1) | z2(t), a(t))
        self.latent1_posterior_step = Gaussian(
            input_dim=latent2_dim + obs_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # q(z2(t+1) | z1(t+1), z2(t), a(t)) = p(z2(t+1) | z1(t+1), z2(t), a(t))
        self.latent2_posterior = self.latent2_prior

        self.latent1_posterior_end = Gaussian(
            input_dim=latent2_dim + obs_dim + skill_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )

        ## p(skill | z1(end), z2(end))
        #self.classifier = Gaussian(
        #    input_dim=latent1_dim + latent2_dim,
        #    output_dim=skill_dim,
        #    hidden_units=hidden_units,
        #    leaky_slope=leaky_slope,
        #)

    def sample_prior(self,
                     obs_seq,
                     ):
        """
        Sample from prior feature encoding

        Args:
            obs_seq             : (N, S, obs_dim) tensor of observations
        Returns:
            latent1_samples     : (N, S+1, L1) tensor of latent samples
            latent2_samples     : (N, S+1, L2) tensor of latent samples
            latent1_dists       : (S+1) list of (N, L1) distributions
            latent2_dists       : (S+1) list of (N, L2) distributions
        """
        batch_dim = 0
        seq_dim = 1
        data_dim = -1
        seq_len = obs_seq.size(seq_dim)
        obs_seq_seqdim_first = torch.transpose(obs_seq, batch_dim, seq_dim)

        latent1_samples = []
        latent2_samples = []
        latent1_dists = []
        latent2_dists = []

        for t in range(seq_len + 1):
            if t == 0:
                # p(z1(0)) = N(0, I)
                latent1_dist = self.latent1_init_prior(obs_seq_seqdim_first[t])
                latent1_sample = latent1_dist.rsample()

                # p(z2(0) | z1(0))
                latent2_dist = self.latent2_init_prior(latent1_sample)
                latent2_sample = latent2_dist.rsample()

            else:
                # p(z1(t) | z2(t-1), a(t-1))
                latent1_dist = self.latent1_prior(
                    [latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # p(z2(t) | z1(t), z2(t-1), a(t-1))
                latent2_dist = self.latent2_prior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            latent1_samples.append(latent1_sample)
            latent2_samples.append(latent2_sample)
            latent1_dists.append(latent1_dist)
            latent2_dists.append(latent2_dist)

        latent1_samples_stacked = torch.stack(latent1_samples, dim=seq_dim)
        latent2_samples_stacked = torch.stack(latent2_samples, dim=seq_dim)

        return {
            'latent1_samples': latent1_samples_stacked,
            'latent2_samples': latent2_samples_stacked,
            'latent1_dists': latent1_dists,
            'latent2_dists': latent2_dists,
        }

    def sample_posterior(self, skill, obs_seq):
        """
        Sample from posterior

        Args:
            skill               : (N, skill_dim) tensors
            obs_seq             : (N, S, obs_dim) tensors of observations
        Returns:
            latent1_samples     : (N, S+1, L1) tensor of sampled latent vectors
            latent2_samples     : (N, S+1, L2) tensor of sampled latent vectors
            latent1_dists       : (S+1) length list of (N, L1) distributions
            latent2_dists       : (S+1) length list of (N, L2) distributions
        """
        batch_dim = 0
        seq_dim = 1
        data_dim = -1
        seq_len = obs_seq.size(seq_dim)

        obs_seq_seqdim_first = torch.transpose(obs_seq, batch_dim, seq_dim)

        latent1_samples = []
        latent2_samples = []
        latent1_dists = []
        latent2_dists = []

        for t in range(seq_len + 1):
            if t == 0:
                # q(z1(0))
                # sample from normal dist
                # (observation is only used for batchsize and device)
                latent1_dist = self.latent1_init_posterior(skill)
                latent1_sample = latent1_dist.rsample()

                # q(z2(0) | z1(0))
                latent2_dist = self.latent2_init_posterior(latent1_sample)
                latent2_sample = latent2_dist.rsample()

            elif t < seq_len:
                # q(z1(t) | z2(t-1), obs(t-1))
                latent1_dist = self.latent1_posterior_step(
                    [latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # q(z2(t) | z1(t), z2(t-1), obs(t-1))
                latent2_dist = self.latent2_posterior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            elif t == seq_len:
                # q(z1(end) | skill, z2(end-1), obs(end-1))
                assert tensor_equality(obs_seq_seqdim_first[-1],
                                       obs_seq_seqdim_first[t-1])
                latent1_dist = self.latent1_posterior_end(
                    [skill,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # q(z2(t) | z1(t), z2(t-1), obs(t-1))
                latent2_dist = self.latent2_posterior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            else:
                raise ValueError

            latent1_samples.append(latent1_sample)
            latent2_samples.append(latent2_sample)
            latent1_dists.append(latent1_dist)
            latent2_dists.append(latent2_dist)

        latent1_samples_stacked = torch.stack(latent1_samples, dim=seq_dim)
        latent2_samples_stacked = torch.stack(latent2_samples, dim=seq_dim)

        return {
            'latent1_samples': latent1_samples_stacked,
            'latent2_samples': latent2_samples_stacked,
            'latent1_dists': latent1_dists,
            'latent2_dists': latent2_dists,
        }


from __future__ import absolute_import, division, print_function

import itertools

from ..components import Actor
from ..schedulers import _ActorRunner, ThreadedScheduler
from future.builtins import super

class Concat(Actor):

    _system_actor = True

    def __init__(self, n_ports, name='concat'):
        super().__init__(name=name)
        for i in range(n_ports):
            self.inports.append('in_{}'.format(i))
        self.outports.append('out')
        self._last_connected = -1

    def get_run_args(self):
        return (), {}

    def run(self, *args, **kwargs):
        res = tuple((port.pop() for port in self.inports))
        return {'out': res}

    def connect_input(self, port, i=None):
        """
        Connects i-th input. If i is None, the last value + 1 is used.

        Args:
            port (OutPort): output port to connect
            i (int): input port index
        """

        if i is None:
            i = self._last_connected + 1
        self._last_connected = i
        self.inports['in_{}'.format(i)].connect(port)

class MultiConcat(Actor):

    _system_actor = True

    def __init__(self, name='multiconcat'):
        super().__init__(name=name)
        self._port_names = []
        self._last_connected = -1

    def get_run_args(self):
        return (), {}

    def run(self, *args, **kwargs):
        res = {}
        for inport, outport in zip(self.inports, itertools.cycle(self.outports)):
            res.setdefault(outport.name, [])
            res[outport.name].append(inport.pop())
        return res

    def add_and_connect(self, actor):
        if self._last_connected > -1:
            assert self._port_names == [port.name for port in actor.outports]
        self._last_connected += 1
        i = self._last_connected
        for port in actor.outports:
            self.inports.append('{}_{}'.format(port.name, i))
            self.inports.at(-1).connect(port)
            if i == 0:
                self.outports.append(port.name)
                self._port_names.append(port.name)

class Map(Actor):
    """
    Maps a given actor on input token elements.

    The mapped actor is assumed to have a single input and a single output port.

    Args:
        actor (Actor): Actor class to be mapped to inputs
        args (list): positional arguments for actor.__init__
        kwargs (dict): keyword arguments for actor.__init__
        scheduler (Scheduler): scheduler to use, default is not to change the scheduler
        name (string): actor name

    Ports:
        in (iterable): contains items to be passed to the mapped actor
        out (tuple): items after applying the map actor
    """

    _system_actor = True

    def __init__(self, actor_class, args=(), kwargs={}, scheduler=None, name='map'):
        super().__init__(name=name)
        self.actor_class = actor_class
        self.actor_args = args
        self.actor_kwargs = kwargs
        self.map_scheduler = scheduler
        # get port names from an actor instance
        actor = self.actor_class(*self.actor_args, **self.actor_kwargs)
        for pname in actor.inports.keys():
            self.inports.append(pname)
        for pname in actor.outports.keys():
            self.outports.append(pname)

    def get_run_args(self):
        return (), {}

    def run(self, *args, **kwargs):
        # run is not a classfunction for Map
        # bacause it needs to change the workflow
        if self.map_scheduler is None:
            # self.schduler is set by the calling scheduler
            map_scheduler = self.scheduler.copy()
        elif isinstance(self.map_scheduler, (_ActorRunner, ThreadedScheduler)):
            # we need a copy of an existing scheduler
            map_scheduler = self.map_scheduler.copy()
        else:
            # in this case, we assume self.map_scheduler is a class
            map_scheduler = self.map_scheduler()
        # destinations = [port for port in self.outports['out'].connections]
        # disconnect the output port
        # for port in destinations:
        #     self.outports['out'].disconnect(port)
        # create the map actors, connect and put inputs
        map_actors = []
        concat_actor = MultiConcat()
        # items will iterate over all ports inputs, i.e. will contain n-th actor input
        # TODO ensure equal input lengths
        for items in zip(*(port.pop() for port in self.inports)):
            # get actor instance
            actor = self.actor_class(*self.actor_args, **self.actor_kwargs)
            map_actors.append(actor)
            concat_actor.add_and_connect(actor)
            # put the input data item
            for port, value in zip(actor.inports, items):
                # value is a single input item of the port
                map_scheduler.put_value(port, value)
        # run the mapping sub-workflows
        # map_workflow = concat_actor.get_workflow()
        # result = map_scheduler.run_workflow(map_workflow)
        map_scheduler.execute()

        result = {port.name: port.pop() for port in concat_actor.outports}
        return result

class PassWID(Actor):
    """
    Pass input argument with hostname and process ID attached
    """
    def __init__(self, name='passwid'):
        super().__init__(name=name)
        self.inports.append('inp')
        self.outports.append('out')

    def get_run_args(self):
        return (self.inports['inp'].pop(), ), {}

    @staticmethod
    def run(*args, **kwargs):
        from os import getpid
        from socket import gethostname
        return {'out':
                    {'inp': args[0],
                     'host': gethostname(),
                     'pid': getpid()}}


import numpy as np
import matplotlib.pyplot as plt

# importando do pacoto sklearn a função de Regresão Linear
from sklearn.linear_model import LinearRegression

# gerando valores com sklearn
from sklearn.datasets import make_regression

# Instanciando a classe de regressão linear
lr = LinearRegression()

# Criando os dados que irão popular o modelo
# função numpy.linspace()
# A função linspace() retorna números uniformemente espaçados em um intervalo especificado [start, stop].
# O ponto final do intervalo pode opcionalmente ser excluído.
# https://www.w3resource.com/numpy/array-creation/linspace.php
X = np.linspace(0, 9, 10).reshape(-1, 1)
Y = np.linspace(0, 27, 10).reshape(-1, 1)
print("X:\n", X)
print("Y:\n", Y)

modelo_linear = lr.fit(X, Y)

# Coeficiente e intercepto
print("Coeficiente:", modelo_linear.coef_)
print("Intercepto:", modelo_linear.intercept_)

# gerando o gráfico do modelo gerado

plt.plot(X, Y)
plt.grid(True)
plt.show()


import time
from picamera2 import Picamera2, Preview
import numpy as np
#from libcamera import controls
#from picamera2.encoders import H264Encoder
from gpiozero import LightSensor
# Set the video size and frame rate
main_size = (1280, 720)
lores_size = (320, 240)
framerate = 30
ldr = LightSensor(12)
import RPi.GPIO as GPIO
# Initialize the camera
picam2 = Picamera2()
#GPIO.setmode(GPIO.BCM)
#GPIO.setwarnings(False)
GPIO.setup(20, GPIO.OUT)
video_config = picam2.create_video_configuration(
    main={"size": main_size, "format": "RGB888"},
    lores={"size": lores_size, "format": "YUV420"})

config = picam2.create_preview_configuration(lores={"size": lores_size, "format": "YUV420"})
picam2.configure(config)

###picam2.configure(video_config)

#encoder = H264Encoder(1000000, framerate)
picam2.set_controls({"ExposureTime": 90000,
                     "AwbEnable": True, "Brightness": 0.0,
                     "Contrast": 1.0,})
                    # "AfMode": controls.AfModeEnum.Continuous})
#picam2.encoder = encoder
picam2.start_preview(Preview.QT)
picam2.start()

# Initialize variables
w, h = lores_size
prev = None
encoding = False
ltime = 0
sent_items = 0
maxium = 5
wt = 1
dif = 20
ex = 90000
#main loop
while True:
    if ldr.value == 0.0:
        ex = 90000
    else:
        ex = 3000
    picam2.set_controls({"ExposureTime": ex,
                     "AwbEnable": True, "Brightness": 0.0,
                     "Contrast": 1.0,})
    #stop if max reached
    if sent_items == maxium:
        GPIO.cleanup()
        break
    cur = picam2.capture_buffer("lores")
    cur = cur[:w*h].reshape(h, w)
    #check thesesomething to comper
    if prev is not None:
        # Measure pixel differences between current and previous frame
        mse = np.square(np.subtract(cur, prev)).mean()
        if mse > dif:
            if not encoding:
                encoding = True
                print(f"New Motion DETECTED VALUE:{mse}")
                GPIO.output(20, GPIO.HIGH)
            ltime = time.time()
        else:
            if encoding and time.time() - ltime > wt:
                encoding = False
                print(f'recording stoped RESET VALUE:{mse}')
                GPIO.output(20, GPIO.LOW)

    prev = cur

#time.sleep(10)
#picam2.capture_file('/home/leddy/%s.jpg' % timestamp)
#picam2.stop_preview()
#picam2.stop()


import music21 as m21
from typing import Optional
z
def create_isorhythm(
    pitches: m21.stream.Stream,
    durations: m21.stream.Stream,
    length: Optional[int] = None,
    color_offset=0,
    talea_offset=0,
) -> m21.stream.Stream:
    stream = m21.stream.Stream()

    color = [str(p.pitch) for p in pitches.flatten().notes]
    talea = [d.duration for d in durations.flatten().notes]

    if length is None:
        length = len(color) * len(talea)

    for i in range(length):
        stream.append(
            m21.note.Note(
                pitch=color[(i + color_offset) % len(color)],
                duration=talea[(i + talea_offset) % len(talea)],
            )
        )

    return stream


from datetime import datetime
from django.forms import ModelForm
from django.forms.widgets import *
from .models import Stock, Trade

# ========== STOCK
class StockForm(ModelForm):
    class Meta:
        model = Stock
        fields = '__all__'
        widgets = {
            'symbol': TextInput(attrs={'class': 'form-control'}),
            'name': TextInput(attrs={'class': 'form-control'}),
            'category': Select(attrs={'class': 'form-control'}),
            'link': TextInput(attrs={'placeholder': 'optional', 'class': 'form-control', 'type': 'url'}),
            'notes': TextInput(attrs={'placeholder': 'optional', 'class': 'form-control'})
        }

# ========== TRADE
class TradeForm(ModelForm):
    class Meta:
        model = Trade
        fields = '__all__'
        widgets = {
            'stock': Select(attrs={'class': 'form-control'}),
            'open': DateInput(attrs={'class': 'form-control', 'type': 'date'}),
            'quantity': NumberInput(attrs={'class': 'form-control'}),
            'open_cost': NumberInput(attrs={'class': 'form-control'}),
            'close': DateInput(attrs={'placeholder': 'optional', 'class': 'form-control', 'type': 'date'}),
            'close_cost': NumberInput(attrs={'class': 'form-control'}),
            'paper': CheckboxInput(attrs={'class': 'form-check-input'}),
        }

    # Logic for raising error if close_date < open_date
    def clean(self):
        cleaned_data = super().clean()
        open_date = cleaned_data.get('open')
        close_date = cleaned_data.get('close')
        close_cost = cleaned_data.get('close_cost')
        try:
            if close_date is not None:
                if close_date < open_date:
                    msg = 'Close date should be greater than open date.'
                    self._errors['close'] = self.error_class([msg])
                if close_cost is None:
                    msg = 'Must have a closing cost to close the trade.'
                    self._errors['close_cost'] = self.error_class([msg])
        except ValueError:
            print('Invalid Close date format')


import numpy as np
import cv2
import matplotlib.pyplot as plt
import sys
import csv

def load_image(path):
    return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)

def image_hsv(img):
    return cv2.cvtColor(img, cv2.COLOR_RGB2HSV)

def display_image(img, color=False):
    if color:
        plt.imshow(img)
    else:
        plt.imshow(img, 'gray')

def get_correct_results():
    results = []
    with open('ditto_count.csv', mode='r') as file:
        csv_reader = csv.reader(file)
        next(csv_reader)
        for row in csv_reader:
            image_name, score = row[0], int(row[1])
            results.append((image_name, score))
    return results

def get_mask_bounds():
    return np.array([120, 50, 86]), np.array([135, 214, 235])

def get_filtered_contours(contours):
    min_area = 3000
    max_area = 130000
    filtered_contours = []
    for contour in contours:
        area = cv2.contourArea(contour)
        if min_area < area < max_area:
            filtered_contours.append(contour)
    return filtered_contours

def get_ditto_count(img):
    hsv = image_hsv(img)

    lower_bound, upper_bound = get_mask_bounds()
    mask = cv2.inRange(hsv, lower_bound, upper_bound)
    # filter contours that fit a certain area
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    filtered_contours = get_filtered_contours(contours)

    total_dittos = len(filtered_contours)

    # displaying the image with red contours
    # img_cont = img.copy()
    # cv2.drawContours(img_cont, filtered_contours, -1, (255, 0, 0), 2)
    # display_image(img_cont)

    return total_dittos

if __name__ == '__main__':
    correct_results = get_correct_results()
    total_differences = []
    print('Picture Name - Correct Answer - My Answer')
    pictures_path = 'pictures/'
    if len(sys.argv) >= 2:
        pictures_path = sys.argv[1]

    for (file_name, value) in correct_results:
        image = load_image(pictures_path + file_name)
        # resizing the image so only the things in the notepad are being looked
        image = image[60:980, 284:900]

        count = get_ditto_count(image)
        total_differences.append(abs(count - int(value)))
        print(file_name + '-' + str(value) + '-' + str(count))

    mae = sum(total_differences) / len(total_differences)
    print('\nMean Absolute error(MAE) is: ' + str(mae))


import logging
import re
from urllib.parse import urlencode
from xml.etree import ElementTree as ET

import osc.core
import osc.oscerr

from ..errors import MissingSourceProjectError
from .assignment import Assignment
from .attribute import Attribute
from .comment import Comment
from .review import GroupReview
from .review import UserReview
from .xmlfactorymixin import XmlFactoryMixin

class Request(osc.core.Request, XmlFactoryMixin):
    """Wrapper around osc request object to add logic required by the
    qam-plugin.
    """

    STATE_NEW = "new"
    STATE_REVIEW = "review"
    STATE_DECLINED = "declined"
    OPEN_STATES = [STATE_NEW, STATE_REVIEW]
    REVIEW_USER = "BY_USER"
    REVIEW_GROUP = "BY_GROUP"
    REVIEW_OTHER = "BY_OTHER"
    COMPLETE_REQUEST_ID_SRE = re.compile(r"(open)?SUSE:Maintenance:\d+:(?P<req>\d+)")

    def __init__(self, remote):
        self.remote = remote
        super().__init__()
        self._comments = None
        self._groups = None
        self._packages = None
        self._assigned_roles = None
        self._priority = None
        self._reviews = []
        self._attributes = {}
        self._issues = []
        self._incident = None

    def active(self):
        return self.state == "new" or self.state == "review"

    @property
    def incident_priority(self):
        if not self._priority:
            self._priority = self.remote.priorities.for_request(self)
        return self._priority

    @property
    def incident(self):
        if not self._incident:
            self._incident = self.src_project.split(":")[-1]
        return self._incident

    @property
    def assigned_roles(self):
        if not self._assigned_roles:
            self._assigned_roles = Assignment.infer(self.remote, self)
        return self._assigned_roles

    @property
    def comments(self):
        if not self._comments:
            self._comments = self.remote.comments.for_request(self) or [Comment.none]
        return self._comments

    @property
    def maker(self):
        for history in self.statehistory:
            if history.description == "Request created":
                self._creator = self.remote.users.by_name(history.who)
                break
        else:
            self._creator = "Unknown"
        return self._creator

    @property
    def issues(self):
        """Bugs that should be fixed as part of this request"""
        if not self._issues:
            self._issues = self.remote.bugs.for_request(self)
        return self._issues

    @property
    def groups(self):
        # Maybe use a invalidating cache as a trade-off between current
        # information and slow response.
        return [
            review.reviewer
            for review in self.review_list()
            if isinstance(review, GroupReview)
        ]

    @property
    def packages(self):
        """Collects all packages of the actions that are part of the request."""
        if not self._packages:
            packages = set()
            for action in self.actions:
                pkg = action.src_package
                if pkg != "patchinfo":
                    packages.add(pkg)
            self._packages = packages
        return self._packages

    @property
    def src_project(self):
        """Will return the src_project or an empty string if no src_project
        can be found in the request.

        """
        for action in self.actions:
            if hasattr(action, "src_project"):
                prj = action.src_project
                if prj:
                    return prj
                else:
                    logging.info("This project has no source project: %s", self.reqid)
                    return ""
        return ""

    def attribute(self, attribute):
        """Load the specified attribute for this request.

        As requests right now can not contain attributes the attribute will be
        loaded from the corresponding source-project.
        """
        if attribute not in self._attributes:
            attributes = self.remote.projects.get_attribute(self.src_project, attribute)
            if len(attributes) == 1:
                attributes = attributes[0]
            self._attributes[attribute] = attributes
        return self._attributes[attribute]

    def review_action(self, params, user=None, group=None, comment=None):
        if not user and not group:
            raise AttributeError("group or user required for this action.")
        if user:
            params["by_user"] = user.login
        if group:
            params["by_group"] = group.name
        url_params = urlencode(params)
        url = "/".join([self.remote.requests.endpoint, self.reqid])
        url += "?" + url_params
        self.remote.post(url, comment)

    def review_assign(self, group, reviewer, comment=None):
        params = {"cmd": "assignreview", "reviewer": reviewer.login}
        self.review_action(params, group=group, comment=comment)

    def review_unassign(self, group, reviewer, comment=None):
        """Will undo the assignment by the group"""
        params = {"cmd": "assignreview", "revert": 1, "reviewer": reviewer.login}
        self.review_action(params, group=group, comment=comment)

    def review_accept(self, user=None, group=None, comment=None):
        comment = self._format_review_comment(comment)
        params = {"cmd": "changereviewstate", "newstate": "accepted"}
        self.review_action(params, user, group, comment)

    def review_add(self, user=None, group=None, comment=None):
        """Will add a new reviewrequest for the given user or group."""
        comment = self._format_review_comment(comment)
        params = {"cmd": "addreview"}
        self.review_action(params, user, group, comment)

    def review_decline(self, user=None, group=None, comment=None, reasons=None):
        """Will decline the reviewrequest for the given user or group.

        :param user: The user declining the request.

        :param group: The group the request should be declined for.

        :param comment: A comment that will be added to describe why the
            request was declined.

        :param reason: A L{oscqam.reject_reasons.RejectReason} that
            explains why the request was declined.
            The reason will be added as an attribute to the Maintenance
            incident.
        """
        if reasons:
            reason = self._build_reject_attribute(reasons)
            self.remote.projects.set_attribute(self.src_project, reason)
        comment = self._format_review_comment(comment)
        params = {"cmd": "changereviewstate", "newstate": "declined"}
        self.review_action(params, user, group, comment)

    def _build_reject_attribute(self, reasons):
        reject_reason = self.attribute(Attribute.reject_reason)
        reason_values = list(
            map(lambda reason: "{0}:{1}".format(self.reqid, reason.flag), reasons)
        )
        if not reject_reason:
            reject_reason = Attribute.preset(
                self.remote, Attribute.reject_reason, *reason_values
            )
        else:
            for r in reason_values:
                reject_reason.value.append(r)
        return reject_reason

    def review_reopen(self, user=None, group=None, comment=None):
        """Will reopen a reviewrequest for the given user or group."""
        params = {"cmd": "changereviewstate", "newstate": "new"}
        self.review_action(params, user, group, comment)

    def _format_review_comment(self, comment):
        if not comment:
            return None
        return "[oscqam] {comment}".format(comment=comment)

    def review_list(self):
        """Returns all reviews as a list."""
        if not self._reviews:
            for review in self.reviews:
                if review.by_group:
                    self._reviews.append(GroupReview(self.remote, review))
                elif review.by_user:
                    self._reviews.append(UserReview(self.remote, review))
        return self._reviews

    def review_list_open(self):
        """Return only open reviews."""
        return [r for r in self.review_list() if r.state in Request.OPEN_STATES]

    def review_list_accepted(self):
        return [r for r in self.review_list() if r.state.lower() == "accepted"]

    def add_comment(self, comment):
        """Adds a comment to this request."""
        endpoint = "/comments/request/{id}".format(id=self.reqid)
        self.remote.post(endpoint, comment)

    def get_template(self, template_factory):
        """Return the template associated with this request."""
        if not self.src_project:
            raise MissingSourceProjectError(self)
        return template_factory(self)

    @classmethod
    def filter_by_project(cls, request_substring, requests):
        return [r for r in requests if request_substring in r.src_project]

    @classmethod
    def parse(cls, remote, xml):
        et = ET.fromstring(xml)
        requests = []
        for request in et.iter(remote.requests.endpoint):
            try:
                req = Request(remote)
                req.read(request)
                requests.append(req)
            except osc.oscerr.APIError as e:
                logging.error(e.msg)
                pass
            except osc.oscerr.WrongArgs as e:
                # Temporary workaround, as OBS >= 2.7 can return requests with
                # acceptinfo-elements that old osc can not handle.
                if not (osc.core.get_osc_version() < "0.152"):
                    raise
                if "acceptinfo" not in str(e):
                    raise
                else:
                    logging.error("Server version too high for osc-client: %s" % str(e))
                    pass
        return requests

    @classmethod
    def parse_request_id(cls, request_id):
        """Extract the request_id from a string if required.

        The method will extract the request-id of a complete request string
        (e.g. SUSE:Maintenance:123:45678 has a request id of 45678) if
        needed.

        """
        reqid = cls.COMPLETE_REQUEST_ID_SRE.match(request_id)
        if reqid:
            return reqid.group("req")
        return request_id

    def __eq__(self, other):
        project = self.actions[0].src_project
        other_project = other.actions[0].src_project
        return self.reqid == other.reqid and project == other_project

    def __hash__(self):
        hash_parts = [self.reqid]
        if self.src_project:
            hash_parts.append(self.src_project)
        hashes = [hash(part) for part in hash_parts]
        return sum(hashes)

    def __str__(self):
        return "{0}".format(self.reqid)


'''
Created on 02-Dec-2015

@author: vijay
'''

from sqlalchemy import Column, DateTime, String, Integer, ForeignKey, func, \
    Column, Integer, String, Column, Integer, String, create_engine, create_engine
from sqlalchemy.ext.declarative import declarative_base, declarative_base
from sqlalchemy.orm import relationship, backref, sessionmaker
import os
import sys
import sqlalchemy
from src.dao.Author import Author
from src.dao.AuthorBookLink import AuthorBookLink
import json
from src.dao.Book import Base, Book
# from src.dao.Book import engine
import shutil
import traceback
# from src.static.constant import Workspace
import datetime
# from src.static.SessionUtil import SingletonSession

import logging

logger = logging.getLogger('extensive')

# if os.path.exists(Workspace().libraryPath):
#     os.chdir(Workspace().libraryPath)
#     listOfDir = os.listdir(Workspace().libraryPath)
#

class CreateDatabase():

    def __init__(self, libraryPath=None, databaseFileName='_opal.sqlite'):
        '''
        Creating database for library.
        '''
        logger.debug('CreateDatabase')
        databasePath = os.path.join(libraryPath, databaseFileName)
        self.libraryPath = libraryPath
        isDatabaseExist = os.path.exists(databasePath)

        databaseFilePath = f'sqlite:///{databasePath}'
        self.engine = create_engine(databaseFilePath , echo=True, connect_args={'check_same_thread': False})
        Session = sessionmaker(autoflush=True, autocommit=False, bind=self.engine)
        self.session = Session()

        os.makedirs(libraryPath, exist_ok=True)
        if not isDatabaseExist:
#             os.mkdir(libraryPath)
            self.creatingDatabase()
        os.chdir(libraryPath)

    def creatingDatabase(self):
        logger.debug('creatingDatabase')
        os.chdir(self.libraryPath)
        Base.metadata.drop_all(self.engine)
        Base.metadata.create_all(self.engine)
        logger.debug('database created blank')

    def addSingleBookData(self, dirName):
        '''
        Using this method you can update single book info into database.
        '''
        logger.debug('addSingleBookData')
        try:
            single = {}
            duplicate = {}
            os.chdir(self.libraryPath)
            duplicateBooks = list()
            addDatabase = True
            b = self.readJsonFile(dirName=dirName)
            if b:
                book = self.createBookFromJson(bookJson=b)
                book.bookPath = os.path.join(self.libraryPath , dirName)
                if book.isbn_13:
                    if not single.has_key(book.isbn_13):
                        single[book.isbn_13] = book

                    else:
                        duplicate[book.isbn_13] = book
                        addDatabase = False
                        duplicateBooks.append(duplicate)
                if addDatabase:
                    self.session.add(book)
                self.session.commit()
        except Exception as e:
            logger.error(e, exc_info=True)
            self.session.rollback();

    def addingData(self):
        logger.debug('addingData')
        directory_name = self.libraryPath
        os.chdir(directory_name)
        listOfDir = list()
#         listOfDir = [ name for name in os.listdir(directory_name) if os.path.isdir(os.path.join(directory_name, name)) ]
        for name in os.listdir(directory_name):
            if os.path.isdir(os.path.join(directory_name, name)) :
                try:
                    if int(name):
                        listOfDir.append(name)
                except Exception as e:
                    logger.error(e, exc_info=True)
        if listOfDir:
            listOfDir.sort(key=int)
        one = ''

        try:
            single = {}
            duplicate = {}
            self.duplicateBooks = list()
            for dirName in listOfDir:
                addDatabase = True
                b = self.readJsonFile(dirName=dirName)
                book = self.createBookFromJson(bookJson=b)
                book.bookPath = os.path.join(directory_name , dirName)
                if book.isbn_13:
                    if not single.has_key(book.isbn_13):
                        single[book.isbn_13] = book

                    else:
                        duplicate[book.isbn_13] = book
                        addDatabase = False
                        self.duplicateBooks.append(duplicate)
                if addDatabase:
                    self.session.add(book)
            self.session.commit()
            logger.debug('number of duplicateBooks: %s', len(self.duplicateBooks))

        except Exception as e:
            logger.error(e, exc_info=True)
            self.session.rollback();
        logger.debug('data loaded')

    def createBookFromJson(self, bookJson=None):
        logger.debug('createBookFromJson')
        book = Book()
        for k in bookJson:
            if not isinstance(bookJson[k], list):
                if k in ['publishedOn', 'createdOn']:
                    if bookJson[k]:
                        book.__dict__[k] = datetime.datetime.strptime(bookJson[k][0:19], "%Y-%m-%d %H:%M:%S")
                else:
                    book.__dict__[k] = bookJson[k]

            else:
                authorList = list()
                for a in bookJson[k]:
                    author = Author()
                    for aKey in a:
                        author.__dict__[aKey] = a[aKey]
                        authorList.append(author)
                book.authors = authorList
        return book

    def readJsonFile(self, dirName=None):
        logger.debug('readJsonFile')
#         print 'readJsonFile----->', os.path.join(Workspace().libraryPath, dirName , 'book.json')
        bookJsonFile = None
        b = None
        try:
            if os.path.exists(os.path.join(self.libraryPath, dirName , 'book.json')):
                bookJsonFile = open(os.path.join(self.libraryPath, dirName , 'book.json'), 'r')
                rep = ''
                for line in bookJsonFile:
                    rep = rep + line
                bookJsonFile.close
                try:
                    b = json.loads(rep)
                except Exception as e:
                    logger.error(e, exc_info=True)
            elif os.path.exists(os.path.join(self.libraryPath, dirName)):
                os.removedirs(os.path.join(self.libraryPath, dirName))
        except Exception as e:
            logger.debug(os.path.join(self.libraryPath, dirName))
            logger.error(e, exc_info=True)

#             print rep
        return b

    def saveAuthorBookLink(self, authorBookLink):
        logger.debug('saveAuthorBookLink')
        self.session.add(authorBookLink)
        self.session.commit()

    def saveBook(self, book):
        self.session.add(book)
        try:
            self.session.commit()
        except Exception as e:
            logger.error(e, exc_info=True)
            self.session.rollback()
            raise

    def countAllBooks(self):
        logger.debug('countAllBooks')
        bookCount = 0
        try:
            bookCount = self.session.query(Book).count()
        except Exception as e:
            logger.error(e, exc_info=True)
        return bookCount

    def findAllBook(self, pageSize=None, offset=0):
        logger.debug('findAllBook pageSize: %s', pageSize)
        bs, count = self.pagination(limit=pageSize, offset=offset)
        return bs, count

    def pagination(self, limit=0, offset=0):
        logger.debug('pagination limit : %s , offset: %s', limit, offset)
        if limit:
            countQuery = self.session.query(Book).count()
            query = self.session.query(Book).limit(limit).offset(offset)
        else:
            query = self.session.query(Book)
        bs = None
        try:
            bs = query.all()
        except Exception as e:
            logger.error(e, exc_info=True)
        return bs, countQuery

    def findBookByIsbn(self, isbn_13):
        logger.debug('findBookByIsbn : %s', isbn_13)
        bs = self.session.query(Book).filter(Book.isbn_13 == isbn_13).first()
        return bs

    def findBookByNextMaxId(self, bookId):
        logger.debug('findBookByNextMaxId bookId: %s', bookId)
        bs = self.session.query(Book).filter(Book.id > bookId).order_by(Book.id.asc()).first()
        return bs

    def findBookByPreviousMaxId(self, bookId):
        logger.debug('findBookByPreviousMaxId bookId: %s', bookId)
        bs = self.session.query(Book).filter(Book.id < bookId).order_by(Book.id.desc()).first()
        return bs

    def removeBook(self, book=None):
        '''
        This method removes entry from database.
        '''
        isBookDeleted = False
        try:
            if book:
                query = self.session.query(Book).filter(Book.id == book.id)
                session_book = query.first()
#                 print book
#                 book = books[0]

                author_id_lst = []
                for author in session_book.authors:
                    author_id_lst.append(author.id)
                    self.session.delete(author)

#                 query = self.session.query(AuthorBookLink).filter(AuthorBookLink.bookId == book.id)
#                 authorBookLinks = query.all()
                self.session.delete(session_book)
#                 for authorBook in authorBookLinks:
#                     self.session.delete(authorBook)

#                 query = self.session.query(Author).filter(Author.id.in_(author_id_lst))
#                 authors = query.all()
#                 for author in authors:
#                     self.session.delete(author)
#                 self.session.delete(book)
                self.session.commit()

#                 path = book.bookPath
#                 if path and os.path.exists(path):
#                     shutil.rmtree(path)
#                     print 'deleting path'
                isBookDeleted = True
        except Exception as e:
            logger.error(e, exc_info=True)
            self.session.flush()
            self.session.close()
            isBookDeleted = False

        return isBookDeleted

    def findByBookName(self, bookName=None, limit=50, offset=0):
        '''
        This method provide search of book name IGNORECASE .
        '''
        logger.debug('findByBookName')
        try:
            if bookName:
                countQuery = self.session.query(Book).filter(func.lower(Book.bookName) == func.lower(bookName)).count()
                query = self.session.query(Book).filter(func.lower(Book.bookName) == func.lower(bookName)).limit(limit).offset(offset)
#                 .order_by(Book.id.desc())
                books = query.all()
                return books, countQuery
        except Exception as e:
            logger.error(e, exc_info=True)

    def findBySimlarBookName(self, bookName=None, limit=50, offset=0):
        '''
        This method provide search of book name IGNORECASE and similar result like.
        '''
        logger.debug('findBySimlarBookName bookName: %s', bookName)
        try:
            if bookName:
                countQuery = self.session.query(Book).filter(Book.bookName.ilike('%' + bookName + '%')).count()
                query = self.session.query(Book).filter(Book.bookName.ilike('%' + bookName + '%')).limit(limit).offset(offset)
#                 .order_by(Book.id.desc())
                books = query.all()
                return books, countQuery
        except Exception as e:
            logger.error(e, exc_info=True)

    def findByIsbn_13Name(self, isbn_13=None):
        logger.debug('findBySimlarBookName isbn_13: %s', isbn_13)
        if isbn_13:
            query = self.session.query(Book).filter(Book.isbn_13.ilike('%' + isbn_13 + '%'))
            books = query.all()
            return books

    def findDuplicateBook(self):
        logger.debug('findDuplicateBook ')
        books = self.session.query(Book).group_by(Book.isbn_13).having(func.count(Book.isbn_13) > 1).order_by(Book.isbn_13.desc())
        return books

    def findBookByFileName(self, bookFileName):
        logger.debug('findBySimlarBookName bookFileName: %s', bookFileName)
        if bookFileName:
            query = self.session.query(Book).filter(Book.bookFileName.ilike('%' + bookFileName + '%'))
            books = query.all()
            return books

    def findBook(self, book=None):
        '''
        This method will find the book in database . It will return true.If book present.

        '''
        logger.debug('findBook')
        books = None
        if book.isbn_13:
            query = self.session.query(Book).filter(Book.isbn_13.ilike('%' + book.isbn_13 + '%'))
            books = query.all()
        if book.bookName:
            query = self.session.query(Book).filter(Book.bookName.ilike('%' + book.bookName + '%'))
            books = query.all()
        return books

    def getMaxBookID(self):
        '''
        getting max bookId from either of database or library folder.
        '''
        sql = f'select max(substr(book_path,{ len(self.libraryPath) + 2}), id) from book order by id desc'
        logger.debug(sql)
        logger.debug(f'getMaxBookID sql: {sql} ')
        maxBookId = self.session.execute(sql).first()
        if maxBookId == None:
            maxBookId = [0]
        logger.debug(f'maxBookId: {maxBookId[0]}')
        return int(maxBookId[0])

if __name__ == '__main__':
#     session = CreateDatabase().creatingDatabase()
#     CreateDatabase().addingData()

#     books = CreateDatabase().findByBookName("java")
    libraryPath = r'/docs/new/library'
    if not os.path.exists(libraryPath):
        print('no workspace')

    try:
        createdb = CreateDatabase(libraryPath=libraryPath)
#         createdb.creatingDatabase()
#         createdb.addingData()
        x = createdb.getMaxBookID()
        page = createdb.pagination(10, 10)
        logger.debug(page)
#         createdb.findAllBook()
    except Exception as e:
        print(e)
#     for b in books:
#         print b.isbn_13, b.id

#         createdb.removeBook(b)

    pass