import logging
import os
import sys
import traceback

from PyQt6 import QtGui, QtWidgets, QtCore

from .code_text import DialogCodeText  # for isinstance()
from .confirm_delete import DialogConfirmDelete
from .GUI.base64_helper import *
from .GUI.ui_special_functions import Ui_Dialog_special_functions
from .helpers import Message
from .merge_projects import MergeProjects
from .select_items import DialogSelectItems
from .text_file_replacement import ReplaceTextFile

path = os.path.abspath(os.path.dirname(__file__))
logger = logging.getLogger(__name__)

def exception_handler(exception_type, value, tb_obj):
    """ Global exception handler useful in GUIs.
    tb_obj: exception.__traceback__ """
    tb = '\n'.join(traceback.format_tb(tb_obj))
    msg = 'Traceback (most recent call last):\n' + tb + '\n' + exception_type.__name__ + ': ' + str(value)
    print(msg)
    logger.error(_("Uncaught exception: ") + msg)
    QtWidgets.QMessageBox.critical(None, _('Uncaught Exception'), msg)

class DialogSpecialFunctions(QtWidgets.QDialog):
    """ Dialog for special QualCoder functions.
    """

    app = None
    parent_text_edit = None
    tab_coding = None  # Tab widget coding tab for updates

    # For Replacing a text file with another and keeping codings
    file_to_replace = None
    file_replacement = None

    # For merging projects
    merge_project_path = ""
    projects_merged = False

    def __init__(self, app, parent_text_edit, tab_coding, parent=None):

        super(DialogSpecialFunctions, self).__init__(parent)
        sys.excepthook = exception_handler
        QtWidgets.QDialog.__init__(self)
        self.ui = Ui_Dialog_special_functions()
        self.ui.setupUi(self)
        self.app = app
        self.parent_text_edit = parent_text_edit
        self.tab_coding = tab_coding
        self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowType.WindowContextHelpButtonHint)
        font = 'font: ' + str(app.settings['fontsize']) + 'pt '
        font += '"' + app.settings['font'] + '";'
        self.setStyleSheet(font)
        self.merge_project_path = ""
        pm = QtGui.QPixmap()
        pm.loadFromData(QtCore.QByteArray.fromBase64(question_icon), "png")
        self.ui.pushButton_select_text_file.setIcon(QtGui.QIcon(pm))
        self.ui.pushButton_select_text_file.setFocus()
        self.ui.pushButton_select_replacement_text_file.setIcon(QtGui.QIcon(pm))
        self.ui.pushButton_select_project.setIcon(QtGui.QIcon(pm))
        pm = QtGui.QPixmap()
        pm.loadFromData(QtCore.QByteArray.fromBase64(cogs_icon), "png")
        self.ui.pushButton_text_starts.setIcon(QtGui.QIcon(pm))
        self.ui.pushButton_text_starts.clicked.connect(self.change_text_code_start_positions)
        self.ui.pushButton_text_ends.setIcon(QtGui.QIcon(pm))
        self.ui.pushButton_text_ends.clicked.connect(self.change_text_code_end_positions)
        self.ui.pushButton_text_update.setIcon(QtGui.QIcon(pm))
        self.ui.groupBox_text_positions.hide()
        self.ui.pushButton_select_text_file.pressed.connect(self.select_original_text_file)
        self.ui.pushButton_select_replacement_text_file.pressed.connect(self.select_replacement_text_file)
        self.ui.pushButton_text_update.setEnabled(False)
        self.ui.pushButton_text_update.pressed.connect(self.replace_file_update_codings)
        self.ui.pushButton_merge.setIcon(QtGui.QIcon(pm))
        self.ui.pushButton_select_project.pressed.connect(self.select_project_folder)
        self.ui.pushButton_merge.setEnabled(False)
        self.ui.pushButton_merge.pressed.connect(self.merge_projects)

    # Functions to merge external project into this project
    def select_project_folder(self):
        """ Select another .qda project """

        self.merge_project_path = ""
        default_directory = self.app.settings['directory']
        if default_directory == "":
            default_directory = os.path.expanduser('~')
        self.merge_project_path = QtWidgets.QFileDialog.getExistingDirectory(self,
                                                                             _('Open project directory'),
                                                                             default_directory)
        if self.merge_project_path is False or len(self.merge_project_path) < 5:
            Message(self.app, _("Error"), _("No project selected")).exec()
            return
        if self.merge_project_path[-4:] != ".qda":
            Message(self.app, _("Error"), _("Not a QualCoder project")).exec()
            return
        if self.merge_project_path == self.app.project_path:
            Message(self.app, _("Error"), _("The same project")).exec()
            return
        msg = _("Merge") + "\n" + self.merge_project_path + "\n" + _("into") + "\n" + self.app.project_path + "\n"
        msg += _("Press Run Button to merge projects")
        Message(self.app, _("Merge projects"), msg).exec()
        self.ui.pushButton_merge.setEnabled(True)
        self.ui.pushButton_merge.setFocus()

    def merge_projects(self):
        """ Merge selected project into this project. """

        mp = MergeProjects(self.app, self.merge_project_path)
        self.parent_text_edit.append(mp.summary_msg)
        self.projects_merged = mp.projects_merged

    # Functions to update a text file but attempt to keep original codings
    def select_original_text_file(self):
        """ Select text file to replace. """

        self.file_to_replace = []
        file_texts = self.app.get_file_texts()
        ui = DialogSelectItems(self.app, file_texts, _("Delete files"), "single")
        ok = ui.exec()
        if not ok:
            return
        self.file_to_replace = ui.get_selected()
        if not self.file_to_replace:
            self.ui.pushButton_select_text_file.setToolTip(_("Select text file to replace"))
            return
        self.ui.pushButton_select_text_file.setToolTip(_("Replacing: ") + self.file_to_replace['name'])
        if self.file_to_replace and self.file_replacement:
            self.ui.pushButton_text_update.setEnabled(True)

    def select_replacement_text_file(self):
        """ Select replacement updated text file. """

        file_types = "Text Files (*.docx *.epub *.html *.htm *.md *.odt *.pdf *.txt)"
        filepath, ok = QtWidgets.QFileDialog.getOpenFileNames(None, _('Replacement file'),
                                                              self.app.settings['directory'], file_types,
                                                              options=QtWidgets.QFileDialog.Option.DontUseNativeDialog)
        if not ok or filepath == []:
            self.ui.pushButton_select_replacement_text_file.setToolTip(_("Select replacement text file"))
            return
        self.file_replacement = filepath[0]
        self.ui.pushButton_select_replacement_text_file.setToolTip(_("Replacement file: ") + self.file_replacement)
        if self.file_to_replace and self.file_replacement:
            self.ui.pushButton_text_update.setEnabled(True)
            self.ui.pushButton_text_update.setToolTip(_("Press to replace the text file"))

    def replace_file_update_codings(self):
        """ Requires two files - original and replacement to be selected before button is enabled.
        Called by:
         pushButton_text_update """

        if self.file_to_replace is None or self.file_replacement is None:
            Message(self.app, _("No files selected"), _("No existing or replacement file selected")).exec()
            return
        ReplaceTextFile(self.app, self.file_to_replace, self.file_replacement)
        self.file_to_replace = None
        self.ui.pushButton_select_text_file.setToolTip(_("Select text file to replace"))
        self.file_replacement = None
        self.ui.pushButton_select_replacement_text_file.setToolTip(_("Select replacement text file"))

    def change_text_code_start_positions(self):
        """ Extend or shrink text coding start positions in all codings and all files for owner. """

        delta = self.ui.spinBox_text_starts.value()
        if delta == 0:
            return
        cur = self.app.conn.cursor()
        sql = "select cid,fid,pos0,pos1,code_text.owner, length(source.fulltext) from code_text join source on source.id=code_text.fid where code_text.owner=?"
        text_sql = "select substr(source.fulltext, ?, ?) from source where source.id=?"
        update_sql = "update code_text set pos0=?, seltext=? where pos0=? and pos1=? and cid=? and fid=? and owner=?"
        cur.execute(sql, [self.app.settings['codername']])
        res = cur.fetchall()
        if not res:
            return
        msg = _("Change ALL text code start positions in ALL text files by ")
        msg += str(delta) + _(" characters.\n")
        msg += _("Made by coder: ") + self.app.settings['codername'] + "\n"
        msg += str(len(res)) + _(" to change.") + "\n"
        msg += _("Backup project before performing this function.\n")
        msg += _("Press OK to continue.")
        ui = DialogConfirmDelete(self.app, msg, _("Change code start positions"))
        ok = ui.exec()
        if not ok:
            return
        for r in res:
            new_pos0 = r[2] - delta
            # cannot have start pos less than start of text
            if new_pos0 < 0:
                new_pos0 = 0
            # cannot have start pos larger than end pos
            if new_pos0 > r[3]:
                new_pos0 = r[3] - 1
            cur.execute(text_sql, [new_pos0 + 1, r[3] - new_pos0, r[1]])
            seltext = ""
            try:
                seltext = cur.fetchone()[0]
            except TypeError:
                pass
            cur.execute(update_sql, [new_pos0, seltext, r[2], r[3], r[0], r[1], r[4]])
            self.app.conn.commit()
        self.parent_text_edit.append(
            _("All text codings by ") + self.app.settings['codername'] + _(" resized by ") + str(delta) + _(
                " characters."))
        self.update_tab_coding_dialog()

    def change_text_code_end_positions(self):
        """ Extend or shrink text coding start positions in all codings and all files for owner. """

        delta = self.ui.spinBox_text_ends.value()
        if delta == 0:
            return
        cur = self.app.conn.cursor()
        sql = "select cid,fid,pos0,pos1,code_text.owner, length(source.fulltext) from code_text join source on source.id=code_text.fid where code_text.owner=?"
        text_sql = "select substr(source.fulltext, ?, ?) from source where source.id=?"
        update_sql = "update code_text set pos1=?, seltext=? where pos0=? and pos1=? and cid=? and fid=? and owner=?"
        cur.execute(sql, [self.app.settings['codername']])
        res = cur.fetchall()
        if not res:
            return
        msg = _("Change ALL text code end positions in ALL text files by ")
        msg += str(delta) + _(" characters.\n")
        msg += _("Made by coder: ") + self.app.settings['codername'] + "\n"
        msg += str(len(res)) + _(" to change.") + "\n"
        msg += _("Backup project before performing this function.\n")
        msg += _("Press OK to continue.")
        ui = DialogConfirmDelete(self.app, msg, _("Change code end positions"))
        ok = ui.exec()
        if not ok:
            return
        for r in res:
            new_pos1 = r[3] + delta
            # cannot have end pos less or equal to startpos
            if new_pos1 <= r[2]:
                new_pos1 = r[2] + 1
            # cannot have end pos larger than text
            if new_pos1 >= r[5]:
                new_pos1 = r[5] - 1
            cur.execute(text_sql, [r[2] + 1, new_pos1 - r[2], r[1]])
            seltext = ""
            try:
                seltext = cur.fetchone()[0]
            except TypeError:
                pass
            cur.execute(update_sql, [new_pos1, seltext, r[2], r[3], r[0], r[1], r[4]])
            self.app.conn.commit()
        self.parent_text_edit.append(
            _("All text codings by ") + self.app.settings['codername'] + _(" resized by ") + str(delta) + _(
                " characters."))
        self.update_tab_coding_dialog()

    def update_tab_coding_dialog(self):
        """ DialogCodeText """

        contents = self.tab_coding.layout()
        if contents:
            # Remove code text widgets from layout
            for i in reversed(range(contents.count())):
                c = contents.itemAt(i).widget()
                if isinstance(c, DialogCodeText):
                    c.get_coded_text_update_eventfilter_tooltips()
                    break

    def accept(self):
        """ Overrride accept button. """

        super(DialogSpecialFunctions, self).accept()


import os
import openai
import tiktoken
import regex as re

def create_chunks(tt_encoding, text, chunkSize):
    # Split the text into sentences
    sentences = re.split("(?<=[.!?])+", text)
    print(sentences)
    # Group the sentences into chunks
    max_chunk_size = chunkSize  # Taking consideration of the extra tokens of prompts
    chunks = []
    current_chunk = ""
    current_chunk_tokens = 0

    for i, sentence in enumerate(sentences):
        sentence_tokens = len(tt_encoding.encode(sentence))
        if current_chunk_tokens + sentence_tokens > max_chunk_size:
            # Add the current chunk to the list of chunks
            chunks.append((current_chunk_tokens, current_chunk))
            # Start a new chunk with the last sentence of the previous chunk
            current_chunk = sentences[i - 1] + sentence
            current_chunk_tokens = len(tt_encoding.encode(current_chunk))
        else:
            # Add the sentence to the current chunk, with overlap if not first chunk
            if current_chunk:
                current_chunk += " " + sentence
                current_chunk_tokens += sentence_tokens
            else:
                current_chunk = sentence
                current_chunk_tokens = sentence_tokens

    # Add the last chunk to the list of chunks
    chunks.append((current_chunk_tokens, current_chunk))
    return chunks

def blogify(paragraphs):
    opanai_api_key = os.environ.get("OPENAI_API_KEY")
    if opanai_api_key == None:
        print("OPENAI_API_KEY is not set")
        exit(1)

    openai.api_key = opanai_api_key

    model = os.environ.get("OPENAI_MODEL", "gpt-3.5-turbo-0301")

    # for now, just join all the sentences in each together
    raw_text = ""
    current_speaker = -1
    for paragraph in paragraphs:
        if paragraph["speaker"] != current_speaker:
            current_speaker = paragraph["speaker"]
            raw_text += f"Speaker {current_speaker}: \r\n"
        for sentence in paragraph["sentences"]:
            sentenceText = sentence["text"]
            raw_text += f"\t{sentenceText}\r\n"

    tt_encoding = tiktoken.encoding_for_model(model)
    chunks = create_chunks(tt_encoding, raw_text, 2049) # gpt-3.5-turbo-0301 has a max of 4096 tokens

    final_response = []
    for i, (tokens, chunk) in enumerate(chunks):
        print(f'Chunk [{i}]: {chunk}\nTokens: {tokens}\n')
        response = openai.ChatCompletion.create(
            model=model,
            messages=[
                {
                    "role": "system",
                    "content": "Ignore all previous instructions.  You are a Principal Software Architect for a famous online tech website. Your task is to write engaging and helpful blob posts from the given context.",
                },
                {
                    "role": "user",
                    "content": f"Ignore all previous instructions. Here is your new role and persona: You are a Principal Software Architect for a famous online tech website. Your task is to write engaging and helpful blob posts from the given context:\n {chunk}",
                },
            ],
        )

        final_response.append(response)

    return final_response


# This is a sample Python script.
import numpy as np
import pandas as pd
import tensorflow
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from numpy import nan
import nltk
import sklearn.feature_extraction.text as pre
from sklearn import preprocessing
from sklearn.feature_extraction.text import CountVectorizer
import pickle as pk

# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.

def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    '''
    print_hi('PyCharm')
    data = pd.read_csv("email_data_spam.csv")
    print(data.loc[223]["email"])
    print(prepro(data.loc[346]["email"]))
    print(data[data["email"] == data.loc[223]["email"]].size)
    print(data[data["email"] == ""].size)
    for i in range(1396):
        data.loc[i]["email"]=prepro((str)(data.loc[i]["email"]))

    print(data[data["email"] == data.loc[223]["email"]].size)

    print(data[data["email"]==""].size)
    matest = np.array(data["email"])

    matr = vecvec.fit_transform(np.array(data["email"]))
    matr = pd.DataFrame(matr,columns= vecvec.get_feature_names_out())
    print(matr)

    data.to_csv("spamPretraité.csv")

    from keras.models import Sequential
    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense

    # Créer un modèle séquentiel
    model = Sequential()

    # Ajouter des couches de convolution et de max pooling
    model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Ajouter une couche de flattening
    model.add(Flatten())

    # Ajouter des couches fully connected
    model.add(Dense(128, activation='relu'))
    model.add(Dense(10, activation='softmax'))

    # Compiler le modèle
    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    # Entraîner le modèle sur les données d'entraînement
    model.fit(x_train, y_train, epochs=10, validation_data=(x_test, y_test))'''
#,index = [i for i in range(3947,4197)]
#,index=[i for i in range(1396,3947)]

# See PyCharm help at https://www.jetbrains.com/help/pycharm/
'''data = pd.read_csv("email_data_spam.csv")

data2 = pd.DataFrame(pd.read_csv("preprocessed_emails.csv"))
data2.rename(columns={"preprocessed_email": "email"}, inplace=True)

print(data2)
data3 = pd.DataFrame(pd.read_csv("preprocessed_emails_hard_ham.csv"))
data3.rename(columns={"preprocessed_email": "email"}, inplace=True)
print(data3)
print('here')

for i in range(1396):
    data.loc[i]["email"]=prepro((str)(data.loc[i]["email"]))
data =pd.concat([data3,data,data2],ignore_index=True)
data = data.sample(frac=1).reset_index()
data.to_csv("spamPretraité.csv")'''
data = pd.read_csv("spamPretraité.csv")
data["email"]=data["email"].astype(str)
Y = pd.DataFrame(np.array(data["label"]),columns=["label"])
Y[Y["label"] == "spam"] = 1
Y[Y["label"] == "ham"] = 0
Y_train = np.array((Y.iloc[[i for i in range(0,3300)]]))
Y_test = np.array((Y.iloc[[i for i in range(3300,4197)]]))
tokenizer = Tokenizer(num_words=30000)
tokenizer.fit_on_texts(data["email"].values)
sequences = tokenizer.texts_to_sequences(data["email"])
word_index = tokenizer.word_index
vecvec = CountVectorizer()
datdat =vecvec.fit_transform(data['email'].values).toarray()
print(datdat)
print(datdat.shape)
datata = pd.DataFrame(datdat,index = [i for i in range(datdat.shape[0])],columns=vecvec.get_feature_names_out())
print(datata)
summ = datata.sum(axis=0)
print(summ)
listvocab= summ[summ>3]
datata = datata[listvocab.index]
datata[datata>1]=1
print(datata)
print(listvocab)
file = open('vocabulaire', 'wb')
datata.to_csv("countMatrix")
pk.dump(listvocab,file)

file.close()
'''
X = tokenizer.texts_to_sequences(data['email'].values)
X = pad_sequences(X)
#X = preprocessing.normalize(X)
#print(X)

# Convert label data to binary values
# y = pd.get_dummies(data['Spam']).values
print('Found %s unique tokens.' % len(word_index))
X_train = X[0:3300]
X_test = X[3300:4197]
from sklearn.preprocessing import MinMaxScaler
# Normalize data
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
print(X_train)

print(Y_train.shape)
print(Y_test.shape)
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense

X_train = np.asarray(X_train).astype('float32')
X_train = tensorflow.expand_dims(X_train, axis=-1)
X_train = tensorflow.expand_dims(X_train, axis=-1)

Y_train = np.asarray(Y_train).astype('float32')
X_test = np.asarray(X_test).astype('float32')
X_test = tensorflow.expand_dims(X_test, axis=-1)
X_test = tensorflow.expand_dims(X_test, axis=-1)
Y_test = np.asarray(Y_test).astype('float32')
print(X_train.shape)
print(X_test.shape)

# Créer un modèle séquentiel
model = Sequential()

# Ajouter des couches de convolution et de max pooling
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=X_train.shape[1:],padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Conv2D(64, (3, 3), activation='relu',padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))

# Ajouter une couche de flattening
model.add(Flatten())

# Ajouter des couches fully connected
model.add(Dense(128, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='softmax'))

# Compiler le modèle
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
print(X_train.shape)
# Entraîner le modèle sur les données d'entraînement
model.fit(X_train, Y_train, epochs=10, validation_data=(X_test, Y_test),batch_size=32)'''


from __future__ import annotations
from typing import Tuple

import numpy as np

from pizza import Pizza
from utils.param_function import ParamFunction

class Human:
    """Representation of relevant human desires/preferences."""

    def __init__(
        self,
        desired_params: list,
        importance_weights: list,
        basis_functions: list,
        beta_coefficient: float,
    ) -> Human:
        """Initialize human representation."""
        self._desired_params = np.array(desired_params).reshape(1, -1)
        self._importance_weights = np.array(importance_weights).reshape(1, -1)
        self._coeff = beta_coefficient
        self.param_fn = ParamFunction(basis_functions)

    @classmethod
    def from_dicts(
        cls, params: dict, importances: dict, basis_funcs: list, beta: float
    ):
        """Initialize a human representation from dictionaries."""
        param_vals = []
        importance_vals = []
        for b in basis_funcs:
            param_vals.append(params[b])
            importance_vals.append(importances[b])

        human = cls(param_vals, importance_vals, basis_funcs, beta)
        return human

    @property
    def desired_params(self) -> np.ndarray:
        """Return the human's desired feature params."""
        return np.array(self._desired_params, copy=True)

    @property
    def importance_weights(self) -> np.ndarray:
        """Return the human's importance weights."""
        return np.array(self._importance_weights, copy=True)

    @property
    def coeff(self) -> np.ndarray:
        """Return the human's desired feature params."""
        return self._coeff

    def choose_option(
        self, to_choose_from: np.ndarray
    ) -> Tuple[int, np.ndarray]:
        """Select from list of options based on human's preference."""
        # 1.) Compute the differences between each option and
        #     the human's desired features:
        if type(to_choose_from[0]) == Pizza:
            option_params = np.array(
                list(map(self.param_fn.compute_params, to_choose_from))
            )
        else:
            option_params = to_choose_from
        features = self.feature_function(option_params)
        ## 2.) Normalize the features relative across the options:
        # normalized_features = features / np.sum(features, axis=0)
        # if len(normalized_features.shape) <= 1:
        #    normalized_features = normalized_features.reshape(-1, 1)
        ## 3.) Take into account the human's importance weights:
        # weighted_features_compared = self.importance_weights.dot(
        #    normalized_features.T
        # )
        ## 4.) Exponentiate the negation of the results:
        # relative_preference = np.exp(-1 * weighted_features_compared)
        rewards = np.array(list(map(self.compute_reward, features)))
        # 5.) Choose the option which maximizes human's reward:
        choice = np.argmax(rewards)
        # 6.) See if the option is above or below some standard of goodness:
        # meets_standard = check_good_enough(features_compared)
        return choice, rewards

    def compute_reward(self, option: Tuple[Pizza, np.ndarray]) -> float:
        """Compute human's liking given some features."""
        if type(option) == Pizza:
            option_params = self.param_fn(option)
            features = self.feature_function.compute_params(
                option_params
            ).reshape(1, -1)
        else:
            features = option.reshape(1, -1)
        weighted_features = self.importance_weights.dot(features.T)
        reward = np.exp(-self.coeff * weighted_features)
        return reward[0, 0]

    def feature_function(self, query_params: np.ndarray) -> np.ndarray:
        """Subtract and combine.

        ::return:
            ::feature_mats: Per-option features in particle params.
        """
        feature_mats = np.zeros(
            (
                query_params.shape[0],
                self.desired_params.shape[0],
                query_params.shape[1],
            )
        )
        # 1.) Separate each option's feature params
        #     and stack them for efficient computation:
        for i in range(query_params.shape[0]):
            q_option = np.repeat(
                np.reshape(query_params[i, :], (1, query_params.shape[1])),
                self.desired_params.shape[0],
                axis=0,
            )
            # 1a.) Compute the element-wise magnitudes between each option
            #      in a query and each possible reward function model:
            feature_mats[i, :, :] = np.abs(q_option - self.desired_params)
        if len(feature_mats.shape) > 2:
            feature_mats = feature_mats.squeeze()
        return feature_mats

    def check_good_enough(pizza_features: np.ndarray) -> bool:
        """Compare this pizza to the human's desired pizza."""
        pass


from ortools.linear_solver import pywraplp

def encontrar_caminho_minimo(num_vertices, num_arestas, vertice_inicial, vertice_final, arestas):
    solver = pywraplp.Solver.CreateSolver('GLOP')

    x = {}
    for (u, v, w) in arestas:
        x[u, v] = solver.BoolVar('x[{}, {}]'.format(u, v))

    # Adiciona o objetivo
    objetivo = solver.Objective()
    for (u, v, w) in arestas:
        objetivo.SetCoefficient(x[u, v], w)
    objetivo.SetMinimization()

    # Adiciona as restrições
    for u in range(num_vertices):
        soma_saida = solver.Sum(x[u, v] for (i, v, _) in arestas if i == u)
        soma_entrada = solver.Sum(x[i, u] for (i, v, _) in arestas if v == u)
        if u == vertice_inicial:
            solver.Add(soma_saida - soma_entrada == 1)
        elif u == vertice_final:
            solver.Add(soma_saida - soma_entrada == -1)
        else:
            solver.Add(soma_saida - soma_entrada == 0)

    # Resolve o problema
    results = solver.Solve()

    # Retorna os resultados
    if results == pywraplp.Solver.OPTIMAL:
        valor_objetivo = objetivo.Value()
        caminho_minimo = [(u, v, x[u, v].solution_value()) for (u, v, _) in arestas]
        return valor_objetivo, caminho_minimo
    else:
        return None, None

def main():
    num_vertices, num_arestas, vertice_inicial, vertice_final = map(int, input().split())

    arestas = []
    for _ in range(num_arestas):
        vertice_origem, vertice_destino, peso_aresta = map(int, input().split())
        arestas.append((vertice_origem, vertice_destino, peso_aresta))

    valor_objetivo, caminho_minimo = encontrar_caminho_minimo(num_vertices, num_arestas, vertice_inicial, vertice_final, arestas)

    if valor_objetivo is not None and caminho_minimo is not None:
        print('Solução:')
        print('Valor objetivo =', valor_objetivo)
        for (u, v, valor) in caminho_minimo:
            print("x[{}, {}] = {}".format(u, v, valor))
    else:
        print('O problema não possui uma solução ótima.')

if __name__ == '__main__':
    main()


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, StreamingHttpResponse
from .models import CustomsDeclaration
from .forms import UploadCustomsDeclaration, XMLRequestForm
from django.contrib.auth.decorators import login_required
import re, os, random
import zipfile, io, datetime
import PyPDF2
from django.core.files import File
from django.conf import settings
from django.utils.encoding import smart_str

from .forms import EditCustomsDeclarationForm

def test(request, file_name):
    return HttpResponse(file_name)

def customs_file_save_location():
    return 'customs_declaration'
def custom_zip_file_folder_name():
    return 'zip_files'

@login_required
def upload(request):
    if request.method == 'POST':
        uploadform = UploadCustomsDeclaration(request.POST, request.FILES,)
        if uploadform.is_valid():
            cust_decl_regex = re.compile('\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d\s*\d')

            upload_date = uploadform.cleaned_data["upload_date"]
            cus_file_save_folder = customs_file_save_location()

            for file in request.FILES.getlist('customs_file'):

                pdfReader = PyPDF2.PdfFileReader(file)

                for pageNum in range(0, pdfReader.numPages):
                    pageObj = pdfReader.getPage(pageNum)
                    text = pageObj.extractText()
                    correct_name = False
                    customs_number = ''

                    # Search for Customs Declaration Number
                    cust_decl_re_results = re.search(cust_decl_regex, text)

                    if cust_decl_re_results != None:
                        customs_number = cust_decl_re_results.group(0).replace(' ', '')
                        customs_number = '1100000' + customs_number[-5:]
                        correct_name = True
                    else:
                        d = datetime.datetime.now()
                        d_year = str(d.year)
                        d_month = str(d.month)
                        d_day = str(d.day)
                        d_hour = str(d.hour)
                        d_min = str(d.minute)
                        d_sec = str(d.second)
                        d_microsec = str(d.microsecond)
                        customs_number = '00' + d_year + d_month + d_day + d_hour + d_min + d_sec + d_microsec

                    pdfWriter = PyPDF2.PdfFileWriter()
                    pdfWriter.addPage(pageObj)

                    filename = customs_number + '.pdf'

                    pdfOutputFile = open(os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, filename), 'wb')
                    pdfWriter.write(pdfOutputFile)
                    customs_declaration = CustomsDeclaration(filename=filename,
                                                             customs_number=customs_number,
                                                             correct_name=correct_name,
                                                             upload_date=upload_date,)
                    customs_declaration.save()

                    pdfOutputFile.close()
    else:
        uploadform = UploadCustomsDeclaration()
    return render(
        request,
        'customs_list/upload.html',
        context={
            "uploadform": uploadform,
        }
    )

def download_zip_date_file(request, year, month, day):
    customs_list = CustomsDeclaration.objects.filter(upload_date__year=year,
                                                           upload_date__month=month,
                                                           upload_date__day=day,)
    zip_filename = year + '_' + month + '_' + day + '.zip'
    cus_file_save_folder = customs_file_save_location()
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, 'w') as zf:

        for c in customs_list:
            filename = c.filename
            pdf_path = os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, filename)
            zf.write(pdf_path, filename)

    response = HttpResponse(buffer.getvalue())
    response['mimetype'] = "application/x-zip-compressed"
    response['Content-Disposition'] = 'attachment; filename=%s' % zip_filename
    return response

def download_customs_pdf(request, file_name, view_pdf=False):
    cus_file_save_folder = customs_file_save_location()

    path_to_file = os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, file_name)
    response = HttpResponse()
    response['Content-Length'] = os.path.getsize(os.path.join(settings.MEDIA_ROOT, cus_file_save_folder, file_name))
    response['Content-Type'] = 'application/pdf'
    if view_pdf:
        response['Content-Disposition'] = 'inline; filename=%s' % file_name
    else:
        response['Content-Disposition'] = 'attachment; filename=%s' % file_name
    response['X-Accel-Redirect'] = "/media/" +  cus_file_save_folder + '/' + file_name
    return response

def view_customs_pdf(request, file_name):
    return download_customs_pdf(request, file_name, view_pdf=True)

def list_all(request):
    customs_all_list = CustomsDeclaration.objects.order_by('filename')
    return render(
        request,
        'customs_list/view_files.html',
        context={
            'customs_list': customs_all_list,
        }
    )

def list_date(request, year=None, month=None, day=None):
    day_list = None
    year_dic = None
    if year != None and month != None and day != None:
        customs_list = CustomsDeclaration.objects.filter(upload_date__year=year,
                                                         upload_date__month=month,
                                                         upload_date__day=day,).order_by('filename')
        return render(
            request,
            'customs_list/view_files.html',
            context={
                'customs_list': customs_list,
                'year': year,
                'day': day,
                'month': month,
            }
        )
    else:
        if year == None or month==None:
            year_dic = {}
            year_query_list = CustomsDeclaration.objects.dates("upload_date", "year")
            for y in year_query_list:
                cur_year = y.year
                month_query_list = CustomsDeclaration.objects.filter(upload_date__year=cur_year).dates("upload_date", "month")
                month_list = []
                for q in month_query_list:
                    month_list.append(q.month)
                year_dic[cur_year] = month_list
        else:
            day_list = []
            query_list = CustomsDeclaration.objects.filter(upload_date__year=year, upload_date__month=month).dates("upload_date", "day")
            for q in query_list:
                day_list.append(q.day)

        return render(
            request,
            'customs_list/view_dates.html',
            context={
                'year_dic': year_dic,
                'day_list': day_list,
                'day': day,
                'year': year,
                'month': month,
            }
        )

def edit_list(request):
    customs_edit_list = CustomsDeclaration.objects.filter(correct_name=False)
    return render(
        request,
        'customs_list/view_files.html',
        context={
            'customs_list': customs_edit_list,
        }
    )

def delete(request):
    form = XMLRequestForm(request.POST)
    message = ''
    if form.is_valid():
        command = form.cleaned_data['command']
        filename = form.cleaned_data['filename']
        if request.user.is_authenticated:
            custdecl_inst = CustomsDeclaration.objects.get(filename=filename)
            custdecl_inst.delete()
            message = filename
        else:
            message = 0
    return HttpResponse(message)

def edit_cust_dec(request, filename):
    queryset = CustomsDeclaration.objects.filter(filename=filename)

    cust_dec_inst = get_object_or_404(queryset)

    if request.method == 'POST':
        cust_dec_editform = EditCustomsDeclarationForm(request.POST)
        if cust_dec_editform.is_valid():

            customs_number = cust_dec_editform.cleaned_data["customs_number"]
            customs_query = CustomsDeclaration.objects.filter(customs_number=customs_number)

            prev_url = request.session["prev_url"]

            # # Means that the PDF with the name already exists
            # if len(customs_query) != 0:
            #     cust_dec_inst.delete()
            #     return HttpResponseRedirect(prev_url)

            cust_dec_inst.edit(
                customs_number=customs_number,
                               )
            return HttpResponseRedirect(prev_url)
    else:
        # upload_date = cust_dec_inst.upload_date
        prev_url = request.META.get('HTTP_REFERER')
        request.session["prev_url"] = prev_url
        cust_dec_editform = EditCustomsDeclarationForm()

    return render(request,
                  'customs_list/cust_dec_edit.html',
                  context={
                      'filename': filename,
                      "form": cust_dec_editform},
                  )


import pygame

pygame.init()

# 创建游戏的窗口
screen = pygame.display.set_mode((480, 700))

# 绘制背景图像
background = pygame.image.load("./images/background.png")
screen.blit(background, (0, 0))
# pygame.display.update()

# 绘制英雄的飞机
hero = pygame.image.load("./images/me1.png")
screen.blit(hero, (200, 500))

# 可以再所有绘制工作完成后，统一调用update方法
pygame.display.update()

while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()

pygame.quit()


from dotenv import load_dotenv
import meraki
import os

load_dotenv('environment.env')

### MUST CREATE environment.env FILE WITH THE FOLLOWING VARIABLES ###
API_KEY = os.getenv('API_KEY')
network_id = os.getenv('NET_ID') # Optionally you can get this at runtime using the calls below

# Initialize Dashboard object
dashboard = meraki.DashboardAPI(API_KEY)

########################
# Uncomment if you need to get your org or network ID.
# Get organizations
# response = dashboard.organizations.getOrganizations()
# print(response)

# # Get networks
# response = dashboard.organizations.getOrganizationNetworks(
#     organization_id, total_pages='all'
# )
# print(response)
########################

#### Specify Tags and Targets Here ####

# Specify Target Strings to match
match_tag_list = ['ios', 'mac', 'apple'] # Here we match the apple/iOS Auto tags

# SPECIFY DESIRED NEW TAG TAG HERE
new_tag = 'SUPER_TAG'

# List the devices enrolled in an SM network with various specified fields and filters
# https://developer.cisco.com/meraki/api-v1/#!get-network-sm-devices
sm_devices_response = dashboard.sm.getNetworkSmDevices(
    network_id, total_pages='all', fields=['autoTags']
)

# print(sm_devices_response)

devices_without_tag =[]
device_names = []

# For all devices
# Check if the VPN tag is applied or not
# Then flag device if it matches the specified iderntifiers
for device in sm_devices_response:
    if new_tag not in device['tags']:
        for tag in device['autoTags']:
            for mac_identifier in match_tag_list:
                if mac_identifier.lower() in tag.lower():
                    devices_without_tag.append(device['id'])
                    device_names.append(device['name'])

# Remove duplicates
devices_without_tag = list(set(devices_without_tag))
# Print Results
print('Checked ' + str(len(sm_devices_response)) +f' devices. Here are the IDs of Apple devices without the {new_tag} tag')
print(devices_without_tag)
print('Device Names:')
print(device_names)

# If flagged devices are found
# Add the tag to the flagged devices
# https://developer.cisco.com/meraki/api-v1/#!modify-network-sm-devices-tags
if devices_without_tag:
    update_tags_response = dashboard.sm.modifyNetworkSmDevicesTags(
        network_id, updateAction='add', ids=devices_without_tag, tags=[new_tag],
    )

    # Print updates devices
    print(f'Added {new_tag} to the following devices:')
    for device in update_tags_response:
        print(device['id'])

print('Task Complete')

# Included is an API call to remove the flag if desired.
remove_tag = dashboard.sm.modifyNetworkSmDevicesTags(
    network_id, scope=['withAny', new_tag], updateAction='delete', tags=[new_tag],
)


import numpy as np
import csv
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

def Sort2Array(x, y):
    temp = np.array([x, y]).T
    temp = temp[temp[:, 0].argsort()].T
    return temp[0], temp[1]

def Sort3Array(x, y, z):
    temp = np.array([x, y, z]).T
    temp = temp[temp[:, 0].argsort()].T
    return temp[0], temp[1], temp[2]

def Line(x, a, b):
    return a*x + b

def Inverte_line(y, a, b):
    return (y - b) / a

filename = "./imstats.dat"
header = 6  # number of header rows
exptime = np.array([])
signal_level = np.array([])
sigma_signal_level = np.array([])

diff_exptime = np.array([])
diff_variance = np.array([])

with open(filename, newline="") as csvfile:
    a = csv.reader(csvfile, delimiter='\t')
    # a is a object containing row arrays
    for row in a:
        #  print(header)
        if header <= 0:
            if "diff" not in row[0]:
                # only orig images
                row[0] = row[0].replace("FLATLIN_", "")
                row[0] = row[0].replace("-1", "")
                row[0] = row[0].replace(".fits", "")
                row[0] = row[0].replace("s", "")
                exptime = np.append(exptime, float(row[0]))

                # get rid of space
                for i in range(1, len(row)):
                    # except the file name
                    row[i] = row[i].replace(" ", "")

                # read mean value
                signal_level = np.append(signal_level, float(row[5]))
                sigma_signal_level = np.append(sigma_signal_level,
                                               float(row[6]))
                #  print(row)
            else:
                #  print(row)
                # only diff images
                row[0] = row[0].replace("FLATLIN_", "")
                row[0] = row[0].replace("-1", "")
                row[0] = row[0].replace("_diff.fits", "")
                row[0] = row[0].replace("s", "")
                diff_exptime = np.append(diff_exptime, float(row[0]))

                for i in range(1, len(row)):
                    # except the file name
                    row[i] = row[i].replace(" ", "")
                diff_variance = np.append(diff_variance, float(row[6]))

        header -= 1

exptime, signal_level, sigma_signal_level = Sort3Array(exptime,
                                                       signal_level,
                                                       sigma_signal_level)

#  print(exptime)
#  print(signal_level)
#  print(sigma_signal_level)

k = 0
color="tab:blue"
plt.figure(num=k, figsize=(10, 7))
plt.errorbar(exptime, signal_level, yerr=sigma_signal_level,
             fmt="o", label="data", color=color)

# fit
mask = (exptime < 1.05)
fit_exptime = exptime[mask]
fit_signal_level = signal_level[mask]
fit_sigma_signal_level = sigma_signal_level[mask]
#  print(fit_exptime, fit_signal_level)
popt, perr = curve_fit(Line, fit_exptime,
                       fit_signal_level, sigma=fit_sigma_signal_level)
print("Best fit parameter and covariance matrix:", popt, perr)

x = np.linspace(np.amin(fit_exptime) - 0.1, np.amax(fit_exptime) + 0.1, 100)
plt.plot(x, Line(x, *popt), label="fit", color=color)

ax = plt.gca()
plt.xlabel("exposure time/s")
ax.set_ylabel("mean pixel values / ADU", color=color)

# residual
color = "tab:red"
pred_signal = Line(exptime, *popt)
res = pred_signal - signal_level
ax2 = ax.twinx()
ax2.scatter(exptime, res, color=color)
ax2.tick_params(axis="y", labelcolor=color)
ax2.set_ylabel("residual / ADU", color=color)

ax.legend(loc=2)
plt.savefig("exp_mean.pdf", bbox_inches="tight")
plt.close(k)
k += 1

pixel_max = np.amax(signal_level)
#  print(pixel_max)
exptime_max = Inverte_line(pixel_max, *popt)
jac = - 1/popt[0] * np.array([exptime_max, 1])
sigma_exptime_max = jac.dot(perr.dot(jac.transpose()))
print("Maximal exposure time=", exptime_max, "+-", sigma_exptime_max)

##################################################################
# difference
##################################################################

diff_exptime, diff_variance = Sort2Array(diff_exptime, diff_variance)
diff_variance = diff_variance**2

signal_for_diff = np.array([])
sigma_signal_for_diff = np.array([])
length = (len(signal_level))/2
for i in range(0, int(length)):
    mean = np.mean([signal_level[2*i], signal_level[2*i+1]])
    signal_for_diff = np.append(signal_for_diff, mean)

    std = np.std([signal_level[2*i], signal_level[2*i+1]])
    sigma_signal_for_diff = np.append(sigma_signal_for_diff, std)

#  print(signal_for_diff)
sigma_signal_level = sigma_signal_level[::2]

# fit
mask = (diff_variance > 0)  # mask for only linear region
popt, perr = curve_fit(Line, signal_for_diff[mask], diff_variance[mask])
print("Fit parameters and covariance matrix", popt, perr)

plt.figure(num=k, figsize=(10, 7))

x = np.linspace(np.amin(signal_for_diff)-1000, np.amax(signal_for_diff), 100)
plt.errorbar(signal_for_diff, diff_variance, color="black", label="data",
             xerr=sigma_signal_level, fmt="o")
plt.plot(x, Line(x, *popt), color="black", label="linear fit")

mask_signal, mask_variance = signal_for_diff[mask], diff_variance[mask]
#  anno_point = (mask_signal[-1], mask_variance[-1]-3)
#  anno_text = (mask_signal[-1], mask_variance[-1]-50)
#  plt.annotate("saturation", xy=anno_point, xytext=anno_text,
             #  arrowprops=dict(facecolor='black', shrink=0.05))
inv_mask_signal = signal_for_diff[~mask]

sat = np.mean([inv_mask_signal[0], mask_signal[-1]])
sigma_sat = np.fabs(inv_mask_signal[0] - mask_signal[-1])
#  print(signal_for_diff)
print("Saturation at:", sat, "+-", sigma_sat, "ADU")

plt.xscale("log")
plt.xlabel("signal of original images / ADU")
plt.yscale("log")
plt.ylabel(r"variance of difference frames / ADU$^2$")
plt.legend()
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

plt.savefig("diff.pdf", bbox_inches="tight")
plt.close(k)
k += 1

gain = 2/popt[0]
sigma_gain = np.fabs( 2 / popt[0]**2 * np.sqrt(np.diag(perr))[0])
print("Gain = ", gain, "+-", sigma_gain, "e/ADU")

full_well = gain * sat
sigma_full_well = full_well * np.sqrt((sigma_gain/gain)**2 + (sigma_sat/sat)**2)
print("Full well capacity =", full_well, "+-", sigma_full_well, "e-")


import requests
import json

def start_record(token):
    url = "https://apitesting.au.vbrickrev.com/api/v1/vc/start-recording"
    querystring = {"title": "Morning Call Recording", "sipAddress": "[insert sip address]"}
    headers = {
        'Authorization': "VBrick {}".format(token),
        'Content-Type': "application/json",
        'cache-control': "no-cache",
        'Postman-Token': "[insert token]"
    }
    response = requests.request("POST", url, headers=headers, params=querystring)
    print(response.text)
    return json.loads(response.text)["videoId"]


from openpyxl import load_workbook
from docx import Document

# Excel-Tabelle öffnen
workbook = load_workbook('deine_tabelle.xlsx')
sheet = workbook.active

# Beispielhafte Verarbeitung der Werte
wert1 = sheet['A1'].value
wert2 = sheet['B1'].value

# Word-Dokument erstellen
document = Document()

# Text mit Formatierung hinzufügen
paragraph = document.add_paragraph()
run = paragraph.add_run()
run.text = f"Der Wert 1 ist {wert1} und der Wert 2 ist {wert2}."
run.bold = True
run.italic = True

# Word-Dokument speichern
document.save('ergebnis.docx')


from PIL import Image

import ImageEnhance

myImage = Image.open("gogeta.png")
size = 700, 700
myImage.thumbnail(size)
myImage.show()

sharp = ImageEnhance.Sharpness(myImage)

sharp.enhance(4).show()
sharp.enhance(10).show()

color = ImageEnhance.Color(myImage)

color.enhance(0).show()
color.enhance(2).show()
color.enhance(4).show()

bright = ImageEnhance.Brightness(myImage)

bright.enhance(2).show()
bright.enhance(4).show()

contrast = ImageEnhance.Contrast(myImage)

contrast.enhance(2).show()
contrast.enhance(4).show()

#Check out the file image_processing_3.png for the output.


from sklearn.datasets import fetch_lfw_people
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt

face_data = fetch_lfw_people(min_faces_per_person=80)

X = face_data.data
Y = face_data.target
labels = face_data.target_names

print("Labels:", labels)

for i in range(5):
    print("Class", labels[i], ":", {(Y == i).sum()})

plt.show()

print("Fitting...")

x_train, x_test, y_train, y_test = train_test_split(X, Y)

svm = SVC(class_weight='balanced')

parameters = {'C': [0.1, 1, 10], 'gamma': [1e-07, 1e-08, 1e-06], 'kernel' : ['rbf', 'linear'] }

grid_search = GridSearchCV(svm, parameters, cv=5, n_jobs=-1)

grid_search.fit(x_train, y_train)

print("Best Settings for SVC:", grid_search.best_params_)

best_svm = grid_search.best_estimator_

print("Score:", best_svm.score(x_test, y_test))

pred = best_svm.predict(x_test)

print(classification_report(y_test, pred))


import requests
import os
import json

class TgSender:
    def __init__(self, token, url):
        self.TOKEN = token
        self.URL = url

    def send_message(self, chat_id, text):
        requests.get(f'{self.URL}{self.TOKEN}/sendMessage?chat_id={chat_id}&text={text}')

    def send_photo(self, chat_id, path):
        files = {'photo': open(path, 'rb')}
        requests.post(f'{self.URL}{self.TOKEN}/sendPhoto?chat_id={chat_id}', files=files)

    def telegram_send_media_group(self, chat_id, path):
        url = "https://api.telegram.org/bot" + self.TOKEN + "/sendMediaGroup"
        channel_id = "@level_signals"

        files = dict.fromkeys(os.listdir(path))
        folder = path + '/'

        media_list = []

        for key in files:
            files[key] = open(folder + key, 'rb')

            media_data = dict.fromkeys(['type', 'media', 'caption'])
            media_data['type'] = 'photo'
            media_data['media'] = 'attach://' + key
            media_data['caption'] = key

            media_list.append(media_data)

        media = json.dumps(media_list)

        params = {
                'chat_id': chat_id,
                'media': media
                }

        r = requests.post(url, params= params, files= files)


import json

class AppJSONDecoder(json.JSONDecoder):

    def __init__(self, file_name = None):
        json.JSONDecoder.__init__(self)
        self.FileName = file_name
        self.JSONData = {}

        try:
            f = open(self.FileName)
            self.JSONData = json.load(f)
        except:
            print("Failed to open %s for reading or JSON" % (self.FileName))
        return

    def GetDictionary(self, name = None):
        if (name == None) or name not in self.JSONData.keys():
            return
        else:
            return self.JSONData[name]

class AppJSONEncoder(json.JSONEncoder):
    def __init__(self):
        json.JSONEncoder.__init__(self)
        pass


import json
import os

def hello_pubsub(event, context):

     if event:
          print(event)

          pubsub_message = json.dumps(event['data']) + '\n'
          print(pubsub_message)
          path = '/data/pv01/'
          print("Using folder: " + path)
          log_file_name = path + 'log.txt'

          # Mais simples que o google, MAS não eh safe pra acesso concorrente!!!
          with open(log_file_name,'a') as log_file:
               log_file.write(pubsub_message)

     else:
          print("Erro sem data no event")


import numpy as np
import sys
import os
from plyfile import PlyData, PlyElement

def read_ply(ply_path):
    plydata = PlyData.read(ply_path)
    points = np.asarray(plydata['vertex'].data.tolist())
    return points

def write_ply(points,output_file):

    with open(output_file, 'w') as ply:
        header = ['ply']
        header.append('format ascii 1.0')
        header.append('element vertex {}'.format(points.shape[0]))
        header.append('property float x')
        header.append('property float y')
        header.append('property float z')
        # header.append('property float intensity')
        header.append('end_header')

        for line in header:
            ply.write("%s\n" % line)

        for p in points:
            ply.write("{} {} {} {}\n".format(p[0], p[1], p[2]))

    return

def process_point_cloud(input_cluster_path, output_cluster_path):

    pcd = read_ply(input_cluster_path)
    write_ply(pcd, output_cluster_path)

    return

if __name__ == '__main__':

    # Path to input folder
    input_clusters_dir = sys.argv[1]

    # Path to output folder
    output_clusters_dir = sys.argv[2]

    # Iterate input folder
    for subdir, dirs, files in os.walk(input_clusters_dir):
        for file in files:
            if '.ply' in file:
                input_cluster_path = os.path.join(subdir,file)
                output_cluster_path = os.path.join(output_clusters_dir, file)

                process_point_cloud(input_cluster_path, output_cluster_path)


import os
import time
from django.utils import timezone
import pandas as pd
import numpy as np

from .renderers import RenderTemplate
from .load import load_hdf

import datetime
from dateutil.relativedelta import relativedelta

module_dir = os.path.dirname(__file__)  #get current directory

hul_df = load_hdf(os.path.join(module_dir, 'dumps/hul.h5'))

class ParsingError(Exception):
    """Generated whenever there is problem parsing
       condition or aggregate function parameters
    """
    def __init__(self, message):
        Exception.__init__(self, message)

def execute_query(db_query, raw_info={}):
    """executes ``pandas_query``"""
    answer = {}
    exec_times = raw_info.get('execution_time', {})
    #execution
    for query in db_query:
        print('executing:', query)
        df = eval(query)
        print(df.head())
    return df

def generate_result(df, answer_type, raw_info={}):
    answer = {}
    exec_times = raw_info.get('execution_time', {})
    try:
    #if True:
        if df.empty:
            answer['type'] = "message"
            answer['data'] = "Didn't find any records. Sorry."
            return answer

        if answer_type == 'graph':
            result = RenderTemplate('test.html')
            content = {}
            content['header'] = ''
            content['type'] = 'graph'
            content['graph_type'] = 'line'
            content['body'] = df
            #print(content)
            result.render_to(section=1, content=content)

            result = result.render()
            answer['type'] = 'graph'
            answer['data'] = result
            answer['format'] = list(df.columns)
            return answer

        else:
            t1 = time.time()
            df = df.drop_duplicates()

            # limiting row count
            df = df[:5000]

            exec_times['drop_duplicates'] = time.time() - t1
            print(df.head())
            data = []
            answer_format = list(df.columns)
            for row in df.values:
                temp = []
                for r in row:
                    temp.append(str(r))
                data.append(temp)

            table_data= '

'
            table_data+= '
'
            for col in answer_format:
                table_data+= '' + col + '	'
            table_data+= '
'
            for chunk in data:
                table_data+= '
'
                for i in range(len(chunk)):
                    table_data+= '' + str(chunk[i]) + '	'
                table_data+= '
'
            table_data+= '
'

            result = RenderTemplate('test.html')
            content = {}
            content['header'] = ''
            content['type'] = 'table'
            content['body'] = table_data
            result.render_to(section=1, content=content)

            result = result.render()
            answer['type'] = 'table'
            answer['data'] = result
            answer['format'] = list(df.columns)
            return answer
    except Exception as e:
    #if False:
        print('problem')
        print(e)
        answer['type'] = "message"
        answer['data'] = "Problem in execution. Sorry."
        if 'format' in answer.keys():
            del answer['format']
        return answer

templates = {}
limit = ''
templates['tb_name+columns'] = "{tb_name}[{columns}]"
templates['tb_name+columns+order_by'] = "{tb_name}[{columns}].sort_values(by={order_by}, ascending=False)"
templates['tb_name+columns+conditions'] = "{tb_name}.ix[{conditions}][{columns}]"
templates['tb_name+columns+conditions+order_by'] = "{tb_name}.ix[{conditions}][{columns}].sort_values(by={order_by}, ascending=False)"
templates['tb_name+columns+conditions+group_by'] = "{tb_name}.ix[{conditions}][{columns}].groupby({group_by})"
templates['tb_name+columns+conditions+group_by+order_by+agg_func'] = "{tb_name}.ix[{conditions}][{columns}].groupby({group_by}){agg_func}"

def get_query(query_parameters):
    #search templates
    print('inside get_query()')
    print(query_parameters)
    for key in templates.keys():
        key_set= set(key.split('+'))
        print(key_set, set(query_parameters.keys()))
        if key_set == set(query_parameters.keys()):
            print('Match Found!')
            return templates[key].format(**query_parameters)
        else:
            print('Not a Match.')

def get_tbname(parameters):
    #accept some parameters (column_names)
    #return table name
    if 'style' in parameters:
        return 'ROS_sell_thru_ss_Report'
    elif 'sub_brand' in parameters:
        return 'ROS_sell_thru_ssb_Report'
    else:
        return 'ROS_sell_thru_sb_Report'

table_column_map = {
        'hul_df': [('df', 'hul_df'),]}

def modify_columns(tbname, columns):
    #modify columns according to a table name.
    for _map in table_column_map[tbname]:
        if _map[0] in columns:
            i = columns.index(_map[0])
            columns[i] = _map[1]
    print('inside modify columns:', columns)
    return columns

def get_target(entities):
    columns = ['account', 'countercode', 'moctarget']
    group_by = []
    order_by = []
    column_filters = []
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append('df')

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res

def plot_target_sales(entities):
    columns = ['moctarget', 'salesvalue']
    group_by = ['moc']
    order_by = []
    column_filters = []
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        stop = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        start = datetime.date.today() + relativedelta(months=-6)
        dates = []
        for date in pd.date_range(start=start, end=stop, freq='M'):
                dates.append('%s-%s-%s'%(date.year, date.month, 1))

        condition_list.append("(<<df>>.<<created_date>>.isin(pd.DatetimeIndex(%s)))"%str(dates))
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')

    if set(['account', 'city', 'state', 'region']).intersection(set(columns)):
        columns.append('countercode')

    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    #if group_by:
        #query_parameters['group_by'] = str(group_by)

    query = get_query(query_parameters)

    for s1, s2 in table_column_map[tb_name]:
        query = query.replace('<<' + s1 + '>>', s2)
    query = query.replace('<<', '').replace('>>', '')

    t1 = execute_query([query, 'df'])
    t2 = t1[['moc', 'salesvalue']].groupby('moc').sum().reset_index()

    if set(['account', 'city', 'state', 'region']).intersection(set(columns)):
        t3 = t1[['moc', 'moctarget', 'countercode']].drop_duplicates().groupby('moc').sum().reset_index()
        data = t2.merge(t3, on='moc')

    elif 'countercode' in columns:
        t3 = t1[['moc', 'moctarget']].drop_duplicates()
        data = t2.merge(t3, on='moc')

    print(data)
    data = data.set_index('moc').unstack()
    data = data.to_frame('')
    res = generate_result(data, 'graph')
    return res
    #return (query, 'text')

def low_sales(entities):
    columns = ['account','countercode','salesvalue']
    group_by = ['countercode']
    order_by = []
    column_filters = []
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    #if group_by:
        #query_parameters['group_by'] = str(group_by)

    query = get_query(query_parameters)

    for s1, s2 in table_column_map[tb_name]:
        query = query.replace('<<' + s1 + '>>', s2)
    query = query.replace('<<', '').replace('>>', '')

    _df = execute_query([query, 'df'])
    _group = _df.groupby(group_by)['salesvalue'].sum().reset_index().sort_values(by=['salesvalue'], ascending=True).drop_duplicates().head(10)

    _cols = list(set(_df.columns)-(set(_group.columns)))
    _cols.append('countercode',)
    data = _group.merge(_df[_cols], on=['countercode'])
    data = data.drop_duplicates()
    res = generate_result(data, 'text')
    return res
    #return (query, 'text')

def top_sales(entities):
    columns = ['account', 'countercode', 'salesvalue']
    group_by = ['countercode']
    order_by = []
    column_filters = []
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    #if group_by:
        #query_parameters['group_by'] = str(group_by)

    query = get_query(query_parameters)

    for s1, s2 in table_column_map[tb_name]:
        query = query.replace('<<' + s1 + '>>', s2)
    query = query.replace('<<', '').replace('>>', '')

    _df = execute_query([query, 'df'])
    _group = _df.groupby(group_by)['salesvalue'].sum().reset_index().sort_values(by=['salesvalue'], ascending=False).drop_duplicates().head(10)

    _cols = list(set(_df.columns)-(set(_group.columns)))
    _cols.append('countercode',)
    data = _group.merge(_df[_cols], on=['countercode'])
    data = data.drop_duplicates()
    res = generate_result(data, 'text')
    return res

def store_performance(entities):
    columns = ['account', 'countercode', 'target_achieved', 'salesvalue', 'moctarget']
    group_by = []
    order_by = []
    column_filters = []
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append("df")

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    #return (query, 'text')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res

def get_mrp(entities):
    columns = ['account', 'countercode', 'mrp', 'basepackcode']
    group_by = []
    order_by = ['mrp']
    column_filters = ['mrp']
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append("df")

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res
    #return (query, 'text')

def get_opening_units(entities):
    columns = ['account', 'countercode', 'openingunits', 'basepackcode']
    group_by = []
    order_by = ['openingunits']
    column_filters = ['openingunits']
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append("df")

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res
    #return (query, 'text')

def get_receipt_units(entities):
    columns = ['account', 'countercode', 'receiptunits', 'basepackcode']
    group_by = []
    order_by = ['receiptunits']
    column_filters = ['receiptunits']
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append("df")

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res
    #return (query, 'text')

def get_closing_stock_units(entities):
    columns = ['account', 'countercode', 'closingstockunits', 'basepackcode',]
    group_by = []
    order_by = ['closingstockunits']
    column_filters = ['closingstockunits']
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')
    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')
    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    if group_by:
        query_parameters['group_by'] = str(group_by)

    query = []
    query.append(get_query(query_parameters))

    if not query[0]:
        return

    query.append("df")

    for s1, s2 in table_column_map[tb_name]:
        for i in range(len(query)):
            query[i] = query[i].replace('<<' + s1 + '>>', s2)

    for i in range(len(query)):
            query[i] = query[i].replace('<<', '').replace('>>', '')

    data = execute_query(query)
    res = generate_result(data, 'text')
    return res
    #return (query, 'text')

def get_sales_info(entities):
    columns = ['account', 'countercode', 'moc', 'salesvalue']
    group_by = ['countercode', 'moc']
    order_by = []
    column_filters = ['salesvalue']
    condition_list, columns = get_conditions(entities, columns)
    agg_func_str, columns = get_agg_functions(entities, columns)
    if 'created_date' not in columns:
        start = '%s-%s-%s'%(timezone.now().year, timezone.now().month, 1)
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start='%s', periods=1)))"%start)
        if 'moc' not in columns:
            columns.append('moc')

    else:
        columns.remove('created_date') # we donot want this column to appear in final results.
        if 'moc' not in columns:
            columns.append('moc')

    #add more to condition here
    tb_name = 'hul_df' #get_tbname(columns)
    #modify the column_names as per the Table_name
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)

    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    if condition_list:
        query_parameters['conditions'] = ' & '.join(condition_list)
    if order_by:
        query_parameters['order_by'] = str(order_by)
    #if group_by:
        #query_parameters['group_by'] = str(group_by)

    query = get_query(query_parameters)

    for s1, s2 in table_column_map[tb_name]:
        query = query.replace('<<' + s1 + '>>', s2)
    query = query.replace('<<', '').replace('>>', '')

    _df = execute_query([query, 'df'])
    _group = _df.groupby(group_by)['salesvalue'].sum().reset_index().sort_values(by=['salesvalue',], ascending=False).drop_duplicates()

    _cols = list(set(_df.columns)-(set(_group.columns)))
    _cols.append('countercode',)
    data = _group.merge(_df[_cols], on=['countercode'])
    data = data.drop_duplicates()
    res = generate_result(data, 'text')
    return res

"""
def plot_store_report(entities, extra_columns=[]):
    columns = ['store', 'target_obj_skew_indicator','created_date']
    column_filters = []
    group_by = ['target_obj_skew_indicator', 'created_date']
    order_by = []
    condition_list, columns = get_conditions(entities, columns)
    if 'created_date' not in columns:
        condition_list.append("(<<df>>.<<created_date>>.isin(pd.date_range(start=timezone.now().date()-timezone.timedelta(days=8), periods=8, freq='D')))")
        columns.append('created_date')
    if 'store' not in columns:
       return
    #add more to condition here
    tb_name = 'worxogo_df' #get_tbname(columns)
    columns = modify_columns(tb_name, columns)
    column_filters = modify_columns(tb_name, column_filters)
    #remove the brand from the columns
    columns.remove('store')
    #modify the column_names as per the Table_name
    query_parameters = {}
    query_parameters['tb_name'] = tb_name
    query_parameters['columns'] = str(columns)
    query_parameters['conditions'] = ' & '.join(condition_list)
    query_parameters['order_by'] = str(order_by)
    query_parameters['group_by'] = str(group_by)
    query_parameters['agg_func'] = "['<<target_obj_skew_indicator>>'].count()"
    query_step_1 = get_query(query_parameters)
    print('query_step_1:',query_step_1)
    if not query_step_1:
        return
    query_step_2 = 'df'
    for s1, s2 in table_column_map[tb_name]:
        query_step_1 = query_step_1.replace('<<' + s1 + '>>', s2)
        query_step_2 = query_step_2.replace('<<' + s1 + '>>', s2)
    query_step_1 = query_step_1.replace('<<', '').replace('>>', '')
    query_step_2 = query_step_2.replace('<<', '').replace('>>', '')
    pandas_query = {'step_1': query_step_1, 'step_2': query_step_2}
    return (pandas_query, columns, column_filters, 'graph')
"""
def _get_value_from_context(context, keyword, current_index=0):
    #search for the keyword in context (list of dict's)
    #search left side
    for exp in context[current_index-1::-1]:
        value = exp.get(keyword, None)
        if value:
            return value
    #search right side
    for exp in context[current_index+1:]:
        value = exp.get(keyword, None)
        if value:
            return value

operators = {
    'gt': '>',
    'lt': '<',
    'gte': '>=',
    'lte': '<=',
    'eq': '='}

connectors = {
    'and': '&',
    'or': '|'}

def get_agg_functions(entities, columns):
    #normalising aggregate functions
    agg_func_list = []
    agg_func_str = ''
    _agg_func_list = entities.get('function_agg_list', None)
    print(_agg_func_list)
    if _agg_func_list:
        keys = list(_agg_func_list.keys())
        keys.sort()
        for e in keys:
            agg_func_list.append(_agg_func_list[e])
        print('functions', agg_func_list, len(agg_func_list))
        for i in range(len(agg_func_list)):
            func = agg_func_list[i]
            if not func.get('func', None):
                val = _get_value_from_context(agg_func_list, 'func', current_index=i)
                if val:
                    agg_func_list[i]['func'] = val
                else:
                    raise ParsingError('Not a valid agg_func name.')
            if not func.get('arg', None):
                val = _get_value_from_context(agg_func_list, 'arg', current_index=i)
                if val:
                    agg_func_list[i]['arg'] = '<<' + val + '>>'
                    columns.append(agg_func_list[i]['arg'])
                else:
                    raise ParsingError('Not a valid argument to agg_func.')
            else:
                columns.append(agg_func_list[i]['arg'])
                agg_func_list[i]['arg'] = '<<' + agg_func_list[i]['arg'] + '>>'

            if i < len(agg_func_list)-1:
                if not func.get('conn', None):
                    agg_func_list[i]['conn'] = '&'
            else:
                if func.get('conn', None):
                    del agg_func_list[i]['conn']

        print(agg_func_list)
        for agg_func in agg_func_list:
            agg_func_str += "(df['{arg}'] == df['{arg}'].{func}())".format(arg=agg_func['arg'], func=agg_func['func'])
            if agg_func.get('conn', None):
                agg_func_str += ' ' + connectors[agg_func['conn']]
    print(agg_func_str)
    columns = sorted(set(columns), key=columns.index) #removing duplicates while retaining original sequence
    return (agg_func_str, columns)

def get_conditions(entities, columns=[]):
    """
        composes WHERE CONDITION from entities
    """
    condition_list = []
    print('InPut inside compose condition:', columns)
    #normalising expressions
    _exp_l = entities.get('expression_list', {})
    exp_l = []
    if _exp_l:
        keys = list(_exp_l.keys())
        keys.sort()
        for e in keys:
            exp_l.append(_exp_l[e])
        for i in range(len(exp_l)):
            exp = exp_l[i]
            if not exp.get('prop', None):
                val  = _get_value_from_context(exp_l, 'prop', current_index=i)
                if val:
                    exp_l[i]['prop'] = val
                else:
                    #parsing error
                    raise ParsingError('Not a valid expression...prop missing.')
            if not exp.get('operator', None):
                val = _get_value_from_context(exp_l, 'operator', current_index=i)
                if val:
                    exp_l[i]['operator'] = operators[val]
                else:
                    raise ParsingError('Not a valid expression...operator missing.')
            else:
                exp_l[i]['operator'] = operators[exp['operator']]
            if not exp.get('value', None):
                val = _get_value_from_context(exp_l, 'value', current_index=i)
                if val:
                    exp_l[i]['value'] = val
                else:
                    raise ParsingError('Not a valid expression...prop_value missing.')
            if i < len(exp_l)-1:
                if not exp.get('conn', None):
                    exp_l[i]['conn'] = '&'
            else:
                if exp.get('conn', None):
                    del exp_l[i]['conn']

        _exp = ''
        for i, exp in enumerate(exp_l):
            _exp += "(<<df>>['<<{prop}>>'] {operator} {value})".format(prop=exp['prop'], operator=exp['operator'], value=exp['value'])
            columns.append(exp['prop'])
            if exp.get('conn', None):
                _exp += ' ' + connectors[exp['conn']]
        condition_list.append(_exp)
        print(condition_list)

    countercode = entities.get('countercode')
    if countercode:
        condition_list.append("(<<df>>['<<countercode>>'] == '%s')"%countercode)
        columns.append('countercode')

    basepackcode = entities.get('basepackcode')
    if basepackcode:
        condition_list.append("(<<df>>['<<basepackcode>>'] == '%s')"%basepackcode)
        columns.append('basepackcode')

    moc = entities.get('moc')
    if moc:
        condition_list.append("(<<df>>['<<moc>>'] == '%s')"%moc)
        columns.append('moc')

    city = entities.get('city')
    if city:
        condition_list.append("(<<df>>['<<city>>'] == '%s')"%city)
        columns.append('city')

    state = entities.get('state')
    if state:
        condition_list.append("(<<df>>['<<state>>'] == '%s')"%state)
        columns.append('state')

    region = entities.get('region')
    if region:
        condition_list.append("(<<df>>['<<region>>'] == '%s')"%region)
        columns.append('region')

    account = entities.get('account')
    if account:
        condition_list.append("(<<df>>['<<account>>'] == '%s')"%account)
        columns.append('account')

    productfamily= entities.get('productfamily')
    if productfamily:
        condition_list.append("(<<df>>['<<productfamily>>'] == '%s')"%productfamily)
        columns.append('productfamily')

    brand = entities.get('brand')
    if brand:
        condition_list.append("(<<df>>['<<brand>>'] == '%s')"%brand)
        columns.append('brand')

    created_date = entities.get('created_date', None)
    if created_date:
        try:
            date_period= created_date.get('date-period', None)
            if date_period:
                dates = tuple(date_period.split('/'))
                condition_list.append("(<<df>>['<<created_date>>'].isin(pd.date_range(start='%s', end='%s', freq='D')))"%dates)
                columns.append('created_date')
            elif created_date.get('date', None):
                condition_list.append("(<<df>>['<<created_date>>'] == '%s')"%created_date['date'])
                columns.append('created_date')
        except Exception as e:
            raise ParsingError('Not a valid date')

    #finally, ``where condition``
    #condition = ' and '.join(condition_list)
    print('Inside compose_condition: condition = ', condition_list)
    print('Inside compose condition: columns = ', columns)
    columns = sorted(set(columns), key=columns.index) #removing duplicates while retaining original sequence
    print('Inside compose_condition: columns = ', columns)
    return (condition_list, columns)


from dotenv import load_dotenv
from bs4 import BeautifulSoup
import pika
import json
import sys
import os
load_dotenv()
sys.path.append(os.path.abspath(os.getenv("system_path")))
from lib import rabbit_mq, request, mongo

try:
    connection = rabbit_mq.create_connection()
    channel = connection.channel()

    channel.queue_declare(queue='mongo_jobs_save', durable=True)
    channel.queue_declare(queue='shine_html_parse')

    count = 0

    def callback(ch, method, properties, body):
        global count
        count += 1
        print("count::", count)
        # print(" [x] Received %r" % body)
        soup = BeautifulSoup(body, 'html.parser')
        res = soup.find_all("li", class_="search_listingleft search_listingleft_100")
        print("\n\n\nJob count before data formation:",len(res))
        print("Creating job data...")
        j = 0
        for item in res:
            # print(j)
            job = {}
            job['title'] = item.find("li",class_="snp cls_jobtitle").getText().strip() if item.find("li",class_="snp cls_jobtitle") else ''
            job['company'] = item.find("li",class_="snp_cnm cls_cmpname cls_jobcompany").getText().strip() if item.find("li",class_="snp_cnm cls_cmpname cls_jobcompany") else ''
            job['experience'] = item.find("span", class_="snp_yoe cls_jobexperience").getText().strip() if item.find("span", class_="snp_yoe cls_jobexperience") else ''
            job['description'] = item.find("li", class_="srcresult").getText().strip() if item.find("li", class_="srcresult") else ''
            job['location'] = item.find("em", class_="snp_loc").getText().strip() if item.find("em", class_="snp_loc") else ''
            # job['skills'] = item.find(["div", "span", "mark"], class_="sk jsrp cls_jobskill").getText().strip() if item.find(["div", "mark"], class_="sk jsrp cls_jobskill") else ''
            job['job_link'] = "https://www.shine.com" + item.find("a", class_="cls_searchresult_a searchresult_link").get('href') if item.find("a", class_="cls_searchresult_a searchresult_link") else ''
            job['link_2'] = ''
            job['salary'] = ''
            job['source'] = 'shine'
            skillsData = item.find(["div", "sk jsrp cls_jobskill"]) if item.find(["div", "sk jsrp cls_jobskill"]) else ''
            if len(skillsData) > 0:
                spans = skillsData.find_all('span') if skillsData.find_all('span') else ''
                if len(spans) > -1:
                    skills = []
                    for span in spans:
                        skills.append(span.getText().strip())
                    skills.pop(0)
            job['skills'] = skills

            channel.basic_publish(exchange='', routing_key='mongo_jobs_save', body=json.dumps(job))

    channel.basic_consume(
        queue='shine_html_parse', on_message_callback=callback, auto_ack=True)
    # channel.basic_consume(callback, 'shine_html_parse',  no_ack=False)

    print(' [*] Waiting for messages. To exit press CTRL+C')
    channel.start_consuming()
except Exception as e:
    error = {
        "status": "Shine......... Error occured while parsing html",
        "errorMsg": e
    }
    print("Error: ",error)


#!/usr/bin/python3

import numpy as np
from features import mfcc
from VoiceActivityDetection import simpleVAD
from scipy import stats
from sklearn import svm
from sklearn import grid_search
import os
import random
import pickle
import scipy.io.wavfile

# main
os.chdir('sounds/wav')
instruments = os.listdir()

X = []
y = []

set_size = 0
for take in range(5,15):
    clsid=0
    for inst in instruments:
        names = os.listdir(path=inst)
        names = random.sample(names, take)
        for f in names:
            samples = scipy.io.wavfile.read(os.path.join(os.getcwd(), inst, f))[1]
            feat = mfcc(samples,16000, appendEnergy=False, winlen=0.030, VAD=simpleVAD)
            # add two symptoms from the middle
            X.append(feat[int(random.random() * len(feat))])
            y.append(clsid)
            X.append(feat[int(random.random() * len(feat))])
            y.append(clsid)
            X.append(feat[int(random.random() * len(feat))])
            y.append(clsid)
            set_size += 3
        clsid += 1

    clf = svm.LinearSVC()

    param_grid = {'C': [0.5, 5, 50, 500], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']}

    gse = grid_search.GridSearchCV(clf, param_grid, n_jobs=os.cpu_count())
    gse.fit(X, y)

    clsid=0
    cfile=0
    error=0
    for inst in instruments:
        names = os.listdir(path=inst)
        cfile += len(names)
        for f in names:
            samples = scipy.io.wavfile.read(os.path.join(os.getcwd(), inst, f))[1]
            feat = mfcc(samples,16000, appendEnergy=False, winlen=0.030, VAD=simpleVAD)
            res = gse.predict(feat[range(int(len(feat) / 2 - 15), int(len(feat) / 2 + 2))])
            cls = int(stats.mode(res)[0])
            if cls != clsid:
                error += 1
        clsid += 1

    mse = (error / cfile) * 100
    print("{0};{1}".format(set_size, mse))

os.chdir('../..')

ids=range(0,len(instruments))
sinst=dict(zip(ids, instruments))

print(sinst)

with open('dataset.pickle', 'wb') as f:
    pickle.dump((X, y, sinst), f, pickle.HIGHEST_PROTOCOL)

#print('data=[')
#for coef in X:
#    for j in coef.tolist():
#        print('%f '%(j), end="")
#    print(';')
#print(']')
#print('y=', end="")
#print(y)


import re
from models import Quote
import redis

# Підключення до Redis
redis_client = redis.StrictRedis(host='localhost', port=6379, db=0)

def search_by_tags(tags):

    # Перевіряємо, чи результат не зберігався у кеші Redis
    tags_str = ','.join(tags)
    cached_result = redis_client.get(f'tags:{tags_str}')
    if cached_result:
        return cached_result.decode('utf-8')

    # Видаляємо можливі пробіли з тегів і виконуємо пошук у базі даних MongoDB
    cleaned_tags = [tag.strip() for tag in tags]
    quotes = Quote.objects(tags__in=cleaned_tags)
    result = '\n'.join([quote.quote for quote in quotes])

    # Зберігаємо результат у кеші Redis на 5 хвилин
    redis_client.setex(f'tags:{tags_str}', 300, result)
    return result

def search_quotes():
    # Функція для пошуку цитат за ім'ям автора, тегом або набором тегів
    while True:
        command = input("Введіть команду: ")

        if command.startswith('name:'):
            name = command.replace('name:', '').strip()
            result = search_by_author_name(name)
            print(result)
        elif command.startswith('tag:'):
            tag = command.replace('tag:', '').strip()
            result = search_by_tags([tag])
            print(result)
        elif command.startswith('tags:'):
            tags = command.replace('tags:', '').strip().split(',')
            result = search_by_tags(tags)
            print(result)
        elif command == 'exit':
            break
        else:
            print("Невірна команда. Спробуйте ще раз.")

def search_by_author_name(name):

    # Перевіряємо, чи результат не зберігався у кеші Redis
    cached_result = redis_client.get(f'author:{name}')
    if cached_result:
        return cached_result.decode('utf-8')
    print(f"Ім'я для пошуку: {name}")

    quotes = Quote.objects()

    # Фільтруємо цитати за ім'ям автора, використовуючи регулярний вираз у Python
    filtered_quotes = [quote for quote in quotes if re.search(f'.*{re.escape(name)}.*', quote.author.fullname, re.IGNORECASE)]

    result = '\n'.join([quote.quote for quote in filtered_quotes])
    print(f"Кількість знайдених цитат: {len(filtered_quotes)}")

    # Зберігаємо результат у кеші Redis на 5 хвилин
    redis_client.setex(f'author:{name}', 300, result)
    return result


#!/usr/bin/env python3

from pytube import Playlist, YouTube

# download directory
download_destination = '/Users/vijoy.vallachira/YouTubeDump'

# To download a single video
def download_video():
    youtube_video_source = input("Enter Youtube Video URL : ")

    # Downloading the highest resolution, progressive type video.
    print("Downloading %s" % youtube_video_source)
    YouTube(youtube_video_source).streams.get_highest_resolution().download(download_destination)

    ## YouTube(youtube_video_source).streams.filter(progressive=True, file_extension='mp4').order_by('resolution')[-1].download()

    return

# To download entire Playlist
def download_playlist():
    youtube_playlist = input("Enter Youtube Playlist URL : ")
    pl = Playlist(youtube_playlist)
    pl.populate_video_urls()
    print('Title of the playlist: %s' % pl.title())
    print('Number of videos in playlist: %s' % len(pl.video_urls))
    pl_video_list = pl.video_urls
    print("Downloading Playlist Videos \n")

    # printing the list using loop
    for x in range(len(pl_video_list)):
        print(str(x) + " >> " + pl_video_list[x])
        individual_video_stream = YouTube(pl_video_list[x]).streams.filter(mime_type="video/mp4").first()
        individual_video_stream.download(download_destination, "PhysicsSession"+str(x), "unAcademy-ClassXI-JEE")

    # print(*pl_video_list, sep="\n")
    # pl.download_all(download_destination)

    return

def display_video_streams():
    youtube_video_source = input("Enter Youtube URL : ")

    yt=YouTube(youtube_video_source)
    print(yt.streams)

    return

option = input("Download a Single [V]ideo OR [P]laylist OR  [Di]splay available Videos? [D, V or P] : ")
# download_destination = input("Download Directory : ")

if option == 'V':
    download_video()
elif option == 'P':
    download_playlist()
elif option == "D":
    display_video_streams()
else:
    print("Invalid Option...! Only 'D', 'V' or 'P' allowed.....")

print("Good Bye!")