"""
Linear classification on the Default data set
"""
# Here we import all we will need
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import statsmodels.api as sm
import statsmodels.formula.api as smf

from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report, precision_score, roc_curve, auc
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import label_binarize

path_to_data = 'https://raw.githubusercontent.com/jeshan49/eemp2/master/Default.csv'

# Let us import the default data

default = pd.read_csv(path_to_data, index_col=0)

# Let us see the summary statistics

default.describe()

# Looks like we need to create dummies for 'default' and 'student'

default['d_default'] = pd.get_dummies(default['default'], drop_first=True)

default['d_student'] = pd.get_dummies(default['student'], drop_first=True)

default['d_student2'] = pd.get_dummies(default['student']).loc[:,'Yes']

default['d_student3'] = default['student'].apply(lambda val: 1 if val == 'Yes' else 0)

default.head()

default.describe()

##########################################################################
##### Here we construct the scatter and box plots from the lecture #######
##########################################################################
fig, axs = plt.subplots(1, 3, gridspec_kw = {'width_ratios':[2, 1, 1]})

# Take a fraction of the samples where target value (default) is 'no'
df_no = default[default['d_default'] == 0].sample(frac=0.15)
# Take all samples  where target value is 'yes'
df_yes = default[default['d_default'] == 1]
# Join the samples in one dataframe
df_ = df_no.append(df_yes)

# Let us construct the scatter plot
axs[0].scatter(df_[df_.default == 'Yes'].balance, df_[df_.default == 'Yes'].income,
   s=40, c='orange', marker='+', linewidths=1, label='Default = Yes')
axs[0].scatter(df_[df_.default == 'No'].balance, df_[df_.default == 'No'].income,
   s=40, marker='o', linewidths='1', edgecolors='lightblue', facecolors='white',
   alpha=.6, label = 'Default = No')

axs[0].set_ylim(ymin=0)
axs[0].set_ylabel('Income')
axs[0].set_xlim(xmin=-100)
axs[0].set_xlabel('Balance')
axs[0].legend()

# Let us now construct the two boxplots
c_palette = {'No':'lightblue', 'Yes':'orange'}
sns.boxplot('default', 'balance', data=default, orient='v', ax=axs[1],
            palette=c_palette)
sns.boxplot('default', 'income', data=default, orient='v', ax=axs[2],
            palette=c_palette)
##########################################################################
##########################################################################

##########################################################################
####### Here we construct the regression plots from the lecture ##########
##########################################################################
fig, axs = plt.subplots(2, 1)
sns.regplot('balance', 'd_default', data=default,
            scatter_kws={'color':'orange'}, line_kws={'color':'blue'}, ax=axs[0])

sns.regplot('balance', 'd_default', data=default, ci=None, logistic=True,
            scatter_kws={'color':'orange'}, line_kws={'color':'blue'}, ax=axs[1])

axs[0].set_title('Linear Regression')
axs[1].set_title('Logistic Regression')

for ax in axs:
    ax.set_ylabel('Prob of Default')
    ax.set_xlabel('Balance')
##########################################################################
##########################################################################

# Let us perform linear regression with statsmodels

y = default['d_default']

X = default[['balance', 'income', 'd_student']]

linreg = sm.OLS(y, X).fit()

linreg.summary()

print(linreg.summary().tables[1])

X = sm.add_constant(X)

linreg2 = sm.OLS(y, X).fit()

linreg2.summary()

# Let us perform logistic regression with statmodels

logreg = sm.Logit(y, X[['const', 'balance']]).fit()

logreg.summary()

print(logreg.summary().tables[1])

logreg2 = sm.Logit(y, X).fit()

logreg2.summary()

# Let us perform the regressions with scikit-learn

X = default[['balance', 'income', 'd_student']]

sk_linreg = LinearRegression().fit(X, y)

print(linreg2.summary().tables[1])
sk_linreg.coef_
sk_linreg.intercept_

sk_logreg = LogisticRegression().fit(X, y)

print(logreg2.summary().tables[1])
sk_logreg.coef_
sk_logreg.intercept_

# Let us try a different algorithm
X = default[['balance', 'income', 'd_student']]
sk_logreg2 = LogisticRegression(solver='newton-cg').fit(X, y)

print(logreg2.summary().tables[1])
sk_logreg2.coef_
sk_logreg2.intercept_

# Let us now perform LDA on the data
lda = LinearDiscriminantAnalysis().fit(X, default['default'])

# We can check the possible classes:
lda.classes_
# We can check the priors, i.e. our estimates of Pr(Y=k):
print(pd.DataFrame(lda.priors_, index=['No', 'Yes'], columns=['Pr(Y)']))
# We can check our estimates of mu_k:
print(pd.DataFrame(lda.means_, index = ['No', 'Yes'],
                   columns = X.columns))

# Let us reproduce the confusion matrix
y_hat = lda.predict(X)

print(pd.DataFrame(confusion_matrix(default['default'], y_hat).T,
                                    index = ['No', 'Yes'], columns = ['No', 'Yes']))

fal_pos_rate = confusion_matrix(default['default'], y_hat).T[1,0]/np.sum(confusion_matrix(default['default'], y_hat).T[:,0])
print(fal_pos_rate)
fal_neg_rate = confusion_matrix(default['default'], y_hat).T[0,1]/np.sum(confusion_matrix(default['default'], y_hat).T[:,1])
print(fal_neg_rate)

# We can also print a classification report:
print(classification_report(default['default'], y_hat))

# Note: Precision is the rate of prediction that corresponds to the truth
## Here: we predict 9645+254=9899 no defaults from which 9645 were no defaults
## Here: we predict 22+79=101 defaults from which 79 where defaults
# Note: Recall is one minus the false positive and false negative rate, respectively

# Let us try a different threshold than 50%, e.g. 20%

prob_hat = lda.predict_proba(X)[:,1]
new_y_hat = []
for i in prob_hat:
    if i>0.2:
        new_y_hat.append('Yes')
    else:
        new_y_hat.append('No')
new_y_hat = np.array(new_y_hat)

print(pd.DataFrame(confusion_matrix(default['default'], new_y_hat).T,
                                    index = ['No', 'Yes'], columns = ['No', 'Yes']))

print(classification_report(default['default'], new_y_hat))

##########################################################################
########## Here we construct the ROC curve from the lecture ##############
##########################################################################
y_score = lda.fit(X, y).decision_function(X)

fpr, tpr, _ = roc_curve(y, y_score)
roc_auc = auc(fpr, tpr)

plt.figure()
plt.plot(fpr, tpr, color='orange',
         label='ROC curve (area = {:0.2f})'.format(roc_auc))
plt.plot([0, 1], [0, 1], color='blue', linestyle='--')
plt.xlim([-0.05, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC curve for our LDA')
plt.legend(loc="lower right")
##########################################################################
##########################################################################

# Let us turn to QDA now

qda = QuadraticDiscriminantAnalysis().fit(X, default['default'])

print(pd.DataFrame(qda.priors_, index=['No', 'Yes'], columns=['Pr(Y)']))

print(pd.DataFrame(qda.means_, index = ['No', 'Yes'],
                   columns = X.columns))

y_hat = qda.predict(X)

print(pd.DataFrame(confusion_matrix(default['default'], y_hat).T,
                                    index = ['No', 'Yes'], columns = ['No', 'Yes']))

print(classification_report(default['default'], y_hat))

# Now, let us split the data into a training and test data set

X_train, X_test, y_train, y_test = train_test_split(X, default['default'],
                                                    test_size=0.3, random_state=181)

logreg = LogisticRegression(solver='newton-cg').fit(X_train, y_train)
logreg_y_hat = logreg.predict(X_test)
precision_score(y_test, logreg_y_hat, labels=['No', 'Yes'], pos_label='No')

lda = LinearDiscriminantAnalysis().fit(X_train, y_train)
lda_y_hat = lda.predict(X_test)
precision_score(y_test, lda_y_hat, labels=['No', 'Yes'], pos_label='No')

qda = QuadraticDiscriminantAnalysis().fit(X_train, y_train)
qda_y_hat = qda.predict(X_test)
precision_score(y_test, qda_y_hat, labels=['No', 'Yes'], pos_label='No')

knn = KNeighborsClassifier(n_neighbors=3).fit(X_train, y_train)
knn_y_hat = knn.predict(X_test)
precision_score(y_test, knn_y_hat, labels=['No', 'Yes'], pos_label='No')


### dense model

from keras import Model
from keras.layers import Dense, Dropout, BatchNormalization, Reshape, Embedding, Reshape, Concatenate, Activation
import numpy as np
from sklearn.model_selection import StratifiedKFold
from metrics import keras_auc,keras_precision,keras_recall,keras_f1,keras_fb
from common import prep, read_data
from keras.callbacks import EarlyStopping
import tensorflow as tf
from keras.optimizers import SGD
from tqdm import tqdm

from keras.constraints import MaxNorm

class SimpleMLP(Model):

    def __init__(self, input_dim, embedding_dim = [128,128], use_bn=False, use_dp=False):
        super(SimpleMLP, self).__init__(name='mlp')
        self.use_bn = use_bn
        self.use_dp = use_dp

        if not isinstance(embedding_dim,list):
            embedding_dim = [embedding_dim] * 2

        constraint = MaxNorm(1,axis=1)

        self.e1 = Embedding(input_dim[0],embedding_dim[0],embeddings_constraint=constraint)
        self.e2 = Embedding(input_dim[1],embedding_dim[1],embeddings_constraint=constraint)
        self.m = Concatenate()

        self.rate = 0.2

        self.ls = [Concatenate(),
                   Dense(32),
                   Dropout(self.rate),
                   BatchNormalization(axis=-1),
                   Activation('relu'),
                   Dense(1,activation='sigmoid'),
                   Reshape((-1,))]

        if self.use_dp:
            self.dp = Dropout(self.rate)
        if self.use_bn:
            self.bn = BatchNormalization(axis=-1)

    def call(self, inputs):
        x1,x2 = inputs[:,0],inputs[:,1]

        x1 = self.e1(x1)
        x2 = self.e2(x2)

        if self.use_dp:
            x1 = self.dp(x1)
            x2 = self.dp(x2)

        if self.use_bn:
            x1 = self.bn(x1)
            x2 = self.bn(x2)

        x = [x1,x2]

        for l in self.ls:
            x = l(x)

        return x

def main(cv=False, verbose = 1, num_cl=10):
    Etr = read_data('./data/endpoints_train.txt')
    Ete = read_data('./data/endpoints_test.txt')

    Cg = read_data('./data/chemical_graph.txt')
    Tg = read_data('./data/taxonomy_graph.txt')

    C = set.union(*[set([a,b]) for a,_,b,_ in Cg])
    T = set.union(*[set([a,b]) for a,_,b,_ in Tg])

    Xc = set([k for k,_,_,_ in Etr if k in C]) | set([k for k,_,_,_ in Ete if k in C])
    Xt = set([k for _,_,k,_ in Etr if k in T]) | set([k for _,_,k,_ in Ete if k in T])
    N = len(Xc)
    M = len(Xt)
    mapping_c = {c:i for c,i in zip(Xc,range(N))}
    mapping_t = {c:i for c,i in zip(Xt,range(M))}

    Xtr = []
    Ytr = []
    Xte = []
    Yte = []

    for c,p,t,e in Etr:
        try:
            tmp = [mapping_c[c],mapping_t[t]]
            if tmp in Xtr: continue
            e = float(e)
            Xtr.append(tmp)
            Ytr.append(e)
        except KeyError:
            pass

    for c,p,t,e in Ete:
        try:
            tmp = [mapping_c[c],mapping_t[t]]
            if tmp in Xte: continue
            e = float(e)
            Xte.append(tmp)
            Yte.append(e)
        except KeyError:
            pass

    Xtr = np.asarray(Xtr)
    Ytr = np.asarray(Ytr)
    Xte = np.asarray(Xte)
    Yte = np.asarray(Yte)

    metrics = ['accuracy', keras_precision, keras_recall, keras_auc,keras_f1, keras_fb]
    callbacks = [EarlyStopping(monitor='loss', mode='min', patience=5, restore_best_weights=True)]
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
    cvscores = []
    epochs = 1000
    if cv:
        with tqdm(total=10,desc='CV') as pbar:
            for train,test in kfold.split(Xtr,Ytr):
                model = SimpleMLP(input_dim=[N,M], embedding_dim=16, use_bn=True,use_dp=True)
                # Compile model
                model.compile(loss='binary_crossentropy', optimizer='adagrad', metrics=metrics)
                # Fit the model
                model.fit(Xtr[train], Ytr[train], epochs=epochs, batch_size=len(Ytr[train]), verbose=verbose, callbacks = callbacks)
                # evaluate the model
                scores = model.evaluate(Xtr[test], Ytr[test], verbose=0,batch_size=len(Ytr[test]))
                cvscores.append(scores)
                pbar.update(1)

            cvscores = np.asarray(cvscores)

        for i,n in enumerate(model.metrics_names):
            print(n,"%.2f (+/- %.2f)" % (np.mean(cvscores[:,i]), np.std(cvscores[:,i])))

    cvscores = []
    for i in tqdm(range(num_cl)):
        model = SimpleMLP(input_dim=[N,M], embedding_dim=16, use_bn=True,use_dp=True)
        model.compile('adagrad', loss='binary_crossentropy',metrics=metrics)
        model.fit(Xtr, Ytr, batch_size = len(Ytr), epochs = epochs, validation_split=0.0, shuffle=True, verbose = verbose, callbacks=callbacks)

        scores = model.evaluate(Xte, Yte, batch_size=len(Yte), verbose=0)
        cvscores.append(scores)

        p = model.predict(Xte)
        with open('./results/MLP/'+str(i)+'.txt', 'w') as f:
            for a,b in zip(Yte,p):
                f.write(str(a)+','+str(b[-1])+'\n')

    cvscores = np.asarray(cvscores)
    for i,n in enumerate(model.metrics_names):
        print(n,"%.2f (+/- %.2f)" % (np.mean(cvscores[:,i]), np.std(cvscores[:,i])))

if __name__ == '__main__':
    main(cv=True,verbose=0,num_cl=10)


import csv
import re
import matplotlib.pyplot as plt
from datetime import datetime

def error_reader(filename):
    with open(filename, "r") as error_log:
        ourlist = []
        for x in error_log:
            if("djovanoska" in x):
                ourlist.append(x)

        ourlist = ''.join(ourlist)
        regexp = r"(?<=\[)(.*?)(?=\])"
        errordata = re.findall(regexp, ourlist)
        return(errordata)

def write_csv(errorlist):
    with open('error_log.csv', 'w') as csvfile:
        writer = csv.writer(csvfile)
        header = ['timestamp ', 'error ', 'pid ', 'requestby ']
        writer.writerow(header)
        count =  0
        for item in errorlist:
            if count == 0:
                time = item
                count += 1
            elif count == 1:
                type = item
                count +=1
            elif count == 2:
                pid = item
                count +=1
            elif count == 3:
                client = item
                writer.writerow((time, type, pid, client))
                count = 0

    print("Write Successful...")

def create_error_timeline(errorlist):
    timestamps = errorlist[::4]
    timestamps = [datetime.strptime(ts, "%a %b %d %H:%M:%S.%f %Y") for ts in timestamps]

    plt.figure(figsize=(10, 6))
    plt.plot(timestamps, range(len(timestamps)), marker='o', linestyle='-', color='red')
    plt.xlabel('Timestamp')
    plt.ylabel('Error Occurrences')
    plt.title('Error Occurrences Timeline')
    plt.xticks(rotation=45)
    plt.tight_layout()
    # plt.show()
    plt.savefig('error_timeline.pdf')
    print("Plot saved as error_timeline.pdf")

if __name__ == "__main__":
    error_list = error_reader('/var/log/apache2/error_log')
    write_csv(error_list)

    create_error_timeline(error_list)


from abc import (
    ABCMeta,
    abstractmethod
)

from lib.utils.misc import load_config

class BaseSourceGenerator(object):
    __metaclass__ = ABCMeta

    def __init__(self, **kwargs):
        self.root_config = load_config()
        self.generator_config = load_config()['source_generator']

    @abstractmethod
    def generate_source_data(self, **kwargs):
        raise NotImplementedError


from os.path import join
import numpy as np
import torch
import pandas as pd

def synthesize_data(f, f_name, project_root, input_dim, N=10000):
    input = torch.rand(N, input_dim)
    output = f(input)
    y = [np.random.choice([1, 0], p=[p.item(), 1 - p.item()]) for p in output]
    df = pd.DataFrame(
        torch.cat((input, output), dim=1).detach().numpy(),
        columns=["x1", "x2", "target_p"],
    )
    df["target_y"] = y
    # write a pandas dataframe to gzipped CSV file

    path = join(project_root, f"data/raw/{f_name}.csv.gz")
    print("saving to :", path)
    df.to_csv(path, index=False, compression="gzip")
    print("saved")
    return df, path


from django.shortcuts import render, redirect

# Create your views here.

def index(request):
    return render(request, "index.html")

def result(request):
    context = {
        "name": request.POST["name"],
        "location": request.POST["location"],
        "language": request.POST["language"],
        "comments": request.POST["comments"],
    }
    return render(request, "result.html", context)


# Copyright (c) 2001-2020 Seagate Technology LLC and/or its Affiliates
#
# This program is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>. For any questions
# about this software or licensing, please email opensource@seagate.com or
# cortx-questions@seagate.com.

"""
 ****************************************************************************
  Description:       Message Handler for requesting node data
 ****************************************************************************
"""

import json
import time
import os

from framework.base.internal_msgQ import InternalMsgQ
from framework.base.module_thread import ScheduledModuleThread
from framework.base.sspl_constants import enabled_products, DATA_PATH
from framework.utils.conf_utils import (GLOBAL_CONF, SSPL_CONF, Conf,
                                        NODE_ID_KEY)
from framework.utils.service_logging import logger
from framework.utils.severity_reader import SeverityReader
from json_msgs.messages.sensors.cpu_data import CPUdataMsg
from json_msgs.messages.sensors.disk_space_alert import DiskSpaceAlertMsg
from json_msgs.messages.sensors.host_update import HostUpdateMsg
from json_msgs.messages.sensors.if_data import IFdataMsg
from json_msgs.messages.sensors.local_mount_data import LocalMountDataMsg
from json_msgs.messages.sensors.node_hw_data import NodeIPMIDataMsg
from json_msgs.messages.sensors.raid_data import RAIDdataMsg
from json_msgs.messages.sensors.raid_integrity_msg import RAIDIntegrityMsg
from framework.messaging.egress_processor import EgressProcessor
from framework.utils.store_factory import file_store
from framework.utils.utility import Utility

# Override default store
store = file_store

class NodeDataMsgHandler(ScheduledModuleThread, InternalMsgQ):
    """Message Handler for generic node requests and generating
        host update messages on a regular interval"""

    MODULE_NAME = "NodeDataMsgHandler"
    PRIORITY    = 2

    # Section and keys in configuration file
    NODEDATAMSGHANDLER = MODULE_NAME.upper()
    TRANSMIT_INTERVAL = 'transmit_interval'
    HIGH_CPU_USAGE_WAIT_THRESHOLD = 'high_cpu_usage_wait_threshold'
    HIGH_MEMORY_USAGE_WAIT_THRESHOLD = 'high_memory_usage_wait_threshold'
    UNITS = 'units'
    DISK_USAGE_THRESHOLD = 'disk_usage_threshold'
    DEFAULT_DISK_USAGE_THRESHOLD = 80
    CPU_USAGE_THRESHOLD = 'cpu_usage_threshold'
    DEFAULT_CPU_USAGE_THRESHOLD = 80
    HOST_MEMORY_USAGE_THRESHOLD = 'host_memory_usage_threshold'
    DEFAULT_HOST_MEMORY_USAGE_THRESHOLD = 80

    SYSTEM_INFORMATION = "SYSTEM_INFORMATION"

    IPMI_RESOURCE_TYPE_PSU = "node:hw:psu"
    IPMI_RESOURCE_TYPE_FAN = "node:hw:fan"
    IPMI_RESOURCE_TYPE_DISK = "node:hw:disk"
    IPMI_RESOURCE_TYPE_TEMPERATURE = "node:sensor:temperature"
    IPMI_RESOURCE_TYPE_VOLTAGE = "node:sensor:voltage"
    IPMI_RESOURCE_TYPE_CURRENT = "node:sensor:current"
    NW_RESOURCE_TYPE = "node:interface:nw"
    NW_CABLE_RESOURCE_TYPE = "node:interface:nw:cable"
    high_usage = {
        'cpu' : False,
        'disk' : False,
        'memory' : False
    }
    usage_time_map = {
        'cpu' : -1,
        'memory' : -1,
        'disk' : -1
    }
    fault_resolved_iterations = {
        'cpu': 0,
        'memory': 0,
        'disk': 0
    }
    prev_nw_if_status = {}
    prev_cable_status = {}
    FAULT = "fault"
    FAULT_RESOLVED = "fault_resolved"

    INTERFACE_FAULT_DETECTED = False

    CACHE_DIR_NAME = "server"

    # Dependency list
    DEPENDENCIES = {
                    "plugins": ["EgressProcessor"],
                    "rpms": []
    }

    @staticmethod
    def name():
        """ @return: name of the module."""
        return NodeDataMsgHandler.MODULE_NAME

    def __init__(self):
        super(NodeDataMsgHandler, self).__init__(self.MODULE_NAME,
                                                  self.PRIORITY)
        # Flag to indicate suspension of module
        self._suspended = False

    @staticmethod
    def dependencies():
        """Returns a list of plugins and RPMs this module requires
           to function.
        """
        return NodeDataMsgHandler.DEPENDENCIES

    def initialize(self, conf_reader, msgQlist, product):
        """initialize configuration reader and internal msg queues"""
        # Initialize ScheduledMonitorThread
        super(NodeDataMsgHandler, self).initialize(conf_reader)

        # Initialize internal message queues for this module
        super(NodeDataMsgHandler, self).initialize_msgQ(msgQlist)

        self._transmit_interval = int(Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.TRANSMIT_INTERVAL}",
                                               60))
        self._high_cpu_usage_wait_threshold = int(Conf.get(SSPL_CONF,
                                                f"{self.NODEDATAMSGHANDLER}>{self.HIGH_CPU_USAGE_WAIT_THRESHOLD}",60))
        self._high_memory_usage_wait_threshold = int(Conf.get(SSPL_CONF,
                                                f"{self.NODEDATAMSGHANDLER}>{self.HIGH_MEMORY_USAGE_WAIT_THRESHOLD}",60))
        self._units = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.UNITS}",
                                                "MB")
        self._disk_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.DISK_USAGE_THRESHOLD}",
                                                self.DEFAULT_DISK_USAGE_THRESHOLD)

        self._cpu_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.CPU_USAGE_THRESHOLD}",
                                                self.DEFAULT_CPU_USAGE_THRESHOLD)

        self._host_memory_usage_threshold = Conf.get(SSPL_CONF, f"{self.NODEDATAMSGHANDLER}>{self.HOST_MEMORY_USAGE_THRESHOLD}",
                                                self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD)

        self.node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01")

        self.bmcNwStatus = None
        self.severity_reader = SeverityReader()
        self._node_sensor    = None
        self._login_actuator = None
        self.disk_sensor_data = None
        self.host_sensor_data = None
        self.if_sensor_data = None
        self.cpu_sensor_data = None
        self.raid_sensor_data = None
        self.sensor_type = None
        self._epoch_time = str(int(time.time()))
        self._raid_drives = []
        self._raid_device = "N/A"
        self.os_sensor_type = {
            "disk_space" : self.disk_sensor_data,
            "memory_usage" : self.host_sensor_data,
            "nw"   : self.if_sensor_data,
            "cpu_usage"  : self.cpu_sensor_data,
            "raid_data" : self.raid_sensor_data
        }

        # UUID used in json msgs
        self._uuid = None

        # Dict of drives by device name from systemd
        self._drive_by_device_name = {}

        # Dict of drive path by-ids by serial number from systemd
        self._drive_byid_by_serial_number = {}

        self._import_products(product)
        self.cache_dir_path = os.path.join(DATA_PATH, self.CACHE_DIR_NAME)

        self.persistent_data = {
            'cpu' : {},
            'disk' : {},
            'memory' : {},
            'nw' : {}
        }
        # Persistent Cache for High CPU usage
        self.init_from_persistent_cache('cpu', 'CPU_USAGE_DATA')
        # Persistent Cache for High Disk Usage
        self.init_from_persistent_cache('disk', 'DISK_USAGE_DATA')
        # Persistent Cache for High Memory Usage
        self.init_from_persistent_cache('memory', 'MEMORY_USAGE_DATA')
        # Persistent Cache for Nework sensor
        self.init_from_persistent_cache('nw', 'NW_SENSOR_DATA')

    def init_from_persistent_cache(self, resource, data_path):
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')

        if os.path.isfile(PER_DATA_PATH):
            self.persistent_data[resource] = store.get(PER_DATA_PATH)
        if self.persistent_data[resource]:
            if resource == 'nw':
                self.prev_nw_if_status = \
                    self.persistent_data[resource].get('prev_nw_if_status', {})
                self.prev_cable_status = \
                    self.persistent_data[resource].get('prev_cable_status', {})
            elif self.persistent_data[resource]\
                [f'high_{resource}_usage'].lower() == "true":
                self.high_usage[resource] = True
            else:
                self.high_usage[resource] = False
        else:
            self.persist_state_data(resource, data_path)

    def persist_state_data(self, resource, data_path):
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')
        if resource == 'nw':
            self.persistent_data[resource] = {
                'prev_nw_if_status': self.prev_nw_if_status,
                'prev_cable_status': self.prev_cable_status
            }
        else:
            self.persistent_data[resource] = {
                f'high_{resource}_usage': str(self.high_usage[resource]),
                f'{resource}_usage_time_map':
                    str(self.usage_time_map[resource]),
                f'{resource}_fault_resolved_iterations':
                    str(self.fault_resolved_iterations[resource])
            }
        store.put(self.persistent_data[resource], PER_DATA_PATH)

    def read_persistent_data(self, data_path):
        """Read resource data from persistent cache."""
        persistent_data = {}
        PER_DATA_PATH = os.path.join(self.cache_dir_path,
                            f'{data_path}_{self.node_id}')
        if os.path.isfile(PER_DATA_PATH):
            persistent_data = store.get(PER_DATA_PATH)
        return persistent_data

    def _import_products(self, product):
        """Import classes based on which product is being used"""
        if product.lower() in [x.lower() for x in enabled_products]:
            from zope.component import queryUtility
            self._queryUtility = queryUtility

    def run(self):
        """Run the module periodically on its own thread."""
        self._log_debug("Start accepting requests")

        # Do not proceed if module is suspended
        if self._suspended == True:
            self._scheduler.enter(1, self._priority, self.run, ())
            return

        # self._set_debug(True)
        # self._set_debug_persist(True)

        try:
            # Query the Zope GlobalSiteManager for an object implementing INodeData
            if self._node_sensor is None:
                from sensors.INode_data import INodeData
                self._node_sensor = self._queryUtility(INodeData)()
                self._log_debug("_node_sensor name: %s" % self._node_sensor.name())

            # Delay for the desired interval if it's greater than zero
            if self._transmit_interval > 0:
                logger.debug("self._transmit_interval:{}".format(self._transmit_interval))
                timer = self._transmit_interval
                while timer > 0:
                    # See if the message queue contains an entry and process
                    jsonMsg, _ = self._read_my_msgQ_noWait()
                    if jsonMsg is not None:
                        self._process_msg(jsonMsg)

                    time.sleep(1)
                    timer -= 1

                # Generate the JSON messages with data from the node and transmit on regular interval
                self._generate_host_update()
                self._generate_cpu_data()
                self._generate_if_data()
                self._generate_disk_space_alert()

            # If the timer is zero then block for incoming requests notifying to transmit data
            else:
                # Block on message queue until it contains an entry
                jsonMsg, _ = self._read_my_msgQ()
                if jsonMsg is not None:
                    self._process_msg(jsonMsg)

                # Keep processing until the message queue is empty
                while not self._is_my_msgQ_empty():
                    jsonMsg, _ = self._read_my_msgQ()
                    if jsonMsg is not None:
                        self._process_msg(jsonMsg)

        except Exception as ae:
            # Log it and restart the whole process when a failure occurs
            logger.exception("NodeDataMsgHandler restarting: %s" % ae)

        self._scheduler.enter(1, self._priority, self.run, ())
        self._log_debug("Finished processing successfully")

    def _process_msg(self, jsonMsg):
        """Parses the incoming message and generate the desired data message"""
        self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg)

        if isinstance(jsonMsg, dict) is False:
            jsonMsg = json.loads(jsonMsg)

        # Parse out the uuid so that it can be sent back in response message
        self._uuid = None
        if jsonMsg.get("sspl_ll_msg_header") is not None and \
           jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
            self._uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
            self._log_debug("_processMsg, uuid: %s" % self._uuid)

        if jsonMsg.get("sensor_request_type") is not None and \
           jsonMsg.get("sensor_request_type").get("node_data") is not None and \
           jsonMsg.get("sensor_request_type").get("node_data").get("sensor_type") is not None:
            self.sensor_type = jsonMsg.get("sensor_request_type").get("node_data").get("sensor_type").split(":")[2]
            self._log_debug("_processMsg, sensor_type: %s" % self.sensor_type)

            if self.sensor_type == "memory_usage":
                self._generate_host_update()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "cpu_usage":
                self._generate_cpu_data()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "nw":
                self._generate_if_data()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "disk_space":
                self._generate_disk_space_alert()
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                          sensor_message_type)
                else:
                    self._log_debug(f"NodeDataMsgHandler, _process_msg, \
                        No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "raid_data":
                self._generate_raid_data(jsonMsg)
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                            sensor_message_type)
                else:
                    self._log_debug("NodeDataMsgHandler, _process_msg " +
                            f"No past data found for {self.sensor_type} sensor type")

            elif self.sensor_type == "raid_integrity":
                self._generate_raid_integrity_data(jsonMsg)
                sensor_message_type = self.os_sensor_type.get(self.sensor_type, "")
                if sensor_message_type:
                    self._write_internal_msgQ(EgressProcessor.name(),
                                            sensor_message_type)
                else:
                    self._log_debug("NodeDataMsgHandler, _process_msg " +
                            f"No past data found for {self.sensor_type} sensor type")

        # Update mapping of device names to serial numbers for global use
        elif jsonMsg.get("sensor_response_type") is not None:
            if jsonMsg.get("sensor_response_type") == "devicename_serialnumber":
                self._update_devicename_sn_dict(jsonMsg)
        elif jsonMsg.get("sensor_request_type") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data").get("info") is not None and \
            jsonMsg.get("sensor_request_type").get("node_data").get("info").get("resource_type") is not None:
                self._generate_node_fru_data(jsonMsg)

        # ... handle other node sensor message types

    def _update_devicename_sn_dict(self, jsonMsg):
        """Update the dict of device names to serial numbers"""
        drive_byid = jsonMsg.get("drive_byid")
        device_name = jsonMsg.get("device_name")
        serial_number = jsonMsg.get("serial_number")

        self._drive_by_device_name[device_name] = serial_number
        self._drive_byid_by_serial_number[serial_number] = drive_byid

        self._log_debug("NodeDataMsgHandler, device_name: %s, serial_number: %s, drive_byid: %s" %
                        (device_name, serial_number, drive_byid))

    def _generate_host_update(self):
        """Create & transmit a host update message as defined
            by the sensor response json schema"""

        current_time = Utility.get_current_time()

        # Notify the node sensor to update its data required for the host_update message
        successful = self._node_sensor.read_data("host_update", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_host_update was NOT successful.")

        self._host_memory_usage_threshold = str(self._host_memory_usage_threshold)
        try:
            if self._host_memory_usage_threshold.isdigit():
                self._host_memory_usage_threshold = int(self._host_memory_usage_threshold)
            else:
                self._host_memory_usage_threshold = float(self._host_memory_usage_threshold)
        except ValueError:
            logger.warn("Host Memory Alert, Invalid host_memory_usage_threshold value are entered in config.")
            # Assigning default value to _memory_usage_threshold
            self._host_memory_usage_threshold = self.DEFAULT_HOST_MEMORY_USAGE_THRESHOLD

        memory_persistent_data = self.read_persistent_data('MEMORY_USAGE_DATA')
        if memory_persistent_data.get('memory_usage_time_map') is not None:
            previous_check_time = int(memory_persistent_data['memory_usage_time_map'])
        else:
            previous_check_time = int(-1)
        if memory_persistent_data\
                .get('memory_fault_resolved_iterations') is not None:
            fault_resolved_iters = int(
                memory_persistent_data['memory_fault_resolved_iterations'])
        else:
            fault_resolved_iters = 0
        try:
            iteration_limit = int(
                self._high_memory_usage_wait_threshold/self._transmit_interval)
        except ZeroDivisionError:
            iteration_limit = 0
        self.usage_time_map['memory'] = current_time

        if self._node_sensor.total_memory["percent"] >= self._host_memory_usage_threshold \
           and not self.high_usage['memory']:
            if previous_check_time == -1:
                previous_check_time = current_time
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

            if self.usage_time_map['memory'] - previous_check_time >= self._high_memory_usage_wait_threshold:
                self.high_usage['memory'] = True
                self.fault_resolved_iterations['memory'] = 0
                # Create the memory data message and hand it over
                # to the egress processor to transmit
                fault_event = "Host memory usage has increased to {}%,"\
                    "beyond the configured threshold of {}% "\
                    "for more than {} seconds.".format(
                        self._node_sensor.total_memory["percent"],
                        self._host_memory_usage_threshold,
                        self._high_memory_usage_wait_threshold
                    )

                logger.warn(fault_event)

                logged_in_users = []
                # Create the host update message and hand it over to the egress processor to transmit
                hostUpdateMsg = HostUpdateMsg(self._node_sensor.host_id,
                                        self._epoch_time,
                                        self._node_sensor.boot_time,
                                        self._node_sensor.up_time,
                                        self._node_sensor.uname, self._units,
                                        self._node_sensor.total_memory,
                                        self._node_sensor.logged_in_users,
                                        self._node_sensor.process_count,
                                        self._node_sensor.running_process_count,
                                        self.FAULT,
                                        fault_event
                                        )
                # Add in uuid if it was present in the json request
                if self._uuid is not None:
                    hostUpdateMsg.set_uuid(self._uuid)
                jsonMsg = hostUpdateMsg.getJson()
                # Transmit it to message processor
                self.host_sensor_data = jsonMsg
                self.os_sensor_type["memory_usage"] = self.host_sensor_data
                self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

        if self._node_sensor.total_memory["percent"] < self._host_memory_usage_threshold:
            if not self.high_usage['memory']:
                self.persist_state_data('memory', 'MEMORY_USAGE_DATA')
            else:
                if fault_resolved_iters < iteration_limit:
                    fault_resolved_iters += 1
                    self.fault_resolved_iterations['memory'] = fault_resolved_iters
                    self.persist_state_data('memory', 'MEMORY_USAGE_DATA')
                elif fault_resolved_iters >= iteration_limit:
                    # Create the memory data message and hand it over
                    # to the egress processor to transmit
                    fault_resolved_event = "Host memory usage has decreased to {}%, "\
                        "lower than the configured threshold of {}%.".format(
                            self._node_sensor.total_memory["percent"],
                            self._host_memory_usage_threshold
                        )
                    logger.info(fault_resolved_event)
                    logged_in_users = []

                    # Create the host update message and hand it over to the egress processor to transmit
                    hostUpdateMsg = HostUpdateMsg(self._node_sensor.host_id,
                                            self._epoch_time,
                                            self._node_sensor.boot_time,
                                            self._node_sensor.up_time,
                                            self._node_sensor.uname, self._units,
                                            self._node_sensor.total_memory,
                                            self._node_sensor.logged_in_users,
                                            self._node_sensor.process_count,
                                            self._node_sensor.running_process_count,
                                            self.FAULT_RESOLVED,
                                            fault_resolved_event
                                            )

                    # Add in uuid if it was present in the json request
                    if self._uuid is not None:
                        hostUpdateMsg.set_uuid(self._uuid)
                    jsonMsg = hostUpdateMsg.getJson()
                    # Transmit it to message processor
                    self.host_sensor_data = jsonMsg
                    self.os_sensor_type["memory_usage"] = self.host_sensor_data
                    self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                    self.high_usage['memory'] = False
                    self.usage_time_map['memory'] = int(-1)
                    self.fault_resolved_iterations['memory'] = 0
                    self.persist_state_data('memory', 'MEMORY_USAGE_DATA')

    def _generate_local_mount_data(self):
        """Create & transmit a local_mount_data message as defined
            by the sensor response json schema"""

        # Notify the node sensor to update its data required for the local_mount_data message
        successful = self._node_sensor.read_data("local_mount_data", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_local_mount_data was NOT successful.")

        # Create the local mount data message and hand it over to the egress processor to transmit
        localMountDataMsg = LocalMountDataMsg(self._node_sensor.host_id,
                                self._epoch_time,
                                self._node_sensor.free_space,
                                self._node_sensor.free_inodes,
                                self._node_sensor.free_swap,
                                self._node_sensor.total_space,
                                self._node_sensor.total_swap,
                                self._units)

        # Add in uuid if it was present in the json request
        if self._uuid is not None:
            localMountDataMsg.set_uuid(self._uuid)
        jsonMsg = localMountDataMsg.getJson()

        # Transmit it to message processor
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

    def _generate_cpu_data(self):
        """Create & transmit a cpu_data message as defined
            by the sensor response json schema"""

        current_time = Utility.get_current_time()

        # Notify the node sensor to update its data required for the cpu_data message
        successful = self._node_sensor.read_data("cpu_data", self._get_debug())
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_cpu_data was NOT successful.")

        self._cpu_usage_threshold = str(self._cpu_usage_threshold)
        try:
            if self._cpu_usage_threshold.isdigit():
                self._cpu_usage_threshold = int(self._cpu_usage_threshold)
            else:
                self._cpu_usage_threshold = float(self._cpu_usage_threshold)
        except ValueError:
            logger.warn("CPU Usage Alert, Invalid host_memory_usage_threshold value are entered in config.")
            # Assigning default value to _cpu_usage_threshold
            self._cpu_usage_threshold = self.DEFAULT_CPU_USAGE_THRESHOLD

        cpu_persistent_data = self.read_persistent_data('CPU_USAGE_DATA')
        if cpu_persistent_data.get('cpu_usage_time_map') is not None:
            previous_check_time = int(cpu_persistent_data['cpu_usage_time_map'])
        else:
            previous_check_time = int(-1)
        if cpu_persistent_data.get('cpu_fault_resolved_iterations') is not None:
            fault_resolved_iters = int(
                cpu_persistent_data['cpu_fault_resolved_iterations'])
        else:
            fault_resolved_iters = 0
        try:
            iteration_limit = int(
                self._high_cpu_usage_wait_threshold/self._transmit_interval)
        except ZeroDivisionError:
            iteration_limit = 0
        self.usage_time_map['cpu'] = current_time

        if self._node_sensor.cpu_usage >= self._cpu_usage_threshold \
           and not self.high_usage['cpu']:
            if previous_check_time == -1:
                previous_check_time = current_time
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')

            if self.usage_time_map['cpu'] - previous_check_time >= self._high_cpu_usage_wait_threshold:

                self.high_usage['cpu'] = True
                self.fault_resolved_iterations['cpu'] = 0
                # Create the cpu usage data message and hand it over
                # to the egress processor to transmit
                fault_event = "CPU usage has increased to {}%, "\
                    "beyond the configured threshold of {}% "\
                    "for more than {} seconds.".format(
                        self._node_sensor.cpu_usage,
                        self._cpu_usage_threshold,
                        self._high_cpu_usage_wait_threshold
                    )
                logger.warn(fault_event)

                # Create the cpu usage update message and hand it over to the egress processor to transmit
                cpuDataMsg = CPUdataMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.csps,
                                    self._node_sensor.idle_time,
                                    self._node_sensor.interrupt_time,
                                    self._node_sensor.iowait_time,
                                    self._node_sensor.nice_time,
                                    self._node_sensor.softirq_time,
                                    self._node_sensor.steal_time,
                                    self._node_sensor.system_time,
                                    self._node_sensor.user_time,
                                    self._node_sensor.cpu_core_data,
                                    self._node_sensor.cpu_usage,
                                    self.FAULT,
                                    fault_event
                                )

                # Add in uuid if it was present in the json request
                if self._uuid is not None:
                    cpuDataMsg.set_uuid(self._uuid)
                jsonMsg = cpuDataMsg.getJson()
                self.cpu_sensor_data = jsonMsg
                self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data

                # Transmit it to message processor
                self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                # Store the state to Persistent Cache.
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')

        if self._node_sensor.cpu_usage < self._cpu_usage_threshold:
            if not self.high_usage['cpu']:
                self.persist_state_data('cpu', 'CPU_USAGE_DATA')
            else:
                if fault_resolved_iters < iteration_limit:
                    fault_resolved_iters += 1
                    self.fault_resolved_iterations['cpu'] = fault_resolved_iters
                    self.persist_state_data('cpu', 'CPU_USAGE_DATA')
                elif fault_resolved_iters >= iteration_limit:

                    # Create the cpu usage data message and hand it over
                    # to the egress processor to transmit
                    fault_resolved_event = "CPU usage has decreased to {}%, "\
                        "lower than the configured threshold of {}%.".format(
                            self._node_sensor.cpu_usage,
                            self._cpu_usage_threshold
                        )
                    logger.info(fault_resolved_event)

                    # Create the cpu usage update message and hand it over to the egress processor to transmit
                    cpuDataMsg = CPUdataMsg(self._node_sensor.host_id,
                                        self._epoch_time,
                                        self._node_sensor.csps,
                                        self._node_sensor.idle_time,
                                        self._node_sensor.interrupt_time,
                                        self._node_sensor.iowait_time,
                                        self._node_sensor.nice_time,
                                        self._node_sensor.softirq_time,
                                        self._node_sensor.steal_time,
                                        self._node_sensor.system_time,
                                        self._node_sensor.user_time,
                                        self._node_sensor.cpu_core_data,
                                        self._node_sensor.cpu_usage,
                                        self.FAULT_RESOLVED,
                                        fault_resolved_event
                                    )

                    # Add in uuid if it was present in the json request
                    if self._uuid is not None:
                        cpuDataMsg.set_uuid(self._uuid)
                    jsonMsg = cpuDataMsg.getJson()
                    self.cpu_sensor_data = jsonMsg
                    self.os_sensor_type["cpu_usage"] = self.cpu_sensor_data

                    # Transmit it to message processor
                    self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
                    self.high_usage['cpu'] = False
                    self.usage_time_map['cpu'] = int(-1)
                    self.fault_resolved_iterations['cpu'] = 0
                    # Store the state to Persistent Cache.
                    self.persist_state_data('cpu', 'CPU_USAGE_DATA')

    def _send_ifdata_json_msg(self, resource_id, resource_type, state, event=""):
        """A resuable method for transmitting IFDataMsg to RMQ and IEM logging"""
        ifDataMsg = IFdataMsg(self._node_sensor.host_id,
                                self._node_sensor.local_time,
                                self._node_sensor.if_data,
                                resource_id,
                                resource_type,
                                state,
                                self.severity_reader.map_severity(state), event)
        # Add in uuid if it was present in the json request
        if self._uuid is not None:
            ifDataMsg.set_uuid(self._uuid)
        jsonMsg = ifDataMsg.getJson()
        self.if_sensor_data = jsonMsg
        self.os_sensor_type["nw"] = self.if_sensor_data

        # Transmit it to message processor
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
        self.persist_state_data('nw', 'NW_SENSOR_DATA')

    def _generate_if_data(self):
        """Create & transmit a network interface data message as defined
            by the sensor response json schema"""
        # Notify the node sensor to update its data required for the if_data message
        successful = self._node_sensor.read_data("if_data", self._get_debug())
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_if_data was NOT successful.")
        interfaces = self._node_sensor.if_data

        cable_alert_desc = "Newtork Cable for interface {} is {}."
        nw_alert_desc = "Network interface {} is {}."

        for interface in interfaces:
            if_name = interface["ifId"]
            nw_if_status = interface["nwStatus"].upper()
            # if nw_fault
            if nw_if_status in ["DOWN", "UNKNOWN"]:
                nw_cable_status = interface["nwCableConnStatus"].upper()
                # if nw_cable_disconneted
                if nw_cable_status == "DISCONNECTED":
                    if self.prev_cable_status.get(if_name) != nw_cable_status:
                        # raise nw_cable_fault_alert
                        self.prev_cable_status[if_name] = nw_cable_status
                        self._send_ifdata_json_msg(
                            if_name, self.NW_CABLE_RESOURCE_TYPE, self.FAULT,
                            cable_alert_desc.format(if_name, nw_cable_status))
                # elif nw_cable_connected
                elif nw_cable_status == "CONNECTED":
                    if self.prev_cable_status.get(if_name) == "DISCONNECTED":
                        # raise nw_cable_fault_resolved_alert
                        self.prev_cable_status[if_name] = nw_cable_status
                        self._send_ifdata_json_msg(
                            if_name, self.NW_CABLE_RESOURCE_TYPE,
                            self.FAULT_RESOLVED,
                            cable_alert_desc.format(if_name, nw_cable_status))
                    if self.prev_nw_if_status.get(if_name) != "DOWN":
                        # raise nw_fault_alert
                        self.prev_nw_if_status[if_name] = "DOWN"
                        self._send_ifdata_json_msg(
                            if_name, self.NW_RESOURCE_TYPE, self.FAULT,
                            nw_alert_desc.format(if_name, "DOWN"))
                # nw_cable_state is unknown and nw_is_faulty
                else:
                    if self.prev_nw_if_status.get(if_name) != "DOWN":
                        # raise nw_fault_alert
                        self.prev_nw_if_status[if_name] = "DOWN"
                        self._send_ifdata_json_msg(
                            if_name, self.NW_RESOURCE_TYPE, self.FAULT,
                            nw_alert_desc.format(if_name, "DOWN"))
            else: # no nw_fault
                if self.prev_nw_if_status.get(if_name) == "DOWN":
                    # raise nw_fault_resolved_alert
                    self.prev_nw_if_status[if_name] = "UP"
                    self._send_ifdata_json_msg(
                        if_name, self.NW_RESOURCE_TYPE, self.FAULT_RESOLVED,
                        nw_alert_desc.format(if_name, "UP"))
                if self.prev_cable_status.get(if_name) == "DISCONNECTED":
                    # raise nw_cable_fault_resolved_alert
                    self.prev_cable_status[if_name] = "CONNECTED"
                    self._send_ifdata_json_msg(
                        if_name, self.NW_CABLE_RESOURCE_TYPE,
                        self.FAULT_RESOLVED,
                        cable_alert_desc.format(if_name, "CONNECTED"))

    def _generate_disk_space_alert(self):
        """Create & transmit a disk_space_alert message as defined
            by the sensor response json schema"""

        # Notify the node sensor to update its data required for the disk_space_data message
        successful = self._node_sensor.read_data("disk_space_alert", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, _generate_disk_space_alert was NOT successful.")
            return

        # Changing disk_usage_threshold type according to what value type entered in config file
        self._disk_usage_threshold = str(self._disk_usage_threshold)
        try:
            if self._disk_usage_threshold.isdigit():
                self._disk_usage_threshold = int(self._disk_usage_threshold)
            else:
                self._disk_usage_threshold = float(self._disk_usage_threshold)
        except ValueError:
            logger.warn("Disk Space Alert, Invalid disk_usage_threshold value are entered in config.")
            # Assigning default value to _disk_usage_threshold
            self._disk_usage_threshold = self.DEFAULT_DISK_USAGE_THRESHOLD

        if self._node_sensor.disk_used_percentage >= self._disk_usage_threshold \
           and not self.high_usage['disk']:

            self.high_usage['disk'] = True
            # Create the disk space data message and hand it over
            # to the egress processor to transmit
            fault_event = "Disk usage has increased to {}%, "\
                "beyond the configured threshold of {}%.".format(
                    self._node_sensor.disk_used_percentage,
                    self._disk_usage_threshold
                )
            logger.warn(fault_event)
            diskSpaceAlertMsg = DiskSpaceAlertMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.total_space,
                                    self._node_sensor.free_space,
                                    self._node_sensor.disk_used_percentage,
                                    self._units,
                                    self.FAULT,fault_event)

            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                diskSpaceAlertMsg.set_uuid(self._uuid)
            jsonMsg = diskSpaceAlertMsg.getJson()
            self.disk_sensor_data = jsonMsg
            self.os_sensor_type["disk_space"] = self.disk_sensor_data

            # Transmit it to message processor
            self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            # Save the new state in Persistent Cache.
            self.persist_state_data('disk', 'DISK_USAGE_DATA')

        if self._node_sensor.disk_used_percentage <= self._disk_usage_threshold \
           and self.high_usage['disk']:

            # Create the disk space data message and hand it over
            # to the egress processor to transmit
            fault_resolved_event = "Disk usage has decreased to {}%, "\
                "lower than the configured threshold of {}%.".format(
                    self._node_sensor.disk_used_percentage,
                    self._disk_usage_threshold
                )
            logger.info(fault_resolved_event)
            diskSpaceAlertMsg = DiskSpaceAlertMsg(self._node_sensor.host_id,
                                    self._epoch_time,
                                    self._node_sensor.total_space,
                                    self._node_sensor.free_space,
                                    self._node_sensor.disk_used_percentage,
                                    self._units,
                                    self.FAULT_RESOLVED,
                                    fault_resolved_event
                                    )

            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                diskSpaceAlertMsg.set_uuid(self._uuid)
            jsonMsg = diskSpaceAlertMsg.getJson()
            self.disk_sensor_data = jsonMsg
            self.os_sensor_type["disk_space"] = self.disk_sensor_data

            # Transmit it to message processor
            self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            self.high_usage['disk'] = False
            # Save the new state in Persistent Cache.
            self.persist_state_data('disk', 'DISK_USAGE_DATA')

    def _generate_raid_data(self, jsonMsg):
        """Create & transmit a RAID status data message as defined
            by the sensor response json schema"""
        successful = self._node_sensor.read_data("raid", self._get_debug(), self._units)
        if not successful:
            logger.error("NodeDataMsgHandler, updating RAID information was NOT successful.")
            return

        # See if status is in the msg; ie it's an internal msg from the RAID sensor
        if jsonMsg.get("sensor_request_type").get("node_data").get("status") is not None:
            sensor_request = jsonMsg.get("sensor_request_type").get("node_data")
            host_name = sensor_request.get("host_id")
            alert_type = sensor_request.get("alert_type")
            alert_id = sensor_request.get("alert_id")
            severity = sensor_request.get("severity")
            info = sensor_request.get("info")
            specific_info = sensor_request.get("specific_info")
            self._raid_device = jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("device")
            self._raid_drives = list(jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("drives"))
            raidDataMsg = RAIDdataMsg(host_name, alert_type, alert_id, severity, info, specific_info)
            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                raidDataMsg.set_uuid(self._uuid)
            jsonMsg = raidDataMsg.getJson()
            self.raid_sensor_data = jsonMsg
            self.os_sensor_type["raid_data"] = self.raid_sensor_data

            # Loop thru each index of drives containing only paths and fill in with s/n
            for drive in self._raid_drives:
                self._log_debug("drive: %s" % str(drive))

                if drive.get("identity") is not None:
                    path = drive.get("identity").get("path")
                    self._log_debug("path: %s" % str(path))

                    # Lookup the serial number from the path
                    serial_number = str(self._drive_by_device_name.get(path))
                    self._log_debug("serial_number: %s" % str(serial_number))
                    if serial_number != "None":
                        drive["identity"]["serialNumber"] = serial_number

                    # Change device path to path-byid
                    drive_byid = str(self._drive_byid_by_serial_number.get(serial_number))
                    if drive_byid != "None":
                        drive["identity"]["path"] = drive_byid

            self._log_debug("_generate_raid_data, host_id: %s, device: %s, drives: %s" %
                    (self._node_sensor.host_id, self._raid_device, str(self._raid_drives)))

    def _generate_raid_integrity_data(self, jsonMsg):
        """Create & transmit a Validate RAID result data message as defined
            by the sensor response json schema"""
        logger.debug("NodeDataMsgHandler, Validating RAID information")

        # See if status is in the msg; ie it's an internal msg from the RAID sensor
        if jsonMsg.get("sensor_request_type").get("node_data").get("status") is not None:
            sensor_request = jsonMsg.get("sensor_request_type").get("node_data")
            host_name = sensor_request.get("host_id")
            alert_type = sensor_request.get("alert_type")
            alert_id = sensor_request.get("alert_id")
            severity = sensor_request.get("severity")
            info = sensor_request.get("info")
            specific_info = sensor_request.get("specific_info")
            self._alert = jsonMsg.get("sensor_request_type").get("node_data").get("specific_info").get("error")
            RAIDintegrityMsg = RAIDIntegrityMsg(host_name, alert_type, alert_id, severity, info, specific_info)
            # Add in uuid if it was present in the json request
            if self._uuid is not None:
                RAIDintegrityMsg.set_uuid(self._uuid)
            jsonMsg = RAIDintegrityMsg.getJson()
            self.raid_integrity_data = jsonMsg
            self.os_sensor_type["raid_integrity"] = self.raid_integrity_data

            self._log_debug("_generate_raid_integrity_data, host_id: %s" %
                    (self._node_sensor.host_id))

    def _generate_node_fru_data(self, jsonMsg):
        """Create & transmit a FRU IPMI data message as defined
            by the sensor response json schema"""

        if self._node_sensor.host_id is None:
            successful = self._node_sensor.read_data("None", self._get_debug(), self._units)
            if not successful:
                logger.error("NodeDataMsgHandler, updating host information was NOT successful.")

        if jsonMsg.get("sensor_request_type").get("node_data") is not None:
            self._fru_info = jsonMsg.get("sensor_request_type").get("node_data")
            node_ipmi_data_msg = NodeIPMIDataMsg(self._fru_info)

        if self._uuid is not None:
            node_ipmi_data_msg.set_uuid(self._uuid)
        jsonMsg = node_ipmi_data_msg.getJson()
        self._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

    def suspend(self):
        """Suspends the module thread. It should be non-blocking"""
        super(NodeDataMsgHandler, self).suspend()
        self._suspended = True

    def resume(self):
        """Resumes the module thread. It should be non-blocking"""
        super(NodeDataMsgHandler, self).resume()
        self._suspended = False

    def shutdown(self):
        """Clean up scheduler queue and gracefully shutdown thread"""
        super(NodeDataMsgHandler, self).shutdown()


from bs4 import BeautifulSoup
from yyyy_mm_dd import *

def test_id(testId):
    return {"data-testid": testId}

def extract_10_day_forecast(html):
    soup = BeautifulSoup(html, features='html.parser')
    daily_forecast = soup.find(attrs=test_id("DailyForecast"))
    summaries = daily_forecast.find_all("summary")

    result = []
    for delta, summary in enumerate(summaries):
        item = extract_day_forecast(summary, delta)
        result.append(item)

    return result

def extract_day_forecast(summary, delta):
    item = {
        "type": "daily",
        "forecast_delta": delta,
        "precipitation": get_precipitation(summary),
        "summary": summary.find(attrs=test_id("wxIcon")).find("span").get_text()
    }
    temperature = summary.find(attrs=test_id("detailsTemperature"))
    item["max"] = parse_temperature(temperature.find(
        attrs=test_id("TemperatureValue")))
    item["min"] = parse_temperature(temperature.find(attrs=test_id(
        "lowTempValue")).find("span"))

    return item

def extract_hourly_forecast(html, now=now()):
    soup = BeautifulSoup(html, features='html.parser')
    daily_forecast = soup.find(attrs=test_id("HourlyForecast"))
    summaries = daily_forecast.find_all("summary")

    delta = 0
    # if it's quarter past the hour TWC shows forecast for next hour
    if to_datetime(now).minute > 15:
        delta = 1
    result = []
    for summary in summaries:
        item = extract_hour_forecast(summary, delta)
        delta = delta + 1
        result.append(item)

    return result

def extract_hour_forecast(summary, delta):
    item = {
        "type": "hourly",
        "forecast_delta": delta,
        "precipitation": get_precipitation(summary),
        "summary": summary.find(attrs=test_id("wxIcon")).find("span").get_text()
    }
    temperature = summary.find(attrs=test_id("detailsTemperature"))
    item["temperature"] = parse_temperature(temperature.find(
        attrs=test_id("TemperatureValue")))

    return item

def get_precipitation(summary):
    precipitation = summary.find(attrs=test_id("Precip"))
    if precipitation:
        precipitation = precipitation.find(
            attrs=test_id("PercentageValue")).get_text()
        precipitation = int(precipitation.replace("%", ""))
        return precipitation

    return 0

def parse_temperature(node):
    temperature = node.get_text().replace("°", "")
    try:
        return int(temperature)
    except:
        return None


"""
Make it more robust.
Stop episode once the finger stop at the final position for 50 steps.
Feature & reward engineering.
"""
#from final.env import ArmEnv
#from final.rl import DDPG
from env import ArmEnv
from rl import DDPG
import vrepInterface

#MAX_EPISODES = 900
MAX_EPISODES = 900
#MAX_EP_STEPS = 200
MAX_EP_STEPS = 100
ON_TRAIN = True

# set env
env = ArmEnv()
s_dim = env.state_dim
a_dim = env.action_dim
a_bound = env.action_bound

# set RL method (continuous)
rl = DDPG(a_dim, s_dim, a_bound)

steps = []
def train():

    vrepInterface.connect()
    vrepInterface.start()

    # start training
    for i in range(MAX_EPISODES):
        env.get_goal()
        s = env.reset()
        # print('the start s is: ',s)
        ep_r = 0.
        for j in range(MAX_EP_STEPS):
            # env.render()

            a = rl.choose_action(s)
            # print('STEP ', j, ' a is: ', a)

            s_, r, done = env.step(a)
            # print('STEP ', j, 's is :', s_)
            # print('STEP ', j, 'r is :', r)

            rl.store_transition(s, a, r, s_)

            ep_r += r
            if rl.memory_full:
                # start to learn once has fulfilled the memory
                rl.learn()
                # print('start to learn')

            s = s_
            if done or j == MAX_EP_STEPS-1:
                print('Ep: %i | %s | ep_r: %.1f | step: %i' % (i, '---' if not done else 'done', ep_r, j))
                break
    rl.save()
    vrepInterface.disconnect()

def eval():
    rl.restore()
    vrepInterface.connect()
    vrepInterface.start()
    # env.render()
    # env.viewer.set_vsync(True)
    env.get_goal()
    s = env.reset()
    while True:
        # env.render()
        a = rl.choose_action(s)
        s, r, done = env.step(a)

if ON_TRAIN:
    train()
else:
    eval()


from django.db import models
import os
from uuid import uuid4

def path_and_rename(instance, filename):
    upload_to = "./input_image"
    ext = filename.split('.')[-1]
    # get filename
    if instance.pk:
        filename = '{}.{}'.format(instance.pk, "png")
    else:
        # set filename as random string
        filename = '{}.{}'.format("input", "png")
    # return the whole path to the file
    return os.path.join(upload_to, filename)

def auto_delete_file_on_change(sender, instance, **kwargs):
    """
    Deletes old file from filesystem
    when corresponding `MediaFile` object is updated
    with new file.
    """
    if not instance.pk:
        return False

    try:
        old_file = Image.objects.get(pk=instance.pk).file
    except Image.DoesNotExist:
        return False

    new_file = instance.file
    if not old_file == new_file:
        if os.path.isfile(old_file.path):
            os.remove(old_file.path)

# Create your models here.
class Image(models.Model):
    title = models.CharField(max_length=200)
    img = models.ImageField(upload_to = path_and_rename,)
    def __str__(self):
        return self.title


from flask import Flask, render_template
from log_analysis.__main__ import answer_all
from log_analysis.database import DBConnection
app = Flask(__name__)

@app.route('/')
@app.route('/answers')
def answers():
    with DBConnection() as conn:
        answers = answer_all(conn)
    return render_template('answers.html', answers=answers)

if __name__ == '__main__':
    app.debug = True
    app.run(host='0.0.0.0', port=5000)


import json

import frappe

def execute():
	reports_data = frappe.get_all(
		"Report",
		filters={
			"json": ["not like", '%%%"order_by": "`tab%%%'],
			"report_type": "Report Builder",
			"is_standard": "No",
		},
		fields=["name"],
	)

	for d in reports_data:
		doc = frappe.get_doc("Report", d.get("name"))

		if not doc.get("json"):
			continue

		json_data = json.loads(doc.get("json"))

		parts = []
		if ("order_by" in json_data) and ("." in json_data.get("order_by")):
			parts = json_data.get("order_by").split(".")

			sort_by = parts[1].split(" ")

			json_data["order_by"] = f"`tab{doc.ref_doctype}`.`{sort_by[0]}`"
			json_data["order_by"] += f" {sort_by[1]}" if len(sort_by) > 1 else ""

			doc.json = json.dumps(json_data)
			doc.save()


# reddit scraper, by subreddit
# results will be saved to csv file named <subreddit>.csv

import csv
import os
import requests

reddit_fields = ['subreddit', 'url']
headers = {'User-agent': 'nlp scraper 0.0.1'}
sort = '?sort=top&t=all&limit=100'

whitelist = []

# read in whitelist
with open('whitelist.txt') as f:
    whitelist = f.readlines()
whitelist = [line.strip() for line in whitelist]

def main():
    REDDIT_BASE = 'https://old.reddit.com/r/'
    after = None

    subreddits = input('Enter subreddits to scrape: ').split(' ')

    for s in subreddits:
        for _ in range(5):
            page = f'&after={after}' if after else ''
            r = requests.get(
                f'{REDDIT_BASE}{s}/top/.json{sort}{page}', headers=headers)
            if r.ok:
                res = r.json()['data']
                posts = process_reddit_response(res['children'])
                write_subreddit_to_csv(s, posts)
                after = res['after']

def process_reddit_response(sub_response):
    posts = []
    for post in sub_response:
        p = post['data']
        if p and not p['stickied'] and accepted(p['url']):
            # get subreddit name and url
            d = {x: p[x] for x in reddit_fields}
            # check if post's url is on whitelist
            posts.append(d)
    return posts

def accepted(url):
    accept = True
    blacklist = ['i.redd.it', 'imgur', 'reddit',
                 'streamable', 'facebook.com',
                 'twitter.com', 'instagram.com', 'redd.it',
                 '.gif', '.jpg', '.mp4', '.png', 'youtube.com',
                 'youtu.be', 'decisiondeskhq.com', 'duckduckgo']

    for b in blacklist:
        accept = accept and b not in url

    return accept

def write_subreddit_to_csv(subreddit, posts):
    filename = f'{subreddit}.csv'
    mode = 'a' if os.path.exists(filename) else 'w'
    with open(filename, mode=mode) as f:
        writer = csv.writer(f)
        if mode == 'w':
            writer.writerow(reddit_fields)
        for p in posts:
            row = [p[x] for x in reddit_fields]
            writer.writerow(row)

def url_in_whitelist(url):
    return any(sitename in url for sitename in whitelist)

if __name__ == '__main__':
    main()


from setuptools import setup, find_packages
import sys

install_requires = dict(
    win32=['pyWinUSB'],
    linux2=['pyUSB'],
    darwin=['hidapi']
)

setup(
    name="pyDAPLink",
    use_scm_version={
        'local_scheme': 'dirty-tag',
        'write_to': 'pyDAPLink/_version.py'
    },
    setup_requires=['setuptools-scm!=1.5.3,!=1.5.4'],
    description="Cross-platform CMSIS-DAP access over USB in Python",
    long_description=open('README.rst', 'Ur').read(),
    author="samux, emilmont",
    author_email="Samuel.Mokrani@arm.com, Emilio.Monti@arm.com",
    url='https://github.com/mbedmicro/pyDAPLink',
    license="Apache 2.0",
    install_requires=install_requires[sys.platform],
    classifiers=[
        "Development Status :: 4 - Beta",
        "License :: OSI Approved :: Apache Software License",
        "Programming Language :: Python",
    ],
    entry_points={
        'console_scripts': [
            'pydaplink-server = pyDAPLink.tools.pydaplink_server:main',
        ],
    },
    use_2to3=True,
    packages=find_packages(),
    include_package_data=True,  # include files from MANIFEST.in
)


import re
import os
import sys
import json
import random
import torch
import pytorch_lightning as pl

from tqdm import tqdm
from pprint import pprint
from dataclasses import dataclass
from collections import Counter
from transformers import BertTokenizerFast, RobertaTokenizerFast
from q_snippets.object import Config, print_config, print_string
from q_snippets.data import BaseData, sequence_padding, save_json
from torch.utils.data import Dataset, DataLoader, random_split
from data.causal_relation.pdtb2.pdtb2 import CorpusReader

FILE_DIR = os.path.dirname(os.path.abspath(__file__))
PROJ_DIR = FILE_DIR[:FILE_DIR.index('src')]
sys.path.append(PROJ_DIR)
os.environ["TOKENIZERS_PARALLELISM"] = "true"

@dataclass
class InputSample:  # 原始数据
    arg1: str
    arg2: str
    connective: str
    sense: str

@dataclass
class InputFeature:  # 处理后的特征数据
    input_ids: list
    attention_mask: list
    # token_type_ids: list
    mask_token_id: int
    label: int

@dataclass
class InputBatch(BaseData):  # 对特征数据进行batch处理
    input_ids: torch.Tensor
    attention_mask: torch.Tensor
    # token_type_ids: torch.Tensor
    mask_token_ids: torch.Tensor
    labels: torch.Tensor

    def __len__(self):
        return self.input_ids.size(0)

    def info(self):
        size = {k: v.size() if type(v) is torch.Tensor else len(v)
                for k, v in self.__dict__.items()}
        print(size)

class LabelMapping(object):
    def __init__(self, mapping_path):
        with open(mapping_path, "r") as f:
            self.label2words:dict = json.load(f)
        self.words2label = self.get_words2label(self.label2words)
        labels = sorted(list(set(self.words2label.values())))
        self.label2id = self.get_label2id(labels)
        self.id2label = {id: label for label, id in self.label2id.items()}

    def get_words2label(self, label2words):
        words2label = {}
        for label, words in label2words.items():
            # if label.startswith("Contingency"):  # causality
            #     for word in words:
            #         words2label[word] = label
            # else:
            for word in words:
                words2label[word] = label.split(".")[0]  # pdtb2 top-level
                # words2label[word] = ".".join(label.split(".")[:2])  # pdtb2 second-level
                # words2label[word] = label  # cross-level
        return words2label

    def get_label2id(self, labels):
        label2id = {"Other": 0}
        for idx, label in enumerate(labels):
            label2id[label] = idx+1  # 1~n
        return label2id

class CoNll16Dataset(Dataset):
    def __init__(self, config, tokenizer, dataset_mode) -> None:
        super().__init__()
        self.tokenizer = tokenizer
        self.config = config
        self.mode = dataset_mode
        self.label_mapping = LabelMapping(config.mapping_path)
        self.label2words = self.label_mapping.label2words
        self.words2label = self.label_mapping.words2label
        self.samples, self.features = self._handle_cache()

    def _handle_cache(self):
        """
            核心是 self.load_data 加载并处理数据，返回原始数据和处理后的特征数据
            需要注意缓存文件与 self.config.cache_dir  self.mode 有关
        Returns:
            samples, features
        """
        os.makedirs(self.config.cache_dir, exist_ok=True)               # 确保缓存文件夹存在
        file = os.path.join(self.config.cache_dir, f"{self.mode}.pt")   # 获得缓存文件的路径
        # 如果已经存在，且没有强制重新生成，则从缓存读取
        if os.path.exists(file) and not self.config.force_reload:
            samples, features = torch.load(file)
            print(f"{len(samples), len(features)} samples, features loaded from {file}")
            return samples, features
        else:
            samples, features = self.load_data()                        # 读取并处理数据
            torch.save((samples, features), file)                       # 生成缓存文件
            return samples, features

    def load_data(self):
        samples, features = [], []
        if self.mode == 'train':
            path = self.config.train_path
        elif self.mode == 'val':
            path = self.config.val_path
        elif self.mode == 'test':
            path = self.config.test_path
        with open(path, 'r', encoding='utf-8') as f:
            all_data = json.load(f)

        sense_list = []
        for dic in tqdm(all_data):
            arg1 = dic["Arg1"]
            arg2 = dic["Arg2"]

            sense = dic["Sense"]
            if sense not in self.label2words.keys():
                continue

            connective = dic["Connective"]
            if connective in self.label2words[sense]:
                mask_word = connective
            else:  # 对于其他连接词用频率最高的连接词进行替换
                mask_word = self.label2words[sense][0]
            label = self.tokenizer.convert_tokens_to_ids(mask_word)

            # prompt = "[CLS] " + arg1 + " [MASK] " + arg2 + ". [SEP]"  # bert prompt
            # prompt = "<s> " + arg1.capitalize() + " <mask> " + arg2.lower() + ". </s>"  # roberta prompt1
            # prompt = "<s> " + arg1.capitalize() + ". That's <mask> " + arg2.lower() + ". </s>"  # roberta prompt2
            prompt3 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The conjunction between Arg1 and Arg2 is <mask>."  # roberta prompt3
            prompt4 = "<s>Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The conjunction between Arg1 and Arg2 is <mask>.</s>"  # roberta prompt4
            inputs = self.tokenizer(
                prompt4,
                max_length=512,
                truncation=True,
                return_offsets_mapping=True
            )

            if self.tokenizer.mask_token_id not in inputs.input_ids: # 如果arg1超过512限制则舍弃
                continue
            mask_token_id = inputs.input_ids.index(self.tokenizer.mask_token_id)

            if self.mode in ["train", "val", "test"]:
                sense_list.append(sense)
                samples.append(InputSample(
                    arg1=arg1, arg2=arg2,
                    connective=connective,
                    sense=sense
                ))
                features.append(InputFeature(
                    input_ids=inputs.input_ids,
                    attention_mask=inputs.attention_mask,
                    # token_type_ids=inputs.token_type_ids,
                    mask_token_id=mask_token_id,
                    label=label
                ))
            else:
                pass

        print(dict(Counter(sense_list)))
        print(f"{str(len(samples))} samples, {str(len(features))} features in total for {self.mode}")

        return samples, features

    def __len__(self):
        return len(self.features)

    def __getitem__(self, index):
        return self.features[index]

    @staticmethod
    def collate_fn(batch):
        test = batch[0].label is None
        max_len = min(max([len(x.input_ids) for x in batch]), 512)

        # 将batch_size条feature转换成Tensor
        input_ids = torch.tensor(sequence_padding([x.input_ids for x in batch], length=max_len))
        attention_mask = torch.tensor(sequence_padding([x.attention_mask for x in batch], length=max_len))
        # token_type_ids = torch.tensor(sequence_padding([x.token_type_ids for x in batch], length=max_len))
        mask_token_ids = torch.tensor([x.mask_token_id for x in batch])
        labels = torch.tensor([x.label for x in batch]) if not test else None
        batch = InputBatch(
            input_ids=input_ids,
            attention_mask=attention_mask,
            # token_type_ids=token_type_ids,
            mask_token_ids=mask_token_ids,
            labels=labels
        )
        return batch

class PDTB2Dataset(Dataset):
    def __init__(self, config, tokenizer, dataset_mode) -> None:
        super().__init__()
        self.tokenizer = tokenizer
        self.config = config
        self.mode = dataset_mode
        self.label_mapping = LabelMapping(config.mapping_path)
        self.label2words = self.label_mapping.label2words
        self.words2label = self.label_mapping.words2label
        self.samples, self.features = self._handle_cache()

    def _handle_cache(self):
        """
            核心是 self.load_data 加载并处理数据，返回原始数据和处理后的特征数据
            需要注意缓存文件与 self.config.cache_dir  self.mode 有关
        Returns:
            samples, features
        """
        os.makedirs(self.config.cache_dir, exist_ok=True)               # 确保缓存文件夹存在
        file = os.path.join(self.config.cache_dir, f"{self.mode}.pt")   # 获得缓存文件的路径
        # 如果已经存在，且没有强制重新生成，则从缓存读取
        if os.path.exists(file) and not self.config.force_reload:
            samples, features = torch.load(file)
            print(f"{len(samples), len(features)} samples, features loaded from {file}")
            return samples, features
        else:
            samples, features = self.load_data()                        # 读取并处理数据
            torch.save((samples, features), file)                       # 生成缓存文件
            return samples, features

    def load_data(self):
        samples, features = [], []
        if self.mode == 'train':
            path = self.config.train_path
        elif self.mode == 'val':
            path = self.config.val_path
        elif self.mode == 'test':
            path = self.config.test_path

        pdtb = CorpusReader(path)
        sense_list = []
        for datum in pdtb.iter_data(display_progress=True):
            if datum.Relation == "Implicit":
                # continue
                arg1:str = datum.Arg1_RawText.strip()
                arg2:str = datum.Arg2_RawText.strip()
                sense = datum.ConnHeadSemClass1
                connective = datum.Conn1.strip()

                if sense not in self.label2words.keys():
                    continue
                if "Ġ"+connective in self.label2words[sense]:
                    mask_word = "Ġ"+connective  # roberta
                    # mask_word = connective  # bert
                else:  # 对于其他连接词用频率最高的连接词进行替换
                    mask_word = self.label2words[sense][0]  # roberta
                    # mask_word = self.label2words[sense][0][1:]  # bert
                label = self.tokenizer.convert_tokens_to_ids(mask_word)

                # prompt = "[CLS] " + arg1.strip().capitalize() + " [MASK] " + arg1.strip().lower() + ". [SEP]"  # bert prompt1
                # prompt = "[CLS] Arg1: " + arg1.strip().capitalize() + ". Arg2: " + arg2.strip().capitalize() + ". [SEP] The conjunction between Arg1 and Arg2 is [MASK]. [SEP]"  # bert prompt2

                roberta_prompt1 = arg1.capitalize() + " <mask> " + arg2.lower() + "."  # roberta prompt1
                roberta_prompt2 = arg1.capitalize() + ". That's <mask> " + arg2.lower() + "."  # roberta prompt2
                roberta_prompt3 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ".</s></s>The conjunction between Arg1 and Arg2 is <mask>."  # roberta prompt3
                roberta_prompt4 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ".</s></s>The connective between Arg1 and Arg2 is <mask>."  # roberta prompt4
                roberta_prompt5 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The conjunction between Arg1 and Arg2 is <mask>."  # roberta prompt5
                roberta_prompt6 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The connective between Arg1 and Arg2 is <mask>."  # roberta prompt6
                roberta_prompt7 = "<s>Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The conjunction between Arg1 and Arg2 is <mask>.</s>"  # roberta prompt7
                roberta_prompt8 = "<s>Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". The connective between Arg1 and Arg2 is <mask>.</s>"  # roberta prompt8
                roberta_prompt9 = "<s>Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". In summary, the discourse relation between Arg1 and Arg2 is <mask>.</s>"  # roberta prompt9
                roberta_prompt10 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + ". In summary, the discourse relation between Arg1 and Arg2 is <mask>."  # roberta prompt10

                inputs = self.tokenizer(roberta_prompt5, max_length=512, truncation=True)

                # if self.tokenizer.mask_token_id not in inputs.input_ids:  # 如果arg1超过512限制则舍弃
                #     continue
                mask_token_id = inputs.input_ids.index(self.tokenizer.mask_token_id)

                if self.mode in ["train", "val", "test"]:
                    # sense_list.append(sense)
                    samples.append(InputSample(
                        arg1=arg1, arg2=arg2,
                        connective=connective,
                        sense=sense
                    ))
                    features.append(InputFeature(
                        input_ids=inputs.input_ids,
                        attention_mask=inputs.attention_mask,
                        # token_type_ids=inputs.token_type_ids,
                        mask_token_id=mask_token_id,
                        label=label
                    ))

            if datum.Relation == "Explicit":
                continue
                # if datum.arg1_contains_arg2() or datum.arg2_contains_arg1():
                #     continue
                arg1:str = datum.Arg1_RawText.strip()
                arg2:str = datum.Arg2_RawText.strip()
                sense:str = datum.ConnHeadSemClass1
                connective:str = datum.Connective_RawText.strip()

                if sense not in self.label2words.keys():
                    continue
                mask_word = self.label2words[sense][0]  # roberta
                # mask_word = self.label2words[sense][0][1:]  # bert
                label = self.tokenizer.convert_tokens_to_ids(mask_word)

                roberta_prompt1 = "Arg1: " + arg1.capitalize() + ". Arg2: " + arg2.capitalize() + \
                                  ". The connective between Arg1 and Arg2 is " + connective.lower() + \
                                  ". In summary, the discourse relation between Arg1 and Arg2 is <mask>."
                                  #  ". In summary, Arg2 is the <mask> of Arg1."

                inputs = self.tokenizer(roberta_prompt1, max_length=512, truncation=True)
                if self.tokenizer.mask_token_id not in inputs.input_ids:  # 如果prompt的总长度超过512限制则舍弃
                    continue
                mask_token_id = inputs.input_ids.index(self.tokenizer.mask_token_id)

                if self.mode in ["train", "val", "test"]:
                    sense_list.append(sense)
                    samples.append(InputSample(
                        arg1=arg1, arg2=arg2,
                        connective=connective,
                        sense=sense
                    ))
                    features.append(InputFeature(
                        input_ids=inputs.input_ids,
                        attention_mask=inputs.attention_mask,
                        # token_type_ids=inputs.token_type_ids,
                        mask_token_id=mask_token_id,
                        label=label
                    ))

        # print(dict(Counter(sense_list)))
        print(f"{str(len(samples))} samples, {str(len(features))} features in total for {self.mode}")

        return samples, features

    def __len__(self):
        return len(self.features)

    def __getitem__(self, index):
        return self.features[index]

    @staticmethod
    def collate_fn(batch):
        test = batch[0].label is None
        max_len = min(max([len(x.input_ids) for x in batch]), 512)

        # 将batch_size条feature转换成Tensor
        input_ids = torch.tensor(sequence_padding([x.input_ids for x in batch], length=max_len))
        attention_mask = torch.tensor(sequence_padding([x.attention_mask for x in batch], length=max_len))
        # token_type_ids = torch.tensor(sequence_padding([x.token_type_ids for x in batch], length=max_len))
        mask_token_ids = torch.tensor([x.mask_token_id for x in batch])
        labels = torch.tensor([x.label for x in batch]) if not test else None
        batch = InputBatch(
            input_ids=input_ids,
            attention_mask=attention_mask,
            # token_type_ids=token_type_ids,
            mask_token_ids=mask_token_ids,
            labels=labels
        )
        return batch

class PromptDataModule(pl.LightningDataModule):

    dataset_mapping = {
        'conll16': CoNll16Dataset,
        'pdtb2': PDTB2Dataset
    }

    def __init__(self, config):
        super().__init__()
        self.config = config.data
        # self.tokenizer = BertTokenizerFast.from_pretrained(config.data.tokenizer)
        self.tokenizer = RobertaTokenizerFast.from_pretrained(config.data.tokenizer)
        print_string("configuration of datamodule")
        print_config(self.config)
        self.DATASET = self.dataset_mapping[self.config.dataset]

    def setup(self, stage=None):
        """
        根据模型运行的阶段，加载对应的数据，并实例化Dataset对象
        """
        if stage == 'fit' or stage is None:
            self.trainset = self.DATASET(self.config, self.tokenizer, dataset_mode='train')
            self.valset = self.DATASET(self.config, self.tokenizer, dataset_mode='val')

        if stage == 'test' or stage is None:
            self.testset = self.DATASET(self.config, self.tokenizer, dataset_mode='test')

    def train_dataloader(self):
        return DataLoader(self.trainset, batch_size=self.config.train_bsz, shuffle=True, collate_fn=self.trainset.collate_fn, num_workers=4)

    def val_dataloader(self):
        return DataLoader(self.valset, batch_size=self.config.val_bsz, shuffle=False, collate_fn=self.valset.collate_fn, num_workers=4)

    def test_dataloader(self):
        return DataLoader(self.testset, batch_size=self.config.test_bsz, shuffle=False, collate_fn=self.testset.collate_fn, num_workers=4)

def test_batch():
    config = Config.create({
        "data": {
            'dataset': 'pdtb2',
            'tokenizer': "roberta-large",
            'train_path': os.path.join(PROJ_DIR, "data/causal_relation/pdtb2/dataset/train.csv"),
            'val_path': os.path.join(PROJ_DIR, "data/causal_relation/pdtb2/dataset/dev.csv"),
            'test_path': os.path.join(PROJ_DIR, "data/causal_relation/pdtb2/dataset/test.csv"),
            'mapping_path': os.path.join(PROJ_DIR, "data/causal_relation/pdtb2/pdtb_label2word_implicit.json"),
            'train_bsz': 32,
            'val_bsz': 32,
            'test_bsz': 32,
            'cache_dir': './cached/data_utils',
            'force_reload': True
        }
    })
    dm = PromptDataModule(config)
    dm.setup('fit')
    print_string("one sample example")
    print(dm.trainset.samples[0])
    print_string("one feature example")
    print(dm.trainset.features[0])
    print_string("one batch example")
    for batch in dm.train_dataloader():
        batch.info()
        break

if __name__ == '__main__':
    # test_batch()
    lm = LabelMapping(os.path.join(PROJ_DIR, "data/causal_relation/pdtb2/pdtb_label2word_implicit.json"))
    pprint(sorted(lm.words2label.items(), key=lambda x:x[1]))


from confluent_kafka import Producer

# Kafka broker configuration
bootstrap_servers = 'localhost:9094'
topic = 'test-topic'

# Test message to be sent
message = 'Hello, Kafka!'

def produce_message():
    # Create Kafka producer
    producer = Producer({'bootstrap.servers': bootstrap_servers})

    # Produce message to Kafka topic
    producer.produce(topic, message.encode())

    producer.flush()

    # Close the producer
    producer.close()

if __name__ == '__main__':
    # Produce a message
    produce_message()


# it allows us to use all pygame commands without using pygame keyword...
import pygame
from pygame.locals import *
from OpenGL.GL import *
from OpenGL.GLU import *

# it should be in tuples
vertices = (
    (1, -1, -1),
    (1, 1, -1),
    (-1, 1, -1),
    (-1, -1, -1),
    (1, -1, 1),
    (1, 1, 1),
    (-1, -1, 1),
    (-1, 1, 1)
)

edges = (
    (0, 1),
    (0, 3),
    (0, 4),
    (2, 1),
    (2, 3),
    (2, 7),
    (6, 3),
    (6, 4),
    (6, 7),
    (5, 1),
    (5, 4),
    (5, 7)
)

surfaces = (
    (0, 1, 2, 3),
    (3, 2, 7, 6),
    (6, 7, 5, 4),
    (4, 5, 1, 0),
    (1, 5, 7, 2),
    (4, 0, 3, 6)
)

def Draw():
    glBegin(GL_LINES)
    for edge in edges:
        for node in edge:
            glColor3fv((1, 0, 0))
            glVertex3fv(vertices[node])
    glEnd()

    glBegin(GL_QUADS)
    for surface in surfaces:
        for node in surface:
            glColor3fv((0, 1, 0))
            glVertex3fv(vertices[node])
    glEnd()

def main():
    pygame.init()
    display = (800, 600)
    # display loads in buffer and openGl is just a parameter
    pygame.display.set_mode(display, DOUBLEBUF|OPENGL)

    #used to cliping the some part of the object...
    gluPerspective(45.0, (display[0] / display[1]), 1 , 50.0)

    # moving backward direction...
    glTranslatef(0.0, 0.0, -5.0)

    # current position of cube...
    glRotatef(40, 20, 20, 0)

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()

            if event.type == MOUSEBUTTONDOWN:
                if event.button == 4:
                    glTranslatef(0.0, 0.0, 1.0)
                if event.button == 5:
                    glTranslatef(0.0, 0.0, -1.0)

        glRotatef(1, 1, 1, 1)

        # basically it clears the pygame background....
        glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)

        Draw()

        pygame.display.flip() # used to update.....
        # just like pygame.display.update()

        pygame.time.wait(10)

# fun call
main()