import os
import torch
import wandb
import argparse
import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedKFold
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, roc_auc_score, average_precision_score, balanced_accuracy_score

os.environ["CUDA_DEVICE_ORDER"] = 'PCI_BUS_ID'
os.environ["CUDA_VISIBLE_DEVICES"] = '0'

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--layer_1', help='layer 1 size', type=int, default=50, required=True)
    parser.add_argument(
        '--layer_2', help='layer 2 size', type=int, default=50, required=True)
    parser.add_argument(
        '--layer_3', help='layer 3 size', type=int, default=50, required=True)
    parser.add_argument(
        '--lr', help='Learning Rate', type=float, default=0.01, required=True)
    parser.add_argument(
        '--weight_decay', help='Weight Decay', type=float, default=0.9, required=True)
    parser.add_argument(
        '--momentum', help='Weight Decay', type=float, default=0.9, required=True)
    parser.add_argument(
        '--no_epoch', help='Number of Epochs', type=int, default=1000, required=True)
    args = parser.parse_args()
    return args

class Model(torch.nn.Module):
    def __init__(self, layer_1, layer_2, layer_3):
        super(Model, self).__init__()
        self.lin1 = torch.nn.Linear(12, layer_1)
        self.lin2 = torch.nn.Linear(layer_1, layer_2)
        self.lin3 = torch.nn.Linear(layer_2, layer_3)
        self.lin4 = torch.nn.Linear(layer_3, 1)
        self.selu = torch.nn.SELU()

    def forward(self, x):
        x = self.selu(self.lin1(x))
        x = self.selu(self.lin2(x))
        x = self.selu(self.lin3(x))
        x = self.lin4(x)
        return x

def load_data():
    cd = os.getcwd()
    x_eicu = pd.read_csv(cd+'../data/x_eicu.csv')
    y_eicu = pd.read_csv(cd+'../data/y_eicu.csv')
    mimic = pd.read_csv(cd+'../data/mimic.csv')
    assert np.all(x_eicu['patientunitstayid'].to_numpy() == y_eicu['patientunitstayid'].to_numpy())
    feature_list = ['lactate', 'oobventday1', 'eyes', 'motor', 'verbal', 'albumin_x',
                    'age', 'creatinine_x', 'BUN', 'PT - INR', 'WBC x 1000', 'meanbp']
    feature_list_mimic = ['Lactate', 'firstdayvent', 'gcseyes', 'gcsmotor', 'gcsverbal', 'Albumin',
                          'Age', 'Creatinine', 'BUN', 'INR', 'WBC', 'MAP']
    x_eicu = x_eicu[feature_list].to_numpy()
    y_eicu = y_eicu['actualicumortality'].to_numpy()
    x_mimic = mimic[feature_list_mimic].to_numpy()
    y_mimic = mimic['Mortality'].to_numpy()
    x = np.vstack((x_eicu, x_mimic))
    y = np.hstack((y_eicu, y_mimic))
    return x, y

def main():
    wandb.init(project='mortality-tool-newfeats')
    args = parse_args()
    x, y = load_data()
    kfold = StratifiedKFold(n_splits=10)
    logits_all = []
    labels_all = []
    counter = 1
    for train_index, test_index in kfold.split(x, y):
        x_train, y_train = x[train_index], y[train_index]
        x_test, y_test = x[test_index], y[test_index]
        imputer = IterativeImputer()
        scaler = StandardScaler()
        x_train = scaler.fit_transform(imputer.fit_transform(x_train))
        x_test = scaler.transform(imputer.transform(x_test))
        x_train, y_train = torch.from_numpy(x_train).float().to('cuda:0'), torch.from_numpy(y_train).float().to('cuda:0')
        model = Model(args.layer_1, args.layer_2, args.layer_3)
        criterion = torch.nn.BCEWithLogitsLoss(pos_weight=torch.tensor([14.80], device='cuda:0'))
        optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=args.momentum, nesterov=True)
        model.train()
        model.to('cuda:0')
        for epoch in range(args.no_epoch):
            optimizer.zero_grad()
            outputs = model.forward(x_train)
            loss = criterion(outputs, y_train.view(-1, 1))
            loss.backward()
            optimizer.step()
        model.eval()
        outputs = model.forward(torch.from_numpy(x_test).float().to('cuda:0'))
        logits = torch.sigmoid(outputs).detach().cpu().numpy()
        logits_all.append(logits.reshape(-1))
        labels_all.append(y_test)
        print('Iter {}/10 done'.format(counter))
        counter += 1
    logits_all = np.hstack(logits_all)
    labels_all = np.hstack(labels_all)
    tn, fp, fn, tp = confusion_matrix(labels_all, np.round(logits_all)).ravel()
    accuracy = (tp + tn) / (tp + tn + fp + fn)
    precision = tp / (tp + fp)
    sensitivity = tp / (tp + fn)
    specificity = tn / (tn + fp)
    roc_auc = roc_auc_score(labels_all, logits_all)
    prc_auc = average_precision_score(labels_all, logits_all)
    balanced_acc = balanced_accuracy_score(labels_all, np.round(logits_all))
    pos_likelihood_ratio = sensitivity / (1 - specificity)
    neg_likelihood_ratio = (1 - sensitivity) / specificity
    class_names = ['ALIVE', 'EXPIRED']
    wandb.log({'accuracy': accuracy, 'precision': precision, 'sensitivity': sensitivity, 'specificitiy': specificity,
               'roc_auc': roc_auc, 'prc_auc': prc_auc, 'balanced_accuracy': balanced_acc,
               'neg_likelihood_ratio': neg_likelihood_ratio, 'pos_likelihood_ratio': pos_likelihood_ratio})

if __name__ == '__main__':
    main()


from __future__ import absolute_import
import os
import sys
import re
import xml.dom.minidom
import random
from sumolib.files.additional import write_additional_minidom
try:
    from typing import Any, List, Tuple, Union
except ImportError:
    # there are python2 versions on MacOS coming without typing
    pass

"""
Creates a vehicle type distribution with a number of representative car-following parameter sets.
"""

class _FixDistribution(object):

    def __init__(self, params, isNumeric=True):
        if isNumeric:
            self._params = tuple([float(p) for p in params])
        else:
            self._params = params
        self._limits = (0, None)
        self._isNumeric = isNumeric
        self._maxSampleAttempts = 10

    def setMaxSamplingAttempts(self, n):
        if n is not None:
            self._maxSampleAttempts = n

    def setLimits(self, limits):
        self._limits = limits

    def sampleValue(self):
        if self._isNumeric:
            value = None
            nrSampleAttempts = 0
            # Sample until value falls into limits
            while nrSampleAttempts < self._maxSampleAttempts \
                    and (value is None or (self._limits[1] is not None and value > self._limits[1]) or
                         (self._limits[0] is not None and value < self._limits[0])):
                value = self._sampleValue()
                nrSampleAttempts += 1
            # Eventually apply fallback cutting value to limits
            if self._limits[0] is not None and value < self._limits[0]:
                value = self._limits[0]
            elif self._limits[1] is not None and value > self._limits[1]:
                value = self._limits[1]
        else:
            value = self._sampleValue()
        return value

    def sampleValueString(self, decimalPlaces):
        if self._isNumeric:
            decimalPattern = "%." + str(decimalPlaces) + "f"
            return decimalPattern % self.sampleValue()
        return self.sampleValue()

    def _sampleValue(self):
        return self._params[0]

class _NormalDistribution(_FixDistribution):

    def __init__(self, mu, sd):
        _FixDistribution.__init__(self, (mu, sd))

    def _sampleValue(self):
        return random.normalvariate(self._params[0], self._params[1])

class _LogNormalDistribution(_FixDistribution):

    def __init__(self, mu, sd):
        _FixDistribution.__init__(self, (mu, sd))

    def _sampleValue(self):
        return random.lognormvariate(self._params[0], self._params[1])

class _NormalCappedDistribution(_FixDistribution):

    def __init__(self, mu, sd, min, max):
        _FixDistribution.__init__(self, (mu, sd, min, max))
        if mu < min or mu > max:
            raise Exception("mean %s is outside cutoff bounds [%s, %s]" % (mu, min, max))

    def _sampleValue(self):
        while True:
            cand = random.normalvariate(self._params[0], self._params[1])
            if cand >= self._params[2] and cand <= self._params[3]:
                return cand

class _UniformDistribution(_FixDistribution):

    def __init__(self, a, b):
        _FixDistribution.__init__(self, (a, b))

    def _sampleValue(self):
        return random.uniform(self._params[0], self._params[1])

class _GammaDistribution(_FixDistribution):

    def __init__(self, alpha, beta):
        _FixDistribution.__init__(self, (alpha, 1.0 / beta))

    def _sampleValue(self):
        return random.gammavariate(self._params[0], self._params[1])

_DIST_DICT = {
    'normal': _NormalDistribution,
    'lognormal': _LogNormalDistribution,
    'normalCapped': _NormalCappedDistribution,
    'uniform': _UniformDistribution,
    'gamma': _GammaDistribution
}

class VehAttribute:

    def __init__(self, name, is_param=False, distribution=None, distribution_params=None,
                 bounds=None, attribute_value=None):
        # type: (str, bool, str, Union[dict, Any], tuple, str) -> None
        """
        This emmulates one line of example config.txt in
            https://sumo.dlr.de/docs/Tools/Misc.html#createvehtypedistributionpy
        Either distribution or attribute_value should be populated
        Args:
            name (str): the name of the attribute. Examples: "tau", "sigma", "length"
            is_param (bool, optional): is the attribute a parameter that should be added as a child element.
            distribution (str, optional): the name of the distribution to use ()
            distribution_params (Union[dict, Any], optional): the parameters corresponding to the distribution
            bounds (tuple, optional): the bounds of the distribution.
            attribute_value (str, optional): if no distribution is given, the fixed value for the attribute
        """
        self.is_param = is_param
        self.name = name
        self.distribution = distribution
        self.distribution_params = distribution_params
        self.bounds = bounds
        self.attribute_value = attribute_value
        if self.attribute_value and self.distribution:
            raise Exception("Only one of distribution or attribute value should be defined, not both")
        self.d_obj = self._dist_helper(distribution, distribution_params, bounds)

    def _dist_helper(self, distribution, dist_params, dist_bounds):
        # type: (...) -> Union[None, _FixDistribution]
        if distribution:
            try:
                d = _DIST_DICT[distribution](**dist_params)
                d.setLimits(dist_bounds) if dist_bounds else d.setLimits(
                    (0, None))
            except KeyError:
                raise KeyError("The distribution %s is not known. Please select one of: \n%s " %
                               (distribution, "\n".join(_DIST_DICT.keys())))
        else:
            isNumeric = False if self.name == "emissionClass" else len(
                re.findall(r'^(-?[0-9]+(\.[0-9]+)?)$', self.attribute_value)) > 0
            d = _FixDistribution((self.attribute_value, ), isNumeric)
        return d

    def add_sampling_attempts(self, attempts):
        if self.d_obj:
            self.d_obj.setMaxSamplingAttempts(attempts)

class CreateVehTypeDistribution:

    def __init__(self, seed=None, size=100, name='vehDist', resampling=100, decimal_places=3):
        # type: (int, int, str, int, int) -> None
        """
        Creates a VehicleType Distribution.
        See https://sumo.dlr.de/docs/Definition_of_Vehicles,_Vehicle_Types,_and_Routes.html#vehicle_type_distributions

        Args:
            seed (int, optional): random seed.
            size (int, optional): number of vTypes in the distribution.
            name (str, optional): alphanumerical ID used for the created vehicle type distribution.
            resampling (int, optional): number of attempts to resample a value until it lies in the specified bounds.
            decimal_places (int, optional): number of decimal places.
        """
        if seed:
            random.seed(seed)

        self.size = size
        self.name = name
        self.resampling = resampling
        self.decimal_places = decimal_places
        self.attributes = []  # type: List[VehAttribute]

    def add_attribute(self, attribute):
        # type: (Union[VehAttribute, dict]) -> None
        """
        Add an instance of the attribute class to the Parameters. Pass the sampling attempts "global" parameter
        Args:
            attribute (VehAttribute or dict): An instance of VehAttribute or
             a dictionary of parameters to be passed to the VehAttribute constructor
        """
        attribute = attribute if isinstance(attribute, VehAttribute) else VehAttribute(**attribute)
        attribute.add_sampling_attempts(self.resampling)
        self.attributes.append(attribute)

    def create_veh_dist(self, xml_dom):
        # type: (xml.dom.minidom.Document) -> xml.dom.minidom.Element
        # create the vehicleDist node
        vtype_dist_node = xml_dom.createElement("vTypeDistribution")
        vtype_dist_node.setAttribute("id", self.name)

        # create the vehicle types
        for i in range(self.size):
            veh_type_node = xml_dom.createElement("vType")
            veh_type_node.setAttribute("id", self.name + str(i))
            self._generate_vehType(xml_dom, veh_type_node)
            vtype_dist_node.appendChild(veh_type_node)

        return vtype_dist_node

    def to_xml(self, file_path):
        # type: (str) -> None
        xml_dom, existing_file = self._check_existing(file_path)
        vtype_dist_node = self.create_veh_dist(xml_dom)
        if existing_file:
            self._handle_existing(xml_dom, vtype_dist_node)
            with open(file_path, 'w') as f:
                dom_string = xml_dom.toprettyxml()
                # super annoying but this makes re-writing the xml a little bit prettier
                f.write(os.linesep.join([s for s in dom_string.splitlines() if s.strip()]))
        else:
            write_additional_minidom(xml_dom, vtype_dist_node, file_path=file_path)
        sys.stdout.write("Output written to %s" % file_path)

    def _handle_existing(self, xml_dom, veh_dist_node):
        # type: (xml.dom.minidom.Document, xml.dom.minidom.Element) -> None
        existingDistNodes = xml_dom.getElementsByTagName("vTypeDistribution")
        replaceNode = None
        for existingDistNode in existingDistNodes:
            if existingDistNode.hasAttribute("id") and existingDistNode.getAttribute("id") == self.name:
                replaceNode = existingDistNode
                break
        if replaceNode is not None:
            replaceNode.parentNode.replaceChild(veh_dist_node, replaceNode)
        else:
            xml_dom.documentElement.appendChild(veh_dist_node)

    def _generate_vehType(self, xml_dom, veh_type_node):
        # type: (xml.dom.minidom.Document, xml.dom.minidom.Element) -> xml.dom.minidom.Node
        for attr in self.attributes:
            if attr.is_param:
                param_node = xml_dom.createElement("param")
                param_node.setAttribute("key", attr.name)
                param_node.setAttribute(
                    "value", attr.d_obj.sampleValueString(self.decimal_places))
                veh_type_node.appendChild(param_node)
            else:
                veh_type_node.setAttribute(
                    attr.name, attr.d_obj.sampleValueString(self.decimal_places))

    @staticmethod
    def _check_existing(file_path):
        # type: (str) -> Tuple[xml.dom.minidom.Document, bool]
        if os.path.exists(file_path):
            try:
                return xml.dom.minidom.parse(file_path), True
            except Exception as e:
                raise Exception("Cannot parse existing %s. Error: %s" %
                                (file_path, str(e)))
        else:
            return xml.dom.minidom.Document(), False

    def save_myself(self, file_path):
        # type: (str) -> None
        """
        This function saves the class to a json format. Used for logging simulation inputs

        Args:
            file_path (str): path to save json to
        """
        import json

        with open(file_path, "w") as f:
            f.write(
                json.dumps(
                    self,
                    default=lambda o: {
                        key: param for key, param in o.__dict__.items() if '_' not in key[0]},
                    sort_keys=True,
                    indent=4,
                )
            )

class CreateMultiVehTypeDistributions(CreateVehTypeDistribution):

    def __init__(self):
        # type: () -> None
        self.distributions = []  # type: List[CreateVehTypeDistribution]

    def register_veh_type_distribution(self, veh_type_dist, veh_attributes):
        # type: (Union[dict, CreateVehTypeDistribution], List[Union[dict, VehAttribute]]) -> None
        veh_type_dist = veh_type_dist if isinstance(
            veh_type_dist, CreateVehTypeDistribution) else CreateVehTypeDistribution(**veh_type_dist)

        for attr in veh_attributes:
            veh_type_dist.add_attribute(attr if isinstance(attr, VehAttribute) else VehAttribute(**attr))

        self.distributions.append(veh_type_dist)

    def write_xml(self, file_path):
        # type: (str) -> None
        """
        This function will overwrite existing files

        Args:
            file_path (str): Path to the file to write to
        """
        xml_dom, _ = self._check_existing(file_path)
        veh_dist_nodes = [dist.create_veh_dist(xml_dom=xml_dom) for dist in self.distributions]
        write_additional_minidom(xml_dom, veh_dist_nodes, file_path=file_path)


import os
from util.IO import IO
from util.Event import Event
import matplotlib.pyplot as plt
from matplotlib import rc
import numpy as np
from datetime import date, timedelta, datetime
import pandas as pd

class Calendar:

    def __init__(self, username):

        self.usernames = username
        io = IO(self.usernames)
        self.mysql = io.dbConnect()

    def plotEvents(self, today, defaultUserName=None):
        '''
        defaultUserName : the current user
        '''
        otherEvents = []
        if defaultUserName != None:

            query = f"""
                    SELECT *
                    FROM REQUESTS
                    WHERE Requestor != '{defaultUserName}'
                    """

            reqTable = pd.read_sql(query, self.mysql)

            for ii, row in reqTable.iterrows():
                event = Event(row['Requestor'], row['EventName'], row['StartTime'], row['EndTime'], row['StartDate'])
                otherEvents.append(event)

        colors = ['firebrick', 'dodgerblue', 'seagreen']

        colorIdx = 0
        #raise ValueError('made it')
        fig, axs = plt.subplots(1, 7, figsize=(30, 15))

        # get new ioObj
        io = IO(self.usernames)

        # generate list of next 7 days
        datesList = [today + timedelta(days=i) for i in range(7)]

        # generate plot of the users schedule for the next 7 days

        strTimes = [f"{ii}:00" for ii in range(24)]
        axs[0].set_ylabel('Time [hh:mm]', fontsize=30)

        x = [0, 1]
        placeTicks = True

        for ax, dd in zip(axs, datesList):
            ax.set_title(dd.strftime("%m/%d"), fontsize=24)
            ax.set_xticks([])
            ax.set_yticks([])
            ax.set_ylim(24)

            for jj in range(24):
                ax.axhline(jj, x[0], x[1], ls='--', color='k', alpha=0.5)

            for event in io.events:
                if event.startTime.strftime("%m/%d") == dd.strftime("%m/%d"):
                    startHr = int(event.startTime.strftime("%H"))
                    startMin = int(event.startTime.strftime("%M"))
                    endHr = int(event.endTime.strftime("%H"))
                    endMin = int(event.endTime.strftime("%M"))
                    ax.fill_between(x, startHr + startMin/60, endHr + endMin/60, color=colors[0], alpha=0.5)
                    midpoint = (startHr + startMin/60 + endHr + endMin/60)/2
                    ax.text(0, midpoint, event.eventName, color='w', fontsize=24)

            for event in otherEvents:
                if event.startTime.strftime("%m/%d") == dd.strftime("%m/%d"):
                    startHr = int(event.startTime.strftime("%H"))
                    startMin = int(event.startTime.strftime("%M"))
                    endHr = int(event.endTime.strftime("%H"))
                    endMin = int(event.endTime.strftime("%M"))
                    ax.fill_between(x, startHr + startMin/60, endHr + endMin/60, color=colors[1], alpha=0.5)
                    midpoint = (startHr + startMin/60 + endHr + endMin/60)/2
                    ax.text(0, midpoint, event.eventName, color='w', fontsize=24)

                    cursor = self.mysql.cursor()

                    # get other User name
                    getNames = f"""
                                    SELECT *
                                    FROM USERNAME
                                    WHERE UserName='{event.userName}'
                                    """
                    userInfo = pd.read_sql(getNames, self.mysql)
                    first = userInfo['FirstName'].tolist()[0]
                    last = userInfo['LastName'].tolist()[0]

                    ax.text(0, midpoint+1, first+" "+last, color='w', fontsize=24)

            if placeTicks:
                ax.set_yticks(np.arange(len(strTimes)), labels=strTimes, fontsize=24)
                placeTicks=False
            else:
                ax.set_yticks(np.arange(len(strTimes)), labels="", fontsize=24)

        fig.suptitle("Year: " + datesList[0].strftime("%Y"), fontsize=36)

        return fig


from calendar import weekday
import os
import zipfile
import numpy as np
import torch
import sklearn.metrics as metrics
import matplotlib.pyplot as plt

# if dataset == "alpha":
        # if (not os.path.isfile("DC_STGCN/data/adj_mat_alpha.npy")
        #         or not os.path.isfile("DC_STGCN/data/node_values_alpha.npy")):
with zipfile.ZipFile("DC_STGCN/data/SCATS_alpha.zip", 'r') as zip_ref:
    zip_ref.extractall("DC_STGCN/data/")

X_alpha = np.load("DC_STGCN/data/alpha_data/node_values_alpha.npy").transpose((1, 2, 0))
X_alpha = X_alpha.astype(np.float32)

    # elif dataset == "bravo":
        # if (not os.path.isfile("DC_STGCN/data/adj_mat_bravo.npy")
        #         or not os.path.isfile("DC_STGCN/data/node_values_bravo.npy")):
with zipfile.ZipFile("DC_STGCN/data/SCATS_bravo.zip", 'r') as zip_ref:
    zip_ref.extractall("DC_STGCN/data/")

# A = np.load("DC_STGCN/data/bravo_data/adj_mat_bravo.npy")
# A = A.astype(np.float32)
X_bravo = np.load("DC_STGCN/data/bravo_data/node_values_bravo.npy").transpose((1, 2, 0))
X_bravo = X_bravo.astype(np.float32)

print(X_alpha.shape)
print(X_bravo.shape)

X_196 = X_alpha[2:5, :, :]
print(X_196.shape)

X_bravoplus = np.concatenate((X_bravo, X_196), axis=0)
print(X_bravoplus.shape)

X_bravoplus = X_bravoplus.transpose((2, 0, 1))

np.save("interpret_csv_bravoplus/node_values_bravoplus", X_bravoplus)

files_string = "TO BE CONFIGURED"

f = open("interpret_csv_bravoplus/nv_info.txt", "w")
info_string = "Num Juncs:\t" + str(X_bravoplus.shape[1]) + "\nNum Channels:\t" + str(X_bravoplus.shape[2]) + "\nNum Days:\t" + str(X_bravoplus.shape[0]/480)
print(info_string)
f.write(info_string)
f.write(files_string)
f.close()

if os.path.isfile("interpret_csv_bravoplus/adj_mat_bravoplus.npy") and os.path.isfile("interpret_csv_bravoplus/adj_info.txt"):
    with zipfile.ZipFile("interpret_csv_bravoplus/SCATS_bravoplus.zip", "w") as zip_object:
        zip_object.write("interpret_csv_bravoplus/node_values_bravoplus.npy", arcname="bravoplus_data/node_values_bravoplus.npy")
        zip_object.write("interpret_csv_bravoplus/adj_mat_bravoplus.npy", arcname="bravoplus_data/adj_mat_bravoplus.npy")
        zip_object.write("interpret_csv_bravoplus/adj_info.txt", arcname="bravoplus_data/adj_info.npy")
        zip_object.write("interpret_csv_bravoplus/nv_info.txt", arcname="bravoplus_data/nv_info.npy")
    print("Zipped")


import scrapy
from sanic.log import logger

reviews_tag_identifier = "//span[@data-hook='review-body']/span/text()"
next_page_tag_identifier = "//li[@class='a-last']/a/@href"

class AmazonSpider(scrapy.Spider):
    name = 'amazon_spider'
    allowed_domains = ['amazon.in']

    def __init__(self, name=None, uid=None, asin=None):
        super().__init__(name)
        if not asin:
            logger.warning('| amazon spider | asin param was null - it should not be null')
            raise Exception('asin should not be null')
        self.asin = asin
        self.uid = uid
        self.start_urls = [
            f'https://www.amazon.in/product-reviews/{asin}'
        ]
        self.reviews = []
        self.base_url = 'https://www.google.com'

    def parse(self, response, **kwargs):
        logger.info('| amazon spider | started parsing the reviews')
        reviews_dom = response.xpath(reviews_tag_identifier)
        if reviews_dom:
            for review in reviews_dom:
                self.reviews.append(
                    review.get()
                )
        else:
            logger.warning('| amazon spider | No reviews element in the html page')

        next_page = response.xpath(next_page_tag_identifier).get()
        if next_page:
            yield scrapy.Request(
                response.urljoin(next_page), self.parse
            )
        else:
            yield {
                'meta-data': {
                    'uid': self.uid,
                    'asin': self.asin,
                    'is_preprocessed': False,
                    'total_reviews': len(self.reviews),
                },
                'reviews': self.reviews
            }


import torch
import torch.nn as nn
import torch.nn.functional as F
import random
from base import BaseModel

class Encoder(nn.Module):
    def __init__(self, embedding, hidden_size, rnn_cell='GRU', bidirectional=False, n_layers=1, dropout=0.0, device='cpu'):
        super(Encoder, self).__init__()
        self.hidden_size = hidden_size
        self.rnn_cell = rnn_cell
        #self.padding_idx = self.embedding.to_index(padding)
        self.bidirectional = bidirectional
        self.n_layers = n_layers
        self.dropout = dropout
        self.device = device
        self.n_init = (2 if bidirectional == True else 1) * n_layers
        self.vocab_size = embedding.vocab_size
        self.emb_size = embedding.emb_size

        self.embedding = nn.Embedding(self.vocab_size, self.emb_size)
        #self.embedding.weight = nn.Parameter(embedding.vectors)
        if rnn_cell == 'GRU': self.rnn = nn.GRU(self.emb_size, self.hidden_size, batch_first=True, dropout=self.dropout, num_layers=self.n_layers)
        #self.freeze_embedding()

    def freeze_embedding(self):
        for param in self.embedding.parameters():
            param.requires_grad = False

    def forward(self, source):
        # source: (batch, seq_len)
        #init_hidden = torch.randn(self.n_init, source.size(0), self.hidden_size).to(self.device) #(n_layer*n_direction, batch, hidden_size)
        source = self.embedding(source) # (batch, seq_len) -> (batch, seq_len, emb_size)
        output, hidden = self.rnn(source, None) #(batch, seq_len, emb_size) -> (batch, seq_len, emb_size*n_direction), (n_layer*n_direction, batch, hidden_size)
        return output, hidden #(n_layer*n_direction, batch, hidden_size)

class Decoder(nn.Module):
    def __init__(self, embedding, hidden_size, rnn_cell='GRU', bidirectional=False, n_layers=1, dropout=0.2, device='cpu', teaching_force_rate=0.0, use_attn=False, method=None, padded_len=None):
        super(Decoder, self).__init__()
        self.hidden_size = hidden_size
        self.rnn_cell = rnn_cell
        self.bidirectional = bidirectional
        self.n_layers = n_layers
        self.dropout = dropout
        self.device = device
        self.n_init = (2 if bidirectional == True else 1) * n_layers
        self.teaching_force_rate = teaching_force_rate
        self.vocab_size = embedding.vocab_size
        self.emb_size = embedding.emb_size
        self.use_attn = use_attn
        self.step = 0

        self.relu = nn.ReLU()
        self.softmax = nn.LogSoftmax(dim=-1)
        self.embedding_org = embedding
        self.embedding = nn.Embedding(self.vocab_size, self.emb_size)
        #self.embedding.weight = nn.Parameter(embedding.vectors)
        self.linear = nn.Linear(self.hidden_size, self.vocab_size)
        if rnn_cell == 'GRU': self.rnn = nn.GRU(self.emb_size, self.hidden_size, batch_first=True, dropout=self.dropout, num_layers=self.n_layers)
        if self.use_attn:
            self.attn = Attention(hidden_size=self.hidden_size, method=method, padded_len=padded_len)
        #self.freeze_embedding()

    def freeze_embedding(self):
        for param in self.embedding.parameters():
            param.requires_grad = False

    def forward(self, label, init_hidden, encoder_output=None):
        if(self.step > 2000): self.teaching_force_rate = 0.2
        self.step += 1
        use_teaching_force = True if random.random() <= self.teaching_force_rate else False
        # source: (batch, seq_len)
        #input = self.relu(self.embedding(input)) # (batch, seq_len) -> (batch, seq_len, emb_size)
        batch, seq_len = label.size(0), label.size(1)
        outputs = []

        hidden = init_hidden

        if use_teaching_force:
            for i in range(seq_len):
                input = label[:, i].unsqueeze(1)
                #print(label)
                #print(str(i) + ': ' + self.embedding_org.indice_to_sentence(label[0].tolist()))
                input = self.relu(self.embedding(input))
                if self.use_attn:
                    attn_output = self.attn(encoder_output, input, hidden)
                    output, hidden = self.rnn(attn_output, hidden)
                else:
                    output, hidden = self.rnn(input, hidden)
                output = self.softmax(self.linear(output))
                last_predict = output.max(2)[1]
                #print(str(i) + ': ' + self.embedding_org.indice_to_sentence(last_predict[0].tolist()))
                outputs.append(output)

        else:
            input = label[:, 0].unsqueeze(1)
            input = self.relu(self.embedding(input))
            for i in range(seq_len):
                if self.use_attn:
                    attn_output = self.attn(encoder_output, input, hidden)
                    output, hidden = self.rnn(attn_output, hidden)
                else:
                    output, hidden = self.rnn(input, hidden)
                output = self.softmax(self.linear(output))
                outputs.append(output)
                last_predict = output.max(2)[1]
                input = self.relu(self.embedding(last_predict))

        outputs = torch.cat(outputs, dim=1)
        return outputs

class Attention(nn.Module):
    def __init__(self, hidden_size, method, padded_len):
        super(Attention, self).__init__()
        self.hidden_size = hidden_size
        self.method = method
        self.attn = nn.Linear(hidden_size*2, padded_len)
        self.attn_combine = nn.Linear(hidden_size*2, hidden_size)
        self.softmax = nn.Softmax()
        self.relu = nn.ReLU()
    def forward(self, encoder_output, decoder_input_embeded, decoder_hidden):
        # encoder_output: [batch, seq_len, hidden_size=embedded_size]
        # decoder_input_embeded: [batch, 1, embedded_size]
        # decoder_hidden: [batch, 1, embedded_size]
        decoder_hidden = decoder_hidden.permute(1, 0, 2)
        # print(encoder_output.size())
        # print(decoder_input_embeded.size())
        # print(decoder_hidden.size())
        similarity = self.attn(torch.cat((decoder_input_embeded, decoder_hidden), dim=-1))
        attn_weight = self.softmax(similarity) # [batch, 1, padded_len]

        attn_applied = torch.bmm(attn_weight, encoder_output) #[batch, 1, hidden_size]
        output = self.relu(self.attn_combine(torch.cat((attn_applied, decoder_input_embeded), dim=-1)))
        return output

class ChatBotModel(BaseModel):
    def __init__(self, embedding, hidden_size, rnn_cell='GRU', bidirectional=False, n_layers=1, dropout=0.2, device='cpu', teaching_force_rate=0.0, use_attn=False, method='concat', padded_len=10):
        super().__init__()
        self.use_attn = use_attn
        self.embedding = embedding
        if self.use_attn:
            self.hidden_size = embedding.emb_size
        else:
            self.hidden_size = hidden_size
        self.encoder = Encoder(self.embedding, self.hidden_size, rnn_cell=rnn_cell, bidirectional=bidirectional, n_layers=n_layers, dropout=dropout, device=device)
        self.decoder = Decoder(self.embedding, self.hidden_size, rnn_cell=rnn_cell, bidirectional=bidirectional, n_layers=n_layers, dropout=dropout, device=device, teaching_force_rate=teaching_force_rate, use_attn=self.use_attn, method=method, padded_len=padded_len)

    def forward(self, source, target):
        # print('> : ' + self.embedding.indice_to_sentence(source[0].tolist()))
        # print('= : ' + self.embedding.indice_to_sentence(target[0].tolist()))
        encoder_output, encoder_hidden = self.encoder(source)
        if self.use_attn:
            output = self.decoder(target, encoder_hidden, encoder_output)
        else:
            output = self.decoder(target, encoder_hidden)

        return output


import copy
import json
import os
import time

from flask import Blueprint, request, Response
import pm4py
from pm4py.algo.filtering.log.attributes import attributes_filter
from pm4py.objects.conversion.process_tree.converter import to_petri_net_transition_bordered as converter
from pm4py.visualization.common.utils import get_base64_from_gviz

from KPIAlgebras.entities import data
from KPIAlgebras.entities import model as model_object
from KPIAlgebras.request_objects import request_objects
from KPIAlgebras.response_objects import response_objects
from KPIAlgebras.serializers import extended_process_tree_serializer as serializer
from KPIAlgebras.use_cases import alignment_computation_use_case as alignment
from KPIAlgebras.use_cases import cycle_time_analysis_use_case as measurement_high_level
from KPIAlgebras.use_cases import decorate_extended_process_tree_use_case as decorate_tree
from KPIAlgebras.use_cases import import_event_log_use_case as import_log
from KPIAlgebras.use_cases import time_range_construction_use_case as measurement_fine_grained
from KPIAlgebras.util import constants, http_response_status_code

blueprint = Blueprint('endpoints', __name__)
alignments = None
log = None
model = None
initial_marking = None
final_marking = None
extended_process_tree = None

@blueprint.route('/measurement', methods=['POST'])
def measurement():
    print("Begining the fine grained analysis")
    t1 = time.perf_counter()
    parameters = dict()
    log_file = request.files['eventLog']
    log_file.save(os.path.join(constants.upload_folder, log_file.filename))

    model_file = request.files['model']
    model_file.save(os.path.join(constants.upload_folder, model_file.filename))

    import_log_use_case = import_log.ImportEventLogUseCase()
    request_object = request_objects.TimeRangeConstructionRequestObject.from_dict({'event_log': log_file.filename})
    global log
    log = data.EventLog(import_log_use_case.import_event_log_from_xes(request_object))
    os.remove(os.path.join(constants.upload_folder, log_file.filename))

    process_tree = pm4py.read_ptml(os.path.join(constants.upload_folder, model_file.filename))
    os.remove(os.path.join(constants.upload_folder, model_file.filename))
    global extended_process_tree
    extended_process_tree = model_object.ExtendedProcessTree(process_tree)
    global model, initial_marking, final_marking
    model, initial_marking, final_marking = converter.apply(extended_process_tree)

    alignment_use_case = alignment.AlignmentComputationUseCase()
    global alignments
    alignments = alignment_use_case.compute(model, initial_marking, final_marking, log)

    high_level_use_case = measurement_high_level.CycleTimeAnalysisUseCase()
    response = high_level_use_case.analyse(log.log, alignments, extended_process_tree, model)
    extended_process_tree = response.value

    lifecycle = attributes_filter.get_attribute_values(log.log, "lifecycle:transition")
    if lifecycle is not None and 'start' in lifecycle:
        fine_grained_use_case = measurement_fine_grained.TimeRangesConstructionUseCase(log, extended_process_tree,
                                                                                model, initial_marking,
                                                                                final_marking, alignments)
        response = fine_grained_use_case.construct_time_ranges(log.log, alignments, model, initial_marking,
                                                               final_marking)

    decoration_use_case = decorate_tree.DecorateExtendedProcessTreeUseCase()
    gviz = decoration_use_case.decorate(extended_process_tree)
    svg = get_base64_from_gviz(gviz)

    extended_process_tree_json = json.dumps(response.value, cls=serializer.ExtendedProcessTreeJsonEncoder)
    json_dict = json.loads(extended_process_tree_json)
    json_dict["svg"] = svg.decode('utf-8')

    extended_process_tree_json = json.dumps(json_dict, cls=serializer.ExtendedProcessTreeJsonEncoder)
    t2 = time.perf_counter()
    print(t2 - t1)
    return Response(extended_process_tree_json, mimetype='application/json',
                    status=http_response_status_code.STATUS_CODES[response.type])

@blueprint.route('/timeshifting', methods=['POST'])
def timeshifting():
    parameters = request.get_json()

    if parameters is None:
        parameters = dict()
        for arg, value in request.args.items():
            parameters[arg] = value

    request_object = request_objects.TimeShiftingRequestObject.from_dict(parameters)

    global log, model, initial_marking, final_marking, extended_process_tree, alignments
    extended_process_tree.states.append(copy.deepcopy(extended_process_tree))

    fine_grained_use_case = measurement_fine_grained.TimeRangesConstructionUseCase(log.log, extended_process_tree,
                                                                                model, initial_marking,
                                                                                final_marking, alignments)
    response = fine_grained_use_case.shift_time_ranges(request_object)

    decoration_use_case = decorate_tree.DecorateExtendedProcessTreeUseCase()
    gviz = decoration_use_case.decorate(extended_process_tree)
    svg = get_base64_from_gviz(gviz)
    extended_process_tree_json = json.dumps(response.value, cls=serializer.ExtendedProcessTreeJsonEncoder)
    json_dict = json.loads(extended_process_tree_json)
    json_dict["svg"] = svg.decode('utf-8')

    extended_process_tree_json = json.dumps(json_dict, cls=serializer.ExtendedProcessTreeJsonEncoder)

    return Response(extended_process_tree_json, mimetype='application/json',
                    status=http_response_status_code.STATUS_CODES[response.type])

@blueprint.route('/undoChange', methods=['GET'])
def undo_change():
    global extended_process_tree
    extended_process_tree = extended_process_tree.states.pop()
    decoration_use_case = decorate_tree.DecorateExtendedProcessTreeUseCase()
    gviz = decoration_use_case.decorate(extended_process_tree)
    svg = get_base64_from_gviz(gviz)
    extended_process_tree_json = json.dumps(extended_process_tree, cls=serializer.ExtendedProcessTreeJsonEncoder)
    json_dict = json.loads(extended_process_tree_json)
    json_dict["svg"] = svg.decode('utf-8')
    extended_process_tree_json = json.dumps(json_dict, cls=serializer.ExtendedProcessTreeJsonEncoder)
    return Response(extended_process_tree_json, mimetype='application/json',
                    status=http_response_status_code.STATUS_CODES[response_objects.ResponseSuccess.SUCCESS])

@blueprint.route('/undoAllChanges', methods=['GET'])
def undo_all_changes():
    global extended_process_tree
    extended_process_tree = extended_process_tree.states[0]
    decoration_use_case = decorate_tree.DecorateExtendedProcessTreeUseCase()
    gviz = decoration_use_case.decorate(extended_process_tree)
    svg = get_base64_from_gviz(gviz)
    extended_process_tree_json = json.dumps(extended_process_tree, cls=serializer.ExtendedProcessTreeJsonEncoder)
    json_dict = json.loads(extended_process_tree_json)
    json_dict["svg"] = svg.decode('utf-8')
    extended_process_tree_json = json.dumps(json_dict, cls=serializer.ExtendedProcessTreeJsonEncoder)
    return Response(extended_process_tree_json, mimetype='application/json',
                    status=http_response_status_code.STATUS_CODES[response_objects.ResponseSuccess.SUCCESS])


from django.urls import path

from posting.views  import (
    PostingView,
    CategoryView,
    PostingLikeView,
    PostingScrapView
)

urlpatterns = [
    path('', PostingView.as_view()),
    path('/category', CategoryView.as_view()),
    path('/like', PostingLikeView.as_view()),
    path('/scrap', PostingScrapView.as_view())
]


from __future__ import annotations

import pickle
import sys
from collections import defaultdict
from unittest.mock import Mock, patch

import pytest

from kombu import Connection, Consumer, Exchange, Producer, Queue
from kombu.exceptions import MessageStateError
from kombu.utils import json
from kombu.utils.functional import ChannelPromise
from t.mocks import Transport

class test_Producer:

    def setup(self):
        self.exchange = Exchange('foo', 'direct')
        self.connection = Connection(transport=Transport)
        self.connection.connect()
        assert self.connection.connection.connected
        assert not self.exchange.is_bound

    def test_repr(self):
        p = Producer(self.connection)
        assert repr(p)

    def test_pickle(self):
        chan = Mock()
        producer = Producer(chan, serializer='pickle')
        p2 = pickle.loads(pickle.dumps(producer))
        assert p2.serializer == producer.serializer

    def test_no_channel(self):
        p = Producer(None)
        assert not p._channel

    @patch('kombu.messaging.maybe_declare')
    def test_maybe_declare(self, maybe_declare):
        p = self.connection.Producer()
        q = Queue('foo')
        p.maybe_declare(q)
        maybe_declare.assert_called_with(q, p.channel, False)

    @patch('kombu.common.maybe_declare')
    def test_maybe_declare_when_entity_false(self, maybe_declare):
        p = self.connection.Producer()
        p.maybe_declare(None)
        maybe_declare.assert_not_called()

    def test_auto_declare(self):
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, auto_declare=True)
        # creates Exchange clone at bind
        assert p.exchange is not self.exchange
        assert p.exchange.is_bound
        # auto_declare declares exchange'
        assert 'exchange_declare' not in channel

        p.publish('foo')
        assert 'exchange_declare' in channel

    def test_manual_declare(self):
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, auto_declare=False)
        assert p.exchange.is_bound
        # auto_declare=False does not declare exchange
        assert 'exchange_declare' not in channel
        # p.declare() declares exchange')
        p.declare()
        assert 'exchange_declare' in channel

    def test_prepare(self):
        message = {'the quick brown fox': 'jumps over the lazy dog'}
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, serializer='json')
        m, ctype, cencoding = p._prepare(message, headers={})
        assert json.loads(m) == message
        assert ctype == 'application/json'
        assert cencoding == 'utf-8'

    def test_prepare_compression(self):
        message = {'the quick brown fox': 'jumps over the lazy dog'}
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, serializer='json')
        headers = {}
        m, ctype, cencoding = p._prepare(message, compression='zlib',
                                         headers=headers)
        assert ctype == 'application/json'
        assert cencoding == 'utf-8'
        assert headers['compression'] == 'application/x-gzip'
        import zlib
        assert json.loads(zlib.decompress(m).decode('utf-8')) == message

    def test_prepare_custom_content_type(self):
        message = b'the quick brown fox'
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, serializer='json')
        m, ctype, cencoding = p._prepare(message, content_type='custom')
        assert m == message
        assert ctype == 'custom'
        assert cencoding == 'binary'
        m, ctype, cencoding = p._prepare(message, content_type='custom',
                                         content_encoding='alien')
        assert m == message
        assert ctype == 'custom'
        assert cencoding == 'alien'

    def test_prepare_is_already_unicode(self):
        message = 'the quick brown fox'
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, serializer='json')
        m, ctype, cencoding = p._prepare(message, content_type='text/plain')
        assert m == message.encode('utf-8')
        assert ctype == 'text/plain'
        assert cencoding == 'utf-8'
        m, ctype, cencoding = p._prepare(message, content_type='text/plain',
                                         content_encoding='utf-8')
        assert m == message.encode('utf-8')
        assert ctype == 'text/plain'
        assert cencoding == 'utf-8'

    def test_publish_with_Exchange_instance(self):
        p = self.connection.Producer()
        p.channel = Mock()
        p.channel.connection.client.declared_entities = set()
        p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient')
        assert p._channel.basic_publish.call_args[1]['exchange'] == 'foo'

    def test_publish_with_expiration(self):
        p = self.connection.Producer()
        p.channel = Mock()
        p.channel.connection.client.declared_entities = set()
        p.publish('hello', exchange=Exchange('foo'), expiration=10)
        properties = p._channel.prepare_message.call_args[0][5]
        assert properties['expiration'] == '10000'

    def test_publish_with_timeout(self):
        p = self.connection.Producer()
        p.channel = Mock()
        p.channel.connection.client.declared_entities = set()
        p.publish('test_timeout', exchange=Exchange('foo'), timeout=1)
        timeout = p._channel.basic_publish.call_args[1]['timeout']
        assert timeout == 1

    def test_publish_with_reply_to(self):
        p = self.connection.Producer()
        p.channel = Mock()
        p.channel.connection.client.declared_entities = set()
        assert not p.exchange.name
        p.publish('hello', exchange=Exchange('foo'), reply_to=Queue('foo'))
        properties = p._channel.prepare_message.call_args[0][5]
        assert properties['reply_to'] == 'foo'

    def test_set_on_return(self):
        chan = Mock()
        chan.events = defaultdict(Mock)
        p = Producer(ChannelPromise(lambda: chan), on_return='on_return')
        p.channel
        chan.events['basic_return'].add.assert_called_with('on_return')

    def test_publish_retry_calls_ensure(self):
        p = Producer(Mock())
        p._connection = Mock()
        p._connection.declared_entities = set()
        ensure = p.connection.ensure = Mock()
        p.publish('foo', exchange='foo', retry=True)
        ensure.assert_called()

    def test_publish_retry_with_declare(self):
        p = self.connection.Producer()
        p.maybe_declare = Mock()
        p.connection.ensure = Mock()
        ex = Exchange('foo')
        p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex])
        p.maybe_declare.assert_called_with(ex)

    def test_revive_when_channel_is_connection(self):
        p = self.connection.Producer()
        p.exchange = Mock()
        new_conn = Connection('memory://')
        defchan = new_conn.default_channel
        p.revive(new_conn)

        assert p.channel is defchan
        p.exchange.revive.assert_called_with(defchan)

    def test_enter_exit(self):
        p = self.connection.Producer()
        p.release = Mock()
        with p as x:
            assert x is p
        p.release.assert_called_with()

    def test_connection_property_handles_AttributeError(self):
        p = self.connection.Producer()
        p.channel = object()
        p.__connection__ = None
        assert p.connection is None

    def test_publish(self):
        channel = self.connection.channel()
        p = Producer(channel, self.exchange, serializer='json')
        message = {'the quick brown fox': 'jumps over the lazy dog'}
        ret = p.publish(message, routing_key='process')
        assert 'prepare_message' in channel
        assert 'basic_publish' in channel

        m, exc, rkey = ret
        assert json.loads(m['body']) == message
        assert m['content_type'] == 'application/json'
        assert m['content_encoding'] == 'utf-8'
        assert m['priority'] == 0
        assert m['properties']['delivery_mode'] == 2
        assert exc == p.exchange.name
        assert rkey == 'process'

    def test_no_exchange(self):
        chan = self.connection.channel()
        p = Producer(chan)
        assert not p.exchange.name

    def test_revive(self):
        chan = self.connection.channel()
        p = Producer(chan)
        chan2 = self.connection.channel()
        p.revive(chan2)
        assert p.channel is chan2
        assert p.exchange.channel is chan2

    def test_on_return(self):
        chan = self.connection.channel()

        def on_return(exception, exchange, routing_key, message):
            pass

        p = Producer(chan, on_return=on_return)
        assert on_return in chan.events['basic_return']
        assert p.on_return

class test_Consumer:

    def setup(self):
        self.connection = Connection(transport=Transport)
        self.connection.connect()
        assert self.connection.connection.connected
        self.exchange = Exchange('foo', 'direct')

    def test_accept(self):
        a = Consumer(self.connection)
        assert a.accept is None
        b = Consumer(self.connection, accept=['json', 'pickle'])
        assert b.accept == {
            'application/json', 'application/x-python-serialize',
        }
        c = Consumer(self.connection, accept=b.accept)
        assert b.accept == c.accept

    def test_enter_exit_cancel_raises(self):
        c = Consumer(self.connection)
        c.cancel = Mock(name='Consumer.cancel')
        c.cancel.side_effect = KeyError('foo')
        with c:
            pass
        c.cancel.assert_called_with()

    def test_enter_exit_cancel_not_called_on_connection_error(self):
        c = Consumer(self.connection)
        c.cancel = Mock(name='Consumer.cancel')
        assert self.connection.connection_errors
        with pytest.raises(self.connection.connection_errors[0]):
            with c:
                raise self.connection.connection_errors[0]()
        c.cancel.assert_not_called()

    def test_receive_callback_accept(self):
        message = Mock(name='Message')
        message.errors = []
        callback = Mock(name='on_message')
        c = Consumer(self.connection, accept=['json'], on_message=callback)
        c.on_decode_error = None
        c.channel = Mock(name='channel')
        c.channel.message_to_python = None

        c._receive_callback(message)
        callback.assert_called_with(message)
        assert message.accept == c.accept

    def test_accept__content_disallowed(self):
        conn = Connection('memory://')
        q = Queue('foo', exchange=self.exchange)
        p = conn.Producer()
        p.publish(
            {'complex': object()},
            declare=[q], exchange=self.exchange, serializer='pickle',
        )

        callback = Mock(name='callback')
        with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
            with pytest.raises(consumer.ContentDisallowed):
                conn.drain_events(timeout=1)
        callback.assert_not_called()

    def test_accept__content_allowed(self):
        conn = Connection('memory://')
        q = Queue('foo', exchange=self.exchange)
        p = conn.Producer()
        p.publish(
            {'complex': object()},
            declare=[q], exchange=self.exchange, serializer='pickle',
        )

        callback = Mock(name='callback')
        with conn.Consumer(queues=[q], accept=['pickle'],
                           callbacks=[callback]):
            conn.drain_events(timeout=1)
        callback.assert_called()
        body, message = callback.call_args[0]
        assert body['complex']

    def test_set_no_channel(self):
        c = Consumer(None)
        assert c.channel is None
        c.revive(Mock())
        assert c.channel

    def test_set_no_ack(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
        assert consumer.no_ack

    def test_add_queue_when_auto_declare(self):
        consumer = self.connection.Consumer(auto_declare=True)
        q = Mock()
        q.return_value = q
        consumer.add_queue(q)
        assert q in consumer.queues
        q.declare.assert_called_with()

    def test_add_queue_when_not_auto_declare(self):
        consumer = self.connection.Consumer(auto_declare=False)
        q = Mock()
        q.return_value = q
        consumer.add_queue(q)
        assert q in consumer.queues
        assert not q.declare.call_count

    def test_consume_without_queues_returns(self):
        consumer = self.connection.Consumer()
        consumer.queues[:] = []
        assert consumer.consume() is None

    def test_consuming_from(self):
        consumer = self.connection.Consumer()
        consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
        consumer._active_tags = {'a': 1, 'b': 2}

        assert not consumer.consuming_from(Queue('c'))
        assert not consumer.consuming_from('c')
        assert not consumer.consuming_from(Queue('d'))
        assert not consumer.consuming_from('d')
        assert consumer.consuming_from(Queue('a'))
        assert consumer.consuming_from(Queue('b'))
        assert consumer.consuming_from('b')

    def test_receive_callback_without_m2p(self):
        channel = self.connection.channel()
        c = channel.Consumer()
        m2p = getattr(channel, 'message_to_python')
        channel.message_to_python = None
        try:
            message = Mock()
            message.errors = []
            message.decode.return_value = 'Hello'
            recv = c.receive = Mock()
            c._receive_callback(message)
            recv.assert_called_with('Hello', message)
        finally:
            channel.message_to_python = m2p

    def test_receive_callback__message_errors(self):
        channel = self.connection.channel()
        channel.message_to_python = None
        c = channel.Consumer()
        message = Mock()
        try:
            raise KeyError('foo')
        except KeyError:
            message.errors = [sys.exc_info()]
        message._reraise_error.side_effect = KeyError()
        with pytest.raises(KeyError):
            c._receive_callback(message)

    def test_set_callbacks(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        callbacks = [lambda x, y: x,
                     lambda x, y: x]
        consumer = Consumer(channel, queue, auto_declare=True,
                            callbacks=callbacks)
        assert consumer.callbacks == callbacks

    def test_auto_declare(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True)
        consumer.consume()
        consumer.consume()  # twice is a noop
        assert consumer.queues[0] is not queue
        assert consumer.queues[0].is_bound
        assert consumer.queues[0].exchange.is_bound
        assert consumer.queues[0].exchange is not self.exchange

        for meth in ('exchange_declare',
                     'queue_declare',
                     'queue_bind',
                     'basic_consume'):
            assert meth in channel
        assert channel.called.count('basic_consume') == 1
        assert consumer._active_tags

        consumer.cancel_by_queue(queue.name)
        consumer.cancel_by_queue(queue.name)
        assert not consumer._active_tags

    def test_consumer_tag_prefix(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, tag_prefix='consumer_')
        consumer.consume()

        assert consumer._active_tags[queue.name].startswith('consumer_')

    def test_manual_declare(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=False)
        assert consumer.queues[0] is not queue
        assert consumer.queues[0].is_bound
        assert consumer.queues[0].exchange.is_bound
        assert consumer.queues[0].exchange is not self.exchange

        for meth in ('exchange_declare',
                     'queue_declare',
                     'basic_consume'):
            assert meth not in channel

        consumer.declare()
        for meth in ('exchange_declare',
                     'queue_declare',
                     'queue_bind'):
            assert meth in channel
        assert 'basic_consume' not in channel

        consumer.consume()
        assert 'basic_consume' in channel

    def test_consume__cancel(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True)
        consumer.consume()
        consumer.cancel()
        assert 'basic_cancel' in channel
        assert not consumer._active_tags

    def test___enter____exit__(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True)
        context = consumer.__enter__()
        assert context is consumer
        assert consumer._active_tags
        res = consumer.__exit__(None, None, None)
        assert not res
        assert 'basic_cancel' in channel
        assert not consumer._active_tags

    def test_flow(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True)
        consumer.flow(False)
        assert 'flow' in channel

    def test_qos(self):
        channel = self.connection.channel()
        queue = Queue('qname', self.exchange, 'rkey')
        consumer = Consumer(channel, queue, auto_declare=True)
        consumer.qos(30, 10, False)
        assert 'basic_qos' in channel

    def test_purge(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        b2 = Queue('qname2', self.exchange, 'rkey')
        b3 = Queue('qname3', self.exchange, 'rkey')
        b4 = Queue('qname4', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
        consumer.purge()
        assert channel.called.count('queue_purge') == 4

    def test_multiple_queues(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        b2 = Queue('qname2', self.exchange, 'rkey')
        b3 = Queue('qname3', self.exchange, 'rkey')
        b4 = Queue('qname4', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1, b2, b3, b4])
        consumer.consume()
        assert channel.called.count('exchange_declare') == 4
        assert channel.called.count('queue_declare') == 4
        assert channel.called.count('queue_bind') == 4
        assert channel.called.count('basic_consume') == 4
        assert len(consumer._active_tags) == 4
        consumer.cancel()
        assert channel.called.count('basic_cancel') == 4
        assert not len(consumer._active_tags)

    def test_receive_callback(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        received = []

        def callback(message_data, message):
            received.append(message_data)
            message.ack()
            message.payload     # trigger cache

        consumer.register_callback(callback)
        consumer._receive_callback({'foo': 'bar'})

        assert 'basic_ack' in channel
        assert 'message_to_python' in channel
        assert received[0] == {'foo': 'bar'}

    def test_basic_ack_twice(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.ack()
            message.ack()

        consumer.register_callback(callback)
        with pytest.raises(MessageStateError):
            consumer._receive_callback({'foo': 'bar'})

    def test_basic_reject(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.reject()

        consumer.register_callback(callback)
        consumer._receive_callback({'foo': 'bar'})
        assert 'basic_reject' in channel

    def test_basic_reject_twice(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.reject()
            message.reject()

        consumer.register_callback(callback)
        with pytest.raises(MessageStateError):
            consumer._receive_callback({'foo': 'bar'})
        assert 'basic_reject' in channel

    def test_basic_reject__requeue(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.requeue()

        consumer.register_callback(callback)
        consumer._receive_callback({'foo': 'bar'})
        assert 'basic_reject:requeue' in channel

    def test_basic_reject__requeue_twice(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])

        def callback(message_data, message):
            message.requeue()
            message.requeue()

        consumer.register_callback(callback)
        with pytest.raises(MessageStateError):
            consumer._receive_callback({'foo': 'bar'})
        assert 'basic_reject:requeue' in channel

    def test_receive_without_callbacks_raises(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        with pytest.raises(NotImplementedError):
            consumer.receive(1, 2)

    def test_decode_error(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        consumer.channel.throw_decode_error = True

        with pytest.raises(ValueError):
            consumer._receive_callback({'foo': 'bar'})

    def test_on_decode_error_callback(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        thrown = []

        def on_decode_error(msg, exc):
            thrown.append((msg.body, exc))

        consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
        consumer.channel.throw_decode_error = True
        consumer._receive_callback({'foo': 'bar'})

        assert thrown
        m, exc = thrown[0]
        assert json.loads(m) == {'foo': 'bar'}
        assert isinstance(exc, ValueError)

    def test_recover(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        consumer.recover()
        assert 'basic_recover' in channel

    def test_revive(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        consumer = Consumer(channel, [b1])
        channel2 = self.connection.channel()
        consumer.revive(channel2)
        assert consumer.channel is channel2
        assert consumer.queues[0].channel is channel2
        assert consumer.queues[0].exchange.channel is channel2

    def test_revive__with_prefetch_count(self):
        channel = Mock(name='channel')
        b1 = Queue('qname1', self.exchange, 'rkey')
        Consumer(channel, [b1], prefetch_count=14)
        channel.basic_qos.assert_called_with(0, 14, False)

    def test__repr__(self):
        channel = self.connection.channel()
        b1 = Queue('qname1', self.exchange, 'rkey')
        assert repr(Consumer(channel, [b1]))

    def test_connection_property_handles_AttributeError(self):
        p = self.connection.Consumer()
        p.channel = object()
        assert p.connection is None


from __future__ import absolute_import
import os
from setuptools import setup, find_packages

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name='pyswf',
    version='1.5.4',
    description='SWF Parsing Utilities',
    long_description=read('README.md'),
    keywords = "swf parser parsing decompile utilities",

    author='Tim Knip',
    author_email='tim@floorplanner.com',
    url='https://github.com/timknip/pyswf',

    install_requires = ["lxml>=3.3.0", "Pillow>=2.3.0", "pylzma>=0.4.6", "six"],
    packages=find_packages(),
    license = "MIT",
    classifiers=[
        "Development Status :: 4 - Beta",
        "Topic :: Utilities",
        "License :: OSI Approved :: MIT License",
    ],
)


from tqdm import tqdm

# Define constant
SHORT  = 15
MIDDLE = 30
LARGE  = 50

def __clear__(times: int = 11, length: int = 123):
    """
        Clear the previous table toward the terminal
    """
    print()
    for i in range(times + 1):
        if i != times:
            print("\033[F" + ' ' * length, end='')
        else:
            print("\033[F", end = '')
    print("\r")

def __getCutLine__(length: int = 50, width_cell: int = 10, equal_symbol: bool = True) -> str:
    """
        Obtain the cut line

        Arg:    length          - The maximun length of the terminal
                width_cell      - The width of the table cell
                equal_symbol    - If use '=' as the symbol or not
        Ret:    The custom cut line string
    """
    cut_string = "+"
    acc_length = 0
    while True:
        if acc_length == 0:
            if equal_symbol:
                cut_string = cut_string + '=' * (width_cell) + '+'
            else:
                cut_string = cut_string + '-' * (width_cell) + '+'
            acc_length += width_cell
        else:
            if equal_symbol:
                cut_string = cut_string + '=' * (width_cell + 2) + '+'
            else:
                cut_string = cut_string + '-' * (width_cell + 2) + '+'
            acc_length += (width_cell + 2)
        if acc_length >= length - width_cell:
            break
    return cut_string

class tqdm_table(tqdm):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.max_length = 100
        self.prev_lines = -1

    def set_table_setting(self, max_length: int = 100):
        """
            Assign the setting of this object

            Arg:    max_length  - The maximun length of the terminal
        """
        self.max_length = max_length

    def set_table_info(self, mapping: dict):
        """
            Set the table information

            Arg:    mapping     - The key-value pair you want to form the table
        """
        key_string = ""
        val_string = ""
        table_string = []
        key_list = sorted(mapping.keys())

        # Clear the previous table information toward stdout
        if self.prev_lines > 0:
            __clear__(self.prev_lines, self.max_length)

        # Determine the width of cell
        if max([max(len(str(val)), len(key)) for key, val in mapping.items()]) <= 15:
            width_length = SHORT
        elif max([max(len(str(val)), len(key)) for key, val in mapping.items()]) <= 30:
            width_length = MIDDLE
        else:
            width_length = LARGE

        # Collect the lines of keys and values
        for key in key_list:
            val = mapping[key]
            single_max_length = max(len(key), len(str(val)))
            if len(key_string) + single_max_length + 2 < self.max_length:
                if width_length == SHORT:
                    key_string += '{:>15} | '.format(key)
                    val_string += '{:>15} | '.format(val)
                elif width_length == MIDDLE:
                    key_string += '{:>30} | '.format(key)
                    val_string += '{:>30} | '.format(val)
                else:
                    key_string += '{:>50} | '.format(key)
                    val_string += '{:>50} | '.format(val)
            else:
                table_string.append(key_string)
                table_string.append(val_string)
                if width_length == SHORT:
                    key_string = '{:>15} | '.format(key)
                    val_string = '{:>15} | '.format(val)
                elif width_length == MIDDLE:
                    key_string = '{:>30} | '.format(key)
                    val_string = '{:>30} | '.format(val)
                else:
                    key_string = '{:>50} | '.format(key)
                    val_string = '{:>50} | '.format(val)

        # Accumulate the rest information if there are some information rest
        if len(key_string) > 0 or len(val_string) > 0:
            table_string.append(key_string)
            table_string.append(val_string)

        # Transfer the containing of queue into string
        cut_string_small = __getCutLine__(length=max([len(_) for _ in table_string]), width_cell=width_length, equal_symbol=False)
        cut_string_large = __getCutLine__(length=max([len(_) for _ in table_string]), width_cell=width_length, equal_symbol=True)
        print_string = cut_string_large
        for i in range(len(table_string) // 2):
            print_string = print_string + '\n' + table_string[2*i] + '\n' + cut_string_small + '\n' + table_string[2*i+1] + '\n' + cut_string_large
        self.prev_lines = 2 * (len(table_string) + 1)

        # Write into tqdm
        self.write(print_string)


from sklearn.datasets import load_breast_cancer
from sklearn.datasets import load_wine
from E2 import sammon
from E1 import bkmeans
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import numpy as np
from sklearn.cluster import KMeans
from scipy.cluster.hierarchy import linkage
from scipy.cluster.hierarchy import dendrogram

sammons_maxiter = 100
e = 0.001
A = 0.3

# Figure 1
fig1, axs1 = plt.subplots(3, 3, figsize=(10, 10))
print(f"Max iter set to {sammons_maxiter}. It might take a total of {sammons_maxiter*3} to run the code. Each iteration takes approx 0.5-2 seconds.")
for i in range(3):
    if i == 0:
        datasetX, datasetLabels = load_breast_cancer().data, load_breast_cancer().target
        dataset_name = 'Breast Cancer Dataset'
    elif i == 1:
        datasetX, datasetLabels = load_wine().data, load_wine().target
        dataset_name = 'Wine Dataset'
    elif i == 2:
        diabetesData = np.genfromtxt('csv_result-diabetes.csv', delimiter=',', skip_header=1)
        datasetX, datasetLabels = diabetesData[:, :-1], diabetesData[:, -1]
        dataset_name = 'Diabetes Dataset'

    for j in range(3):
        if j == 0:
            print(f"Computing sammon mapping for {dataset_name} ({i+1}/3)")
            result = sammon(datasetX, sammons_maxiter, e, A)
            plot_title = 'Sammon Mapping'
        elif j == 1:
            pca = PCA(n_components=2)
            result = pca.fit_transform(datasetX)
            plot_title = 'PCA'
        elif j == 2:
            tsne = TSNE(n_components=2)
            result = tsne.fit_transform(datasetX)
            plot_title = 't-SNE'

        axs1[i, j].scatter(result[:, 0], result[:, 1], c=datasetLabels, s=5)
        axs1[i, j].set_title(f'{dataset_name} - {plot_title}')

plt.tight_layout()

fig, axes = plt.subplots(3, 3, figsize=(10, 10))

for i in range(3):
    if i == 0:
        datasetX, datasetLabels = load_breast_cancer().data, load_breast_cancer().target
        dataset_name = 'Breast Cancer Dataset'
    elif i == 1:
        datasetX, datasetLabels = load_wine().data, load_wine().target
        dataset_name = 'Wine Dataset'
    elif i == 2:
        diabetesData = np.genfromtxt('csv_result-diabetes.csv', delimiter=',', skip_header=1)
        datasetX, datasetLabels = diabetesData[:, :-1], diabetesData[:, -1]
        dataset_name = 'Diabetes Dataset'

    pca = PCA(n_components=2)
    X_pca = pca.fit_transform(datasetX)

    for j in range(3):
        ax = axes[i, j]
        if j == 0:
            bkmeans_result = bkmeans(X_pca, 2, 30)
            c = bkmeans_result
            algorithm_name =  'bk means'
        elif j == 1:
            kmeans_result = KMeans(n_init = 10, n_clusters=2, random_state=0).fit_predict(X_pca)
            c = kmeans_result
            algorithm_name =  'classic k means'
        elif j == 2:
            hierarchical_result = linkage(X_pca, method='ward')
            dendro = dendrogram(hierarchical_result, ax=ax)
            c = datasetLabels
            algorithm_name =  'hierarchical'

        ax.scatter(X_pca[:, 0], X_pca[:, 1], c=c)
        ax.set_title(f'{dataset_name} - {algorithm_name}')

plt.tight_layout()
plt.show()


from django.db import models
from django.contrib.auth.models import AbstractUser
# Create your models here.
from django.dispatch import receiver
from rest_framework.authtoken.models import Token
from django.db.models.signals import post_save
from django.conf import settings

class User(AbstractUser):
	is_teacher = models.BooleanField(default = False)
	is_student = models.BooleanField(default = False)

	def __Str__(self):
		return self.username

#class Teacher(models.Model):
#	user = models.OneToOneField(User, on_delete = models.CASCADE, primary_key = True)
#
#class Student(modesl.Model):
#	user = models.OneToOneField(User, on_delete = models.CASCADE, primary_key = True)

class Doubt(models.Model):
	questions = models.TextField(max_length = 200)
	picture = models.ImageField(blank = True, null = True, upload_to='images')
	student = models.ForeignKey(User, related_name = 'student',  on_delete = models.CASCADE)
	teacher = models.ForeignKey(User, related_name = 'teacher',  on_delete = models.CASCADE)

@receiver(post_save, sender=settings.AUTH_USER_MODEL)
def create_auth_token(sender, instance=None, created=False, **kwargs):
    if created:
        Token.objects.create(user=instance)


#!/usr/bin/env python3
"""OpenCV-based frame viewer that replays recordings and assign time-based labels"""
import argparse
from pathlib import Path
import time
import cv2
import numpy as np
import derp.util

class Labeler:
    """OpenCV-based frame viewer that replays recordings and assign time-based labels"""

    def __init__(self, folder, scale=1, bhh=40):
        """Load the topics and existing labels from the folder, scaling up the frame"""
        self.folder = folder
        self.scale = scale
        self.bhh = bhh
        self.config_changed = False
        self.quality = None
        self.config_path = self.folder / "config.yaml"
        self.window_name = "Labeler %s" % self.folder
        self.config = derp.util.load_config(self.config_path)
        self.quality_colors = [(0, 0, 255), (0, 128, 255), (0, 255, 0)]
        self.topics = derp.util.load_topics(folder)
        self.frame_id = 0
        self.n_frames = len(self.topics["camera"])
        self.seek(self.frame_id)
        self.f_h = self.frame.shape[0]
        self.f_w = self.frame.shape[1]
        self.l_h = int(self.bhh // 5)
        self.window = np.zeros(
            [self.f_h + self.bhh * 2 + self.l_h + 2, self.f_w, 3], dtype=np.uint8
        )
        self.paused = True
        self.show = False

        # Prepare labels
        self.autonomous_bar = np.ones((self.f_w, 3), dtype=np.uint8) * (255, 255, 255)
        self.quality_bar = np.ones((self.f_w, 3), dtype=np.uint8) * (128, 128, 128)
        if "quality" in self.topics and len(self.topics["quality"]) >= self.n_frames:
            self.qualities = [str(msg.quality) for msg in self.topics["quality"]]
        else:
            self.qualities = ["junk" for _ in range(self.n_frames)]
        for i, quality in enumerate(self.qualities):
            self.update_quality(i, i, quality)

        # Prepare state messages
        self.camera_times = [msg.publishNS for msg in self.topics["camera"]]
        self.camera_autos = []
        auto = False
        for timestamp, topic, msg in derp.util.replay(self.topics):
            if topic == 'controller':
                auto = msg.isAutonomous
            elif topic == 'camera':
                self.camera_autos.append(auto)

        actions = derp.util.extract_car_actions(self.topics)
        self.camera_speeds = derp.util.extract_latest(self.camera_times,
                                                      actions[:, 0], actions[:, 1])
        self.camera_steers = derp.util.extract_latest(self.camera_times,
                                                      actions[:, 0], actions[:, 2])
        window_Xs = np.linspace(self.camera_times[0], self.camera_times[-1], self.f_w)
        self.window_speeds = np.array(np.interp(window_Xs, self.camera_times, self.camera_speeds)
                                      * -self.bhh, dtype=np.int)
        self.window_steers = np.array(np.interp(window_Xs, self.camera_times, self.camera_steers)
                                      * -self.bhh, dtype=np.int)
        self.autonomous_bar *= np.array(np.interp(window_Xs, self.camera_times,
                                                  self.camera_autos), dtype=np.uint8)[:, None]
        self.window_steers[self.window_steers > self.bhh] = self.bhh
        self.window_steers[self.window_steers < -self.bhh] = -self.bhh

        cv2.namedWindow(self.window_name)
        cv2.setMouseCallback(self.window_name, self.click_handler)
        # Print some statistics
        duration = (self.camera_times[-1] - self.camera_times[0]) / 1e9
        fps = (len(self.camera_times) - 1) / duration
        print("Duration of %.0f seconds at %.0f fps" % (duration, fps))

    def __del__(self):
        """Deconstructor to close window"""
        cv2.destroyAllWindows()

    def click_handler(self, event, x, y, flags, param):
        """ Handle clicks on the window """
        if event == cv2.EVENT_LBUTTONDOWN:
            if y > self.f_h:
                frame_id = int((x / self.f_w) * self.n_frames)
                self.seek(frame_id)
                self.show = True

    def update_quality(self, first_index, last_index, quality=None):
        """Update the label bar to the given quality"""
        if quality is None:
            return False
        first_index, last_index = min(first_index, last_index), max(first_index, last_index)
        for index in range(first_index, last_index + 1):
            self.qualities[index] = quality
        beg_pos = self.frame_pos(first_index)
        end_pos = self.frame_pos(last_index + (self.n_frames < len(self.quality_bar)))
        self.quality_bar[beg_pos : end_pos + 1] = self.bar_color(quality)
        return True

    def seek(self, frame_id=None):
        """Update the current frame to the given frame_id, otherwise advances by 1 frame"""
        if frame_id is None:
            frame_id = self.frame_id + 1
        if frame_id < 0:
            frame_id = 0
            self.paused = True
        if frame_id >= self.n_frames:
            frame_id = self.n_frames - 1
            self.paused = True
        self.update_quality(self.frame_id, frame_id, self.quality)
        self.frame = cv2.resize(
            derp.util.decode_jpg(self.topics["camera"][frame_id].jpg),
            None,
            fx=self.scale,
            fy=self.scale,
            interpolation=cv2.INTER_AREA,
        )
        self.frame_id = frame_id
        return True

    def bar_color(self, quality):
        """Figure out the color for the given quality"""
        if quality is None:
            return (128, 128, 128)
        return self.quality_colors[derp.util.TOPICS["quality"].QualityEnum.__dict__[quality]]

    def display(self):
        """Blit all the status on the screen"""
        self.window[: self.frame.shape[0], :, :] = self.frame
        horizon_percent = self.config["camera"]["pitch"] / self.config["camera"]["vfov"] + 0.5
        # Horizon line
        self.window[int(self.f_h * horizon_percent), :, :] = (255, 0, 255)
        # Clear status buffer
        self.window[self.f_h :, :, :] = 0
        # Draw label bar
        self.window[self.f_h : self.f_h + self.l_h // 2, :, :] = self.autonomous_bar
        self.window[self.f_h + self.l_h // 2 : self.f_h + self.l_h, :, :] = self.quality_bar
        # Draw current timestamp vertical line
        current_x = self.frame_pos(self.frame_id)
        self.window[self.f_h + self.l_h :, current_x, :] = self.bar_color(self.quality)
        # Draw zero line
        self.window[self.f_h + self.l_h + self.bhh, :, :] = (96, 96, 96)
        offset = self.f_h + self.bhh + self.l_h
        self.window[self.window_speeds + offset, np.arange(self.f_w), :] = (255, 64, 255)
        self.window[self.window_steers + offset, np.arange(self.f_w), :] = (64, 255, 255)
        text = "%05i %07.3f %06.3f %06.3f" % (self.frame_id,
                                            (self.camera_times[self.frame_id] / 1E9) % 100,
                                            self.camera_steers[self.frame_id],
                                            self.camera_speeds[self.frame_id])
        font = cv2.FONT_HERSHEY_SIMPLEX
        pink = (255, 128, 255)
        offset = (0, int(self.scale * 30))
        cv2.putText(self.window, text, offset, font, self.scale, pink, 1, cv2.LINE_AA)
        cv2.imshow(self.window_name, self.window)

    def save_labels(self):
        """Write all of our labels to the folder as messages"""
        with derp.util.topic_file_writer(self.folder, "quality") as quality_fd:
            for quality_i, quality in enumerate(self.qualities):
                msg = derp.util.TOPICS["quality"].new_message(
                    createNS=derp.util.get_timestamp(),
                    publishNS=self.topics["camera"][quality_i].publishNS - 1,
                    writeNS=derp.util.get_timestamp(),
                    quality=quality,
                )
                msg.write(quality_fd)
        print("Saved quality labels in", self.folder)
        if self.config_changed:
            derp.util.dump_config(self.config, self.config_path)
            print("Saved changes to config")

    def handle_keyboard_input(self):
        """Fetch a new keyboard input if one exists"""
        key = cv2.waitKey(1) & 0xFF
        if key == 255:
            return True
        if key == 27:
            return False  # ESC
        if key == ord(" "):
            self.paused = not self.paused
        elif key == ord("g"):
            self.quality = "good"
        elif key == ord("r"):
            self.quality = "risk"
        elif key == ord("t"):
            self.quality = "junk"
        elif key == ord("c"):
            self.quality = None
        elif key == ord("s"):
            self.save_labels()
        elif key == 82:
            self.seek(self.frame_id + 10)  # up
        elif key == 84:
            self.seek(self.frame_id - 10)  # down
        elif key == 81:
            self.seek(self.frame_id - 1)  # left
        elif key == 83:
            self.seek(self.frame_id + 1)  # right
        elif key == 85:
            self.config["camera"]["pitch"] -= 0.1  # page up
            self.config_changed = True
        elif key == 86:
            self.config["camera"]["pitch"] += 0.1  # page down
            self.config_changed = True
        elif ord("1") <= key <= ord("5"):
            self.seek(int(self.n_frames * (key - ord("0") - 1) / 4))
        elif key != 255:
            print("Unknown key press: [%s]" % key)
        self.show = True
        return True

    def frame_pos(self, frame_id):
        """Position of current camera frame on the horizontal status bars"""
        return min(self.f_w - 1, int(frame_id / self.n_frames * self.f_w))

    def run(self):
        """Run the labeling program in a forever loop until the user quits"""
        self.display()
        while True:
            if not self.paused:
                self.show = self.seek()
            if self.show:
                self.display()
                self.show = False
            if not self.handle_keyboard_input():
                break
            time.sleep(0.01)

def main():
    """Initialize the labeler based on user args and run it"""
    print(
        """
This labeling tool interpolates the data based on camera frames and then lets you label each.
To exit press ESCAPE
To save press s
To navigate between frames:
    Left/Right: move in 1 frame increments
    Up/Down: move in 10 frame increments
    1: goes to beginning
    2: goes to 25% in
    3: goes to 50% in
    4  goes to 25% in
    5: goes to end
To adjust horizon line press PAGE_UP or PAGE_DOWN
To change the quality label of this frame
    g: good (use for training)
    r: risk (advanced situation not suitable for classic training)
    t: junk (don't use this part of the video, aka trash)
    c: clear, as in don't change the quality label
"""
    )
    parser = argparse.ArgumentParser()
    parser.add_argument("paths", type=Path, nargs="*", metavar="N", help="recording path location")
    parser.add_argument("--scale", type=float, default=1.0, help="frame rescale ratio")
    args = parser.parse_args()
    if not args.paths:
        recordings = (derp.util.DERP_ROOT / "recordings").glob("recording-*")
        args.paths = [r for r in recordings if not (r / "quality.bin").exists()]
    for path in args.paths:
        print("Labeling", path)
        labeler = Labeler(folder=path, scale=args.scale)
        labeler.run()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed May  6 15:04:40 2020

@author: Rijk

Scipy signal sos filter toolbox test
"""

import numpy as np
from scipy import signal
import matplotlib.pyplot as plt

plt.close('all')

b, a = signal.butter(4, 100, 'low', analog=True)
w, h = signal.freqs(b, a)

plt.figure()
plt.semilogx(w, 20 * np.log10(abs(h)))

plt.title('Butterworth filter frequency response')
plt.xlabel('Frequency [radians / second]')
plt.ylabel('Amplitude [dB]')
plt.margins(0, 0.1)
plt.grid(which='both', axis='both')
plt.axvline(100, color='green') # cutoff frequency
#plt.show()

t = np.linspace(0, 1, 1000, False)  # 1 second
sig = np.sin(2*np.pi*10*t) + np.sin(2*np.pi*20*t)

fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
ax1.plot(t, sig)
ax1.set_title('10 Hz and 20 Hz sinusoids')
ax1.axis([0, 1, -2, 2])

sos = signal.butter(10, 15, 'hp', fs=1000, output='sos')
filtered = signal.sosfilt(sos, sig)

ax2.plot(t, filtered)
ax2.set_title('After 15 Hz high-pass filter')
ax2.axis([0, 1, -2, 2])
ax2.set_xlabel('Time [seconds]')
plt.tight_layout()
plt.show()

# Compare FFT before and after
fft_before  = np.fft.fft(sig)
fft_after   = np.fft.fft(filtered)

sample_time = np.mean(np.diff(t))
f           = np.fft.fftfreq(len(t), sample_time)
half        = int(len(t)/2)
plt.figure()
plt.plot(f[1:half], fft_before[1:half], label='Original')
#plt.plot(f[1:half], fft_after[1:half], label='Filtered')
plt.legend()

## Measurement data filter
#f_axis = 1.12
#nyquist_f = f_axis
#
## Define filter
#lower_f = 0.025
#upper_f = 0.065
#bandpass_f = 2*np.pi * np.array([lower_f, upper_f]) / nyquist_f
#butter_low  = signal.butter(2, lower_f, btype='lowpass', output='sos')
#butter_high  = signal.butter(2, upper_f, btype='lowpass', output='sos')
#
#b, a = signal.butter(2, bandpass_f, btype='bandstop', output='ba')
#w, h = signal.freqz(b, a)
#
#plt.figure()
#plt.plot(w, 20 * np.log10(abs(h)))

Measurements/20200324 WO3196dev9/Hydrogen R_T correction/0324_1904_WO3196dev9_H2ToAir/signal_toolbox_test.py


import json
import os
import subprocess
from collections import OrderedDict
from copy import deepcopy
from sys import platform

from tabulate import tabulate

from openwpm.config import ConfigEncoder

def parse_http_stack_trace_str(trace_str):
    """Parse a stacktrace string and return an array of dict."""
    stack_trace = []
    frames = trace_str.split("\n")
    for frame in frames:
        try:
            func_name, rest = frame.split("@", 1)
            rest, async_cause = rest.rsplit(";", 1)
            filename, line_no, col_no = rest.rsplit(":", 2)
            stack_trace.append(
                {
                    "func_name": func_name,
                    "filename": filename,
                    "line_no": line_no,
                    "col_no": col_no,
                    "async_cause": async_cause,
                }
            )
        except Exception as exc:
            print("Exception parsing the stack frame %s %s" % (frame, exc))
    return stack_trace

def get_firefox_binary_path():
    """
    If ../../firefox-bin/firefox-bin or os.environ["FIREFOX_BINARY"] exists,
    return it. Else, throw a RuntimeError.
    """
    if "FIREFOX_BINARY" in os.environ:
        firefox_binary_path = os.environ["FIREFOX_BINARY"]
        if not os.path.isfile(firefox_binary_path):
            raise RuntimeError(
                "No file found at the path specified in "
                "environment variable `FIREFOX_BINARY`."
                "Current `FIREFOX_BINARY`: %s" % firefox_binary_path
            )
        return firefox_binary_path

    root_dir = os.path.dirname(__file__) + "/../.."
    if platform == "darwin":
        firefox_binary_path = os.path.abspath(
            root_dir + "/Nightly.app/Contents/MacOS/firefox-bin"
        )
    else:
        firefox_binary_path = os.path.abspath(root_dir + "/firefox-bin/firefox-bin")

    if not os.path.isfile(firefox_binary_path):
        raise RuntimeError(
            "The `firefox-bin/firefox-bin` binary is not found in the root "
            "of the  OpenWPM directory (did you run the install script "
            "(`install.sh`)?). Alternatively, you can specify a binary "
            "location using the OS environment variable FIREFOX_BINARY."
        )
    return firefox_binary_path

def get_version():
    """Return OpenWPM version tag/current commit and Firefox version"""
    try:
        openwpm = subprocess.check_output(
            ["git", "describe", "--tags", "--always"]
        ).strip()
    except subprocess.CalledProcessError:
        ver = os.path.join(os.path.dirname(__file__), "../../VERSION")
        with open(ver, "r") as f:
            openwpm = f.readline().strip()

    firefox_binary_path = get_firefox_binary_path()
    try:
        firefox = subprocess.check_output([firefox_binary_path, "--version"])
    except subprocess.CalledProcessError as e:
        raise RuntimeError("Firefox not found. " " Did you run `./install.sh`?") from e

    ff = firefox.split()[-1]
    return openwpm, ff

def get_configuration_string(manager_params, browser_params, versions):
    """Construct a well-formatted string for {manager,browser}params

    Constructs a pretty printed string of all parameters. The config
    dictionaries are split to try to avoid line wrapping for reasonably
    size terminal windows.
    """

    config_str = "\n\nOpenWPM Version: %s\nFirefox Version: %s\n" % versions
    config_str += "\n========== Manager Configuration ==========\n"

    config_str += json.dumps(
        manager_params.to_dict(),
        sort_keys=True,
        indent=2,
        separators=(",", ": "),
        cls=ConfigEncoder,
    )
    config_str += "\n\n========== Browser Configuration ==========\n"

    print_params = [deepcopy(x.to_dict()) for x in browser_params]
    table_input = list()
    profile_dirs = OrderedDict()
    archive_dirs = OrderedDict()
    js_config = OrderedDict()
    profile_all_none = archive_all_none = True
    for item in print_params:
        browser_id = item["browser_id"]

        # Update print flags
        if item["seed_tar"] is not None:
            profile_all_none = False
        if item["profile_archive_dir"] is not None:
            archive_all_none = False

        # Separate out long profile directory strings
        profile_dirs[browser_id] = str(item.pop("seed_tar"))
        archive_dirs[browser_id] = str(item.pop("profile_archive_dir"))
        js_config[browser_id] = item.pop("cleaned_js_instrument_settings")

        # Copy items in sorted order
        dct = OrderedDict()
        dct["browser_id"] = browser_id
        for key in sorted(item.keys()):
            dct[key] = item[key]
        table_input.append(dct)

    key_dict = OrderedDict()
    counter = 0
    for key in table_input[0].keys():
        key_dict[key] = counter
        counter += 1
    config_str += "Keys:\n"
    config_str += json.dumps(key_dict, indent=2, separators=(",", ": "))
    config_str += "\n\n"
    config_str += tabulate(table_input, headers=key_dict)

    config_str += "\n\n========== JS Instrument Settings ==========\n"
    config_str += json.dumps(js_config, indent=None, separators=(",", ":"))

    config_str += "\n\n========== Input profile tar files ==========\n"
    if profile_all_none:
        config_str += "  No profile tar files specified"
    else:
        config_str += json.dumps(profile_dirs, indent=2, separators=(",", ": "))

    config_str += "\n\n========== Output (archive) profile dirs ==========\n"
    if archive_all_none:
        config_str += "  No profile archive directories specified"
    else:
        config_str += json.dumps(archive_dirs, indent=2, separators=(",", ": "))

    config_str += "\n\n"
    return config_str


import pykka
import re
import json

from gpt_connection import GPT_Connection
from tools import ToolRunner
from frontend_utils import initial_request

from bots.Dispatcher import Dispatcher
from bots.Workflow import Workflow

from prompts import DISPATCHER_PROMPT_TEMPLATE, INITIAL_PROMPT_TEMPLATE, ITERATING_ACTION_PROMPT_TEMPLATE, FORCE_END_ITERATION_PROMPT, \
    PLANT_HEALTH_BOT_DESCRIPTION, PRODUCTION_OUTPUT_BOT_DESCRIPTION, DISTRIBUTION_BOT_DESCRIPTION, \
    ONT_PRODUCTION_PLANT, ONT_MACHINES, ONT_WORK_ORDERS, ONT_PRODUCTION_ALLOCATION_PLAN, ONT_DISTRIBUTION_WAREHOUSE, ONT_TRANSIT_ORDER, \
    GET_OBJECTS, MODIFY_OBJECT, CREATE_OBJECT

from string import Template

"""
Main Execution Code
"""

gpt_connection = GPT_Connection()
tool_runner = ToolRunner()

plant_health_ref = Workflow.start(
    name="Plant Health Bot",
    id=2,
    dispatcher_id=1,
    bot_description=PLANT_HEALTH_BOT_DESCRIPTION,
    initial_prompt_template=INITIAL_PROMPT_TEMPLATE,
    iteration_prompt_template=ITERATING_ACTION_PROMPT_TEMPLATE,
    force_end_prompt_template=FORCE_END_ITERATION_PROMPT,
    information=[ONT_PRODUCTION_PLANT, ONT_MACHINES, ONT_WORK_ORDERS],
    readtools=[GET_OBJECTS],
    writetools=[MODIFY_OBJECT, CREATE_OBJECT],
    gpt_connection=gpt_connection,
    tool_runner=tool_runner
)
production_output_ref = Workflow.start(
    name="Production Output Bot",
    id=3,
    dispatcher_id=1,
    bot_description=PRODUCTION_OUTPUT_BOT_DESCRIPTION,
    initial_prompt_template=INITIAL_PROMPT_TEMPLATE,
    iteration_prompt_template=ITERATING_ACTION_PROMPT_TEMPLATE,
    force_end_prompt_template=FORCE_END_ITERATION_PROMPT,
    information=[ONT_PRODUCTION_PLANT, ONT_MACHINES, ONT_PRODUCTION_ALLOCATION_PLAN],
    readtools=[GET_OBJECTS],
    writetools=[MODIFY_OBJECT, CREATE_OBJECT],
    gpt_connection=gpt_connection,
    tool_runner=tool_runner
)
distribution_ref = Workflow.start(
    name="Distribution Bot",
    id=4,
    dispatcher_id=1,
    bot_description=DISTRIBUTION_BOT_DESCRIPTION,
    initial_prompt_template=INITIAL_PROMPT_TEMPLATE,
    iteration_prompt_template=ITERATING_ACTION_PROMPT_TEMPLATE,
    force_end_prompt_template=FORCE_END_ITERATION_PROMPT,
    information=[ONT_PRODUCTION_PLANT, ONT_DISTRIBUTION_WAREHOUSE, ONT_PRODUCTION_ALLOCATION_PLAN, ONT_TRANSIT_ORDER],
    readtools=[GET_OBJECTS],
    writetools=[MODIFY_OBJECT, CREATE_OBJECT],
    gpt_connection=gpt_connection,
    tool_runner=tool_runner
)
workflows = {
    "Plant Health Bot": plant_health_ref,
    "Production Output Bot": production_output_ref,
    "Distribution Bot": distribution_ref
}
workflow_ids = {
    "Plant Health Bot": 2,
    "Production Output Bot": 3,
    "Distribution Bot": 4
}
dispatcher_ref = Dispatcher.start(
    id=1,
    prompt_template=DISPATCHER_PROMPT_TEMPLATE,
    workflows=workflows,
    workflow_ids=workflow_ids,
    gpt_connection=gpt_connection)

initial_ask = "We just got a message that the ocelot making machine has broken at Stuffed Animal Plant 8"
# initial_ask = "Our trucks carrying transit order T0030 got into an accident."
# initial_ask = "Alice has left the company"
initial_request(initial_ask)
dispatcher_ref.ask(initial_ask)

dispatcher_ref.stop()