from abc import ABC

from pyspark.ml.classification import LinearSVC, NaiveBayes, OneVsRest
from pyspark.ml.classification import RandomForestClassifier as RandomForest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder

class AbstractClassifier(ABC):
    def __init__(self, dataset, train_ratio=0.8, fix_imbalance=True):
        self.dataset = dataset
        self.classifier = None
        self.evaluator = MulticlassClassificationEvaluator()
        self.train_set, self.test_set = dataset.randomSplit([train_ratio, 1-train_ratio])
        self.train_set = self.undersample_data(self.train_set) if fix_imbalance else self.train_set

    def train(self, k_folds=7):
        """Trains a classification model using k-fold cross validation

        Keyword Arguments:
            k_folds {int} -- Number of folds (default: {7})
        """
        param_grid = self._get_param_grid()
        cross_validator = CrossValidator(estimator=self.classifier, estimatorParamMaps=param_grid, evaluator=self.evaluator, numFolds=k_folds)
        self.model = cross_validator.fit(self.train_set)

    def undersample_data(self, df):
        """Fixes data imbalances by undersampling overrepresented classes

        Returns:
            DataFrame -- Dataframe representation of an article
        """
        class_distribution = df.groupBy(df.label).count()
        distribution_dict = class_distribution.rdd.map(lambda x: (x['label'], x['count'])).collectAsMap()
        lowest_count = min(distribution_dict.values())
        sampling_fractions = {label: lowest_count/distribution_dict[label] for label in distribution_dict}
        return df.sampleBy('label', fractions=sampling_fractions)

    def evaluate(self):
        """Evaluates the trained model

        Raises:
            AttributeError: Raises exception if model is not trained before evaluating

        Returns:
            Tuple -- A tuple containing the following metrics: f1-score, accuracy, precision, recall
        """
        if not hasattr(self, 'model'):
            raise AttributeError('Please train the model before evaluation')
        predictions = self.model.transform(self.test_set)

        f1 = self.evaluator.evaluate(predictions)
        accuracy = self.evaluator.evaluate(predictions, {self.evaluator.metricName: 'accuracy'})
        precision = self.evaluator.evaluate(predictions, {self.evaluator.metricName: 'weightedPrecision'})
        recall = self.evaluator.evaluate(predictions, {self.evaluator.metricName: 'weightedRecall'})

        return f1, accuracy, precision, recall

    def _get_param_grid(self):
        pass

class NaiveBayesClassifier(AbstractClassifier):
    def __init__(self, dataset, train_ratio=0.8):
        super().__init__(dataset)
        self.classifier = NaiveBayes(labelCol='label', featuresCol='features')

    def _get_param_grid(self):
        return ParamGridBuilder() \
               .addGrid(self.classifier.smoothing, [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]) \
               .build()

class RandomForestClassifier(AbstractClassifier):
    def __init__(self, dataset, train_ratio=0.8):
        super().__init__(dataset)
        self.classifier = RandomForest(labelCol='label', featuresCol='features', impurity='gini',
                                              maxBins=31)

    def _get_param_grid(self):
        return ParamGridBuilder() \
            .addGrid(self.classifier.maxDepth, [5, 7, 11, 13]) \
            .addGrid(self.classifier.numTrees, [5, 7, 11, 13, 17]) \
            .build()

class SVMClassifier(AbstractClassifier):
    def __init__(self, dataset, train_ratio=0.8):
        super().__init__(dataset)
        self.classifier = OneVsRest(classifier=LinearSVC(maxIter=10))

    def _get_param_grid(self):
        self.classifier.getClassifier
        return ParamGridBuilder() \
            .addGrid(self.classifier.getClassifier().regParam, [0.1, 0.2, 0.4, 0.6, 0.8, 1]) \
            .build()


# Imports necessary libraries
import streamlit as st

# Sets a title "Wine Quality Predictor" for the web-application using streamlit
st.title("Wine Quality Predictor")

# Creates 10 objects with the same widths, where the individual variable-names indicate their position
row1_col1, row1_col2 = st.columns([1, 1])
row2_col1, row2_col2 = st.columns([1, 1])
row3_col1, row3_col2 = st.columns([1, 1])
row4_col1, row4_col2 = st.columns([1, 1])
row5_col1, row5_col2 = st.columns([1, 1])

# Sets individual titles for each variable
row1_col1.write("Wine type")
row1_col2.write("Fixed acidity")
row2_col1.write("Volatile acidity")
row2_col2.write("Citric acid")
row3_col1.write("Residual sugar")
row3_col2.write("Chlorides")
row4_col1.write("Free sulfur dioxide")
row4_col2.write("Total sulfur dioxide")
row5_col1.write("Sulphates ")
row5_col2.write("Alcohol")

# Displays a radio button widget for the user to enter the wine-type
Wine_type = row1_col1.radio(
    "Enter wine type:",
    options=['Red wine', 'White wine'], )
row1_col1.write(Wine_type)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 16 for the fixed acidity
Fixed_acidity = row1_col2.slider(
    "Enter the amount of fixed acidity:",
    min_value=0.0,
    max_value=16.0,
    value=8.0)  # Sets a default value of 8
row1_col2.write(Fixed_acidity)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 1 for the volatile acidity
Volatile_acidity = row2_col1.slider(
    "Enter the amount of volatile acidity:",
    min_value=0.0,
    max_value=1.0,
    value=0.5)  # Sets a default value of 0.5
row2_col1.write(Volatile_acidity)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 2 for the citric acidity
Citric_acid = row2_col2.slider(
    "Enter the amount of citric acid:",
    min_value=0.0,
    max_value=2.0,
    value=1.0)  # Sets a default value of 1.0
row2_col2.write(Citric_acid)  # Shows the user the entered input

# Displays a slider widget for the user to enter integer values from 0 to 70 for the residual sugar
Residual_sugar = row3_col1.slider(
    "Enter the amount of residual sugar:",
    min_value=0,
    max_value=70,
    value=35)  # Sets a default value of 35
row3_col1.write(Residual_sugar)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 0.4 for the amount of chlorides
Chlorides = row3_col2.slider(
    "Enter the amount of chlorides:",
    min_value=0.0,
    max_value=0.4,
    value=0.2)  # Sets a default value of 0.2
row3_col2.write(Chlorides)  # Shows the user the entered input

# Displays a slider widget for the user to enter integer values from 0 to 289 for the amount of free sulfur dioxide
Free_sulfur_dioxide = row4_col1.slider(
    "Enter the amount of free sulfur dioxide:",
    min_value=0,
    max_value=289,
    value=144)  # Sets a default value of 144
row4_col1.write(Free_sulfur_dioxide)  # Shows the user the entered input

# Displays a slider widget for the user to enter integer values from 0 to 450 for the amount of total sulfur dioxide
Total_sulfur_dioxide = row4_col2.slider(
    "Enter the amount of total sulfur dioxide:",
    min_value=0,
    max_value=450,
    value=225)  # Sets a default value of 225
row4_col2.write(Total_sulfur_dioxide)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 1.2 for the amount of sulphates
Sulphates = row5_col1.slider(
    "Enter the amount of sulphates:",
    min_value=0.0,
    max_value=1.2,
    value=0.6)  # Sets a default value of 0.6
row5_col1.write(Sulphates)  # Shows the user the entered input

# Displays a slider widget for the user to enter float values from 0 to 1.2 for the amount of alcohol
Alcohol = row5_col2.slider(
    "Enter the amount of alcohol:",
    min_value=7.0,
    max_value=15.0,
    value=11.0)  # Sets a default value of 11.0
row5_col2.write(Alcohol)  # Shows the user the entered input

# Calculates the quality of red wine using the parameters of a OLS regression method trained on a training data set
Red_quality = 2.8245 + (0.0364*Fixed_acidity) - (1.1789*Volatile_acidity) - (0.0910*Citric_acid)
Red_quality += (0.0053*Residual_sugar) - (1.7123*Chlorides) + (0.0028*Free_sulfur_dioxide)
Red_quality += - (0.0026*Total_sulfur_dioxide) + (0.8231*Sulphates) + (0.2695*Alcohol)

# Calculates the quality of white wine using the parameters of a OLS regression method trained on a training data set
White_quality = 2.5787 - (0.0602*Fixed_acidity) - (1.9103*Volatile_acidity) - (0.0126*Citric_acid)
White_quality += (0.0238*Residual_sugar) - (1.7935*Chlorides) + (0.0047*Free_sulfur_dioxide)
White_quality += - (0.0005*Total_sulfur_dioxide) + (0.4161*Sulphates) + (0.3703*Alcohol)

# Returns a rounded quality-value for red wine if the entered wine-type is red
if Wine_type == "Red wine":
    st.subheader(f"Your red wine's quality is: {round(Red_quality,2)}")

# Returns a rounded quality-value for white wine if the entered wine-type is white
elif Wine_type == "White wine":
    st.subheader(f"Your white wine's quality is: {round(White_quality,2)}")

# Returns an error message in case the code doesn't work.
else:
    st.write("Excuse me, an error has occurred.")


import copy

import numpy as np
import torch
import torch.cuda.random
import torch.distributed as dist
import torch.optim as optim

try:
    from apex import amp
    import pyrannc.amp
except ImportError:
    print("Failed to import apex. Tests with FP16 will fail.")
import pyrannc

RELATIVE_TOLERANCE = 1e-2
ABSOLUTE_TOLERANCE = 0
LOSS_SCALE = 2 ** 8
MAX_NORM = 1.0

torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False

def get_dataset_default(dataset_size, input_dim, output_dim):
    ds_x = torch.randn((dataset_size,) + input_dim)
    ds_tgt = torch.randn((dataset_size,) + output_dim)
    return ds_x, ds_tgt

def get_loader(batch_size_per_proc, input_dim, output_dim, num_iter, get_dataset, gather_inputs):
    DATASET_SIZE = pyrannc.get_world_size() * batch_size_per_proc * num_iter
    if get_dataset is None:
        get_dataset = get_dataset_default
    ds_x, ds_tgt = get_dataset(DATASET_SIZE, input_dim, output_dim)
    ds = torch.utils.data.TensorDataset(ds_x, ds_tgt)
    sampler = torch.utils.data.distributed.DistributedSampler(ds, num_replicas=pyrannc.get_world_size(),
                                                              rank=pyrannc.get_rank(),
                                                              shuffle=False) if gather_inputs else torch.utils.data.SequentialSampler(
        ds)
    data_loader = torch.utils.data.DataLoader(ds, batch_size=batch_size_per_proc, sampler=sampler)
    return data_loader

def compare_tensors(v1, v2, rtol, atol):
    np.testing.assert_equal(v1.size(), v2.size())
    np.testing.assert_allclose(v1.tolist(), v2.tolist(), rtol=rtol, atol=atol)

def compare_dist_params(model_exp, model_act, rtol, atol):
    if hasattr(model_exp, "module"):  # ddp model
        model_exp = model_exp.module

    expected_params = {n: p for n, p in model_exp.named_parameters()}
    actual_params = {n: p for n, p in model_act.named_parameters()}

    dist_param_ranges = {n: pyrannc.get_dist_param_range(id(p)) for n, p in model_act.named_parameters()}

    for n, rp in actual_params.items():
        p = expected_params[n]
        v_a = rp.flatten()
        v_e = p.flatten()[dist_param_ranges[n]]
        compare_tensors(v_a, v_e, rtol, atol)

def do_compare_params(expected_params, actual_params, rtol, atol):
    for n, rp in actual_params.items():
        p = expected_params[n]
        compare_tensors(rp, p, rtol, atol)

def compare_params(model_exp, model_act, rtol, atol, fp16, zero=False, opt_exp=None, opt_act=None):
    if hasattr(model_exp, "module"):  # ddp model
        model_exp = model_exp.module

    expected_params = {n: p for n, p in pyrannc.amp.named_master_params(model_exp, opt_exp)} if fp16 \
        else {n: p for n, p in model_exp.named_parameters()}
    actual_params = {n: model_act.get_param(n, fp16) for n in sorted(model_act.name_to_param.keys())}
    do_compare_params(expected_params, actual_params, rtol, atol)

def compare_grads(model_exp, model_act, rtol, atol, fp16, zero=False, opt_exp=None, opt_act=None):
    if hasattr(model_exp, "module"):  # ddp model
        model_exp = model_exp.module

    expected_grads = {n: p.grad for n, p in pyrannc.amp.named_master_params(model_exp, opt_exp)} if fp16 \
        else {n: p.grad for n, p in model_exp.named_parameters()}
    actual_grads = {n: model_act.get_param_grad(n, fp16) for n in sorted(model_act.name_to_param.keys())}
    do_compare_params(expected_grads, actual_grads, rtol, atol)

def reset_running_stats(model):
    for name, _module in model.named_modules():
        if hasattr(_module, "reset_running_stats"):
            _module.reset_running_stats()

def backward_loss(loss, optimizer, fp16):
    if fp16:
        with amp.scale_loss(loss, optimizer) as scaled_loss:
            scaled_loss.backward()
    else:
        loss.backward()

def bwd_with_criterion(out, tgt, optimizer, fp16):
    criterion = torch.nn.MSELoss()
    loss = criterion(tgt, out)
    backward_loss(loss, optimizer, fp16)

def bwd_loss_output(loss, tgt, optimizer, fp16):
    backward_loss(loss, optimizer, fp16)

def convert_dtype(t, dtype):
    if t.is_floating_point():
        return t.to(dtype)
    return t

def do_run(model_cls, batch_size_per_proc, num_iter,
           trace, fwd, aggregate, bwd, dtype, preprocess, gradient_accumulation_steps,
           use_amp, allreduce_amp_master_params,
           enable_zero, dist_params, offload_params, rtol, atol, get_dataset,
           **kwargs):
    print("Starting test using {}".format(model_cls.__name__))

    model_base = model_cls()
    input_dim = model_base.INPUT_DIM
    output_dim = model_base.OUTPUT_DIM

    device = torch.cuda.current_device()

    model = copy.deepcopy(model_base)
    model = model.to(device)
    ddp_model = None
    lr = 0.01
    opt = None
    has_param = len(list(model.parameters())) > 0
    if has_param:
        opt = optim.Adam(model.parameters(), lr=lr)
        if use_amp:
            model, opt = amp.initialize(model, opt, opt_level="O2",
                                        max_loss_scale=LOSS_SCALE,
                                        min_loss_scale=1)
        if preprocess:
            preprocess(model)

    if dist_params:
        with pyrannc.DistributeModelParams(model_cls):
            rmodel = model_cls()

        base_named_params = {n: p for n, p in model.named_parameters()}
        for n, p in rmodel.named_parameters():
            pyrannc.set_dist_param(id(p), base_named_params[n])
    else:
        # We copy the model for verification of parameter update.
        rmodel = copy.deepcopy(model_base)

    rmodel.to(dtype)

    r_opt = None
    if has_param:
        r_opt = optim.Adam(rmodel.parameters(), lr=lr)
        if use_amp:
            rmodel = rmodel.to(device)
            rmodel, r_opt = amp.initialize(rmodel, r_opt, opt_level="O2",
                                           max_loss_scale=LOSS_SCALE,
                                           min_loss_scale=1)
        if preprocess:
            preprocess(rmodel)

    module_args = {}
    if "check_unused_values" in kwargs:
        module_args["check_unused_values"] = kwargs["check_unused_values"]

    if "offload_params" in kwargs:
        module_args["offload_params"] = kwargs["offload_params"]

    gather_inputs = True
    if "gather_inputs" in kwargs:
        gather_inputs = module_args["gather_inputs"] = kwargs["gather_inputs"]

    rmodel = pyrannc.RaNNCModule(rmodel, r_opt, enable_apex_amp=use_amp, enable_zero=enable_zero,
                                 allreduce_amp_master_params=allreduce_amp_master_params,
                                 offload_params=offload_params, **module_args)

    if dist_params:
        compare_dist_params(model, rmodel, rtol, atol)
    else:
        compare_params(model, rmodel, rtol, atol, False)

    data_loader = get_loader(batch_size_per_proc, input_dim, output_dim, num_iter, get_dataset, gather_inputs)
    for step, (x, tgt) in enumerate(data_loader):

        run_update = step % gradient_accumulation_steps == 0

        # Create test input
        x = x.to(device)
        tgt = tgt.to(device)

        if not ddp_model:
            jit_model = trace(model, x, tgt)
            reset_running_stats(model)
            if has_param:
                if gather_inputs:
                    ddp_model = torch.nn.parallel.DistributedDataParallel(jit_model, device_ids=[device],
                                                                          output_device=device)
                else:
                    ddp_model = jit_model
            else:
                ddp_model = jit_model

        with torch.random.fork_rng(devices=[device]):
            if run_update or (not has_param):
                p_out = fwd(ddp_model, x, tgt)
            else:  # delay allreduce by ddp
                with ddp_model.no_sync():
                    p_out = fwd(ddp_model, x, tgt)
            agg_out = aggregate(p_out)

        r_out = fwd(rmodel, convert_dtype(x, dtype), convert_dtype(tgt, dtype))

        # Verify the equality of outputs
        if gather_inputs or pyrannc.get_rank() == 0:
            compare_tensors(r_out, agg_out, rtol, atol)

        # Create test target
        if has_param:
            bwd(p_out, tgt, opt, use_amp)

            pyrannc.delay_grad_allreduce(not run_update)
            bwd(r_out, convert_dtype(tgt, dtype), r_opt, use_amp)

            if run_update:
                compare_grads(model, rmodel, rtol, atol, use_amp, zero=enable_zero, opt_exp=opt, opt_act=r_opt)

                opt.step()
                r_opt.step()
                opt.zero_grad()
                r_opt.zero_grad()

        compare_params(model, rmodel, rtol, atol, has_param and use_amp, zero=enable_zero, opt_exp=opt,
                       opt_act=r_opt)

    if gather_inputs or pyrannc.get_rank() == 0:
        compare_params(model, rmodel, rtol, atol, has_param and use_amp, zero=enable_zero, opt_exp=opt, opt_act=r_opt)

    if not has_param:
        print("Done")
        return

    # Save model & opt
    # state_dict should run on all ranks
    model_state_dict = rmodel.state_dict()
    global_opt_state_dict = r_opt.state_dict(from_global=True)

    if pyrannc.get_rank() == 0:
        torch.save(model_state_dict, 'model.pt')
        torch.save(global_opt_state_dict, 'opt_state.pt')

    pyrannc.barrier()

    ld_model = copy.deepcopy(model_base)

    loaded_state_dict = torch.load('model.pt', map_location='cpu')
    ld_model.load_state_dict(loaded_state_dict)
    ld_opt = optim.Adam(ld_model.parameters(), lr=lr)

    if use_amp:
        ld_model = ld_model.to(device)
        ld_model, ld_opt = amp.initialize(ld_model, ld_opt, opt_level="O2",
                                          max_loss_scale=2. ** 4,
                                          min_loss_scale=1)

    ld_model = pyrannc.RaNNCModule(ld_model, ld_opt, enable_apex_amp=use_amp, **module_args)

    # Verify parameters
    r_params = {n: rmodel.get_param(n, use_amp) for n, p in rmodel.named_parameters(from_global=True)}
    ld_params = {n: p for n, p in ld_model.named_parameters()}

    for n, rp in r_params.items():
        ld_p = ld_params[n]
        compare_tensors(rp, ld_p, rtol, atol)

    global_opt_state_dict = torch.load('opt_state.pt')
    opt_state_dict = opt.state_dict()

    for ld_grp, pt_grp in zip(global_opt_state_dict['param_groups'], opt_state_dict['param_groups']):
        np.testing.assert_(ld_grp.keys(), pt_grp.keys())
        for k in pt_grp.keys():
            if k == 'params':
                np.testing.assert_equal(len(ld_grp['params']), len(pt_grp['params']))
            else:
                np.testing.assert_(ld_grp[k] == pt_grp[k])

        for ld_pid, pt_pid in zip(ld_grp['params'], pt_grp['params']):
            ld_param_state = global_opt_state_dict['state'][ld_pid]
            pt_param_state = opt_state_dict['state'][pt_pid]
            np.testing.assert_(ld_param_state.keys() == pt_param_state.keys())
            for k in pt_param_state.keys():
                ldv = ld_param_state[k]
                pv = pt_param_state[k]
                if isinstance(ldv, torch.Tensor):
                    compare_tensors(ldv, pv, rtol, atol)
                else:
                    np.testing.assert_(ldv == pv)

    r_opt.load_state_dict(global_opt_state_dict, from_global=True)

    pyrannc.clear()
    pyrannc.barrier()
    print("Done")

def run(model_base, batch_size_per_proc, num_iter,
        dtype=torch.float, loss_out=False, preprocess=False, gradient_accumulation_steps=1,
        use_amp=False, allreduce_amp_master_params=False, enable_zero=False,
        dist_params=False, offload_params=False, rtol=RELATIVE_TOLERANCE, atol=ABSOLUTE_TOLERANCE,
        get_dataset=None, **kwargs):
    if loss_out:
        def aggregate_out_loss(out):
            tmp_loss = out.clone()
            torch.distributed.all_reduce(tmp_loss)
            tmp_loss /= pyrannc.get_world_size()
            return tmp_loss

        do_run(model_base, batch_size_per_proc, num_iter,
               lambda model, x, tgt: torch.jit.trace(model, (x, tgt)),
               lambda model, x, tgt: model(x, tgt),
               aggregate_out_loss,
               bwd_loss_output,
               dtype, preprocess, gradient_accumulation_steps,
               use_amp, allreduce_amp_master_params, enable_zero, dist_params, offload_params,
               rtol, atol, get_dataset, **kwargs)

    else:
        do_run(model_base, batch_size_per_proc, num_iter,
               lambda model, x, tgt: torch.jit.trace(model, (x,)),
               lambda model, x, tgt: model(x),
               lambda out: out,
               bwd_with_criterion,
               dtype, preprocess, gradient_accumulation_steps,
               use_amp, allreduce_amp_master_params, enable_zero, dist_params, offload_params,
               rtol, atol, get_dataset, **kwargs)


# -*- coding: utf-8 -*-

import binascii
from scapy.all import sniff
from scapy.layers.inet import TCP
from scapy.layers.inet6 import IPv6
import scapy

flow_info = {}

# 定义回调函数来处理捕获的数据包
def packet_callback(packet):
    if IPv6 in packet and packet.haslayer("IPv6ExtHdrHopByHop"):
        # 获取HBH信息
        hbh_header = packet["IPv6ExtHdrHopByHop"]
        options = hbh_header.options

        # 查找自定义的HBH选项
        for option in options:
            if option.otype == 0x3c:
                # print("Scapy6 Unknown Option:")
                # print("otype: {}".format(option.otype))
                # print("optlen: {}".format(option.optlen))
                # print("optdata: {}".format(binascii.hexlify(option.optdata)))
                num = int(binascii.hexlify(option.optdata),16)
                # print("optdata:{}".format(bin(num)[2:]))

                #获取到的二进制APN信息
                QoE_usr       = num >> 93       #用户类型
                QoE_sex       = (num >> 92) & 0b1       #性别
                QoE_edu       = (num >> 88) & 0b1111      #教育程度
                QoE_major     = (num >> 84) & 0b1111     #专业
                QoE_age       = (num >> 80) & 0b1111       #年龄
                QoE_app       = (num >> 76) & 0b1111       #应用类型
                QoE_qoe_i     = (num >> 72) & 0b1111     #QoE得分整数部分
                QoE_loss      = (num >> 64) & 0b11111111      #丢包率
                QoE_delay     = (num >> 48) & 0b1111111111111111     #时延
                QoE_qoe_f     = (num >> 32) & 0b1111111111111111           #QoE得分小数部分
                QoE_bandwidth = num & 0b11111111111111111111111111111111  #带宽

                #打印二进制APN信息
                # print("QoE_usr: {}".format(bin(QoE_usr)[2:]))
                # print("QoE_sex: {}".format(bin(QoE_sex)[2:]))
                # print("QoE_edu: {}".format(bin(QoE_edu)[2:]))
                # print("QoE_major: {}".format(bin(QoE_major)[2:]))
                # print("QoE_age: {}".format(bin(QoE_age)[2:]))
                # print("QoE_app: {}".format(bin(QoE_app)[2:]))
                # print("QoE_qoe_i: {}".format(bin(QoE_qoe_i)[2:]))
                # print("QoE_bandwith: {}".format(bin(QoE_bandwidth)[2:]))
                # print("QoE_delay: {}".format(bin(QoE_delay)[2:]))
                # print("QoE_loss: {}".format(bin(QoE_loss)[2:]))
                # print("QoE_qoe_f: {}".format(bin(QoE_qoe_f)[2:]))
                # print("--------------------------------")

                #处理QoE得分的函数
                def handle_qoe(qoe_i, qoe_f):
                    qoe_i = int(bin(QoE_qoe_i), 2)

                    qoe_f = int(bin(QoE_qoe_f), 2)
                    qoe_f = float(qoe_f)
                    if qoe_f < 10:
                       qoe_f = qoe_f / 10
                    elif 10 <= qoe_f < 100:
                        qoe_f = qoe_f / 100
                    elif 100 <= qoe_f < 1000:
                        qoe_f = qoe_f / 1000
                    elif 1000 <= qoe_f < 10000:
                        qoe_f = qoe_f / 10000
                    elif 10000 <= qoe_f < 100000:
                        qoe_f = qoe_f / 100000

                    qoe = qoe_i + qoe_f
                    return qoe

                #处理网络参数的函数
                def handle_net(net):
                    value = net & 0b00001111
                    #print(value)
                    unit = net >> 4
                    #print(unit)
                    net = int(bin(value),2)
                    if unit == 0b0000:
                        net = net
                    elif unit == 0b0001:
                        net = net * 10
                    elif unit == 0b0010:
                        net = net * 100
                    elif unit == 0b0100:
                        net = net * 1000
                    elif unit == 0b1000:
                        net = net * 10000
                    return net

                #将二进制APN信息转为十进制
                usr         = int(bin(QoE_usr), 2)
                sex         = int(bin(QoE_sex), 2)
                edu         = int(bin(QoE_edu), 2)
                major       = int(bin(QoE_major), 2)
                age         = int(bin(QoE_age), 2)
                app         = int(bin(QoE_app), 2)
                bandwidth   = int(bin(QoE_bandwidth), 2)
                delay       = int(bin(QoE_delay), 2)
                loss        = handle_net(QoE_loss)
                qoe         = handle_qoe(QoE_qoe_i, QoE_qoe_f)

                print("用户类型: {}".format(usr))
                print("性别: {}".format(sex))
                print("受教育程度: {}".format(edu))
                print("专业: {}".format(major))
                print("年龄: {}".format(age))
                print("应用类型: {}".format(app))
                print("带宽: {}".format(bandwidth))
                print("时延: {}".format(delay))
                print("丢包率: {}".format(loss))
                print("qoe分数: {}".format(qoe))
                print("--------------------------------")

                #打印IPv6的地址和TCP端口
                src_ipv6 = packet[IPv6].src
                dst_ipv6 = packet[IPv6].dst
                # print("Packet:")
                # print("Source IPv6:", src_ipv6)
                # print("Destination IPv6:", dst_ipv6)
                # if packet.haslayer(TCP):
                #     print("TCP Source Port:", packet[TCP].sport)
                #     print("TCP Destination Port", packet[TCP].dport)
                # print("--------------------------------")

                #数据包头中的apn信息存储在字典中
                apn_info = {
                    "src_ipv6":src_ipv6,
                    "dst_ipv6":dst_ipv6,
                    "usr":usr,
                    "sex":sex,
                    "edu":edu,
                    "major":major,
                    "age":age,
                    "app":app,
                    "bandwidth":bandwidth,
                    "delay":delay,
                    "loss":loss,
                    "qoe":qoe
                }

                session = (src_ipv6, dst_ipv6)
                if session not in flow_info:
                    flow_info[session] = []
                    flow_info[session].append(apn_info)

                # if qoe < 3:

                #     command = "ovs-vsctl set queue"
                #     uuid = "eeb42583-7e72-4cd1-8f69-5a9dc4afea94"
                #     other_config = "other-config:min-rate=600000 other-config:max-rate=1000000"
                #     send_queue_command(command, uuid, other_config)

                #     command = "ovs-ofctl add-flow"
                #     bridge = "s2"
                #     other_config = "ipv6,ipv6_dst=2001:250:1001:1044::33,in_port=s2-eth3,actions=set_queue:2,output=eno2"
                #     send_flow_command(command,bridge,other_config)

                # print("--------------------------------")

pcap_name = "capture_302.pcap"
packets = scapy.utils.rdpcap(pcap_name)
packet_callback(packets[1])


# __author__ = "Amos"
# Email: 379833553@qq.com

import configparser,re
from os.path import join
from scrapyd_api import ScrapydAPI
from django.core.mail import send_mail,BadHeaderError
import os
from django.conf import settings

def config(path, section, option, name='scrapy.cfg', default=None):
    try:
        cf = configparser.ConfigParser()
        cfg_path = join(path, name)
        cf.read(cfg_path)
        return cf.get(section, option)
    except configparser.NoOptionError:
        return default

def get_scrapyd(client):
    if not client.auth:
        return ScrapydAPI(scrapyd_url(client.ip, client.port))
    return ScrapydAPI(scrapyd_url(client.ip, client.port), auth=(client.username, client.password))

def scrapyd_url(ip, port):
    url = 'http://{ip}:{port}'.format(ip=ip, port=port)
    return url

def scrapyd_status_url(ip,port):
    url = 'http://{ip}:{port}/daemonstatus.json'.format(ip=ip, port=port)
    return url

def send_email(request):
    subject = request.POST.get('subject')
    message = request.POST.get('message')
    from_email = request.POST.get('from_email')
    to_email = request.POST.get('to_email')
    if subject and message and from_email:
        try:
            send_mail(subject, message, from_email, to_email)
        except BadHeaderError:
            return {'status':'error','message':'Invalid header found'}
        return {'status':'ok','message':'Success send email'}
    else:
        return {'status':'error','message':'Make sure all fields are entered and valid'}

def get_schedulerjob_info(job):
    a = re.findall(r"'(.+?)'", str(job.trigger))
    b = a.pop(2)
    a.reverse()
    a.append(b)
    crontab = ' '.join(a)
    info = {
        'status': 'ok',
        'name': job.id,
        'crontab': crontab,
        'next_run_time': job.next_run_time,
    }
    return info

def get_project_path(project_name=None):
    path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
                        settings.PROJECTS_FOLDER)
    if project_name:
        return os.path.join(path, project_name)
    else:
        return path


import math
import datetime
import db

def update_testTime(answer, userId):
    new_testTime = datetime.datetime.now()
    new_testTime_str  = new_testTime.strftime('%Y-%m-%d %H:%M:%S')
    sql = 'UPDATE forgettingRate_'+userId+ ' SET testTime = %s WHERE meaning = %s'###
    db.setDB(sql, (new_testTime_str, answer))

def fx(testTime, forgettingStage):

    K = 1.48
    C = 1.25

    now = datetime.datetime.now()

    diff = now - testTime
    days, seconds = diff.days, diff.seconds
    hours = days*24 + seconds/3600
    t = hours
    x_move = 12*(forgettingStage -1)

    if t <= x_move:
        y = 100000 # 발산 # 기억 잘 남
    else:
        log = math.log10(t - x_move)
        y = (100*K)/(C*log + K)

    return y

def cal_forgettingRate(forgettingStage, testTime):
    if (forgettingStage >= 5): # 5단계는 memoryRate 항상 20000(장기기억)
        new_forgettingRate = 200000
    else: # 1-4단계
        if (forgettingStage == 1):
            n = 1
        elif (forgettingStage == 2):
            n = 1.5
        elif (forgettingStage == 3):
            n = 2.0
        elif (forgettingStage == 4):
            n = 2.5

        #n = 1 + 0.5*(forgettingStage -1)
        new_forgettingRate = fx(testTime, forgettingStage)*n
        if new_forgettingRate > 100000: # 10000이상은 다 10000처리. forgettingRate 0-10000까지..(중복 최소화)
            new_forgettingRate = 100000

    return new_forgettingRate

def update_forgettingRate(answer, userId):
    # sql = 'SELECT word, meaning FROM ForgettingRate_' + userId
    sql_f = 'SELECT forgettingStage FROM forgettingRate_' +userId+ ' WHERE meaning = %s'
    sql_t = 'SELECT testTime FROM forgettingRate_'+userId+' WHERE meaning = %s'
    forgettingStage = db.getDB(sql_f, (answer))
    testTime = db.getDB(sql_t, (answer))

    defaultTestTime  = '2000-10-10 10:10:10'

    if testTime[0][0] == defaultTestTime: ## testTime == NULL 일 때
    #study만 하고 exam 거치지 않은 경우.(오늘 처음 study한 단어들) -> forgettingRate=0해서 exam에 뽑히게
        new_forgettingRate = 0
    #elif forgettingStage[0][0] == 0: ## study 안 하고 바로 test(forgettigStage=0)
    #    new_forgettingRate = 0
    #    update_SubWordSet_studied(subWordSet)
    #    update_Word_studied(wordSetId, subWordSetId)
    else:
        new_forgettingRate = cal_forgettingRate(forgettingStage[0][0], testTime[0][0])

    #sql2 = """UPDATE Word SET forgettingRate = %f WHERE id = %d"""
    sql2 = 'UPDATE forgettingRate_'+userId+ ' SET forgettingRate = %s WHERE meaning = %s'

    if new_forgettingRate > 200000:
        new_foregettingRate = 100000

    db.setDB(sql2, (new_forgettingRate, answer))

def update_forgettingStage(answer, correct, userId):
    if correct == 1:
        sql1 = 'SELECT forgettingStage FROM forgettingRate_'+userId+' WHERE meaning = %s'###
        forgettingStage = db.getDB(sql1, (answer))
        forgettingStage_up = forgettingStage[0][0] + 1

        sql2 = 'UPDATE forgettingRate_'+userId+ ' SET forgettingStage = %s WHERE meaning = %s'###
        db.setDB(sql2, (forgettingStage_up, answer))
        # forgettingStage +1
        # correct == 0 이면 forgettingStage 유지
    elif correct == 0:
        pass

def update_SubWordSet_studied(wordSetId, subWordSetId):
    # sql = 'UPDATE SubWordSet SET studied = 1 WHERE name = %s'
    sql = 'UPDATE Word SET studied = 1 WHERE wordSetId = %s AND subWordSetId = %s'
    db.setDB(sql, (wordSetId, subWordSetId))

"""
def update_Word_studied(wordSetId, subWordSetId):
    #sql = 'UPDATE Word SET studied = 1 WHERE wordSetId = %d AND subWordSetId = %d'
    sql = 'UPDATE Word SET studied = 1 WHERE wordSetId = %s AND subWordSetId = %s' ## %s
    db.setDB(sql, (wordSetId, subWordSetId))

def update_ForgettingRate_studied(wordSetId, subWordSetId):###
    sql = 'UPDATE ForgettingRate SET studied = 1 WHERE wordSetId = %s AND subWordSetId = %s'
    db.setDB(sql, (wordSetId, subWordSetId))
"""

def init_forgettingStage(wordSetId, subWordSetId, userId):
    sql2 = 'SELECT forgettingStage FROM forgettingRate_' +userId+ ' WHERE wordSetId = %s AND subWordSetId = %s'
    forgettingStage = db.getDB(sql2, (wordSetId, subWordSetId))
    print("@#!@#!@$!@$!@#!@$!@$!@#!@$!@$!@$")
    print(forgettingStage)
    print("@#!@#!@$!@$!@#!@$!@$!@#!@$!@$!@$")
    if(forgettingStage == 0):
        sql = 'UPDATE forgettingRate_' + userId + ' SET forgettingStage = 1 WHERE wordSetId = %s AND subWordSetId = %s'
        db.setDB(sql, (wordSetId, subWordSetId))
    elif(forgettingStage[0][0] == 0):
        sql = 'UPDATE forgettingRate_' +userId+ ' SET forgettingStage = 1 WHERE wordSetId = %s AND subWordSetId = %s'
        db.setDB(sql, (wordSetId, subWordSetId))

#test stage 끝난 후 exam 비우기 전! testTime 먼저 업데이트


import unittest
from Player_file import Player
from Player_file import Gambler

class TestPlayer(unittest.TestCase):

    def setUp(self):
        players = []
        self.dealer = Gambler('Dealer')
        self.player1 = Player('John')
        self.player2 = Player('Arya')
        self.player3 = Player('Sansa')

        players.append(self.dealer)
        players.append(self.player1)
        players.append(self.player2)
        players.append(self.player3)

        for player in players:
            player.blackjack = False
            player.hand = []
            player.doublehand = []
            player.score = 0
            player.doublescore = 0
            player.bust = False
            player.doublebust = False
            player.init_count = True
            player.bet = 0
            player.doublebet = 0
            player.split = False
            player.insured = False

    def test_count_scores(self):

        self.player1.hand = ['Ah', '10c']
        self.player2.hand = ["Ah", "8c", "2c"]
        self.player2.init_count = False
        self.player3.doublehand = ['10c', '2c', '10h']
        self.dealer.hand = ['Ac', '8c', '10c']

        self.player1.count_scores()
        self.player2.count_scores()
        self.player3.count_scores(True)
        self.dealer.count_scores()

        self.assertEqual(self.player1.score, 21)
        self.assertEqual(self.player2.score, 21)
        self.assertEqual(self.player3.doublescore, 22)
        self.assertEqual(self.dealer.score, 19)
        self.assertEqual(self.player1.blackjack, True)
        self.assertEqual(self.player2.blackjack, False)

    def test_draw_a_card(self):
        deck = ['Ah', 'Jc', '5h']
        self.player1.draw_a_card(deck)
        self.player2.draw_a_card(deck, True)

        self.assertEqual(self.player1.hand, ['Ah'])
        self.assertEqual(self.player2.doublehand, ['Jc'])
        self.assertEqual(deck, ['5h'])

if __name__ == '__main__':
    if __name__ == '__main__':
        unittest.main()


# BFS(Breadth First Search): 너비 우선 탐색
# 1. 탐색 시작 노드를 큐에 삽입하고 방문 처리.
# 2. 큐에서 노드를 꺼내 해당 노드의 인접 노드의 인접 노드 중에서 방문하지 않은 노드를 모두 큐에 삽입하고 방문 처리
# 3. 2번 과정을 더 이상 수행할 수 없을 때까지 반복.
from collections import deque

def bfs(graph, start, visited):
    queue = deque([start])
    visited[start] = True
    while queue:
        v = queue.popleft()
        print(v, end=' ')
        for i in graph[v]:
            if not visited[i]:
                queue.append(i)
                visited[i] = True


# footer text
SPONSORED_TEXT = None

# For S3-compatible Object Stores, setup access_key, secret_key
# bucket_name, calling_format, host and port as StorageBoxOptions

NAME = 'Sequencing Facility'

SITE_TITLE = 'Sequencing Facility'

INDEX_VIEWS = {1: 'tardis.apps.sequencing_facility.views.'
                  'SequencingFacilityIndexView'}

DATASET_VIEWS = [
    ('http://www.tardis.edu.au/schemas/ngs/project/raw_reads',
     'tardis.apps.sequencing_facility.views.FastqDatasetView'),
    ('http://www.tardis.edu.au/schemas/ngs/project/fastqc',
     'tardis.apps.sequencing_facility.views.FastqcDatasetView')
]

FACILITY_MANAGER_GROUPS = ['mhtp-facility-managers']
TRASH_USERNAME = '__trashman__'
TRASH_GROUP_NAME = '__trashcan__'

NGS_PUBLICATION_DATASET_SCHEMA = 'http://www.tardis.org.au/pub/ngs/datasets/'

PUBLICATION_FORM_MAPPINGS = [
    {'dataset_schema': r'^http://www.tardis.edu.au/schemas/ngs/project/raw_reads$',
     'publication_schema': NGS_PUBLICATION_DATASET_SCHEMA,
     'form_template':
         '/static/publication-form/ngs-extra-info-dataset-template.html'},
]


#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from setuptools import setup, find_packages

# Variables ===================================================================
changelog = open('CHANGES.rst').read()
long_description = "\n\n".join([
    open('README.rst').read(),
    changelog
])

# Functions ===================================================================
def allSame(s):
    return not any(filter(lambda x: x != s[0], s))

def hasDigit(s):
    return any(char.isdigit() for char in s)

def getVersion(data):
    """
    Parse version from changelog written in RST format.
    """
    data = data.splitlines()
    return next((
        v
        for v, u in zip(data, data[1:])  # v = version, u = underline
        if len(v) == len(u) and allSame(u) and hasDigit(v) and "." in v
    ))

# Actual setup definition =====================================================
setup(
    name='normalize_cz_unicode',
    version=getVersion(changelog),
    description='Take unicode string, leave czech characters, normalize rest.',
    long_description=long_description,
    url='https://github.com/Bystroushaak/normalize_cz_unicode',

    author='Bystroushaak',
    author_email='bystrousak@kitakitsune.org',

    classifiers=[
        'Intended Audience :: Developers',
        "Programming Language :: Python :: 2",
        'Programming Language :: Python :: 2.7',
        "License :: OSI Approved :: MIT License",
        "Development Status :: 5 - Production/Stable",
        "Natural Language :: Czech",
        "Topic :: Text Processing",
        "Topic :: Text Processing :: General",
    ],
    license='MIT',

    packages=find_packages('src'),
    package_dir={'': 'src'},
    include_package_data=True,
    zip_safe=True,

    # install_requires=open("requirements.txt").read().splitlines(),

    test_suite='py.test',
    tests_require=["pytest"],
    extras_require={
        "test": [
            "pytest"
        ],
    },
)


import sys
sys.path.append('./')
from utils.utils import calc_AUC
from copy import deepcopy
import matplotlib.pyplot as plt
import numpy as np
import utils.hsdmdl2_nml as hsdmdl2_nml
import utils.hsdmdl1_nml as hsdmdl1_nml
import utils.sdmdl_nml as sdmdl_nml
import tsmdl.aw2s_mdl as aw2s_mdl
import tsmdl.fw2s_mdl as fw2s_mdl
import dmdl.hsdmdl2 as hsdmdl2
import dmdl.hsdmdl1 as hsdmdl1
import dmdl.sdmdl as sdmdl
from functools import partial
import pandas as pd
import optuna
import changefinder
import bocpd
import datetime as dt
from multiprocessing import Pool
import multiprocessing as multi
import boto3
from settings import AWS_S3_BUCKET_NAME
import os

print(os.cpu_count())

df = pd.read_csv('./data/factory_sensor.csv')

X = np.zeros((8, 26450))
for i in range(8):
    # X[i]=np.array(df.iloc[:,i])
    for j in range(1, 26450):
        X[i, j] = df.iloc[j, i] - df.iloc[j - 1, i]
    X[i] -= np.mean(X[i, 0:250])
    X[i] /= np.std(X[i, 0:250])
changepoints = np.where(np.array(df.iloc[:, 8] == 1))[0]
changepoints = [667,  2633,  9662, 13229, 17371, 17831, 20130, 25440]
print("Starting Points of Changes: ", changepoints)

# parameters
tolerance_delay = 500
n_trials = 200
both = True

# BOCPD

def _objective_BOCPD(trial, train, changepoints, tolerance_delay):
    lam = trial.suggest_int('lam', 100, 1000)
    alpha = trial.suggest_uniform('alpha', 1e-8, 10)
    beta = trial.suggest_uniform('beta', 1e-8, 0.01)
    kappa = trial.suggest_uniform('kappa', 1e-8, 0.01)
    mu = 0
    #mu = trial.suggest_uniform('mu', -1, 1)

    h = partial(bocpd.constant_hazard, lam)
    lik = bocpd.StudentT(alpha, beta, kappa, mu)

    retrospective = bocpd.Retrospective(hazard_func=h, likelihood_func=lik)

    scores = np.zeros((8, 26450))

    for i in range(8):
        scores[i] = retrospective.calc_scores(train[i])

    scores = np.nanmax(scores, axis=0)

    AUC = calc_AUC(scores, changepoints, tolerance_delay, both=both)

    return -AUC

# hyperparameter tuning
objective_BOCPD = partial(_objective_BOCPD, train=X,
                          changepoints=changepoints, tolerance_delay=tolerance_delay)
study = optuna.create_study()
study.optimize(objective_BOCPD, n_trials=n_trials, n_jobs=-1)

# calculate scores
opt_lam = study.best_params['lam']
opt_alpha = study.best_params['alpha']
opt_beta = study.best_params['beta']
opt_kappa = study.best_params['kappa']
#opt_mu = study.best_params['mu']
opt_mu = 0

h = partial(bocpd.constant_hazard, opt_lam)
lik = bocpd.StudentT(opt_alpha, opt_beta, opt_kappa, opt_mu)
retrospective = bocpd.Retrospective(hazard_func=h, likelihood_func=lik)

bocpd_scores = np.zeros((8, 26450))

for i in range(8):
    bocpd_scores[i] = retrospective.calc_scores(X[i])

bocpd_scores = np.nanmax(bocpd_scores, axis=0)

# CF

def _objective_CF(trial, train, changepoints, tolerance_delay):
    # hyperparameters
    r = trial.suggest_uniform('r', 0.01, 0.99)
    order = trial.suggest_int('order', 1, 20)
    smooth = trial.suggest_int('smooth', 3, 20)

    retrospective = changefinder.Retrospective(r=r, order=order, smooth=smooth)

    scores = np.zeros((8, 26450))

    for i in range(8):
        scores[i] = retrospective.calc_scores(train[i])

    scores = np.nanmax(scores, axis=0)

    AUC = calc_AUC(
        scores, changepoints, tolerance_delay, both=both)
    return -AUC

# hyperparameter tuning
objective_CF = partial(_objective_CF, train=X,
                       changepoints=changepoints, tolerance_delay=tolerance_delay)
study = optuna.create_study()
study.optimize(objective_CF, n_trials=n_trials, n_jobs=-1)
opt_r = study.best_params['r']
opt_order = study.best_params['order']
opt_smooth = study.best_params['smooth']

retrospective = changefinder.Retrospective(
    r=opt_r, order=opt_order, smooth=opt_smooth)

cf_scores = np.zeros((8, 26450))

for i in range(8):
    cf_scores[i] = retrospective.calc_scores(X[i])

cf_scores = np.nanmax(cf_scores, axis=0)

# SDMDL 0th

def _objective_SDMDL(trial, train, changepoints, tolerance_delay, params):

    nml_multgaussian = partial(sdmdl_nml.nml_multgaussian, R=params[
                               "R"], lambda_min=params["lambda_min"])
    complexity_multgaussian = partial(sdmdl_nml.complexity_multgaussian, m=8, R=params[
                                      "R"], lambda_min=params["lambda_min"])

    window_size = trial.suggest_int('window_size', 100, 300)
    retrospective = sdmdl.Retrospective(h=window_size, encoding_func=nml_multgaussian,
                                        complexity_func=complexity_multgaussian, order=params["order"])

    scores = retrospective.calc_scores(train)
    n = len(scores)
    scores[2 * window_size - 1: n] = scores[window_size - 1: n - window_size]
    scores[0:2 * window_size] = np.nan

    AUC = calc_AUC(scores, changepoints,  tolerance_delay, both=both)
    return -AUC

R = 60
lambda_min = 1e-5

params = {"R": R, "lambda_min": lambda_min, "order": 0}

# 0th D-MDL
# hyperparameter tuning
objective_SDMDL = partial(_objective_SDMDL, train=X.T,
                          changepoints=changepoints, tolerance_delay=tolerance_delay, params=params)
study = optuna.create_study()
study.optimize(objective_SDMDL, n_trials=n_trials, n_jobs=-1)
opt_window_size = study.best_params['window_size']

nml_multgaussian = partial(sdmdl_nml.nml_multgaussian, R=params[
                           "R"], lambda_min=params["lambda_min"])
complexity_multgaussian = partial(sdmdl_nml.complexity_multgaussian, m=8, R=params[
                                  "R"], lambda_min=params["lambda_min"])

retrospective = sdmdl.Retrospective(h=opt_window_size, encoding_func=nml_multgaussian,
                                    complexity_func=complexity_multgaussian, order=params["order"])

sdmdl_0_scores = retrospective.calc_scores(X.T)
n = len(sdmdl_0_scores)
sdmdl_0_scores[2 * opt_window_size -
               1: n] = sdmdl_0_scores[opt_window_size - 1: n - opt_window_size]
sdmdl_0_scores[0:2 * opt_window_size] = np.nan

# 1st D-MDL
params = {"R": R, "lambda_min": lambda_min, "order": 1}

# hyperparameter tuning
objective_SDMDL = partial(_objective_SDMDL, train=X.T,
                          changepoints=changepoints, tolerance_delay=tolerance_delay, params=params)
study = optuna.create_study()
study.optimize(objective_SDMDL, n_trials=n_trials, n_jobs=-1)
opt_window_size = study.best_params['window_size']

retrospective = sdmdl.Retrospective(h=opt_window_size, encoding_func=nml_multgaussian,
                                    complexity_func=complexity_multgaussian, order=params["order"])

sdmdl_1_scores = retrospective.calc_scores(X.T)
n = len(sdmdl_1_scores)
sdmdl_1_scores[2 * opt_window_size -
               1: n] = sdmdl_1_scores[opt_window_size - 1: n - opt_window_size]
sdmdl_1_scores[0:2 * opt_window_size] = np.nan

# 2nd D-MDL
params = {"R": R, "lambda_min": lambda_min, "order": 2}

# hyperparameter tuning
objective_SDMDL = partial(_objective_SDMDL, train=X.T,
                          changepoints=changepoints, tolerance_delay=tolerance_delay, params=params)
study = optuna.create_study()
study.optimize(objective_SDMDL, n_trials=n_trials, n_jobs=-1)
opt_window_size = study.best_params['window_size']

retrospective = sdmdl.Retrospective(h=opt_window_size, encoding_func=nml_multgaussian,
                                    complexity_func=complexity_multgaussian, order=params["order"])

sdmdl_2_scores = retrospective.calc_scores(X.T)
n = len(sdmdl_2_scores)
sdmdl_2_scores[2 * opt_window_size -
               1: n] = sdmdl_2_scores[opt_window_size - 1: n - opt_window_size]
sdmdl_2_scores[0:2 * opt_window_size] = np.nan

# FW2S_MDL

def _objective_FW2S_MDL(trial, train, changepoints, tolerance_delay, params):
    nml_multgaussian = partial(sdmdl_nml.nml_multgaussian, R=params[
                               "R"], lambda_min=params["lambda_min"])
    complexity_multgaussian = partial(sdmdl_nml.complexity_multgaussian, m=8, R=params[
                                      "R"], lambda_min=params["lambda_min"])

    nml_gaussian = partial(sdmdl_nml.nml_gaussian, mu_max=params[
                           "mu_max"], div_min=params["div_min"], div_max=params["div_max"])
    complexity_gaussian = partial(sdmdl_nml.complexity_gaussian, mu_max=params[
                                  "mu_max"], div_min=params["div_min"], div_max=params["div_max"])
    delta_0 = trial.suggest_uniform('delta_0', 0.001, 0.499)
    window_size_1 = trial.suggest_int('window_size_1', 100, 300)
    window_size_2 = trial.suggest_int('window_size_2', 100, 300)

    retrospective_first = sdmdl.Retrospective(h=window_size_1, encoding_func=nml_multgaussian,
                                              complexity_func=complexity_multgaussian, delta_0=delta_0, order=0)
    retrospective_second = sdmdl.Retrospective(h=window_size_2, encoding_func=nml_gaussian,
                                               complexity_func=complexity_gaussian, order=0)
    retrospective = fw2s_mdl.Retrospective(
        retrospective_first, retrospective_second)
    scores = retrospective.calc_scores(train)

    n = len(scores)
    scores[2 * (window_size_1 + window_size_2): n] = scores[window_size_1 +
                                                            window_size_2: n - (window_size_1 + window_size_2)]
    scores[0:2 * (window_size_1 + window_size_2)] = np.nan

    AUC = calc_AUC(scores, changepoints,  tolerance_delay, both=both)
    return -AUC

R = 60
lambda_min = 1e-5
mu_max = 10
div_min = 1e-2
div_max = 1e2

params = {"R": R, "lambda_min": 1e-5, "mu_max": mu_max,
          "div_min": div_min, "div_max": div_max}

# hyperparameter tuning
objective_FW2S_MDL = partial(_objective_FW2S_MDL, train=X.T,
                             changepoints=changepoints, tolerance_delay=tolerance_delay, params=params)
study = optuna.create_study()
study.optimize(objective_FW2S_MDL, n_trials=n_trials, n_jobs=-1)
opt_delta_0 = study.best_params['delta_0']
opt_window_size_1 = study.best_params['window_size_1']
opt_window_size_2 = study.best_params['window_size_2']

nml_multgaussian = partial(sdmdl_nml.nml_multgaussian, R=params[
                           "R"], lambda_min=params["lambda_min"])
complexity_multgaussian = partial(sdmdl_nml.complexity_multgaussian, m=8, R=params[
                                  "R"], lambda_min=params["lambda_min"])
nml_gaussian = partial(sdmdl_nml.nml_gaussian, mu_max=params[
                       "mu_max"], div_min=params["div_min"], div_max=params["div_max"])
complexity_gaussian = partial(sdmdl_nml.complexity_gaussian, mu_max=params[
                              "mu_max"], div_min=params["div_min"], div_max=params["div_max"])

retrospective_first = sdmdl.Retrospective(h=opt_window_size_1, encoding_func=nml_multgaussian,
                                          complexity_func=complexity_multgaussian, delta_0=opt_delta_0, order=0)
retrospective_second = sdmdl.Retrospective(h=opt_window_size_2, encoding_func=nml_gaussian,
                                           complexity_func=complexity_gaussian, order=0)
retrospective = fw2s_mdl.Retrospective(
    retrospective_first, retrospective_second)

fw2s_mdl_scores = retrospective.calc_scores(X.T)
n = len(fw2s_mdl_scores)
fw2s_mdl_scores[2 * (opt_window_size_1 + opt_window_size_2): n] = fw2s_mdl_scores[opt_window_size_1 +
                                                                                opt_window_size_2: n - (opt_window_size_1 + opt_window_size_2)]
fw2s_mdl_scores[0:2 * (opt_window_size_1 + opt_window_size_2)] = np.nan

sdmdl_0_scores[np.isnan(sdmdl_0_scores)] = np.nanmin(sdmdl_0_scores)
sdmdl_1_scores[np.isnan(sdmdl_1_scores)] = np.nanmin(sdmdl_1_scores)
sdmdl_2_scores[np.isnan(sdmdl_2_scores)] = np.nanmin(sdmdl_2_scores)
fw2s_mdl_scores[np.isnan(fw2s_mdl_scores)] = np.nanmin(fw2s_mdl_scores)

print("SDMDL 0th: ", calc_AUC(sdmdl_0_scores,
                              changepoints,  tolerance_delay, both=both))
print("SDMDL 1st: ", calc_AUC(sdmdl_1_scores,
                              changepoints,  tolerance_delay, both=both))
print("SDMDL 2nd: ", calc_AUC(sdmdl_2_scores,
                              changepoints,  tolerance_delay, both=both))
print("FW2S-MDL: ", calc_AUC(fw2s_mdl_scores,
                             changepoints,  tolerance_delay, both=both))
print("ChangeFinder: ", calc_AUC(
    cf_scores, changepoints,  tolerance_delay, both=both))
print("BOCPD: ", calc_AUC(bocpd_scores, changepoints,  tolerance_delay, both=both))

df_result = pd.DataFrame()
row = pd.DataFrame({"method: ": ["SDMDL 0th"], "AUC: ": calc_AUC(
    sdmdl_0_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)
row = pd.DataFrame({"method: ": ["SDMDL 1st"], "AUC: ": calc_AUC(
    sdmdl_1_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)
row = pd.DataFrame({"method: ": ["SDMDL 2nd"], "AUC: ": calc_AUC(
    sdmdl_2_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)
row = pd.DataFrame({"method: ": ["FW2S-MDL"], "AUC: ": calc_AUC(
    fw2s_mdl_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)
row = pd.DataFrame({"method: ": ["ChangeFinder"], "AUC: ": calc_AUC(
    cf_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)
row = pd.DataFrame({"method: ": ["BOCPD"], "AUC: ": calc_AUC(
    bocpd_scores, changepoints,  tolerance_delay, both=both)})
df_result = pd.concat([df_result, row], axis=0)

df_result.to_csv("production_condition.csv", index=False)

plt.clf()
fig, axes = plt.subplots(nrows=7, ncols=1, figsize=(20, 20))

fontsize = 18

axes[0].plot(X[0], label='Raw Data')
axes[0].set_ylabel('Return', fontsize=fontsize)
axes[0].vlines(changepoints, np.min(X[0]), np.max(X[0]), color="black")
axes[0].set_ylim(np.min(X[0]), np.max(X[0]))
axes[0].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[0].grid(True)

axes[1].plot(sdmdl_0_scores, label='SD-MDL 0th')
axes[1].set_ylabel('Description Length', fontsize=fontsize)
axes[1].vlines(changepoints, np.nanmin(sdmdl_0_scores) - 1,
               np.nanmax(sdmdl_0_scores) + 1, color="black")
axes[1].set_ylim(np.nanmin(sdmdl_0_scores) - 1, np.nanmax(sdmdl_0_scores) + 1)
axes[1].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[1].grid(True)

axes[2].plot(sdmdl_1_scores, label='SD-MDL 1st')
axes[2].set_ylabel('Description Length', fontsize=fontsize)
axes[2].vlines(changepoints, 0, np.nanmax(sdmdl_1_scores), color="black")
axes[2].set_ylim(0, np.nanmax(sdmdl_1_scores))
axes[2].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[2].grid(True)

axes[3].plot(sdmdl_2_scores, label='SD-MDL 2nd')
axes[3].set_ylabel('Description Length', fontsize=fontsize)
axes[3].vlines(changepoints, 0, np.nanmax(sdmdl_1_scores), color="black")
axes[3].set_ylim(0, np.nanmax(sdmdl_1_scores))
axes[3].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[3].grid(True)

axes[4].plot(fw2s_mdl_scores, label='FW2S-MDL')
axes[4].set_ylabel('Description Length', fontsize=fontsize)
axes[4].vlines(changepoints, np.nanmin(fw2s_mdl_scores) - 1,
               np.nanmax(fw2s_mdl_scores) + 1, color="black")
axes[4].set_ylim(np.nanmin(fw2s_mdl_scores) - 1,
                 np.nanmax(fw2s_mdl_scores) + 1)
axes[4].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[4].grid(True)

axes[5].plot(cf_scores, label='ChangeFinder')
axes[5].set_ylabel('Change Score', fontsize=fontsize)
axes[5].vlines(changepoints, np.nanmin(cf_scores) - 1,
               np.nanmax(cf_scores) + 1, color="black")
axes[5].set_ylim(np.nanmin(cf_scores) - 1, np.nanmax(cf_scores) + 1)
axes[5].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[5].grid(True)

axes[6].plot(bocpd_scores, label='BOCPD')
axes[6].set_ylabel('Change Score', fontsize=fontsize)
axes[6].vlines(changepoints, np.nanmin(bocpd_scores),
               np.nanmax(bocpd_scores), color="black")
axes[6].set_ylim(np.nanmin(bocpd_scores), np.nanmax(bocpd_scores))
axes[6].legend(bbox_to_anchor=(1, 1), loc='upper right',
               borderaxespad=0, fontsize=fontsize)
axes[6].grid(True)

fig.savefig("production_condition.png")

# upload to s3
s3 = boto3.resource('s3')
bucket = s3.Bucket(AWS_S3_BUCKET_NAME)

bucket.upload_file("production_condition.png",
                   "production_condition/" + "production_condition.png")
bucket.upload_file("production_condition.csv",
                   "production_condition/" + "production_condition.csv")


"""       Reads a number from the user and prints out the names of each digit
"""

number = int(input ("Please type a number "))
digitNames = ('zero','one','two','three','four','five','six','seven','eight','nine')
result = []
while number >0:
    currentDigit = number % 10
    # modulus operator gives remainder after integer division
    # in this case it is the rightmost digit
    result.append( digitNames[currentDigit])
    number = number // 10
    # integer division throws away the remainder

# my code
result.reverse()
for n in result:
    print (n)
'''
Modify the program above so that:
- the names of the digits are printed in the correct order, and
- the names of the digits are printed without the list notation around them.
'''


from flask import Flask, request, jsonify
import product_data_access_object
from sql_connection import get_sql_connection

app = Flask(__name__)
connection = get_sql_connection()

@app.route('/getProducts', methods=['GET'])
def get_products():
    products = product_data_access_object.get_all_products(connection)
    response = jsonify(products)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

if __name__ == "__main__":
    print("Starting Python Flask Server For Grocery Store Management System")
    app.run(port=5000)


class FileNode:
    def __init__(self, val, parent = None, children = [], _type = None, size = 0):
        self.children = children
        self.size = size
        self.val = val
        self.parent = parent
        self.type = _type

def parseInput(filename, runner):
    with open(filename) as f:
        line = f.readline()
        while line:
            if '$' in line:
                CMD = line[2:4]
                ARG = line[5:]
                if CMD == 'cd':
                    if '..' in ARG:
                        runner = runner.parent

                    elif '/' not in ARG:
                        for node in runner.children:
                            if node.val == ARG.strip(' '):
                                runner = node
                                break

                elif CMD == 'ls':
                    line = f.readline()
                    while len(line) > 1 and '$' not in line:
                        a = line.split(' ')
                        if a[0] == 'dir':
                            runner.children.append(
                                FileNode(
                                    a[1],
                                    runner,
                                    [],
                                    'dir'
                                )
                            )
                        elif a[0].isdigit():
                            runner.children.append(
                                FileNode(
                                    a[1],
                                    runner,
                                    None,
                                    'file',
                                    int(a[0])
                                )
                            )
                        line = f.readline()
                    continue
            else:
                assert(False, "UNKNOWN TOKEN")
            line = f.readline()

def getSumDirs(Tree, res = [0]):
    if Tree.children == None or Tree.children == []:
        return
    for i in range(len(Tree.children)):
        node = Tree.children[i]
        if node.type == 'dir' and node.size < 100000:
            res[0] += node.size
        getSumDirs(Tree.children[i], res)

def smallestDirToFreeUpEnoughSpace(Tree, res, neededSpace):
    if Tree.children == None or Tree.children == []:
        return
    for i in range(len(Tree.children)):
        node = Tree.children[i]
        if node.type == 'dir':
            if node.size >= neededSpace:
                res[0] = min(res[0], node.size)
        smallestDirToFreeUpEnoughSpace(Tree.children[i], res, neededSpace)

def makeSizeForTreeDirs(Tree):
    if Tree.size > 0:
        return Tree.size

    if Tree.children == None or Tree.children == []:
        return 0

    res = 0
    for i in range(len(Tree.children)):
        res += makeSizeForTreeDirs(Tree.children[i])
        if Tree.type == 'dir':
            Tree.size = res
    return res

def day7():
    # Create parser and initialize TreeNode
    filename = 'day-7.txt'
    Tree = FileNode('*')
    Tree.type = 'dir'
    Root = Tree
    parseInput(filename, Root)

    # Compute size for all Dirs
    makeSizeForTreeDirs(Root)

    # Get size sum of Dirs
    sumOfDirs = [0]
    getSumDirs(Root, sumOfDirs)
    yield sumOfDirs[0]

    # Get smallest Dirs needed to be deleted
    neededSpace = 30000000 - (70000000 -  Root.size)
    smallestDir = [float('inf')]
    smallestDirToFreeUpEnoughSpace(Root, smallestDir, neededSpace)
    yield smallestDir[0]

def main():
    for sol in day7():
        print(sol)

main()


import os

this_file_path = os.path.abspath(__file__)
BASE_DIR = os.path.dirname(this_file_path)

email = os.path.join(BASE_DIR, 'templates', 'email.txt')
content = ""

with open(email, 'r') as file:
    content = file.read()

print(content.format(name='Saurabh'))

Files - create, read & download/file_manager.py


#Define a global dictionary. Iterate through that dictionary and print the output in the following format.
#Sample Output
#a -- 2
#x -- 8
#z -- 1

Dictionary1 = {"a" : "2","x" : "8","z" : "1"}
for i in Dictionary1:
    print(i,"--",Dictionary1[i])


# coding: utf8

import random
from typing import Dict, List
from discrete_model import DiscreteModel, InfluenceGraph, Gene, Transition, State
import matplotlib.pyplot as plt
import numpy as np

class Simulation:
    def __init__(self, model: DiscreteModel):
        self.model = model
        self.random = random.Random()
        self.steps = 100
        self.initial_state = {gene.name: self.random.choice(gene.states)
                              for gene in self.model.genes}

    def run(self) -> 'Result':
        states = [self._convert_initial_state_to_state()]
        for _ in range(self.steps-1):
            states.append(self._next_step(states[-1]))
        return Result(self, states)

    def _convert_initial_state_to_state(self) -> State:
        return State({self.model.find_gene_by_name(gene): state
                      for gene, state in self.initial_state.items()})

    def _next_step(self, state: State):
        next_states = self.model.available_state(state)
        if next_states:
            return self.random.choice(next_states)
        return state

class Result:
    def __init__(self, simulation: Simulation, states: List[State]):
        self.simulation = simulation
        self.states = states

    def plot_evolution(self):
        genes = self.simulation.model.genes
        t = np.arange(0, len(self.states), 1)
        _, ax = plt.subplots()
        for gene in genes:
            ax.plot(t, [state[gene] for state in self.states], label=gene.name)
        ax.set(xlabel='step', ylabel='state', title='Simulation')
        ax.grid()
        plt.legend()
        plt.show()


from __future__ import print_function
import argparse, os, sys, random, time, datetime
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Ellipse
#
import torch
import torch.nn as nn
import torch.nn.functional as F
#
from custom_models import *
from custom_datasets import *
from custom_transforms import *
from utils import *

# sklearn
from sklearn.metrics import confusion_matrix
from sklearn.manifold import TSNE
# RAPIDS AI (much faster t-SNE we used in the paper)
#from cuml import TSNE

def get_args():
    parser = argparse.ArgumentParser(description='Visualization of AutoDO')
    parser.add_argument('--data', default='./local_data', type=str, metavar='NAME',
                        help='folder to save all data')
    parser.add_argument('--dataset', default='MNIST', type=str, metavar='NAME',
                        help='dataset MNIST/CIFAR10/CIFAR100/SVHN/SVHN_extra/ImageNet')
    parser.add_argument('--workers', default=4, type=int, metavar='NUM',
                        help='number of data loading workers (default: 4)')
    parser.add_argument("--gpu", default='0', type=str, metavar='NUM',
                        help='GPU device number')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--log-interval', type=int, default=500, metavar='NUM',
                        help='how many batches to wait before logging training status')
    parser.add_argument('-ir', '--imbalance-ratio', type=int, default=1, metavar='N',
                        help='ratio of [1:C/2] to [C/2+1:C] labels in the training dataset drawn from uniform distribution')
    parser.add_argument('-sr', '--subsample-ratio', type=float, default=1.0, metavar='N',
                        help='ratio of selected to total labels in the training dataset drawn from uniform distribution')
    parser.add_argument('-nr', '--noise-ratio', type=float, default=0.0, metavar='N',
                        help='ratio of noisy (randomly flipped) labels (default: 0.0)')
    parser.add_argument('-r', '--run-folder', default='run0', type=str,
                        help='dir to save run')
    parser.add_argument('--plot-cmx', action='store_true', help='confusion matrix or TSNE')

    args = parser.parse_args()

    return args

def main(args):
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    use_cuda = not args.no_cuda and torch.cuda.is_available()
    init_seeds(seed=int(time.time()))
    device = torch.device("cuda" if use_cuda else "cpu")
    kwargs = {'num_workers': args.workers, 'pin_memory': True} if use_cuda else {}
    args.hyper_est = True
    dataset = args.dataset
    imbalance_ratio = args.imbalance_ratio
    subsample_ratio = args.subsample_ratio
    noise_ratio = args.noise_ratio
    model_postfix = 'ir_{}_sr_{}_nr_{}'.format(imbalance_ratio, subsample_ratio, noise_ratio)
    run_folder = args.run_folder
    save_folder = '{}/{}'.format(args.data, dataset)
    model_folder = '{}/{}'.format(save_folder, run_folder)
    assert os.path.isdir(model_folder), 'Error: {} model folder is not found!'.format(model_folder)
    print('model_folder:', model_folder)
    plot_cmx = args.plot_cmx
    results_folder = './visualizations'
    if not os.path.isdir(results_folder):
        os.mkdir(results_folder)
    #
    if   dataset == 'MNIST':
        total_images = 60000
        valid_images = 10000
        train_images = total_images - valid_images
        num_classes = 10
        num_channels = 1
        # model:
        model_name = 'resnet18'
        encoder = EncoderResNet(dataset=dataset, depth=18, num_classes=num_classes).to(device)
        decoder = SupCeResNet(dataset=dataset, depth=18, num_classes=num_classes).to(device)
        # dataloaders
        train_batch_size = 1000
        test_data  = MNIST(save_folder, train=False, transform=transform_test_mnist)
        train_data = MNIST(save_folder, train=True,  transform=transform_train_mnist)
    elif dataset == 'SVHN':
        total_images = 73257
        valid_images = 23257
        train_images = total_images - valid_images
        num_classes = 10
        num_channels = 3
        # model:
        train_batch_size = 256
        model_name = 'wresnet28_10'
        encoder = EncoderWideResNet(depth=28, widen_factor=10, num_classes=num_classes).to(device)
        decoder = SupCeWideResNet(name=model_name, num_classes=num_classes).to(device)
        # data:
        test_data  = SVHN(save_folder, split='test',  transform=transform_test_svhn,  download=True)
        train_data = SVHN(save_folder, split='train', transform=transform_train_svhn, download=True)
    else:
        raise NotImplementedError('{} is not supported dataset!'.format(dataset))
    # dataloaders:
    data_file = '{}/data_{}.pt'.format(model_folder, model_postfix)
    if os.path.isfile(data_file):
        _, train_sub_indices, train_targets = torch.load(data_file) # load saved indices
    else:
        raise NotImplementedError('{} is missing!'.format(data_file))
    #
    valid_loader = torch.utils.data.DataLoader(test_data, batch_size=train_batch_size, shuffle=False, **kwargs)
    #train_sub_data = torch.utils.data.Subset(train_data, train_sub_indices)
    #train_loader = torch.utils.data.DataLoader(train_sub_data, batch_size=train_batch_size, shuffle=False, **kwargs)
    #
    args.train_batch_size = train_batch_size
    args.num_classes = num_classes
    # list model layers
    for n, p in encoder.named_parameters():
        print (n, p.data.shape)
    F = p.data.shape[0] # last layer dimension
    for n, p in decoder.named_parameters():
        print (n, p.data.shape)
    # hyper models
    C = num_classes
    T = total_images
    M = len(valid_loader.dataset)
    N = 0 #len(train_loader.dataset)
    print('Train/Test Split: {}/{} out of total {} images'.format(M, N, T))
    # CFGS:
    cfgs = list()
    # format: [hyper_opt, hyper_est, aug_model, los_model]
    #cfgs.append(['RAND', True, 'RAND', 'NONE'])
    cfgs.append(['NONE', True, 'NONE', 'NONE'])
    cfgs.append(['NONE', True, 'AUTO', 'NONE'])
    cfgs.append([ 'HES', True,  'SEP', 'BOTH'])
    E = len(cfgs)
    # experiment name
    exp_name = 'debug'
    if plot_cmx:
        results_file = '{}_cmat_{}'.format(dataset, exp_name)
    else:
        results_file = '{}_tsne_{}'.format(dataset, exp_name)
    fvs_2d_file = '{}/fvs_2d_tsne_{}.npy'.format(results_folder, exp_name)
    gts_file    = '{}/gts_tsne_{}.npy'.format(results_folder, exp_name)
    prs_file    = '{}/prs_tsne_{}.npy'.format(results_folder, exp_name)
    szs_file    = '{}/szs_tsne_{}.npy'.format(results_folder, exp_name)
    cmx_file    = '{}/cmx_tsne_{}.npy'.format(results_folder, exp_name)
    if os.path.isfile(fvs_2d_file):
        fvs_2d = np.load(fvs_2d_file)
        gts    = np.load(gts_file)
        prs    = np.load(prs_file)
        szs    = np.load(szs_file)
        cmx    = np.load(cmx_file)
    else:
        fvs    = np.empty([E,M+N,F])
        fvs_2d = np.empty([E,M+N,2])
        gts = np.empty([E,M+N], dtype=int)
        prs = np.empty([E,M+N])
        szs = np.empty([E,M+N])
        cmx = np.empty([E,C,C])
        for i, cfg in enumerate(cfgs):
            hyper_opt, hyper_est, aug_model, los_model = cfg
            run_name = '{}_opt_{}_est_{}_aug_model_{}_los_model_{}_{}'.format(
                model_name, hyper_opt, hyper_est, aug_model, los_model, model_postfix)
            checkpoint_file = '{}/best_{}.pt'.format(model_folder, run_name)
            if os.path.isfile(checkpoint_file):
                print('Loading pretrained models...', checkpoint_file)
                checkpoint = torch.load(checkpoint_file)
                encoder.load_state_dict(checkpoint['encoder_state_dict'])
                decoder.load_state_dict(checkpoint['decoder_state_dict'])
            else:
                raise NotImplementedError('{} checkpoint is missing!'.format(checkpoint_file))
            # test data
            fv, gt, pr, mi = vizStat(args, encoder, decoder, device, valid_loader, M, F)
            sz = torch.zeros(M).to(device)
            sz[mi] = 1.0
            fv, gt, pr, sz = fv.cpu().numpy(), gt.cpu().numpy(), pr.cpu().numpy(), sz.cpu().numpy()
            fvs[i,:M] = fv
            gts[i,:M] = gt
            prs[i,:M] = pr
            szs[i,:M] = sz
            cm = confusion_matrix(gt, pr)
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            cmx[i] = cm
            ## train data
            #fv, gt, pr, _ = vizStat(args, encoder, decoder, device, train_loader, T, F)
            #train_idx = torch.tensor(train_sub_indices, dtype=torch.long, device=device)
            ##print(train_idx.shape)
            #fv, gt, pr, sz = fv[train_idx].cpu().numpy(), gt[train_idx].cpu().numpy(), pr[train_idx].cpu().numpy(), sz.cpu().numpy()
            ##print(fv.shape, gt.shape, pr.shape)
            #fvs[i,M:M+N] = fv
            #gts[i,M:M+N] = gt
            #prs[i,M:M+N] = pr
            #szs[i,M:M+N] = sz
            fvs_2d[i] = TSNE(n_components=2, verbose=1).fit_transform(fvs[i])
            print('{}-{} : {}/{}/{}/{}'.format(i, run_name, fvs.shape, gts.shape, prs.shape, szs.shape))
        # save results
        #fvs_2d = TSNE(n_components=2, verbose=1).fit_transform(fvs.reshape((E*(M+N),F))).reshape((E,M+N,2))
        np.save(fvs_2d_file, fvs_2d)
        np.save(gts_file, gts)
        np.save(prs_file, prs)
        np.save(szs_file, szs)
        np.save(cmx_file, cmx)
    #
    print('Check output:', fvs_2d.shape)
    pdf = PdfPages('{}/{}'.format(results_folder, results_file+'.pdf'))
    np.set_printoptions(precision=2)
    fontText = 16
    S = 9
    plt.rcParams.update({'font.size': 16})
    plt.rcParams.update({'figure.dpi': 600})
    plt.rcParams.update({'savefig.dpi': 600})
    fig, ax = plt.subplots(1, E, sharex=True, sharey=True)
    fig.set_size_inches(np.array([E*5+1,6]), forward=True)
    if plot_cmx:
        print('Plotting confusion matrix')
        fmt = '.1f'
        for e, cfg in enumerate(cfgs):
            cm = 100.0*cmx[e]
            th = cm.max() / 2.0
            ax[e].imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
            for i in range(C):
                for j in range(C):
                    ax[e].text(j, i, format(cm[i,j], fmt), ha="center", va="center", color="white" if cm[i,j] > th else "black", fontsize = 10)
            #ax[e].text(0.25, 10.4, t[e], rotation = 0, fontsize = fontText)
            plt.tight_layout()
        ax[0].set(xticks=np.arange(C), yticks=np.arange(C), xlabel='Predicted label, (a)', ylabel='True label')
        ax[1].set(xticks=np.arange(C), yticks=np.arange(C), xlabel='Predicted label, (b)')
        ax[2].set(xticks=np.arange(C), yticks=np.arange(C), xlabel='Predicted label, (c)')
    else:
        print('Plotting t-sne')
        t = ['(a)', '(b)', '(c)']
        for e, cfg in enumerate(cfgs):
            im = ax[e].scatter(fvs_2d[e,:,0], fvs_2d[e,:,1], cmap=plt.get_cmap('tab10'), c=gts[e], s=S*szs[e]+1)
            ax[e].text(-240, -245, t[e], rotation = 0, fontsize = fontText)
            plt.tight_layout()
            plt.xlim(-255, 255)
            plt.ylim(-260, 260)
        #fig.colorbar(im, ax=ax.ravel().tolist(), cmap=plt.get_cmap('tab10'), orientation='horizontal', shrink = 0.25, spacing = 'proportional', drawedges = True)
        fig.subplots_adjust(right=0.9)
        cbar_ax = fig.add_axes([0.91, 0.1, 0.01, 0.84])
        fig.colorbar(im, cax=cbar_ax, cmap=plt.get_cmap('tab10'), orientation='vertical')
        #5
        ellipse = Ellipse(xy=(-60.0, -25.0), width=250.0, height=250.0, angle=  0.0, edgecolor='k', fc='None', lw=2, ls='--')
        ax[0].add_patch(ellipse)
        ellipse = Ellipse(xy=( 35.0, -15.0), width=210.0, height=210.0, angle=  0.0,  edgecolor='k', fc='None', lw=2, ls='--')
        ax[1].add_patch(ellipse)
        ellipse = Ellipse(xy=(-90.0, -10.0), width=125.0, height=125.0, angle=  0.0,  edgecolor='k', fc='None', lw=2, ls='--')
        ax[2].add_patch(ellipse)
        #
    plt.savefig('{}/{}'.format(results_folder, results_file+'.png'), bbox_inches='tight')
    #plt.savefig('{}/{}'.format(results_folder, results_file+'.svg'), format="svg", bbox_inches='tight')
    pdf.savefig(bbox_inches='tight')
    pdf.close()
    print('Plot done!')

if __name__ == '__main__':
    args = get_args()
    main(args)


#!/usr/bin/python

import win32con
import win32api
import win32security

import wmi
import sys
import os

def log_to_file( message ):
	fd = open("process_monitor_log.csv", "a+")
	fd.write(message + "\n")
	fd.close()
	return

def get_process_privileges(pid):
	try:
		#OBTAIN A HANDLE TO THE TARGET
		hproc = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION,False,pid)

		#OPRN THE MAIN PROCESS TOKEN
		htok = win32security.OpenProcessToken(hproc,win32con.TOKEN_QUERY)

		#RETRIEVE THE LIST OF PRIVLEGES ENABLED
		privs = win32security.GetTokenInformation(htok,win32security.TokenPrivileges)

		#ITERATE OVER PRIVELEGES AND OUTPUT THE ENABLED
		priv_list = ""
		for i in privs:
			#CHECK IF PRIVILEGE IS ENABLED
			if i[1] == 3:
				priv_list += "%s|" % win32security.LookupPrivilegeName(None,i[0])

	except:
		priv_list = "N/A"
	return priv_list

#CREATE A LOG FILE HEADER
header = "Time,User,Executable,CommandLine,PID,Parent PID,Privileges"
log_to_file(header)

#INSTANTIATE THE WMI INTERFACE
c = wmi.WMI()

#CREATE OUR PROCESS MONITOR
process_watcher = c.Win32_Process.watch_for("creation")

while True:
	try:
		new_process = process_watcher()
		proc_owner = new_process.GetOwner()
		proc_owner = "%s\\%s" % (proc_owner[0],proc_owner[2])
		create_date = new_process.CreationDate
		executable = new_process.ExecutablePath
		cmdline = new_process.CommandLine
		pid = new_process.ProcessId
		parent_pid = new_process.ParentProcessId
		privileges = get_process_privileges(pid)
		process_log_message = "%s,%s,%s,%s,%s,%s,%s" % (create_date,proc_owner,executable,cmdline,pid,parent_pid,privileges)

		print (process_log_message)

		log_to_file(process_log_message)
	#EXIT OUT ON KEYBOARD CTRL-C
	except KeyboardInterrupt:
		print ("Keyboard Exit Command - Exiting...")
		sys.exit()
	#CATCH ALL OTHER ERRORS
	except:
		#PRINT ERRORS
		#pass

		e = sys.exc_info()
		print(e)


from django.shortcuts import render, redirect

def list_names(request):
    # either take existing names or empty list
    if 'names' in request.session:
        names = request.session['names']
    else:
        names = []

    if request.method == "POST":
        # add name to list
        fullname = request.POST['fullname']
        names.append(fullname)
        # create key names in the session
        request.session['names'] = names

    return render(request, 'demo/session_names.html',
                  {'names': names , 'length' : len(names)})