#  This file is part of RADAR.
#  Copyright (C) 2019 Cole Daubenspeck
#
#  RADAR is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  RADAR is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with RADAR.  If not, see <https://www.gnu.org/licenses/>.

import re

from typing import Tuple

# used if anything wants to sort/use numeric values instead of strings
VALUE_STANDARD_CONVERSIONS = {
    "unknown": 0,
    "low": 1,
    "medium": 2,
    "high": 3,
    "very-high": 4
}

# format
#   device_type_string
#       - list of services (if it might not appear in consistent ports) (THESE ARE REGULAR EXPRESSIONS!)
#       - list of ports that correspond to hosts of that type (prefered unless service tends to run on different ports)
HOST_TYPE = {
    "webserver": {
        "value": "high",
        "service_names": [
            "werkzeug",
            "httpd",
            "nginx",
            "apache"
        ],
        "ports": [
            80,
            443,
            3000,  # node webserver
            8000,
            8443
        ]
    },
    "database": {
        "value": "very-high",
        "service_names": [
        ],
        "ports": [
            1433,  # mssql
            3306,  # mysql
            6379,  # redis
            27017, # mongo

        ]
    },
    "fileserver": {
        "value": "high",
        "service_names": [
        ],
        "ports": [
            21,
            990
        ]
    },
    "mailserver": {
        "value": "medium",
        "service_names": [
        ],
        "ports": [
            25,  # smtp ports
            468,
            587,
            2525,
            110,  # pop3 ports
            993,
            143,  # imap ports
            995
        ]
    },
    "ics": {  # industrial control system
        "value": "very-high",
        "service_names": [
            "modbus"
        ],
        "ports": [
            502
        ]
    },
    "domain_controller": {
        "value": "very-high",
        "service_names": [
        ],
        "ports": [
            88  # kerberos
        ]
    }
}

def get_info(target: dict) -> Tuple[str, str]:
    """ For a given target, returns information about the priority and best-guess type of host

    arguments:
    target: a dictionary that conforms to RADAR target specifications

    returns:
    a tuple of strings (priority, type). First string is the value of the device (e.g. "high"), second is the type of device (e.g. "webserver").
    Multiple device types will be seperated with a semicolon (e.g. 'webserver;database').
    """
    services = target.get("services")

    if not services:  # no running services, we don't care
        return "unknown", "generic"

    device_value = "unknown"
    device_type = ""

    global HOST_TYPE
    global VALUE_STANDARD_CONVERSIONS
    # for every service on the target
    for service in services:
        port = int(service.get("port"))
        name = service.get("service")
        # check if any of the host types matches the target...
        for host_type, details in HOST_TYPE.items():
            # skip checking the type if it's already flagged (e.g. it has multiple services related to being a webserver)
            if host_type in device_type:
                continue
            type_value = details.get("value")
            # by seeing if the port is in one of the lists
            if port in details.get("ports"):
                device_value = device_value if VALUE_STANDARD_CONVERSIONS[type_value] < VALUE_STANDARD_CONVERSIONS[device_value] else type_value
                device_type += f";{host_type}"
            # or by seeing if any of the patterns matches
            else:
                for check_names in details.get("service_names", []):
                    if re.search(check_names, name):
                        device_value = device_value if VALUE_STANDARD_CONVERSIONS[type_value] < VALUE_STANDARD_CONVERSIONS[device_value] else type_value
                        device_type += f";{host_type}"
                        break

    return device_value, device_type[1:] or "unknown"


#! /bin/python

import os
import sys
import json

import luigi
import nifty.tools as nt

import elf.skeleton.io as skelio
from elf.skeleton import skeletonize as skel_impl, get_method_names

import cluster_tools.utils.volume_utils as vu
import cluster_tools.utils.function_utils as fu
from cluster_tools.cluster_tasks import SlurmTask, LocalTask, LSFTask
from cluster_tools.utils.task_utils import DummyTask

#
# skeletonize tasks
#

class SkeletonizeBase(luigi.Task):
    """ Skeletonize base class
    """

    task_name = 'skeletonize'
    src_file = os.path.abspath(__file__)
    allow_retry = False

    # input and output volumes
    input_path = luigi.Parameter()
    input_key = luigi.Parameter()
    morphology_path = luigi.Parameter()
    morphology_key = luigi.Parameter()
    output_path = luigi.Parameter()
    output_key = luigi.Parameter()
    number_of_labels = luigi.IntParameter()
    resolution = luigi.ListParameter()
    size_threshold = luigi.IntParameter(default=None)
    method = luigi.Parameter(default='thinning')
    dependency = luigi.TaskParameter(default=DummyTask())

    methods = get_method_names()

    # expose skeletonization parameter if we support more parameter
    @staticmethod
    def default_task_config():
        # we use this to get also get the common default config
        config = LocalTask.default_task_config()
        config.update({'chunk_len': 1000, 'method_kwargs': {}})
        return config

    def requires(self):
        return self.dependency

    def _prepare_output(self, config):
        # make the blocking
        block_len = min(self.number_of_labels, config.get('chunk_len', 1000))
        block_list = vu.blocks_in_volume((self.number_of_labels,),
                                         (block_len,))
        n_jobs = min(len(block_list), self.max_jobs)
        # require output dataset
        with vu.file_reader(self.output_path) as f:
            f.require_dataset(self.output_key, shape=(self.number_of_labels,),
                              chunks=(1,), compression='gzip', dtype='uint64')
        # update the config
        config.update({'number_of_labels': self.number_of_labels,
                       'block_len': block_len})
        return config, n_jobs, block_list

    def run_impl(self):
        assert self.method in self.methods,\
            "Method %s is not supported, must be one of %s" % (self.method, str(self.methods))

        # TODO support roi
        # get the global config and init configs
        shebang, block_shape, _, _ = self.global_config_values()
        self.init(shebang)

        # load the skeletonize config
        # update the config with input and output paths and keys
        config = self.get_task_config()
        config.update({'input_path': self.input_path, 'input_key': self.input_key,
                       'morphology_path': self.morphology_path,
                       'morphology_key': self.morphology_key,
                       'output_path': self.output_path, 'output_key': self.output_key,
                       'resolution': self.resolution, 'size_threshold': self.size_threshold,
                       'method': self.method})
        config, n_jobs, block_list = self._prepare_output(config)

        # prime and run the jobs
        self.prepare_jobs(n_jobs, block_list, config)
        self.submit_jobs(n_jobs)

        # wait till jobs finish and check for job success
        self.wait_for_jobs()
        self.check_jobs(n_jobs)

class SkeletonizeLocal(SkeletonizeBase, LocalTask):
    """
    skeletonize on local machine
    """
    pass

class SkeletonizeSlurm(SkeletonizeBase, SlurmTask):
    """
    skeletonize on slurm cluster
    """
    pass

class SkeletonizeLSF(SkeletonizeBase, LSFTask):
    """
    skeletonize on lsf cluster
    """
    pass

#
# Implementation
#

# not parallelized for now
def _skeletonize_id_block(blocking, block_id, ds_in, ds_out,
                          sizes, bb_min, bb_max, resolution, size_threshold,
                          method):

    fu.log("start processing block %i" % block_id)
    block = blocking.getBlock(block_id)
    id_begin, id_end = block.begin[0], block.end[0]
    # we don't compute the skeleton for id 0, which is reserved for the ignore label
    id_begin = 1 if id_begin == 0 else id_begin

    # we increase the bounding box with a small halo, otherwise there
    # semms to be boundary inconsistencies
    halo = (2, 2, 2)
    shape = ds_in.shape

    # skeletonize ids in range and serialize skeletons
    for seg_id in range(id_begin, id_end):
        if size_threshold is not None:
            if sizes[seg_id] < size_threshold:
                continue
        bb = tuple(slice(max(int(mi - ha), 0),
                         min(int(ma + ha), sh)) for mi, ma, sh, ha in zip(bb_min[seg_id],
                                                                          bb_max[seg_id],
                                                                          shape, halo))
        fu.log("skeletonize id %i from bb %s" % (seg_id, str(bb)))
        obj = ds_in[bb] == seg_id

        # try to skeletonize the object, skip if any exception is thrown
        try:
            nodes, edges = skel_impl(obj, resolution=resolution, method=method)
        except Exception:
            continue

        offsets = [b.start * res for b, res in zip(bb, resolution)]
        skelio.write_n5(ds_out, seg_id, nodes, edges, offsets)
    fu.log_block_success(block_id)

def skeletonize(job_id, config_path):
    fu.log("start processing job %i" % job_id)
    fu.log("reading config from %s" % config_path)
    with open(config_path, 'r') as f:
        config = json.load(f)

    # read the input cofig
    input_path = config['input_path']
    input_key = config['input_key']
    morphology_path = config['morphology_path']
    morphology_key = config['morphology_key']
    output_path = config['output_path']
    output_key = config['output_key']
    size_threshold = config['size_threshold']
    resolution = config['resolution']
    method = config['method']

    # morphology feature-columns
    # 0    = label-id
    # 1    = pixel size
    # 2:5  = center of mass
    # 5:8  = min coordinate
    # 8:11 = max coordinate
    with vu.file_reader(morphology_path) as f:
        morpho = f[morphology_key][:]
        sizes = morpho[:, 1].astype('uint64')
        bb_min = morpho[:, 5:8].astype('uint64')
        bb_max = morpho[:, 8:11].astype('uint64') + 1

    block_list = config['block_list']
    block_len = config['block_len']
    n_labels = config['number_of_labels']
    blocking = nt.blocking([0], [n_labels], [block_len])

    # skeletonize this id block
    with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out:
        ds_in = f_in[input_key]
        ds_out = f_out[output_key]
        for block_id in block_list:
            _skeletonize_id_block(blocking, block_id, ds_in, ds_out,
                                  sizes, bb_min, bb_max, resolution, size_threshold,
                                  method)

    # log success
    fu.log_job_success(job_id)

if __name__ == '__main__':
    path = sys.argv[1]
    assert os.path.exists(path), path
    job_id = int(os.path.split(path)[1].split('.')[0].split('_')[-1])
    skeletonize(job_id, path)


from utils import pickle_load
from matplotlib import cm
import matplotlib.pyplot as plt
import collections

def show_results(res_paths):
    results = {}
    for path in res_paths:
        result = pickle_load(path)
        for k, v in result.items():
            if k not in results.keys():
                results[k] = result[k]
    results = collections.OrderedDict(sorted(results.items()))
    fig, ax = plt.subplots(figsize=(9, 5.5))
    colors = cm.Dark2(np.linspace(0, 1, len(results)))
    count = 0
    for k, res in results.items():
        mean, std = np.nanmean(res, axis=0), np.nanstd(res, axis=0)
        # ax.errorbar(np.arange(mean.shape[0]), mean, yerr=std, color=colors[count], label=k, fmt='-o')
        plt.plot(np.arange(mean.shape[0]) + 1, mean, '-o', color=colors[count], label=k)
        count += 1
        print(np.array_str(mean[8:], precision=3))
        print("Average precision of %s for future prediction: %f" % (k, mean[8:].mean()))

    # Now add the legend with some customizations.
    legend = ax.legend(loc='upper right')

    ax.set_xlabel("time step")
    ax.set_ylabel("average precision")

    plt.axvline(x=8.5, color='r', linestyle='--')
    plt.text(3, 0.1, 'tracking', fontsize=18, color='grey')
    plt.text(11, 0.1, 'prediction', fontsize=18, color='grey')

    plt.show()

def show_best(filename, metric, k=1):

    def line_to_list(line):
        exclude_next_line = lambda x: x[:-1] if x.endswith('\n') else x
        entries = map(exclude_next_line, line.split(','))
        return entries

    items = []

    def print_dict(dic, attrs=None):

        if attrs is None:
            attrs = ['omega', 'noise_var', 'extent', metric, metric + ' mean']
            if 'keep_motion' in dic and dic['keep_motion']:
                attrs += ['window_size', 'initial_motion_factor', 'keep_motion_factor']
            if 'blur_spatially' in dic and dic['blur_spatially']:
                attrs += ['blur_extent', 'blur_var']

        for k, v in dic.items():
            if attrs is not None and k not in attrs:
                continue
            print("{}: {}".format(k, v))

    with open(filename, 'r') as f:
        line = f.readline()
        #print(line)
        attrs = line_to_list(line)

        for i, line in enumerate(f):
            #print(line)
            values = line_to_list(line)
            #print(values)
            dict_ = {k: v for (k, v) in zip(attrs, values)}
            items.append(dict_)
    #print(items[0])

    items = sorted(items, key=lambda item: item[metric + ' mean'])
    if metric == 'f1_score' or metric == 'average_precision':
        items = items[::-1]

    for i in range(k):
        print("------- {}th best ------- ".format(i+1))
        print_dict(items[i])


import pandas as pd
import numpy as np
import io
import requests
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn import metrics
from beta_encoder import BetaEncoder
import category_encoders as ce
from utils import *
import csv
import xgboost as xgb

def run_ls_experiments():

    print("Loading Data")
    df = load_data()

    continuous = ['company_size', 'interested_desks']
    categorical = ['industry','location', 'lead_source']
    #columns:
    print("continuous columns: ",continuous)
    print("categorical columns: ",categorical)

    # plot increasing dimensionality va computation time
    sample_sizes = [2000,
    4000,
    6000,
    8000,
    10000,
    12000,
    14000,
    16000,
    18000,
    20000,
    22000,
    24000,
    26000,
    28000,
    30000,
    32000,
    34000,
    36000,
    38000,
    40000,
    42000,
    44000,
    46000,
    48000,
    50000]
    sample_sizes = [42000,
    44000,
    46000,
    48000,
    50000]

    results = [['model','Encoder','Accuracy','STD','Training Time','Sparsity','Dimensions','sample_size']]
    for sample_size in sample_sizes:

        print("")
        print("----------------------")
        print("Sample Size: ",sample_size)
        print("----------------------")

        if not sample_size < len(df):
            sample_size = len(df)
        sample = df.sample(sample_size)

        X = sample[continuous+categorical]
        y = sample[['converted']]

        successes = y.sum()[0]
        alpha_prior = float(successes / len(y))

        model = xgb.XGBClassifier(n_jobs=4) #[GradientBoostingClassifier(max_depth=8, n_estimators=64)]

        #BetaEncoder (mean)
        print("Beta Encoder (mean) Results:")
        acc, std, time, sparsity, dimensions = cv_lead_scoring_classification(model, X, y, continuous, categorical, encoder=BetaEncoder(alpha=alpha_prior, beta=1-alpha_prior))
        results.append([type(model), 'BetaEncoder (m)', acc, std, time, sparsity, dimensions,sample_size])

        #OneHotEncoder
        print("OneHotEncoder Results:")
        acc, std, time, sparsity, dimensions = cv_lead_scoring_classification(model, X, y, continuous, categorical, encoder=OneHotEncoder(handle_unknown='ignore', sparse=False))
        results.append([type(model), 'OneHotEncoder', acc, std, time, sparsity, dimensions,sample_size])

    file = 'lead_scoring_experiments_comp_time_2.csv'
    with open(file, "w") as output:
        writer = csv.writer(output, lineterminator='\n')
        writer.writerows(results)
    try:
        upload_file(file)
    except:
        print("File Not Uploaded")

def load_data():
    df = pd.read_csv('lead_scoring_1mil.csv')
    df = df.fillna('null')
    industries = df.industry.str.split(',', n=-1, expand=True)
    df['industry'] = industries[0]
    #training_df['sector'] = industries[1]

    return df

if __name__ == '__main__':
    run_ls_experiments()


import math
import boto3
from aws_cdk import (
    core,
    aws_ec2 as ec2,
    aws_ecs as ecs,
    aws_cloudwatch as cw
    )

from cdklocust.locust_container import locustContainer

class CdklocustStack(core.Stack):

    def __init__(self, scope: core.Construct, id: str, vpc, **kwargs) -> None:
        super().__init__(scope, id, **kwargs)

        self.get_cdk_context()
        self.vpc=vpc

        #ECS cluster for the loadgen
        self.loadgen_cluster = ecs.Cluster(
            self, "Loadgen-Cluster",
            vpc=self.vpc
        )

        #Just using base ENI count, not caring about having ENI trunking turned on
        client = boto3.client('ec2')
        response = client.describe_instance_types(InstanceTypes=[self.ecs_instance_type])
        eni_per_instance = response['InstanceTypes'][0]['NetworkInfo']['MaximumNetworkInterfaces']

        number_of_instances = math.ceil((self.number_of_workers + 1) / (eni_per_instance-1))
        self.loadgen_cluster.add_capacity("AsgSpot",
                                          max_capacity=number_of_instances * 2,
                                          min_capacity=number_of_instances,
                                          instance_type=ec2.InstanceType(self.ecs_instance_type),
                                          spot_price="0.07",
                                          spot_instance_draining=True
                                         )
        #cloudmap for service discovery so workers can lookup mast via dns
        self.loadgen_cluster.add_default_cloud_map_namespace(name = self.cloudmap_namespace)

        #Create a graph widget to track reservation metrics for our cluster
        ecs_widget = cw.GraphWidget(
            left=[self.loadgen_cluster.metric_cpu_reservation()],
            right=[self.loadgen_cluster.metric_memory_reservation()],
            title="ECS - CPU and Memory Reservation",
            )

        #CloudWatch dashboard to monitor our stuff
        self.dashboard = cw.Dashboard(self, "Locustdashboard")
        self.dashboard.add_widgets(ecs_widget)

        if not self.distributed_locust:
            role = "standalone"
            locustContainer(self, "locust" + role, self.vpc, self.loadgen_cluster, role, self.target_url)
        else:
            role = "master"
            master_construct = locustContainer(self, "locust" + role, self.vpc,
                                               self.loadgen_cluster, role, self.target_url)

            lb_widget = cw.GraphWidget(
                left=[master_construct.lb.metric_active_connection_count(),
                      master_construct.lb.metric_target_response_time()],
                right=[master_construct.lb.metric_request_count()],
                title="Load Balancer")

            self.dashboard.add_widgets(lb_widget)

            role = "worker"
            worker_construct = locustContainer(self, "locust" + role, self.vpc,
                                               self.loadgen_cluster, role, self.target_url,
                                               self.number_of_workers)
            worker_construct.node.add_dependency(master_construct)

    def get_cdk_context(self):
    # grab stuff from context

        self.number_of_workers = int(self.node.try_get_context("number_of_workers"))
        self.ecs_instance_type = self.node.try_get_context("ecs_instance_type")
        self.vpc_cidr = self.node.try_get_context("vpc_cidr")
        self.distributed_locust = self.node.try_get_context("distributed_locust")
        self.cloudmap_namespace = self.node.try_get_context("cloudmap_namespace")
        self.target_url = self.node.try_get_context("target_url")


from sklearn.datasets import load_boston

from  sklearn.model_selection import train_test_split           # model_selection模型选择过程中各种数据分割的类与函数

from sklearn.preprocessing import StandardScaler

from sklearn.linear_model import LinearRegression, SGDRegressor, LogisticRegression           # 线性回归
# externals是外部的、外部扩展的意思
from sklearn.externals import joblib             # 模型的保存和提取  ==》  https://blog.csdn.net/YZXnuaa/article/details/80694372

from sklearn.metrics import mean_squared_error, classification_report    # 均方误差的评估
import pandas as pd
import numpy as np

# 通过线性回归预测房价
# （数据量小的时候，可以用linearRegression，也就是最小二乘法之正规方程，可以直接求得误差最小点，但是计算量特别大，数据多时不推荐使用）
# 这个也就是高中学的线性方程的求解k和b的值类似的方法
def mylinear():
    # 获取数据
    lb = load_boston()
    # 特征数值化

    # 分割数据集
    x_train, x_test, y_train, y_test = train_test_split(lb.data, lb.target, test_size=0.25)

    # 归一化处理
    """因为特征值和目标值对应的数组维度不一样， 所以无法用同一个转换器进行归一化，而是分别创造一个转换器"""
    std_x = StandardScaler()
    x_train = std_x.fit_transform(x_train)
    x_test = std_x.transform(x_test)
    # 对于目标的
    std_y = StandardScaler()
    y_train = std_y.fit_transform(y_train.reshape(-1, 1))
    y_test = std_y.transform(y_test.reshape(-1, 1))

    # 建立模型
    lr = LinearRegression()
    lr.fit(x_train, y_train)
    print(lr.coef_)

    # 模型保存，可以在别的地方直接加载出来这次保存好的模型用来继续学习或者预测使用
    joblib.dump(lr, './通过joblib保存线性回归模型.pkl')

    # 预测结果评估
    y_predict = lr.predict(x_test)
    # 线性回归模型的好坏通过均方误差来评测
    print("产生的均方误差为：", mean_squared_error(y_test, y_predict))
    # 将数据变回归一化之前的大小
    y_predict_orign = std_y.inverse_transform(y_predict)
    print(y_predict, "==================", y_predict_orign)    # ==》产生的均方误差为： 0.42782165417388635

# 用梯度下降来预测房价,在数据量大的时候推荐使用这个
def sgd_regression():
    # 下面这些是直接从上面的函数中copy下来的
    # 获取数据
    lb = load_boston()
    # 特征数值化
    # 分割数据集
    x_train, x_test, y_train, y_test = train_test_split(lb.data, lb.target, test_size=0.25)
    # 归一化处理
    """因为特征值和目标值对应的数组维度不一样， 所以无法用同一个转换器进行归一化，而是分别创造一个转换器"""
    std_x = StandardScaler()
    x_train = std_x.fit_transform(x_train)
    x_test = std_x.transform(x_test)
    # 对于目标的
    std_y = StandardScaler()
    y_train = std_y.fit_transform(y_train.reshape(-1, 1))
    y_test = std_y.transform(y_test.reshape(-1, 1))

    # 建立模型(基于随机梯度下降法估计参数的SGDRegressor)
    sgd = SGDRegressor()

    # 预测并评估
    sgd.fit(x_train, y_train)
    y_predict = sgd.predict(x_test)
    print(mean_squared_error(y_test, y_predict))  # ==》产生的均方误差为： 0.21901470588593194

# 岭回归：也就是逻辑回归
# 只能用于解决二分类问题，属于判别模型（与前面所学的生成模型相区别，生成模型是需要从历史中总结出某些概率）
# 但我感觉对于判别模型和生成模型还是不太对

# 岭回归解觉二分类问题的思路 ==》 https://blog.csdn.net/weixin_39445556/article/details/83930186
# 先生成回归曲线，再从y轴找一个合理的值设置为阈值（临界值），求解的y值大于这个阈值的为一个结果，y小于这个阈值是另一个结果
# 阈值的选择可以根据二分类中的一类占总数量的比例
def LingHuiGui():
    # 对于癌症的预测
    # 读取数据
    column = ['Sample code number','Clump Thickness', 'Uniformity of Cell Size','Uniformity of Cell Shape','Marginal Adhesion', 'Single Epithelial Cell Size','Bare Nuclei','Bland Chromatin','Normal Nucleoli','Mitoses','Class']
    data = pd.read_csv(r"E:\chrome下载的东西\breast-cancer-wisconsin.data", names=column)   # 这个指定标签名的方式一定要学一学
    print(data)

    # 缺失值处理
    data = data.replace("?", np.nan)
    data = data.dropna()

    # 数据分离                                            # data中的column是前面第89行的那个标签名列表
    x_train, x_test, y_train, y_test = train_test_split(data[column[1:10]], data[column[10]], test_size=0.25)

    # 数据标准化（归一化）
    std_x = StandardScaler()
    x_train = std_x.fit_transform(x_train)
    x_test = std_x.transform(x_test)

    # 这里用的逻辑回归是个二分类方法，是解决二分类问题的
    # std_y = StandardScaler()
    # y_train = std_y.fit_transform(y_train)
    # y_test = std_y.transform(y_test)

    # 建立模型
    lg = LogisticRegression(C=1.0)  # c为正则化参数（正则化用于解决过拟合问题，可以减少模型的复杂度）
    lg.fit(x_train, y_train)
    y_predict = lg.predict(x_test)

    # 误差评估
    print("精确率：", lg.score(x_test, y_test))
    # 将结果为2、4两个数值换为名字良性和恶性
    print("召回率", classification_report(y_test, y_predict, labels=[2, 4], target_names=["良性", "恶性"]))

print("加油！")
# mylinear()
# sgd_regression()
LingHuiGui()

机器学习/19_线性回归.py


from __future__ import annotations

import dataclasses
from typing import TYPE_CHECKING

from randovania.bitpacking import bitpacking
from randovania.bitpacking.bitpacking import BitPackDecoder, BitPackValue
from randovania.game_description import default_database

if TYPE_CHECKING:
    from collections.abc import Iterator

    from randovania.game_description.pickup.ammo_pickup import AmmoPickupDefinition

@dataclasses.dataclass(frozen=True)
class AmmoPickupState(BitPackValue):
    ammo_count: tuple[int, ...] = (0,)
    pickup_count: int = 0
    requires_main_item: bool = True

    def check_consistency(self, ammo: AmmoPickupDefinition):
        db = default_database.resource_database_for(ammo.game)

        if len(self.ammo_count) != len(ammo.items):
            raise ValueError(f"Ammo state has {len(self.ammo_count)} ammo counts, expected {len(ammo.items)}")

        for count, ammo_name in zip(self.ammo_count, ammo.items):
            ammo_item = db.get_item(ammo_name)
            minimum_count = -ammo_item.max_capacity if ammo.allows_negative else 0
            if not (minimum_count <= count <= ammo_item.max_capacity):
                raise ValueError(
                    f"Ammo count for item {ammo_name} of value {count} is not "
                    f"in range [{minimum_count}, {ammo_item.max_capacity}]."
                )

        if self.pickup_count < 0:
            raise ValueError(f"Pickup count must be at least 0, got {self.pickup_count}")

    def bit_pack_encode(self, metadata) -> Iterator[tuple[int, int]]:
        ammo: AmmoPickupDefinition = metadata["ammo"]
        db = default_database.resource_database_for(ammo.game)

        for count, ammo_name in zip(self.ammo_count, ammo.items):
            ammo_item = db.get_item(ammo_name)
            yield from bitpacking.encode_int_with_limits(
                abs(count),
                (ammo_item.max_capacity // 2, ammo_item.max_capacity + 1),
            )
            if ammo.allows_negative:
                yield from bitpacking.encode_bool(count < 0)  # Negative?

        yield from bitpacking.encode_big_int(self.pickup_count)
        if ammo.unlocked_by is not None:
            yield from bitpacking.encode_bool(self.requires_main_item)

    @classmethod
    def bit_pack_unpack(cls, decoder: BitPackDecoder, metadata) -> AmmoPickupState:
        ammo: AmmoPickupDefinition = metadata["ammo"]
        db = default_database.resource_database_for(ammo.game)

        # Ammo Count
        ammo_count = []
        for ammo_name in ammo.items:
            ammo_item = db.get_item(ammo_name)
            count = bitpacking.decode_int_with_limits(
                decoder,
                (ammo_item.max_capacity // 2, ammo_item.max_capacity + 1),
            )
            if ammo.allows_negative and bitpacking.decode_bool(decoder):  # Negative?
                count *= -1
            ammo_count.append(count)

        # Pickup Count
        pickup_count = bitpacking.decode_big_int(decoder)

        # Require Main Item
        requires_main_item = True
        if ammo.unlocked_by is not None:
            requires_main_item = bitpacking.decode_bool(decoder)

        return cls(
            ammo_count=tuple(ammo_count),
            pickup_count=pickup_count,
            requires_main_item=requires_main_item,
        )

    @property
    def as_json(self) -> dict:
        result: dict = {}

        for field in dataclasses.fields(self):
            value = getattr(self, field.name)
            result[field.name] = value

        result["ammo_count"] = list(result["ammo_count"])

        return result

    @classmethod
    def from_json(cls, value: dict) -> AmmoPickupState:
        kwargs = {}

        for field in dataclasses.fields(cls):
            if field.name in value:
                kwargs[field.name] = value[field.name]

        if "ammo_count" in kwargs:
            kwargs["ammo_count"] = tuple(kwargs["ammo_count"])

        return cls(**kwargs)


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from .routers import post, user, auth, vote

############################################

#models.Base.metadata.create_all(bind=engine)

app = FastAPI()

origins = ["*"]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

app.include_router(post.router)
app.include_router(user.router)
app.include_router(auth.router)
app.include_router(vote.router)

@app.get("/")
def root():
    return {"message": "Hello World"}

"""
my_posts = [{"title": "title of post 1", "content": "content of post 1", "id":1},
            {"title": "favorite foods", "content": "I like pizza", "id":2}]

def find_post(id):
    for p in my_posts:
        if p["id"] == id:
            return p

def find_index_post(id):
    for i, p in enumerate(my_posts):
        if p['id'] == id:
            return i
"""

"""
#Path Operations
@app.get("/")
#IT IS THE DECORATOR: IT CREATES THE ENDPOINT OF THE FUNCTION. It is called on the already created FastAPI INSTANCE (app)
#Within the brackets we have the PATH, the path that we must access from the URL
#the get method: is one of the possible HTTP methods

def root():

    1): Async it is optional (so we can delete it)
    2): the name of the fun should be as much descriptive as possible
    3)RETURN:  is the message that is returned back to the user

    return {"message": "Hello World"}
"""


# USAGE
# python knn.py --dataset ../../SolutionDL4CV/SB_code/datasets/animals
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from pyimagesearch.preprocessing import SimplePreprocessor
from pyimagesearch.datasets import SimpleDatasetLoader
from imutils import paths
import argparse

ap = argparse.ArgumentParser()
ap.add_argument('-d', '--dataset', required=True, help='path to input dataset')
ap.add_argument('-k', '--neighbors', type=int, default=1, help='# of nearest neighbors for classification')
ap.add_argument('-j', '--jobs', type=int, default=-1, help='# of jobs for k-NN distance (-1 uses all available cores)')
args = vars(ap.parse_args())

print('[INFO] loading images...')
img_paths = list(paths.list_images(args['dataset']))

sp = SimplePreprocessor(32, 32)
sdl = SimpleDatasetLoader(preprocessors=[sp])
data, labels = sdl.load(img_paths, verbose=500)
data = data.reshape((len(data), -1))

print(f'[INFO] feature matrix: {data.nbytes/(1024*1000.0):.1f}MB')

# Encode the labels as integer
le = LabelEncoder()
labels = le.fit_transform(labels)

# Split to train and test set
X_train, X_test, y_train, y_test = train_test_split(data, labels, test_size=0.25, random_state=42)

print("[INFO] evaluating k-NN classifier...")
model = KNeighborsClassifier(n_neighbors=args['neighbors'], n_jobs=args['jobs'])
model.fit(X_train, y_train)
print(classification_report(y_test, model.predict(X_test),  target_names=le.classes_))


import numpy as np
import matplotlib.pyplot as plt
import activation_functions as acfunc

inp_a = np.arange(-1.0, 1.0, 0.2)
inp_b = np.arange(-1.0, 1.0, 0.2)

outputs = np.zeros((10, 10))

weight_a = 2.5
weight_b = 3

bias = 0.1

for i in range(10):
    for j in range(10):
        u_single = inp_a[i] * weight_a + inp_b[j] * weight_b + bias
        outputs[i][j] = acfunc.sigmoid_func(u_single)

plt.imshow(outputs, "gray", vmin=0.0, vmax=1.0)
plt.colorbar()
plt.show()


import os
import random
import math
import seaborn
import matplotlib.pyplot as plt

num_train_samples = 1
threshold = 0.25
dtw_window = 50

# thresholds: 0.15, 0.2, ...

def read_gesture(path):
    with open(path, "r") as file:
        lines = [line.rstrip() for line in file]
        gesture = [[float(value) for value in data.split(',')] for data in lines]
        return gesture

labels = ['circle_ccw', 'circle_cw', 'heart_cw', 'square_ccw', 'triangle_cw', 'junk']

paths = os.listdir('gestures')
circle_ccw = [('circle_ccw', read_gesture('gestures/' + path)) for path in paths if path.startswith('circle_ccw')]
circle_cw = [('circle_cw', read_gesture('gestures/' + path)) for path in paths if path.startswith('circle_cw')]
heart_cw = [('heart_cw', read_gesture('gestures/' + path)) for path in paths if path.startswith('heart_cw')]
square_ccw = [('square_ccw', read_gesture('gestures/' + path)) for path in paths if path.startswith('square_ccw')]
triangle_cw = [('triangle_cw', read_gesture('gestures/' + path)) for path in paths if path.startswith('triangle_cw')]
junk = [('junk', read_gesture('gestures/' + path)) for path in paths if path.startswith('junk')]

def fir_lowpass_first(a):
    q = 0.95
    b = [a[0]]
    for i in range(1, len(a)):
        x = (1.0 - q) * a[i - 1][0] + q * a[i][0]
        y = (1.0 - q) * a[i - 1][1] + q * a[i][1]
        z = (1.0 - q) * a[i - 1][2] + q * a[i][2]
        b.append([x, y, z])
    return b

def calc_distance(a, b) -> float:
    ax = a[0]
    ay = a[1]
    az = a[2]
    bx = b[0]
    by = b[1]
    bz = b[2]
    dir = (ax * bx + ay * by + az * bz) / (normalize(ax, ay, az) * normalize(bx, by, bz) + 0.0000001)

    return (1.0 - 0.5 * dir) * normalize(ax - bx, ay - by, az - bz)

def normalize(x, y, z) -> float:
    return math.sqrt(x * x + y * y + z * z)

def calc_dtw(a, b) -> float:
    a = fir_lowpass_first(a)
    b = fir_lowpass_first(b)

    dtw = [[0.0 for _ in range(50)] for _ in range(50)]
    dtw[0][0] = calc_distance(a[0], b[0])

    for i in range(1, 50):
        dtw[i][0] = calc_distance(a[i], b[0]) + dtw[i - 1][0]
        dtw[0][i] = calc_distance(a[0], b[i]) + dtw[0][i - 1]

    for i in range(1, 50):
        for j in range(1, 50):
            dtw[i][j] = calc_distance(a[i], b[j]) + min(dtw[i - 1][j], dtw[i][j - 1], dtw[i - 1][j - 1])

    i = 49
    j = 49
    distance = [0.0 for _ in range(100)]
    length = 0

    while i > 0 and j > 0:
        if dtw[i - 1][j] <= dtw[i][j - 1] and dtw[i - 1][j] <= dtw[i - 1][j - 1] and (j - i) <= dtw_window:
            distance[length] = dtw[i][j] - dtw[i - 1][j]
            i -= 1
        elif dtw[i][j - 1] < dtw[i - 1][j - 1] and (i - j) <= dtw_window:
            distance[length] = dtw[i][j] - dtw[i][j - 1]
            j -= 1
        else:
            distance[length] = dtw[i][j] - dtw[i - 1][j - 1]
            i -= 1
            j -= 1
        length += 1

    while i > 0:
        distance[length] = dtw[i][0] - dtw[i - 1][0]
        i -= 1
        length += 1

    while j > 0:
        distance[length] = dtw[0][j] - dtw[0][j - 1]
        j -= 1
        length += 1

    distance[length] = dtw[0][0]
    length += 1

    mean = 0.0

    for i in range(length):
        mean += distance[i]

    mean = mean / float(length)

    return mean

confusion_matrix = {}
num_trails = {}

for true_label in labels:
    confusion_matrix[true_label] = {}
    num_trails[true_label] = 0.0

    for predicted_label in labels:
        confusion_matrix[true_label][predicted_label] = 0.0

for _ in range(25):
    random.shuffle(circle_ccw)
    random.shuffle(circle_cw)
    random.shuffle(heart_cw)
    random.shuffle(square_ccw)
    random.shuffle(triangle_cw)

    circle_ccw_train = circle_ccw[:num_train_samples]
    circle_ccw_test = circle_ccw[num_train_samples:]

    circle_cw_train = circle_cw[:num_train_samples]
    circle_cw_test = circle_cw[num_train_samples:]

    heart_cw_train = heart_cw[:num_train_samples]
    heart_cw_test = heart_cw[num_train_samples:]

    square_ccw_train = square_ccw[:num_train_samples]
    square_ccw_test = square_ccw[num_train_samples:]

    triangle_cw_train = triangle_cw[:num_train_samples]
    triangle_cw_test = triangle_cw[num_train_samples:]

    train = circle_ccw_train + circle_cw_train + heart_cw_train + square_ccw_train + triangle_cw_train
    test = circle_ccw_test + circle_cw_test + heart_cw_test + square_ccw_test + triangle_cw_test + junk

    for (predicted_label, gesture) in test:
        means = [(a, calc_dtw(gesture, tr)) for (a, tr) in train]
        means.sort(key=lambda x:x[1])

        true_label = means[0][0]
        mean = means[0][1]

        if mean > threshold:
            true_label = 'junk'

        confusion_matrix[true_label][predicted_label] += 1.0
        num_trails[predicted_label] += 1.0

cf_plot = [[0 for _ in labels] for _ in labels]

for (i, true_label) in enumerate(['circle_ccw', 'circle_cw', 'heart_cw', 'square_ccw', 'triangle_cw', 'junk']):
    for (j, predicted_label) in enumerate(['circle_ccw', 'circle_cw', 'heart_cw', 'square_ccw', 'triangle_cw', 'junk']):
        cf_plot[j][i] = confusion_matrix[true_label][predicted_label] / num_trails[predicted_label]

plt.tick_params(labeltop=True, labelbottom=False)
seaborn.heatmap(cf_plot, cmap='rocket_r', annot=True, vmin=0.0, vmax=1.0, xticklabels=labels, yticklabels=labels, cbar=False)
plt.show()


# import transformers
# import datasets
# from pprint import pprint

# # with pipeline

# model = transformers.AutoModelForSequenceClassification.from_pretrained("") # load model from local directory
# tokenizer = transformers.AutoTokenizer.from_pretrained("TurkuNLP/bert-base-finnish-cased-v1")

# test_pipe = transformers.pipeline(task="text-classification", model=model, tokenizer=tokenizer, function_to_apply="sigmoid", top_k=None) # return_all_scores=True is deprecated

# test = [""] # add examples to test

# results = test_pipe(test)

# for zipped in zip(test, results):
#   pprint(zipped)

import transformers
import torch
import numpy as np
import argparse
from pprint import PrettyPrinter
import json
import datasets
import pandas as pd
import csv

""" This script is meant for looking at multi-label predictions for raw text data and saving the probabilities with id. """

# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('--model', required=True,
    help="the model name")
parser.add_argument('--data', required=True,
    help="the file name of the raw text to use for predictions")
parser.add_argument('--tokenizer', required=True,
    help="the tokenizer to use for tokenizing new text")
parser.add_argument('--filename', required=True,
    help="the file name to give file resulting from the predictions")
args = parser.parse_args()
print(args)

pprint = PrettyPrinter(compact=True).pprint

# read the data in
data = args.data

if ".json" in data:
    with open(data, 'r') as json_file:
            json_list = list(json_file)
    lines = [json.loads(jline) for jline in json_list]

    # use pandas to look at each column
    df=pd.DataFrame(lines)

# # TODO might have to change this depending on the data type
# elif ".tsv" in data:
#     with open(data, "rt", encoding="utf-8") as f:
#         lines = f.readlines()
#     lines = lines[1:]
#     for i in range(len(lines)):
#         lines[i] = lines[i].replace("\n", "")
#         lines[i] = lines[i].split("\t")
#         assert len(lines[i]) == 3

#     df=pd.DataFrame(lines, columns = ['id', 'label', 'text'])

elif ".tsv" in data:
    with open(data, "rt", encoding="utf-8") as f:
        lines = f.readlines()
    lines = lines[1:]
    for i in range(len(lines)):
        lines[i] = lines[i].replace("\n", "")
        lines[i] = lines[i].split("\t")
        assert len(lines[i]) == 2

    df=pd.DataFrame(lines, columns = ['label', 'text'])

# instantiate model, this is pretty simple
model=transformers.AutoModelForSequenceClassification.from_pretrained(args.model)

trainer = transformers.Trainer(
    model=model
)

tokenizer = transformers.AutoTokenizer.from_pretrained(args.tokenizer)

def tokenize(example):
    return tokenizer(
        example["text"],
        padding='max_length', # this got it to work, data_collator could have helped as well?
        max_length=512,
        truncation=True,
    )

dataset = datasets.Dataset.from_pandas(df)

#map all the examples
dataset = dataset.map(tokenize)

labels = dataset["label"]
# oh right I would have to change the labels for the test set to match the upper ones if I wanted easily readable results

dataset = dataset.remove_columns("label")
texts = dataset["text"]
#ids = dataset["id"]

# see how the labels are predicted
test_pred = trainer.predict(dataset)
predictions = test_pred.predictions # these are the logits

sigmoid = torch.nn.Sigmoid()
probs = sigmoid(torch.Tensor(predictions))
probs = probs.numpy()

unique_labels = ["IN", "NA", "HI", "LY", "IP", "SP", "ID", "OP", "QA_NEW"] # upper labels plus qa_new

with open(args.filename, 'w') as outfile:
    header = ["text", "gold_labels", *unique_labels] #maybe I should put the text last

    writer = csv.writer(outfile, delimiter="\t")
    writer.writerow(header)

    for i in range(len(texts)):

        text = texts[i]
        gold = labels[i]
        line = [text, gold]
        pred_list = [str(val) for val in probs[i]]
        line = [*line, *pred_list]

        writer.writerow(line)


'''
Loss functions.
'''

import copy
import torch
import torch.nn as nn
from torch.autograd import Variable
import numpy as np

import utils

class NLLLoss(nn.Module):
    """Self-Defined NLLLoss Function

    Args:
        weight: Tensor (num_class, )
    """
    def __init__(self, weight):
        super(NLLLoss, self).__init__()
        self.weight = weight

    def forward(self, prob, target):
        """
        Args:
            prob: (N, C)
            target : (N, )
        """
        N = target.size(0)
        C = prob.size(1)
        weight = Variable(self.weight).view((1, -1))
        weight = weight.expand(N, C)  # (N, C)
        if prob.is_cuda:
            weight = weight.cuda()
        prob = weight * prob

        one_hot = torch.zeros((N, C))
        if prob.is_cuda:
            one_hot = one_hot.cuda()
        one_hot.scatter_(1, target.data.view((-1,1)), 1)
        one_hot = one_hot.type(torch.ByteTensor)
        one_hot = Variable(one_hot)
        if prob.is_cuda:
            one_hot = one_hot.cuda()
        loss = torch.masked_select(prob, one_hot)

        return -torch.sum(loss)

class GANLoss(nn.Module):

    """Reward-Refined NLLLoss Function for adversial training of Generator"""

    def __init__(self):
        super(GANLoss, self).__init__()

    def forward_reinforce(self, prob, target, reward, cuda=False):
        """
        Forward function used in the SeqGAN implementation.
        Args:
            prob: (N, C), torch Variable
            target : (N, ), torch Variable
            reward : (N, ), torch Variable
        """
        N = target.size(0)
        C = prob.size(1)
        one_hot = torch.zeros((N, C))
        if cuda:
            one_hot = one_hot.cuda()
        one_hot.scatter_(1, target.data.view((-1,1)), 1)
        one_hot = one_hot.type(torch.ByteTensor)
        one_hot = Variable(one_hot)
        if cuda:
            one_hot = one_hot.cuda()
        loss = torch.masked_select(prob, one_hot)
        loss = loss * reward
        loss =  -torch.sum(loss)

        return loss

    def forward_reward(self, i, samples, prob, rewards, BATCH_SIZE, g_sequence_len, VOCAB_SIZE, cuda=False):
        """
        Returns what is used to get the gradient contribution of the i-th term of the batch.

        """
        conditional_proba = Variable(torch.zeros(BATCH_SIZE, VOCAB_SIZE))
        if cuda:
            conditional_proba = conditional_proba.cuda()
        for j in range(BATCH_SIZE):
            conditional_proba[j, int(samples[j, i])] = 1
            conditional_proba[j, :] = - (rewards[j]/BATCH_SIZE * conditional_proba[j, :])

        return conditional_proba

    def forward_reward_grads(self, samples, prob, rewards, g, BATCH_SIZE, g_sequence_len, VOCAB_SIZE, cuda=False):
        """
        Returns a list of gradient contribution of every term in the batch

        """
        conditional_proba = Variable(torch.zeros(BATCH_SIZE, g_sequence_len, VOCAB_SIZE))
        batch_grads = []
        if cuda:
            conditional_proba = conditional_proba.cuda()
        for j in range(BATCH_SIZE):
            for i in range(g_sequence_len):
                conditional_proba[j, i, int(samples[j, i])] = 1
            conditional_proba[j, :, :] = - (rewards[j] * conditional_proba[j, :, :])
        for j in range(BATCH_SIZE):
            j_grads = []
            # since we want to isolate each contribution, we have to zero the generator's gradients here.
            g.zero_grad()
            prob[j, :, :].backward(conditional_proba[j, :, :], retain_graph=True)
            for p in g.parameters():
                j_grads.append(p.grad.clone())
            batch_grads.append(j_grads)

        return batch_grads

class VarianceLoss(nn.Module):

    """Loss for the control variate annex network"""

    def __init__(self):
        super(VarianceLoss, self).__init__()

    def forward(self, grad, cuda = False):
        """
        Used to get the gradient of the variance.

        """
        bs = len(grad)
        ref = 0
        for j in range(bs):
            for i in range(len(grad[j])):
                ref += torch.sum(grad[j][i]**2).item()
        total_loss = np.array([ref/bs])
        total_loss = Variable(torch.Tensor(total_loss), requires_grad=True)
        if cuda:
            total_loss = total_loss.cuda()

        return total_loss

    def forward_variance(self, grad, cuda=False):
        """
        Used to get the variance of one single parameter.
        In this case, we take look at the last layer, then take the variance of the first parameter of this last layer in main.py

        """
        bs = len(grad)
        n_layers = len(grad[0])
        square_term = torch.zeros((grad[0][n_layers-1].size()))
        normal_term = torch.zeros((grad[0][n_layers-1].size()))
        if cuda:
            square_term = square_term.cuda()
            normal_term = normal_term.cuda()
        for j in range(bs):
            square_term = torch.add(square_term, grad[j][n_layers-1]**2)
            normal_term = torch.add(normal_term, grad[j][n_layers-1])
        square_term /= bs
        normal_term /= bs
        normal_term = normal_term ** 2

        return square_term - normal_term


#-------------------------------------------------------------------------
# Script en python que se encarga de conectarse a un recurso Event Hub de
# Microsoft Azure y leer todos los mensajes disponibles, al mismo tiempo
# que deja un checkpoint de lo que ha leído para no repetir mensajes
# la siguiente vez que arranque el programa.
#
# Instrucciones para utilizarla.
#   1-. Tener Python 3.4 o mayor.
#   2-. Tener el instalador de paquetes "pip".
#   3-. Ingresar el comando "pip install azure-eventhub"
#
# Autor: Noé Amador Campos Castillo.
# E-mail: noecampos@tec.mx
#--------------------------------------------------------------------------

import os
import sys
import json
import signal
import logging
import asyncio
import functools
from azure.eventprocessorhost import (
    AbstractEventProcessor,
    AzureStorageCheckpointLeaseManager,
    EventHubConfig,
    EventProcessorHost,
    EPHOptions)

class EventProcessor(AbstractEventProcessor):

    # Constructor de un Event Processor
    def __init__(self, params=None):
        super().__init__(params)
        self._msg_counter = 0

    # Función que se puede hacer override
    # Es para inicializar un Procesador de Eventos
    async def open_async(self, context):
        print("Connection established {}".format(context.partition_id))

    # Función que se puede hacer override
    # Sirve para detener el Procesador de Eventos.
    async def close_async(self, context, reason):

        print("Connection closed (reason {}, id {}, offset {}, sq_number {})".format(
            reason,
            context.partition_id,
            context.offset,
            context.sequence_number))

    # Función que se puede hacer override
    """
        Se llama cuando el EPH recibe un nuevo batch de eventos.
        Es donde se programa las acciones a realizar.
        Parametros:
            context     = Información sobre la partición
            messages    = El batch de eventos a procesar
    """
    async def process_events_async(self, context, messages):
        # Por cada evento...
        for Event in messages:
            # Se imprime el número de secuencia
            print("Mensaje: {}".format(Event.sequence_number))

            # Se parsea el json recibido en el mensaje del evento
            parsedMessage = json.loads(Event.body_as_str())
            # Se imprime de manera más estetica
            print(json.dumps(parsedMessage, indent=2, sort_keys=True))

        # Deja un checkpoint del evento recibido
        await context.checkpoint_async()

    # Función que se puede hacer override
    """
        Se llama cada que el cliente experimenta algún error al recibir eventos.
        El Event Proccessor Host se recupera recibiendo desde donde se quedo.
        ( A menos de que se haya matado el programa )
        Parametros:
            context     = Información sobre la partición
            messages    = El batch de eventos a procesar
    """
    async def process_error_async(self, context, error):
        print("Event Processor Error {!r}".format(error))

# Recibir eventos por dos minutos y luego apagarlo
async def wait_and_close(host):
    await asyncio.sleep(60)
    await host.close_async()

# Se conecta y recibe mensajes
try:
    # Regresa un loop asincrono
    ephLoop = asyncio.get_event_loop()

    # Nombre del Storage Account
    stgName = "-"
    # Key del storage
    stgKey = "-"
    # Nombre del Blob
    blobName = "-"

    # Nombre del namespace de Event Hubs
    ehNamespace = "-"
    # Nombre del Event Hub
    ehName = "-"
    # Nombre del SAS Policy del Event Hub
    SASUser = "-"
    # Llave del SAS Policy del Event Hub
    SASKey = "-"

    """
    Configuración del Event Hub
    Párametros:
        sb_name   = Nombre del namespace de Event Hubs
        eh_name   = Nombre del Event Hub
        policy    = Nombre del SAS Policy
        key       = Llave de la SAS Policy
    """
    ehConfig = EventHubConfig(ehNamespace, ehName, SASUser, SASKey)

    # Opciones por default
    ehOptions = EPHOptions()
    # Set algunas opciones
    ehOptions.release_pump_on_timeout = True
    ehOptions.debug_trace = False

    """
    Configuración del Storage
    Párametros:
        storage_account_name    = Nombre del storage
        storage_account_key     = Llave del storage
        lease_container_name    = Nombre del contenedor
    """
    stgManager = AzureStorageCheckpointLeaseManager(
        stgName, stgKey, blobName)

    # Host del Event Hub Processor
    ehHost = EventProcessorHost(
        EventProcessor,
        ehConfig,
        stgManager,
        ep_params = ["param1", "param2"],
        eph_options = ehOptions,
        loop = ephLoop)

    # Prepara los procedimientos a ejecutar en loop
    ephTasks = asyncio.gather(
        ehHost.open_async(),
        wait_and_close(ehHost))

    # Corre el loop
    ephLoop.run_until_complete(ephTasks)

# En caso de ocurrri excepciones de teclado
except KeyboardInterrupt:
    # Cancela las tareas y el loop
    for task in asyncio.Task.all_tasks():
        task.cancel()
    ephLoop.run_forever()
    ephTasks.exception()

# Cierra el loop
finally:
    ephLoop.stop()


from django.urls import path
from django.conf.urls import url

from . import views

app_name = 'choistick'
urlpatterns = [
	path('', views.index, name='index'),
	path('map/', views.map, name='map'),
	path('join/', views.signup, name='join'),
	path('pick/', views.pick, name='pick'),
	path('warn/', views.warn, name='warn'),
	# url(r'^login/$', views.signin, name='login'),
	url(r'^login/$', views.signin, name='login'),
]


from django import forms
from django.core.exceptions import ValidationError

from semester.models import Semester, CourseOffered, CourseDistribution, DistributedSectionDetail
from tempus_dominus.widgets import TimePicker, DatePicker
from django.contrib import messages
from django.shortcuts import redirect

class SemesterForm(forms.ModelForm):
    start_effect_date = forms.DateField(
        widget=DatePicker(
            options={
                'collapse': False,
                'format': 'L',
            },
            attrs={
                'append': 'fa fa-calendar',
                'icon_toggle': True,
                'input_toggle': True,
                'placeholder': 'Start Effect Date',
                'required': 'true',
                'autocomplete': 'off',
            },
        )
    )

    end_effect_date = forms.DateField(
        widget=DatePicker(
            options={
                'collapse': False,
                'format': 'L',
            },
            attrs={
                'append': 'fa fa-calendar',
                'icon_toggle': True,
                'input_toggle': True,
                'placeholder': 'End Effect Date',
                'required': 'true',
                'autocomplete': 'off',
            },
        )
    )

    class Meta:
        model = Semester
        fields = ('name', 'year', 'start_effect_date', 'end_effect_date')

class CourseOfferingForm(forms.ModelForm):
    class Meta:
        model = CourseOffered
        fields = '__all__'

class CourseDistributionForm(forms.ModelForm):
    starting_id = forms.CharField()
    ending_id = forms.CharField()

    class Meta:
        model = CourseDistribution
        fields = ('offered', 'section', 'teacher', 'parent_dist', 'starting_id', 'ending_id')

    def clean(self):
        cd = self.cleaned_data
        qs = CourseDistribution.objects.filter(offered=cd.get('offered'), section=cd.get('section'))
        if qs:
            for obj in qs:
                try:
                    ms = DistributedSectionDetail.objects.get(distribution=obj)
                    if ms.starting_id == "*" or cd.get('starting_id') == "*":
                        raise ValidationError("Course already distributed to Section.")
                    else:
                        if not (cd.get('ending_id') < ms.starting_id or cd.get('starting_id') > ms.ending_id):
                            raise ValidationError("Course already distributed to Section.")
                # raise ValidationError("Course already distributed to Section.")
                except:
                    return redirect("/distribution/create/")

        parent = cd.get('parent_dist')
        if parent:
            while parent.parent_dist != parent:
                cd['parent_dist'] = parent.parent_dist
                parent = cd.get('parent_dist')

        if parent and cd.get('teacher') != parent.teacher:
            raise ValidationError("Merged sections has different teacher")

        return cd

    def save(self, commit=True):
        cd = self.cleaned_data
        instance = super(CourseDistributionForm, self).save(commit=True)
        if not instance.parent_dist:
            instance.parent_dist = instance
            instance.save()

        DistributedSectionDetail.objects.update(
            distribution=instance,
            starting_id=cd.get('starting_id'),
            ending_id=cd.get('ending_id')
        )

        return instance

class CourseDistributionUpdateForm(forms.ModelForm):
    class Meta:
        model = CourseDistribution
        fields = ('offered', 'section', 'teacher', 'parent_dist')
    def clean(self):
        cd = self.cleaned_data
        parent = cd.get('parent_dist')
        if parent:
            while parent.parent_dist != parent:
                cd['parent_dist'] = parent.parent_dist
                parent = cd.get('parent_dist')

        if parent and cd.get('teacher') != parent.teacher:
            raise ValidationError("Merged sections has different teacher")

        return cd


import requests
import re
from bs4 import BeautifulSoup
from openpyxl import load_workbook

DIRECTORY_URL = "https://directory.tufts.edu/searchresults.cgi"
WORKBOOK_NAME = "DirectoryResults_2017-2018.xlsx"
NAME_SHEET = "DirectoryResults"

# This script works on Excel Sheets with a single column in the A column of
# a unique listing of names. It searches for every name in the tufts directory,
# then strips out a department if it is found and places it in the B column next
# to the search query.

def getDirectoryPage(name):
    rawPage = requests.post(DIRECTORY_URL, data={"type": "Faculty", "search": name})
    soup = BeautifulSoup(rawPage.text, "html.parser")
    for child in soup.find_all(href=re.compile("department.cgi")):
        return child.contents[0].strip()

dataBook = load_workbook(WORKBOOK_NAME)
nameSheet = dataBook[NAME_SHEET]

for index in range(2, nameSheet.max_row + 1):
    currentName = nameSheet["A{0}".format(index)].value
    affiliation = getDirectoryPage(currentName)
    nameSheet["B{0}".format(index)] = affiliation

    # Rudimentary backup every 100 entries
    if index % 100 == 0:
        print("Progress: {:.2%}".format(index / (nameSheet.max_row + 1)))
        dataBook.save(WORKBOOK_NAME)

dataBook.save(WORKBOOK_NAME)


from anomaly_detection import Amean
from multiprocessing import Process, Queue
from database import DataBase
from datetime import datetime, timedelta
import time
import traceback

class AnomalyDomain (Process):
    # initilize data
    def __init__(self, name, host) :
        super(AnomalyDomain, self).__init__()
        self.name = name
        self.db = DataBase(host=host, database="anomaly", collection="data")
        self.last_update = ""
        self.last_value = 0

        self.timeline = []
        for i in range(24) :
            hour_detection = Amean()
            self.timeline.append(hour_detection)

        self.timeline_weekend = []
        for i in range(24) :
            hour_detection = Amean()
            self.timeline_weekend.append(hour_detection)
        return

    def __predict (self, hour=0, current=0, angle=0, date="") :
        """predict and return value for new data point"""
        date = datetime.strptime(date, "%Y-%m-%d")
        code = self.timeline[hour].predict(current=current, angle=angle)
        if code == '' :
            return 10
        else :
            return code

    def __log (self, msg) :
        with open("log.txt","a") as f :
            f.write(msg + '\n')
        return

    def __check_new_data (self, name) :
        """check if there is new data in repo
            if yes, return all new data"""
        # check if repo is null (start-date = null)
        if self.last_update == "" :
            start_date = self.db.get_start_date(name=name)
            if start_date != '' :
                self.last_update = start_date

        # check last update
        # go to database and get last_update, then update data in anomaly class (this class)
        db_last_update = self.db.get_last_update(name=name)
        print("db_last_update: ",db_last_update)
        if db_last_update == '' or not db_last_update:
            return []
        else :
            db_last_update = datetime.strptime(db_last_update, "%Y-%m-%d %H")
        last_update = datetime.strptime(self.last_update,  "%Y-%m-%d %H")
        result = []
        while last_update < db_last_update :
            print("db_last_update: ", name," ", db_last_update)
            last_update += timedelta(seconds=3600)
            print("check last update :", last_update)
            date = last_update.strftime("%Y-%m-%d")
            hour = last_update.hour
            data_value = self.db.get_data_by_hour(name=name, date=date, hour=hour)
            self.__log(date + ' ' + str(hour) + ' ' + str(data_value))
            data = {'angle':float(data_value)-float(self.last_value),
                    'current':data_value,
                    'date':date,
                    'hour':hour}
            result.append(data)
            self.last_value = data_value

        self.last_update = datetime.strftime(last_update, '%Y-%m-%d %H')
        return result

    def __save_result (self, name, date, result) :
        self.db.insert_result(name=name, date=date, value=result)
        return

    #=========================  RUN   ==============================
    # Run process method
    # start per process by calling run()
    def run(self) :
        name = self.name
        try :
            while True :
                time.sleep(10)

                data = self.__check_new_data (name)
                # data :
                # [] : no new data
                # [ {date:, hour:, current:, angle:)]
                print("--------------AnomalyDomain is running1--------------")
                if data != [] :
                    print("--------------AnomalyDomain is running2--------------")
                    # predict new data
                    for hour_data in data :
                        result_prediction = self.__predict(hour=hour_data['hour'],
                                                           current=hour_data['current'],
                                                           angle=hour_data['angle'],
                                                           date=hour_data["date"])
                        # save result to db
                        self.__save_result(name=name,
                                           date=hour_data['date']+' '+str(hour_data['hour']),
                                           result=result_prediction)

                #continue waiting
        except Exception as e:
            with open("log.txt","a") as f :
                f.write(str(e) + '\n')
                traceback.print_exc()


from jetracer.nvidia_racecar import NvidiaRacecar
import time
import sys
from multiprocessing import Process, Value
import zmq
import Jetson.GPIO as GPIO

pinrun = 'DAP4_SCLK' #12
pinbouton = 'SPI2_SCK' #13
pinau = 'SPI2_CS1' #16
autrepin = 'SPI2_CS0' #18

GPIO.setmode(GPIO.TEGRA_SOC)
GPIO.setup(pinrun, GPIO.OUT)
GPIO.output(pinrun, 0)

GPIO.setup(pinau, GPIO.OUT)
GPIO.output(pinau, 0)

GPIO.setup(pinbouton, GPIO.IN)

context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("tcp://*:5555")

currentlyRunning = Value('b', False)

def RunScript(script, data):
	#print("running " + script)
	GPIO.output('DAP4_SCLK', 1)
	data.value = True
	exec(script)
	print(f"currentlyRunning from process {currentlyRunning.value}")
	print("Script terminé avec succès")
	data.value = False
	print(f"currentlyRunning from process {currentlyRunning.value}")
	GPIO.output('DAP4_SCLK', 0)

def BumperChock(data):
	print("Detection pare-chocs")
	if currentlyRunning.value:
		runThread.terminate() # sends a SIGTERM
		currentlyRunning.value = False;
		car.throttle = 0.001
		car.throttle = 0
		GPIO.output(pinrun, 0)
		GPIO.output(pinau, 1)
		data.value = False

car = NvidiaRacecar()
car.steering_gain = -0.65
car.steering_offset = -0.25
if car.steering_offset != -0.25 : exit()

print("Car ready")

GPIO.add_event_detect(pinbouton, GPIO.FALLING, callback=lambda x: BumperChock(currentlyRunning), bouncetime=10)

while True:
	try:
		print("en attente recv...")
		message = socket.recv()
		GPIO.output(pinau, 0)
		socket.send(b"OK")
		message = message.decode("utf-8")

		#print("Received request: %s" % message)
		f = open("/KDesir_Tests/logging.txt", "a")

		t = time.strftime('%d/%m/%Y-%H:%M:%S', time.localtime()) + ","
		log = message.replace("\n", "\n" + t)
		f.write(t + log + "\n")
		f.close()

		#print(message)

		if "ArretUrgence" in message:
			runThread.terminate() # sends a SIGTERM
			#socket.send(b"AU_Done")
			print("Arrêt d'urgence déclenché")
			currentlyRunning.value = False;
			raise
		else:
			print(f"currentlyRunning from main script {currentlyRunning.value}")
			if not currentlyRunning.value:
				print(f"currentlyRunning {currentlyRunning.value}")
				runThread=Process(target=RunScript,args=(message, currentlyRunning))
				runThread.start()
			else:
				print("Impossible d'exécuter le script car un autre est déjà en cours")

	except Exception as e:
		print(e)

		car.throttle = 0.001
		car.throttle = 0
		GPIO.output(pinrun, 0)
		GPIO.output(pinau, 1)

	#finally:
#		GPIO.cleanup()

sys.exit("Fin du programme")


import hearts.model.game as m
from hearts.game_master import GameMaster
import logging

class GameBackend(object):
    def __init__(self, player_svc):
        self._next_game_id = 1
        self._game_masters = {}
        self._players = {}
        self._player_mapping = {}
        self._player_svc = player_svc
        self.logger = logging.getLogger(__name__)

    def create_game(self, players):
        game_id = self._next_game_id
        self._next_game_id += 1

        model = m.HeartsGame()
        model.start()

        master = GameMaster(model, game_id)
        self._game_masters[game_id] = master
        master.add_observer(self)

        self._players[game_id] = list(players)

        for idx, player_id in enumerate(players):
            self._player_mapping[player_id] = (game_id, idx)

        return game_id

    def get_game_master(self, game_id):
        return self._game_masters[game_id]

    def try_get_player_game(self, player_id):
        data = self._player_mapping.get(player_id)
        if data is None:
            return None

        return data[0]

    def try_get_game_info(self, player_id):
        data = self._player_mapping.get(player_id)
        if data is None:
            return None

        return data

    def is_in_game(self, player_id):
        return player_id in self._player_mapping

    def on_game_finished(self, game_id):
        self.logger.info("Game %d has finished.", game_id)
        self._destruct_game(game_id)

    def on_game_abandoned(self, game_id):
        self.logger.info("Game %d has been abandoned.", game_id)
        self._destruct_game(game_id)

    def _destruct_game(self, game_id):
        for player in self._players[game_id]:
            del self._player_mapping[player]
            self._player_svc.remove_player(player)

        del self._players[game_id]
        del self._game_masters[game_id]


from pathlib import Path
from sphinx.directives import SphinxDirective
from docutils.parsers.rst import directives
from docutils import nodes
from sphinx.util.logging import getLogger
import yaml
import json

logger = getLogger(__name__)

class PyConfig(SphinxDirective):

    has_content = True

    def run(self):

        if self.content:
            data = "\n".join(self.content)
            data = json.dumps(yaml.safe_load(data), indent=2)

            return [nodes.raw("", f'<py-config type="json">\n{data}\n</py-config>\n', format="html")]

class PyRepl(SphinxDirective):

    has_content = True
    option_spec = {
        "auto-generate": directives.unchanged,
        "output": directives.unchanged,
        "src": directives.path,
    }

    def run(self):
        attrs: str = ""
        code: str = ""

        for key, value in self.options.items():
            attrs += f' {key}="{value}"'

        if self.content:
            code = "\n".join(self.content)

        py_repl = f'''
            <py-repl {attrs}>
            {code}
            </py-repl>
        '''
        return [nodes.raw("", py_repl, format="html")]

class PyScript(SphinxDirective):

    has_content = True
    option_spec = {
        "file": directives.path,
        "output": directives.unchanged
    }

    def run(self):

        if "file" in self.options:
            path = self.env.relfn2path(self.options['file'])[1]
            try:

                with open(path, 'r') as f:
                    code = f.read()
                self.env.note_dependency(path)

            except (FileNotFoundError, Exception) as err:
                logger.warn('reading error: %s, %s', path, err)
                return []

        elif self.content:
            code = "\n".join(self.content)
        else:
            raise logger.error("Must provide either content or the 'file' option")
        return [nodes.raw("", f"<py-script>\n{code}\n</py-script>\n", format="html")]

class PyTerminal(SphinxDirective):

    option_spec = {
        "auto": directives.flag,
        "false": directives.flag
    }

    def run(self):
        attrs: str = ""

        for key, _ in self.options.items():
            attrs += f' {key}'

        py_terminal = f'''
            <py-terminal {attrs}></py-terminal>
        '''

        return [nodes.raw("", py_terminal, format="html")]


#%%
import pandas as pd
import numpy as np
import datetime as dt
import xarray as xr
import cftime
import dask
from glob import glob

#%%
'''SUBSET RH DATA'''

data = pd.read_csv("preprocessing/inputdata/AMF_US-MBP_BASE_HH_2-5.csv",
    skiprows = 2,
    na_values = -9999)

data['TIMESTAMP_START'] = pd.to_datetime(data.TIMESTAMP_START, format = '%Y%m%d%H%M')
data['TIMESTAMP_END'] = pd.to_datetime(data.TIMESTAMP_END, format = '%Y%m%d%H%M')

data = data.rename(columns = {'TIMESTAMP_START':'TIMESTAMP'})

'''
#TRY EDI Data
data = pd.read_csv('preprocessing/inputdata/BogLake_Met_data_30min.csv', skiprows = 1, sep = ",",
    names=["TIMESTAMP", "Air_TempC_Avg", "RH", "Soil_TempC_5cm", "Soil_TempC_10cm", "Soil_TempC_20cm",
            "Soil_TempC_30cm", "Soil_TempC_40cm", "Soil_TempC_50cm", "Soil_TempC_100cm",  "Soil_TempC_200cm",
            "WS_Tot",  "WindDir_D",  "WindDir_SD",  "PAR_Den_Avg"],
    parse_dates = ['TIMESTAMP'],
    na_values = {'RH':['NA',]})
'''

# Subset to the right variables
RH = data[['TIMESTAMP', 'RH']]

# Sort into Year-Month combos
RH['Year'] = RH.TIMESTAMP.dt.year
RH['Month'] = RH.TIMESTAMP.dt.month
RH['Year-Month'] = RH.TIMESTAMP.dt.strftime('%Y') + '-' + RH.TIMESTAMP.dt.strftime('%m')

#Subset to proper years
RH = RH[(RH.Year > 2010) & (RH.Year < 2018)]

#Patch long NA period in August 2017 with data from the NADP site in the EDI data
patch_data = pd.read_csv('preprocessing/inputdata/NADP_Met_data_30min.csv', skiprows = 1, sep = ",",
    names=["TIMESTAMP", "Air_TempC_Avg", "RH", "Soil_TempC_Avg", "WS_Tot", "WindDir_D", "WindDir_SD", "PAR_Den_Avg", "Soil_VWC_Avg"],
    parse_dates = ['TIMESTAMP'],
    na_values = {'RH':['NA',]})

RH_patch = patch_data[['TIMESTAMP', 'RH']]
RH_patch = RH_patch[(RH_patch.TIMESTAMP.dt.year == 2017) & (RH_patch.TIMESTAMP.dt.month == 8)]

# %%
'''OPEN ALL NCDF DATA'''

vals = set(RH['Year-Month'])

for val in vals:
    #Open sample data set
    test = xr.open_mfdataset(val + '.nc', decode_times = False)

    #Select RH data:
    if val == '2017-08':
        test_RH = RH_patch
    else:
        test_RH = RH[RH['Year-Month'] == val]

    #Subset netdcf spatially
    test2 = test.drop_sel(lon = 1)

    #If leap Feb and leap year remove Feb29 data
    r = test_RH['RH']
    if (val.endswith('02')) & (len(r) > len(test2.time)):
        #remove feb 9
        r = r[:len(test2.time)]

    #print(val + ": " + str(min(r)))

    #print("NEW" + val + ": " + str(min(r)))

    # Interpolate
    r3 = pd.Series(r).interpolate(method = "linear")

    print("NAN" + val + ": " + str(any(np.isnan(r3))))

    #Reshape
    r4 = np.reshape(list(r3),(-1,1,1))

    #Add RH Data
    test2['RH'] = xr.DataArray(r4,
        dims = ['time', 'lat', 'lon'],
        attrs = {'FillValue': np.NaN,
                'long_name': 'relative humidity at the lowest atm level (RH)',
                'units': '%' })

    #Write sample
    test2.to_netcdf('preprocessin/forcings-modified' + val + '.nc')

# %%
'''CHECK'''
#Open file to check
dat = xr.open_mfdataset('preprocessing/forcings-modified/2017-08.nc')
# %%


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
import sys
import json
import mainwindow, mystock, recepe, compute

class MyMainWindow(mainwindow.Ui_MainWindow):
    def setupUi(self, mw, database):
        super().setupUi(mw)
        self.tabWidget = QtWidgets.QTabWidget()
        mw.setCentralWidget(self.tabWidget)
        self.ms = QtWidgets.QWidget()
        self.mystock = mystock.MyStock()
        self.mystock.setupUi(self.ms, database["bernard"]["stock"])
        self.tabWidget.addTab(self.ms, "STOCK")
        self.mr = QtWidgets.QWidget()
        self.myRecepe = recepe.Ui_TabRecepe()
        self.myRecepe.setupUi(self.mr)
        self.tabWidget.addTab(self.mr, "Recepe")
        self.mc = QtWidgets.QWidget()
        self.myCompute = compute.Ui_TabCompute()
        self.myCompute.setupUi(self.mc)
        self.tabWidget.addTab(self.mc, "Compute")

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    mw = QtWidgets.QMainWindow()
    ui = MyMainWindow()
    ui.setupUi(mw)
    mw.show()
    sys.exit(app.exec_())


from numpy import linspace, sin

from chaco.api import ArrayPlotData, Plot
from chaco.tools.api import PanTool, ZoomTool
from enable.api import ComponentEditor
from traits.api import Enum, HasTraits, Instance
from traitsui.api import Item, Group, View

class PlotEditor(HasTraits):

    plot = Instance(Plot)
    plot_type = Enum("scatter", "line")
    orientation = Enum("horizontal", "vertical")
    traits_view = View(
        Item("orientation", label="Orientation"),
        Item("plot", editor=ComponentEditor(), show_label=False),
        width=500,
        height=500,
        resizable=True,
    )

    def __init__(self, *args, **kw):
        HasTraits.__init__(self, *args, **kw)
        # Create the data and the PlotData object
        x = linspace(-14, 14, 100)
        y = sin(x) * x ** 3
        plotdata = ArrayPlotData(x=x, y=y)
        # Create the scatter plot
        plot = Plot(plotdata)
        plot.plot(("x", "y"), type=self.plot_type, color="blue")
        plot.tools.append(PanTool(plot))
        plot.tools.append(ZoomTool(plot))
        self.plot = plot

    def _orientation_changed(self):
        if self.orientation == "vertical":
            self.plot.orientation = "v"
        else:
            self.plot.orientation = "h"
        self.plot.request_redraw()

# ===============================================================================
# demo object that is used by the demo.py application.
# ===============================================================================
class Demo(HasTraits):

    # Scatter plot.
    scatter_plot = Instance(PlotEditor)

    # Line plot.
    line_plot = Instance(PlotEditor)

    traits_view = View(
        Group(Item("@scatter_plot", show_label=False), label="Scatter"),
        Group(Item("@line_plot", show_label=False), label="Line"),
        title="Chaco Plot",
        resizable=True,
    )

    def __init__(self, *args, **kws):
        super(Demo, self).__init__(*args, **kws)
        # Hook up the ranges.
        self.scatter_plot.plot.range2d = self.line_plot.plot.range2d

    def _scatter_plot_default(self):
        return PlotEditor(plot_type="scatter")

    def _line_plot_default(self):
        return PlotEditor(plot_type="line")

demo = Demo()
if __name__ == "__main__":
    demo.configure_traits()


import os
import time
import requests
import functools
from concurrent.futures import ThreadPoolExecutor
import click
import yaml

def _get_connection_urls(workload_yaml):
    with open(workload_yaml) as f:
        workload = yaml.safe_load(f)
    uris = workload.get("EnvironmentDetails", {}).get("MongosyncConnectionURIs")
    if not uris:
        raise Exception(
            f"This actor requires setting EnvironmentDetails: MongosyncConnectionURIs to use this script"
        )
    return uris

def poll(workload_yaml, predicate, key):
    """
    Wait for all mongosyncs to reach a certain state (e.g. predicate returns False)
    based on a value returned by the /progress endpoint
    """
    connection_urls = _get_connection_urls(workload_yaml)

    def get_progress():
        res = requests.get(f"{url}/api/v1/progress")
        return res.json()["progress"][key]

    for url in connection_urls:
        info = get_progress()
        while predicate(info):
            time.sleep(1)
            print(f"Polling {url} for {key}, current value = {info}", flush=True)
            info = get_progress()

def _change_one_mongosync_state(route, body, url):
    """
    Change state of a given mongosync running at the provided url
    """
    resp = requests.post(f"{url}{route}", json=body)
    print(resp.json(), flush=True)
    success = resp.json()["success"]
    if not success:
        raise Exception(f"State change failed at route {route}")
    return success

def change_state(workload_yaml, route, body):
    """
    Helper function to change state of mongosync. This must
    send all requests in parallel, as some commands block until
    all instances recieve them
    """

    connection_urls = _get_connection_urls(workload_yaml)

    fn = functools.partial(_change_one_mongosync_state, route, body)
    with ThreadPoolExecutor() as executor:
        futures = []
        # Using executor.map swallows exceptions from the task,
        # using .submit and then accessing the future's .result
        # will cause exceptions to be rethrown
        for url in connection_urls:
            futures.append(executor.submit(fn, url))
        for f in futures:
            f.result()

@click.group(name="MongosyncActor", context_settings=dict(help_option_names=["-h", "--help"]))
def cli():
    pass

@cli.command(
    "start",
    help=("Issue /start to all mongosync processes"),
)
@click.argument("workload_yaml", nargs=1)
def start(workload_yaml):
    change_state(workload_yaml, "/api/v1/start", {"Source": "cluster0", "Destination": "cluster1"})

@cli.command(
    "poll_for_cea",
    help=("Poll all available instances for the CEA stage"),
)
@click.argument("workload_yaml", nargs=1)
def poll_for_cea(workload_yaml):
    poll(workload_yaml, lambda x: x != "change event application", "info")

@cli.command(
    "poll_for_commit_point",
    help=("Wait till all the instances canCommit = true and lagTimeSeconds < 120"),
)
@click.argument("workload_yaml", nargs=1)
def poll_for_commit_point(workload_yaml):
    poll(workload_yaml, lambda x: bool(x) == False, "canCommit") or poll(
        workload_yaml, lambda x: int(x) > 120, "lagTimeSeconds"
    )

@cli.command(
    "drain_writes",
    help=("Wait till all writes have been drained to the destination cluster"),
)
@click.argument("workload_yaml", nargs=1)
def drain_writes(workload_yaml):
    poll(workload_yaml, lambda x: int(x) > 5, "lagTimeSeconds")

@cli.command(
    "commit",
    help=("Commit the migration"),
)
@click.argument("workload_yaml", nargs=1)
def commit(workload_yaml):
    change_state(workload_yaml, "/api/v1/commit", {})

@cli.command(
    "wait_for_commit",
    help=("Wait until all mongosyncs are finished commiting the migration"),
)
@click.argument("workload_yaml", nargs=1)
def wait_for_commit(workload_yaml):
    poll(workload_yaml, lambda x: x != "COMMITTED", "state")

@cli.command(
    "pause",
    help=("Pause the migration"),
)
@click.argument("workload_yaml", nargs=1)
def pause(workload_yaml):
    change_state(workload_yaml, "/api/v1/pause", {})

@cli.command(
    "resume",
    help=("Resume the migration"),
)
@click.argument("workload_yaml", nargs=1)
def resume(workload_yaml):
    change_state(workload_yaml, "/api/v1/resume", {})

if __name__ == "__main__":
    cli()


"""This file is to run the model inference here's the command
python run_inference.py -i trainval/images/image_000000001.jpg -m model/model.pt"""

# import the necessary packages
import argparse
import cv2
import numpy as np
from PIL import Image
import torch
from torchvision import transforms
import config
from utils import get_model_instance_segmentation

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--input_image", required=True,
    help="path to input image")
ap.add_argument("-m", "--model", required=True,
    help="path to trained pytorch model")
ap.add_argument("-c", "--confidence", type=float, default=0.85,
    help="minimum probability to filter weak detections")

args = vars(ap.parse_args())

model_path = args["model"]
input_image = args["input_image"]
confidence = args["confidence"]

# classes which our model will detect and the color object of the bounding box it will create
CLASSES=["Background","Person","Car"]

# reading the image with pillow and converion into the numpy arrays
img = Image.open(input_image)
open_cv_image = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)

# pytorch will work on the suitable device wheather it's CPU or GPU
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

# getting the model instance and loading the pytorch model
model = get_model_instance_segmentation(config.num_classes)
model.load_state_dict(torch.load(model_path))

# move model to the right device
model.to(device)
model.eval()

trans =transforms.Compose([transforms.ToTensor()])
img = trans(img).cuda()

# getting the all the detections generated by the trained model
detections = model([img])

# seperating out all the bounding boxes, labels, and scores we get
_bboxes, _labels, _scores = detections[0]['boxes'], detections[0]['labels'], detections[0]['scores']

# loop over the detections
for i in range(0, len(_bboxes)):
    # extract the confidence (i.e., probability) associated with the
    # prediction
    pred_confidence = _scores[i]
    # filter out weak detections by ensuring the confidence is
    # greater than the minimum confidence
    if pred_confidence > confidence:
        # extract the index of the class label from the detections,
        # then compute the (x, y)-coordinates of the bounding box
        # for the object
        idx = int(_labels[i])
        box = _bboxes[i].detach().cpu().numpy()
        (startX, startY, endX, endY) = box.astype("int")
        # display the prediction to our terminal
        label = "{}: {:.2f}%".format(CLASSES[idx], pred_confidence * 100)
        print("[INFO] {}".format(label))

        # draw the bounding box and label on the image
        cv2.rectangle(open_cv_image, (startX, startY), (endX, endY),
            (0,0,255) if idx==1 else (0,255,0), 1)
        y = startY - 15 if startY - 15 > 15 else startY + 15
        cv2.putText(open_cv_image, label, (startX, y),
            cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255) if idx==1 else (0,255,0), 2)

# show the output image
cv2.imshow("output", open_cv_image)
cv2.waitKey(0)


from rest_framework.authentication import TokenAuthentication
from rest_framework.exceptions import AuthenticationFailed

from .models import AuthToken
from utils.exceptions import *

def expire_token(user):
    try:
        for auth_token in user.auth_tokens.all():
            auth_token.delete()
    except AuthToken.DoesNotExist:
        pass

def get_auth_token_by(raise_exception=True, only_deleted=False, **kwargs):
    key = kwargs.get('key')
    if only_deleted:
        auth_token = AuthToken.objects.deleted_only().filter(**kwargs).first()
    else:
        auth_token = AuthToken.objects.filter(key=key).first()
    if not auth_token and raise_exception:
        raise ObjectNotFound
    return auth_token

def create_token(user):
    auth_token = AuthToken.objects.create(user=user)
    return auth_token.key

def token_expire_handler(auth_token):
    if auth_token.is_expired:
        auth_token = create_token(user=auth_token.user)
    return auth_token.is_expired, auth_token

class ExpiringTokenAuthentication(TokenAuthentication):
    def authenticate_credentials(self, key):
        try:
            auth_token = AuthToken.objects.get(key=key)
        except AuthToken.DoesNotExist:
            raise AuthenticationFailed
        is_expired, auth_token = token_expire_handler(auth_token)

        if is_expired:
            raise AuthenticationFailed
        return auth_token.user, auth_token


import telebot
import config
from ibm_watson import LanguageTranslatorV3
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
import json
from telebot import types
import pyowm

owm = pyowm.OWM('c8548689b28b1916f78403fb9c92e4f3', language='ru')

bot = telebot.TeleBot(config.TOKEN)

authenticator = IAMAuthenticator('9n-ZTrznhrAKV0YAJIWIM-fwico0pbNeHp9Wek67nt6V')
language_translator = LanguageTranslatorV3(
    version='2018-05-01',
    authenticator=authenticator
)
language_translator.set_service_url('https://api.eu-gb.language-translator.watson.cloud.ibm.com/instances'
                                    '/1bec2e12-6251-4b94-8d80-fcead8ec6d68')
languages = language_translator.list_languages().get_result()

user_dict = {}

# Формируем класс, для сохранения в него переменных (то что вводит кандидат)!
class User:
    def __init__(self, name):
        self.name = name
        self.application = None
        self.writer_lang = None
        self.phrases = None
        self.place = None

# Обработать / start и / help
@bot.message_handler(commands=['start', 'help'])
def command_start(message):
    chat_id = message.chat.id
    text = message.text
    msg = bot.send_message(chat_id,
                           "Здравствуйте, я полезный бот! Мое предназначение помогать в тех функциях которые в меня "
                           "встроенные! "
                           "В них входит перевод текста на необходимый Вам язык! Подскажите как я могу к Вам обращатся?")
    bot.register_next_step_handler(msg, process_name_step)

# Бот предлагает свои услуги\возможности!
@bot.message_handler(content_types=['text'])
def process_name_step(message):
    chat_id = message.chat.id
    name = message.text
    user = User(name)
    user_dict[chat_id] = user
    markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
    markup.add('Переводчик', 'Погода', 'Список покупок')
    msg = bot.send_message(chat_id, 'Приятно познакомится ' + user.name + '! Выберите приложение, которым хотите '
                                                                          'воспользоваться!', reply_markup=markup)
    bot.register_next_step_handler(msg, how_can_i_help)

# Действие выполняется при выборе приложения!
@bot.message_handler(content_types=['text'])
def how_can_i_help(message):
    chat_id = message.chat.id
    application = message.text
    if application == '/start' or application == '/help':
        return command_start
    if application == u'Переводчик':
        user = user_dict[chat_id]
        user.application = application
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('английский', 'русский', 'испанский', 'немецкий', 'итальянский', 'французский', 'китайский',
                   'Погода')
        msg = bot.send_message(chat_id, "Укажите на каком языке будете писать (если такого языка нет в выпадающем "
                                        "списке, напишите этот язык в сообщении!): ", reply_markup=markup)
        bot.register_next_step_handler(msg, translater_func1)
    elif application == u'Погода':
        user = user_dict[chat_id]
        user.application = application
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('Переводчик', 'Погода')
        msg = bot.send_message(chat_id, "Введите в каком городе/стране?: ", reply_markup=markup)
        bot.register_next_step_handler(msg, weather_bot)
    elif application == u'Список покупок':
        pass

# Тут запускается приложение для перевода текста!
@bot.message_handler(content_types=['text'])
def translater_func1(message):
    global lang1  # Создаем переменную lang1 (язык ввода текста)
    try:  # Создаем исключение для того чтобы различать кнопки и команду /start
        chat_id = message.chat.id
        writer_lang = message.text
        if writer_lang == u'Погода':  # Создаем условие нажатия на кнопку: если это погода выполняем код - ниже!
            user = user_dict[chat_id]
            user.writer_lang = writer_lang
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('Переводчик')
            msg = bot.send_message(chat_id, "Введите в каком городе/стране?: ", reply_markup=markup)  # Спрашиваем в
            # каком городе проверить и при этом крепим кнопку переводчика!
            bot.register_next_step_handler(msg, weather_bot)
        # Дальше условия касаются выбора языка с помощью кнопок! Если выбрали одну из кнопок ниже :
        if writer_lang == u'английский':
            lang1 = 'en'
        elif writer_lang == u'русский':
            lang1 = 'ru'
        elif writer_lang == u'украинский':
            lang1 = 'uk'
        elif writer_lang == u'испанский':
            lang1 = 'es'
        elif writer_lang == u'немецкий':
            lang1 = 'de'
        elif writer_lang == u'итальянский':
            lang1 = 'it'
        elif writer_lang == u'французский':
            lang1 = 'fr'
        elif writer_lang == u'китайский':
            lang1 = 'zh'
        user = user_dict[chat_id]
        user.writer_lang = writer_lang
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('английский', 'русский', 'испанский', 'немецкий', 'итальянский', 'французский', 'китайский')
        # При этом спрашиваем на какой язык переводить и крепим кнопки с соответствующими языками
        msg = bot.send_message(chat_id, "На какой язык переводить (если такого языка нет в выпадающем "
                                        "списке, напишите этот язык в сообщении!): ", reply_markup=markup)
        bot.register_next_step_handler(msg, translater_func2)
    except:
        if writer_lang == '/start' or writer_lang == '/help': # Если написали команду старт, то возвращаемся в начало кода!
            msg = bot.send_message(chat_id,
                                   "Здравствуйте, я полезный бот! Мое предназначение помогать в тех функциях которые "
                                   "в меня "
                                   "встроенные! "
                                   "В них входит перевод текста на необходимый Вам язык! Подскажите как я могу к Вам "
                                   "обращатся?")
            bot.register_next_step_handler(msg, command_start)
        else:
            msg = bot.send_message(chat_id, "Oooops!")
            bot.register_next_step_handler(msg, translater_func1)

def translater_func2(message):
    global lang # Создаем глобальную переменную для языка на который будет осуществлятся перевод!
    try:
        chat_id = message.chat.id
        translation_lang = message.text
        if translation_lang == u'Погода': # Создаем условие нажатия на кнопку: если это погода выполняем код - ниже!
            user = user_dict[chat_id]
            user.translation_lang = translation_lang
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('Переводчик')
            msg = bot.send_message(chat_id, "Введите в каком городе/стране?: ", reply_markup=markup)  # Спрашиваем в
            # каком городе проверить и при этом крепим кнопку переводчика!
            bot.register_next_step_handler(msg, weather_bot)
        # Условия для выбора языка на который будет перевод!
        if translation_lang == u'английский':
            lang = 'en'
        elif translation_lang == u'русский':
            lang = 'ru'
        elif translation_lang == u'украинский':
            lang = 'uk'
        elif translation_lang == u'испанский':
            lang = 'es'
        elif translation_lang == u'немецкий':
            lang = 'de'
        elif translation_lang == u'итальянский':
            lang = 'it'
        elif translation_lang == u'французский':
            lang = 'fr'
        elif translation_lang == u'китайский':
            lang = 'zh'
        user = user_dict[chat_id]
        user.translation_lang = translation_lang
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('Смена языков для перевода', 'Смена языка для перевода', 'Погода')
        msg = bot.send_message(chat_id, "Введите фразу для перевода : ", reply_markup=markup) # После определения
        # языков просим ввести текст для перевода. А так же крепим кнопки!
        bot.register_next_step_handler(msg, translate_phrases)
    except:
        if translation_lang == '/start' or translation_lang == '/help':  # Если написали команду старт, то возвращаемся в начало кода!
            msg = bot.send_message(chat_id,
                                   "Здравствуйте, я полезный бот! Мое предназначение помогать в тех функциях которые "
                                   "в меня "
                                   "встроенные! "
                                   "В них входит перевод текста на необходимый Вам язык! Подскажите как я могу к Вам "
                                   "обращатся?")
            bot.register_next_step_handler(msg, command_start)
        else:
            msg = bot.send_message(chat_id, 'Oooops')
            bot.register_next_step_handler(msg, translater_func2)

def translate_phrases(message):
    #global translation

    chat_id = message.chat.id
    phrases = message.text
    # Устанавливаем цикл: если нажаты кнопки текст не переводится, а выполняется переход по приложениям или смена языков!
    while phrases == u'Погода' or phrases == u'Смена языков для перевода' or phrases == u'Смена языка для перевода':
        # Тут как раз условия которые работают если нажали на одну из кнопок!
        if phrases == u'Погода':
            user = user_dict[chat_id]
            user.phrases = phrases
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('Переводчик')
            msg = bot.send_message(chat_id, "Введите в каком городе/стране?: ", reply_markup=markup)
            bot.register_next_step_handler(msg, weather_bot)
        elif phrases == u'Смена языков для перевода':
            user = user_dict[chat_id]
            user.phrases = phrases
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('английский', 'русский', 'испанский', 'немецкий', 'итальянский', 'французский', 'китайский',
                       'Погода')
            msg = bot.send_message(chat_id, "Укажите на каком языке будете писать (если такого языка нет в выпадающем "
                                            "списке, напишите этот язык в сообщении!): ", reply_markup=markup)
            bot.register_next_step_handler(msg, translater_func1)
        elif phrases == u'Смена языка для перевода':
            user = user_dict[chat_id]
            user.phrases = phrases
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('английский', 'русский', 'испанский', 'немецкий', 'итальянский', 'французский', 'китайский',
                       'Погода')
            msg = bot.send_message(chat_id, "На какой язык переводить (если такого языка нет в выпадающем "
                                            "списке, напишите этот язык в сообщении!): ", reply_markup=markup)
            bot.register_next_step_handler(msg, translater_func2)
        break
    else:
        translation = language_translator.translate(
            phrases,
            source=lang1, target=lang).get_result()
        msg = bot.send_message(chat_id, json.dumps(translation, indent=2, ensure_ascii=False))
        bot.register_next_step_handler(msg, translate_phrases)

# Здесь запускается приложение погода!
@bot.message_handler(content_types=['text'])
def weather_bot(message):
    global place
    try:
        chat_id = message.chat.id
        place = message.text
        user = user_dict[chat_id]
        user.place = place
        observation = owm.weather_at_place(place)
        w = observation.get_weather()
        markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
        markup.add('Переводчик')
        temp = w.get_temperature('celsius')["temp"]
        temp = round(temp)
        msg = bot.send_message(chat_id,
                               'В городе ' + place + ' сейчас ' + w.get_detailed_status() + ' Температура в этом '
                                                                                'городе: ' + str(
                                   temp), reply_markup=markup)

        if temp < 10 and temp >= 0:
            msg = bot.send_message(chat_id, 'Сейчас пипец как холодно, одевайся как танк!')
        elif temp >= 10 and temp < 20:
            msg = bot.send_message(chat_id, 'Тепло конечно, но загорать еще рано!')
        elif temp >= 20 and temp < 25:
            msg = bot.send_message(chat_id, 'Ну еще чуть чуть и загорать можно идти!')
        elif temp > 25:
            msg = bot.send_message(chat_id, 'Можно смело загорать!')
        else:
            msg = bot.send_message(chat_id, 'Снеговики наступааааают!!!')

        bot.register_next_step_handler(msg, weather_bot)

    except:
        if place == u'Переводчик':
            user = user_dict[chat_id]
            user.place = place
            markup = types.ReplyKeyboardMarkup(one_time_keyboard=True)
            markup.add('английский', 'русский', 'испанский', 'немецкий', 'итальянский', 'французский', 'китайский',
                       'Погода')
            msg = bot.send_message(chat_id, 'Да, давайте переводить. Укажите на каком языке будете писать!', reply_markup=markup)
            bot.register_next_step_handler(msg, translater_func1)
        elif place == '/start' or place == '/help':
            msg = bot.send_message(chat_id,
                                   "Здравствуйте, я полезный бот! Мое предназначение помогать в тех функциях которые в меня "
                                   "встроенные! "
                                   "В них входит перевод текста на необходимый Вам язык! Подскажите как я могу к Вам обращатся?")
            bot.register_next_step_handler(msg, command_start)
        else:
            msg = bot.send_message(chat_id, 'Такого города нет... Уточните пожалуйста название!')
            bot.register_next_step_handler(msg, weather_bot)

bot.polling()
while True:
    pass