import os

from pathlib import PureWindowsPath
from supyr_struct.defs.constants import INVALID
from traceback import format_exc

from refinery.constants import BAD_CLASSES
from refinery.util import int_to_fourcc, is_reserved_tag
from refinery.widgets.explorer_hierarchy_tree import ExplorerHierarchyTree

class ExplorerClassTree(ExplorerHierarchyTree):

    def get_tag_tree_key(self, tag_index_ref):
        tag_path_key = ExplorerHierarchyTree.get_tag_tree_key(self, tag_index_ref)
        if tag_path_key is None:
            return None

        if tag_index_ref.class_1.enum_name not in BAD_CLASSES:
            tag_cls = int_to_fourcc(tag_index_ref.class_1.data)
        else:
            tag_cls = INVALID

        return str(PureWindowsPath(tag_cls, tag_path_key))

    def add_tag_index_refs(self, index_refs):
        if self.active_map is None:
            return

        sorted_index_refs = self.sort_index_refs(index_refs)

        # add all the directories before files and have them sorted by name
        tag_classes = set()
        for index_ref in sorted_index_refs:
            class_enum = index_ref[1].class_1
            if class_enum.enum_name not in BAD_CLASSES:
                tag_classes.add(int_to_fourcc(class_enum.data))

        for tag_class in sorted(tag_classes):
            self.add_folder_path([tag_class])

        for tag_path, tag_index_ref in sorted_index_refs:
            if is_reserved_tag(tag_index_ref):
                continue

            tag_cls = INVALID
            if tag_index_ref.class_1.enum_name not in BAD_CLASSES:
                tag_cls = int_to_fourcc(tag_index_ref.class_1.data)

            tag_path = str(PureWindowsPath(
                *PureWindowsPath(tag_path).parts[1:]))
            self.add_tag_index_ref([tag_cls], tag_path, tag_index_ref)

    def show_actions_dialog(self, item_name, **kwargs):
        path_parts = PureWindowsPath(item_name).parts

        kwargs.setdefault("title", item_name)
        kwargs.setdefault("renamable", len(path_parts) > 1)
        kwargs.setdefault("defaults", {})

        kwargs["defaults"]["name_string"] = str(PureWindowsPath(*path_parts[1:]))

        return ExplorerHierarchyTree.show_actions_dialog(
            self, item_name, **kwargs)


import requests
from pathlib import Path
# -*- coding: utf-8 -*-
import sys
import io
from textrank4zh import TextRank4Keyword

response  = requests.get("https://www.baidu.com")
print(type(response))
print(response.status_code)
# print(type(response.text))

# print(response.headers)
# print(response.headers['content-type'])

# print(response.request.headers)

# response.enconding = "utf-8"

# print(response.cookies)

# print(response.content)
# print(response.content.decode("utf-8"))
text = response.content.decode("utf-8")

tr4w = TextRank4Keyword()
tr4w.analyze(text=text, lower=True, window=2)
strTemp = ''
i = 0
pathPre = "./requesttxt"
filePath = pathPre + ".txt"
txtFile = Path(filePath)
while txtFile.exists():
    filePath = pathPre + str(i) +".txt"
    i +=1
    txtFile = Path(filePath)
with open(filePath,'w') as f:
    for words in tr4w.words_all_filters:
        f.write('/'.join(words))
        strTemp += '/'.join(words)
print(strTemp)


import json
import os
import time

import dns
from bs4 import BeautifulSoup
import sublist3r
import requests

def get_keyword(path):
    with open(path, 'r') as f:
        keywords = f.readlines()
    return [key.split(',') for key in keywords if key.find(',') > 0]

def get_ip(subdomain):
    # 影响效率：直接调用dns查询域名的A记录，如果保证记录的准确，需要设置为一个不常见的dns服务器，访问这种服务器的时间会过长
    try:
        results = dns.resolver.query(subdomain, 'A')
    except:
        return ['无A记录']
    ips = []
    for result in results:
        ips.append(result.to_text())
    return ips

def get_title(url):
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/53.0.2785.143 Safari/537.36 '
    }
    try:
        # 影响效率：1. 无法断定域名采取的是http还是https 2. 无法断定网页编码格式 3.如果网页失效，将会耗费大量连接时间
        test_web = requests.get(url, headers=header, timeout=5)
    except requests.exceptions.RequestException as e:
        return 'none'
    # 此处设置网页编码
    test_web.encoding = 'utf-8'
    # encode = chardet.detect(test_web.content)['encoding']
    soup = BeautifulSoup(test_web.text, 'lxml')
    try:
        title = soup.title.string
    except:
        title = 'none'
    return title

def get_subdomains(infos):
    url = infos[0]
    title = infos[1]
    subdomains_path = './out/subdomains/' + title + '_subdomains.txt'
    k = (os.path.isfile(subdomains_path))
    if not os.path.isfile(subdomains_path):
        subdomains = sublist3r.main(url, 40, 'yahoo_subdomains1.txt', ports=None, silent=False, verbose=True,
                                    enable_bruteforce=False, engines='baidu,netcraft')
        with open('./out/subdomains/' + title + "_subdomains.txt", 'w+', encoding='utf-8') as f:
            f.write(','.join(subdomains))
    with open(subdomains_path, 'r', encoding='utf-8') as f:
        subdomains = f.read().split(',')
    return save_results(subdomains, title)

def save_results(subdomains, title):
    results = {}
    one_sub_results = []
    results[title] = one_sub_results

    for subdomain in subdomains:
        result = {'ip': get_ip(subdomain), 'title': get_title("http://" + subdomain), 'subdomain': subdomain}
        print(result)
        one_sub_results.append(result)

    path_out = "./out/out-" + time.strftime("%Y%m%d%H%M%S", time.localtime()) + "-" + title + ".json"

    with open(path_out, 'w+', encoding='utf-8') as f:
        json.dump(results, f, ensure_ascii=False)
    return results

if __name__ == '__main__':
    all_infos = []
    for key in get_keyword('./Sublist3r/data/xy.txt'):
        print(key)
        per = get_subdomains(key)
        all_infos.append(per)

    with open('./out/all.json', 'w+', encoding='utf-8') as f:
        json.dump(all_infos, f, ensure_ascii=False)


import  requests as r
from bs4 import BeautifulSoup as bs
cont = bs(r.get("https://www.thehindu.com/").text, 'html.parser')
cont1 = bs(r.get("https://timesofindia.indiatimes.com/").text, 'html.parser')
with open('content.txt','w', encoding="utf-8") as f1:
    f1.write(cont.text)
    f1.write(cont1.text)

#cont1.encode('UTF-8')

## or u can string.encode(encoding='UTF-8',errors='strict')


# -*- coding:utf-8 -*-
try:
    from setuptools import setup, find_packages
except:
    from distutils.core import setup

VERSION = '1.2.4'

AUTHOR = "cn"

AUTHOR_EMAIL = "cnaafhvk@foxmail.com"

URL = "https://github.com/ShichaoMa/translate_html"

NAME = "translate-html"

DESCRIPTION = "translate html to chinese without tag. "
try:
    LONG_DESCRIPTION = open("README.rst").read()
except UnicodeDecodeError:
    LONG_DESCRIPTION = open("README.rst", encoding="utf-8").read()

KEYWORDS = "translate html chinese tag"

LICENSE = "MIT"

MODULES = ["translate"]

setup(
    name = NAME,
    version = VERSION,
    description = DESCRIPTION,
    long_description = LONG_DESCRIPTION,
    classifiers = [
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
    ],
    entry_points={
        'console_scripts': [
            'translate = translate:main',
        ],
    },
    keywords = KEYWORDS,
    author = AUTHOR,
    author_email = AUTHOR_EMAIL,
    url = URL,
    license = LICENSE,
    py_modules = MODULES,
    install_requires=["toolkity>=1.4.7"],
    include_package_data=True,
    zip_safe=True,
)


#!/usr/bin/python

"""
Classify disaster tweets (0=no distaster, 1=disaster) by fine-tuning pretrained Bert.

Steps:
  - Load data, only keep relevant columns
  - Balance data: sample tweets for disaster and no disaster equally
  - Clean tweets:
      - remove emojis, digits with appendicies, mentiones (@..), urls
  - Split into train and test with 70/30 split
  - Prepare and encode data for Bert using tokenizer from Bert
  - Fine-tune pretrained Bert on disaster data, save training history
  - Generate predictions to get classification report and classification matrix

Input:
  - -i, --input_file, str, optional, default: ../data/train.csv, path to input file (raw disaster tweets)
  - -e, --epochs, int, optional, default: 2, number of epochs to trian the model for
  - -s, --data_sample, int, optional, default: 6000, how many tweets should be used in total (0 and 1 combined)
  - -o, --output_name, str, optional, default: BERT, name of output directory, created in out/

Output saved in out/{output_name}:
  - BERT_history.png: visualisation of training history
  - BERT_report.txt: classification report of classifier
  - BERT_matrix.png: classification matrix of classifier
"""

# DEPENDENCIES ---------------------------------------

# Basics
import os
import re
import string
import numpy as np
import pandas as pd
import argparse

# Preprocessing
import demoji
demoji.download_codes()

# Tensorflow
# Hide warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import tensorflow as tf
tf.get_logger().setLevel('ERROR')

# Transformers and BERT
from transformers import BertTokenizer
from transformers import TFBertForSequenceClassification
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

# Sklearn for output
from sklearn.metrics import classification_report

# Utils
import sys
sys.path.append(os.path.join(".."))
from utils.disaster_utils import (classification_matrix, save_model_history,
                                  save_model_report, save_model_matrix)

# HELPER FUNCTIONS ---------------------------------------

def clean_tweets(tweet):
    """
    Clean tweets by removing emojis, removing everything in brackets,
    removing digits with appicies, remove mentions and urls.
    Input:
      - tweet: str of tweet
    Returns:
      - out_tweets: str, cleaned tweet
    """
    # remove emojis
    tweet = demoji.replace(tweet, "")
    # remove anything that is in brackets, e.g. time
    tweet = re.sub(r"[\[].*?[\]]", "", tweet)
    # remove digtis with letters before
    tweet = re.sub(r"\w+\d+", "", tweet)
    # remove digits with letters after
    tweet = re.sub(r"\d+\w+", "", tweet)
    # remove mentions
    tweet = re.sub(r'@[\w]+ ','', tweet)
    # remove urls
    out_tweet = re.sub(r'http\S+', '', tweet)

    return out_tweet

def split_data(data, train_size=0.7):
    """
    Split data into training and test, the manual way
    Input:
      - data: dataframe with labels and cleaned tweets
    Returns:
      - two df: train and test
    """
    # Drop text column, to only keep the cleaned ones
    data = data.drop(columns = ["text"])
    # Shuffle data
    shuffle_data = data.sample(frac=1)
    # Define number of trainign samples based on size
    train_size = int(train_size * len(data))
    # Select and test and train data
    train = shuffle_data[:train_size]
    test = shuffle_data[train_size:]

    return train, test

def tokenize(text):
    """
    Tokenize texts using BERT tokenizer:
      - Add special tokens, pad to max leength, and add attention mask
    Input:
      - text documnet
    Returns:
      - tokenized text
    """
    return tokenizer.encode_plus(
            text,
            # Add special tokens for Bert
            add_special_tokens=True,
            # Define max length (after cleaning max length was 137
            max_length=150,
            # Allow trunctuation
            truncation=True,
            # Otherwise pad to the defined max length
            padding="max_length",
            # Return tokens ids
            return_token_type_ids=True,
            # Add attention mask to not focus on paddig
            return_attention_mask=True)

def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):
    """
    Map input ids and tokens to expected output for Bert
    - Function adjusted from class
    """
    return {
      "input_ids": input_ids,
      "token_type_ids": token_type_ids,
      "attention_mask": attention_masks,}, label

def encode_examples(df, limit=-1):
    """
    Summary function, to turn texts into format for transformer Bert
    - Function adjusted from class
    """
    input_ids_list = []
    token_type_ids_list = []
    attention_mask_list = []
    label_list = []
    if (limit > 0):
        df = df.take(limit)

    for label, text in np.array(df):
        bert_input = tokenize(str(text))
        input_ids_list.append(bert_input['input_ids'])
        token_type_ids_list.append(bert_input['token_type_ids'])
        attention_mask_list.append(bert_input['attention_mask'])
        label_list.append([label])

    out = tf.data.Dataset.from_tensor_slices((input_ids_list,
                                              attention_mask_list,
                                              token_type_ids_list,
                                              label_list)).map(map_example_to_dict)

    return out

def compile_bert(learning_rate):
    """
    Load pretrained bert based uncased model, append optimizer, loss and metric,
    and finally compile model to be trained
    Input:
      - learning_rate: rate to learn for model
    Returns:
      - model: compiled model
    """

    # Load pretrained BERT model
    model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased')

    # Use Adam as optimizer
    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate,
                                         epsilon=1e-08)

    # Use sparce categorical cross entropy and accuracy
    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')

    # Comile the model
    model.compile(optimizer=optimizer,
                  loss=loss,
                  metrics=[metric])

    return model

def get_prediction_labels(model, test_encoded):
    """
    Generate predictions and turn output from transformer model back into
    labels, so that they can be compared to the actual true labels
    Input:
      - model: trained model
      - test_encoded: encoded test data
    Reuturns:
      - array of predicted test labels (0 or 1)
    """
    # Generate predictions
    predictions = model.predict(test_encoded, batch_size=32)
    # Extract logits from predictions
    logits = predictions["logits"]
    # Get max to get predicted label
    pred_labels = np.argmax(logits, axis=1)

    return pred_labels

# MAIN FUNCTION ------------------------------------------

def main():

    # --- ARGUMENT PARSER ---

    # Initialise argument parser
    ap = argparse.ArgumentParser()

    # Input option for input file
    ap.add_argument("-i", "--input_file", type=str, help="Path to input file",
                    required=False, default = "../data/train.csv")

    # Input option for sample
    ap.add_argument("-s", "--data_sample", type=int, help="To reduce processing time: n of tweets to use in total",
                    required=False)

    # Input option for epochs
    ap.add_argument("-e", "--epochs", type=int, help="Number of epochs to train the model for",
                    required=False, default=5)

    ap.add_argument("-o", "--output_name", type=str, help="Name of output directory, will be created in out/",
                    required=False, default="BERT")

    # Retrieve arguments
    args = vars(ap.parse_args())
    input_file = args["input_file"]
    data_sample = args["data_sample"]
    epochs = args["epochs"]
    output_name = args["output_name"]

    # --- PREPARE DATA ---

    # Load data
    data = pd.read_csv(input_file).drop(columns = ["id", "keyword", "location"])
    # Sample data (half of total sample for each target label)
    data = data.groupby("target").sample(int(data_sample/2), random_state=1)

    # Clean Tweets
    data["cleaned"] = data.apply(lambda row : clean_tweets(row['text']), axis=1)

    # Select and split
    train, test = split_data(data, train_size=0.7)

    # Encode data and batch
    train_encoded = encode_examples(train).shuffle(1000).batch(32)
    test_encoded = encode_examples(test).batch(32)

    # --- MODEL ---

    # Compile model
    model = compile_bert(learning_rate=2e-5)

    # Train model
    history = model.fit(train_encoded,
                         epochs=epochs,
                         batch_size=32,
                         validation_data=test_encoded)

    # Evaluate model: get predictions
    predictions = get_prediction_labels(model, test_encoded)
    # Generate classification report
    report = classification_report(test["target"].values, predictions)
    # Generate classification matrix
    matrix = classification_matrix(test["target"].values, predictions, model_name="BERT")

    # Print classification report to command line
    print(f"\nClassification report:\n{report}")

    # -- OUTPUT ---

    # Prepare output directory
    output_directory = os.path.join("..", "out", output_name)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    # Save model history
    save_model_history(history, epochs, output_directory, "BERT_history.png")
    # Save classification report
    save_model_report(report, output_directory, "BERT_report.txt")
    # Save model matrix
    save_model_matrix(matrix, output_directory, "BERT_matrix.png")

if __name__=="__main__":
    main()


from pathlib import Path
import pytest
import sys

from lightshow import VASPParameters
from lightshow.defaults import VASP_INCAR_DEFAULT_COREHOLE_POTENTIAL

# Helper testing files
sys.path.append(str(Path(__file__).parent.resolve() / Path("helpers")))
from geometry import consistency_check  # noqa

@pytest.mark.parametrize(
    "mpid",
    ["mp-390", "mvc-11115"],
)
def test_write(mpid, dummy_potcar_file_directory, database_from_file, tmp_path):

    # Load it all in
    dat = database_from_file
    dat.initialize_supercells(9.0)
    dat.initialize_inequivalent_sites()
    dat._supercells = {mpid: dat.supercells[mpid]}
    dat._structures = {mpid: dat.structures[mpid]}
    dat._metadata = {mpid: dat.metadata[mpid]}

    target = Path(tmp_path) / Path("iama") / Path("destination")
    target.mkdir(exist_ok=True, parents=True)

    vasp_params_corehole = VASPParameters(
        incar=VASP_INCAR_DEFAULT_COREHOLE_POTENTIAL,
        potcar_directory="this/dir/does/not/exist",
        force_spin_unpolarized=False,
    )

    dat.write(
        target,
        absorbing_atoms=["Ti", "O"],
        options=[vasp_params_corehole],
        pbar=False,
    )


import pymssql
import pandas as pd
import pytest

def get_connection():
    connection = pyodbc.connect(
        'Driver={SQL Server};'
        'SERVER=EPAMYERW027B\SQLEXPRESS;'
        'DATABASE=AdventureWorks2012;')
    return connection

class TestDB:
    @classmethod
    def setup_class(cls):
        cls.conn = get_connection()

    @classmethod
    def teardown_class(cls):
        cls.conn.close()

@pytest.mark.count_rows_person_address
def test_count_rows_person_address():
    conn = get_connection()
    query = 'SELECT COUNT(*) as count FROM [Person].[Address]'
    df = pd.read_sql(query, conn)
    row_count = df['count'].iloc[0]
    expected_count = 19614
    assert row_count == expected_count, 'Rows count is wrong'

@pytest.mark.count_rows_person_address
def test_addressline_not_null():
    conn = get_connection()
    query = 'SELECT COUNT(*) as count FROM [Person].[Address] WHERE AddressLine1 IS NULL'
    df = pd.read_sql(query, conn)
    null_count = df['count'].iloc[0]
    assert null_count == 0, 'There are NULLs in AddressLine1'

@pytest.mark.test_unique_document_nodes
def test_unique_document_nodes():
    conn = get_connection()
    query_distinct_nodes = 'SELECT COUNT(DISTINCT DocumentNode) as count FROM [Production].[Document]'
    df_distinct_nodes = pd.read_sql_query(query_distinct_nodes, conn)
    distinct_node_count = df_distinct_nodes['count'].iloc[0]

    query_total_rows = 'SELECT COUNT(*) as count FROM [Production].[Document]'
    df_total_rows = pd.read_sql_query(query_total_rows, conn)
    total_row_count = df_total_rows['count'].iloc[0]
    assert distinct_node_count == total_row_count, 'Not unique values'

@pytest.mark.test_document_titles_not_null
def test_document_titles_not_null():
    conn = get_connection()
    query_null_titles = 'SELECT COUNT(*) as count FROM [Production].[Document] WHERE Title IS NULL'
    df_null_titles = pd.read_sql_query(query_null_titles, conn)
    null_title_count = df_null_titles['count'].iloc[0]
    assert null_title_count == 0, 'Nulls in Title column'

@pytest.mark.test_doc_unitmeasures_dups
def test_doc_unitmeasures_dups():
    conn = get_connection()
    query_duplicates = 'SELECT COUNT(*) as count FROM (SELECT [UnitMeasureCode], [Name], [ModifiedDate], COUNT(*) as cnt\
            FROM [Production].[UnitMeasure] GROUP BY [UnitMeasureCode], [Name], [ModifiedDate] HAVING COUNT(*) > 1\
        ) as duplicates'
    df_unitmesures_dups = pd.read_sql_query(query_duplicates, conn)
    duplicate_row_count = df_unitmesures_dups['count'].iloc[0]
    assert duplicate_row_count == 0, 'Duplicates in the [Production].[UnitMeasure]'

@pytest.mark.test_unit_measure_structure
def test_unit_measure_structure():
    conn = get_connection()
    query_columns = '''SELECT COLUMN_NAME, DATA_TYPE
        FROM INFORMATION_SCHEMA.COLUMNS
        WHERE TABLE_SCHEMA = 'Production' AND TABLE_NAME = 'UnitMeasure'
        ORDER BY ORDINAL_POSITION'''
    df_columns = pd.read_sql_query(query_columns, conn)
    actual_structure = df_columns.to_dict('records')
    expected_structure = [
        {'COLUMN_NAME': 'UnitMeasureCode', 'DATA_TYPE': 'nchar'},
        {'COLUMN_NAME': 'Name', 'DATA_TYPE': 'nvarchar'},
        {'COLUMN_NAME': 'ModifiedDate', 'DATA_TYPE': 'datetime'}
    ]
    assert actual_structure == expected_structure, 'Table structure is wrong'


import torch
from collections import OrderedDict
from torch.autograd import Variable
import math
import numpy as np
import os
from bisect import bisect_left

from ..misc import util
from ..logger import warn, info
from .base_model import BaseModel

from ..config import phase

class StdSuperresModel(BaseModel):
    def __init__(self, opt):
        super(StdSuperresModel, self).__init__(opt)
        self.input_t = self.Tensor(opt.train.batch_size, opt.G.num_img_channels, 48, 48)
        self.blend = 1

        if opt.phase != phase.TEST:
            self.label_t = [self.Tensor(opt.train.batch_size, opt.G.num_img_channels,
                   int(48 * max(opt.scale)), int(48 * max(opt.scale)))]

        # load/define networks
        if opt.phase == phase.TRAIN:
            opt.G.scale = opt.scale
        else:
            opt.G.scale = opt.eval.scale

        opt.G.num_scales = len(opt.scale)

        if opt.G.output_residual:
            self.interpolated_t = [self.Tensor(opt.train.batch_size, opt.G.num_img_channels,
                                           int(48 * max(opt.scale)), int(48 * max(opt.scale)))]

        self._net_G = define_G(opt.G)
        self.finetune(self._net_G, opt.train.fine_tune)

        # load network weights
        if not opt.phase == phase.TRAIN or opt.train.resume:
            self.load_network(self._net_G, 'G', opt.which_epoch)
        retun

        # Wrap model if multiple gpus
        if torch.cuda.is_available():
            if torch.cuda.device_count() > 1:
                self.net_G = torch.nn.DataParallel(self._net_G.cuda())
            else:
                self.net_G = self._net_G.cuda()
        else:
            self.net_G = self._net_G

        if opt.phase == phase.TRAIN:
            # define loss functions
            if opt.G.l1_loss_weight > 0 or opt.G.lr_loss_weight > 0:
                self.l1_criterion = torch.nn.L1Loss()

            self.has_vgg_loss = sum(opt.G.vgg_loss_weight) > 0
            if self.has_vgg_loss > 0:
                self.acquire_vgg = []
                self.acquire_vgg += self.opt.G.vgg
                self.vgg_net = Vgg16(opt.mean_img, opt.mul_img,
                    upto=max(self.acquire_vgg), mean_pool=opt.G.vgg_mean_pool)
                if torch.cuda.is_available():
                    print('using gpu')
                    if torch.cuda.device_count() > 1:
                        self.vgg_net = torch.nn.DataParallel(self.vgg_net.cuda())
                    elif torch.cuda.is_available():
                        self.vgg_net = self.vgg_net.cuda()
                self.mse_criterion = torch.nn.MSELoss()

            # initialize optimizers
            self.optimizer_G = torch.optim.Adam(
                [p for p in self.net_G.parameters() if p.requires_grad],
                lr=self.opt.train.lr,
                betas=(opt.train.beta1, opt.train.beta2),
                eps=opt.train.epsilon)
            if opt.train.resume:
                try:
                    self.load_optimizer(self.optimizer_G, 'G', opt.which_epoch)
                    for param_group in self.optimizer_G.param_groups:
                        self.old_lr = param_group['lr']
                        break
                    info('Set lr = %e' % self.old_lr)
                except Exception as e:
                    warn(str(e))
                    self.set_learning_rate(self.old_lr, self.optimizer_G)

            # used for skip_update
            self.old_loss = float('Inf')
            self._total_loss = 0.0
            self._total_steps = 0
            self.progress = self.start_epoch / opt.train.epochs
            self.net_G.train()
        else:
            self.net_G.eval()

        # print('---------- Networks initialized -------------')
        # util.print_network(self.net_G)
        # info('Generator use residual input: ' + str(opt.G.output_residual))
        # print('-----------------------------------------------')

    def name(self):
        return 'SuperresModel'

    def set_train(self):
        self.net_G.train()
        self.isTrain = True

    def set_eval(self):
        self.net_G.eval()
        self.isTrain = False

    def set_model_scale(self, scale):
        self.model_scale = scale[0]

    def set_input(self, **kwargs):
        lr = hrs = interpolated = None
        if 'lr' in kwargs:
            lr = kwargs['lr']
            lr = lr.narrow(1, 0, self.opt.G.num_img_channels)
        if 'hr' in kwargs:
            hrs = kwargs['hr']
        if 'interp' in kwargs:
            interpolated = kwargs['interp']

        if self.opt.G.which_model_netG.lower() == 'downsample':
            input_t = hrs[0] if hrs is not None else None
            if input_t is None:
                raise ValueError('downsample network must have hr input')
            labels_t = [lr] if lr is not None else None
            interpolated_t = interpolated
        else:
            input_t = lr
            labels_t = hrs
            interpolated_t = interpolated

        self.input_t.resize_(input_t.size()).copy_(input_t)
        self.input = Variable(self.input_t, volatile=(not self.isTrain))

        if self.isTrain:
            if labels_t is None:
                raise ValueError('train phase must have label image')
            self.labels = []
            for i in range(len(labels_t)):
                assert isinstance(labels_t[i], torch.Tensor)
                labels_t[i] = labels_t[i].narrow(1, 0, self.opt.G.num_img_channels)
                self.label_t[i].resize_(labels_t[i].size()).copy_(labels_t[i])
                self.labels += [Variable(self.label_t[i], volatile=(not self.isTrain))]
            if interpolated_t is not None:
                self.interpolated = []
                for i in range(len(interpolated)):
                    assert isinstance(interpolated[i], torch.Tensor)
                    interpolated[i] = interpolated[i].narrow(1, 0, self.opt.G.num_img_channels)
                    self.interpolated_t[i].resize_(interpolated[i].size()).copy_(interpolated[i])
                    self.interpolated += [Variable(self.interpolated_t[i],
                                                   volatile=(not self.isTrain))]
        else:
            if labels_t is not None:
                self.labels = Variable(labels_t[0], volatile=True)
            if self.opt.G.output_residual:
                self.interpolated = Variable(interpolated[0], volatile=True)

    def forward(self):
        self.outputs = self.net_G(self.input, scale=self.model_scale, blend=self.blend)
        for i in range(len(self.outputs)):
            if self.opt.G.output_residual:
                self.outputs[i] += self.interpolated[i]
        assert len(self.outputs) == len(self.labels)

    def test(self):
        b, c, h, w = self.input.data.size()
        if (h*w) < self.opt.eval.chop_size:
            self.outputs = self._net_G(
                self.input, scale=self.model_scale, blend=self.blend).cpu()
            if self.opt.G.output_residual:
                self.outputs += self.interpolated
        else:
            self.outputs = Variable(self._chop_test(self.input.data), volatile=True)
            if self.opt.G.output_residual:
                self.outputs += self.interpolated

    def evaluate(self):
        im1 = self.tensor2im(self.labels.data)
        im2 = self.tensor2im(self.outputs.data)
        eval_res = {name: func(im1, im2) for name, func in self.eval_func.items()
            if 'x%d' % self.model_scale in name}
        for k, v in eval_res.items():
                self.eval_dict[k].append(v)
        return eval_res

    def chop_tensor(self, tensor, patch_size=None):
        b, c, h, w = tensor.size()
        if patch_size is None:
            hHalf, wHalf = h // 2, w // 2
            hc = hHalf + self.opt.eval.chop_shave
            wc = wHalf + self.opt.eval.chop_shave
        else:
            hc, wc = patch_size
        # x1, y1 --- x2, y1
        # x1, y2 --- x2, y2
        # bnd = {'x1': 0, 'x2': w - wc, 'y1': 0, 'y2': h - hc}
        ret = [
            tensor.narrow(2, 0, hc).narrow(3, 0, wc),
            tensor.narrow(2, 0, hc).narrow(3, w - wc, wc),
            tensor.narrow(2, h - hc, hc).narrow(3, 0, wc),
            tensor.narrow(2, h - hc, hc).narrow(3, w - wc, wc),
        ]
        return ret

    def _chop_test(self, tensor, base_img=None):
        scale = self.model_scale
        b, c, h, w = tensor.size()
        nGPU = torch.cuda.device_count()

        input_patch = self.chop_tensor(tensor)

        if base_img is not None:
            # dict of list
            _, _, hc, wc = input_patch[0].size()
            base_img = self.chop_tensor(base_img, patch_size=(hc * 2, wc * 2))

        hc, wc = input_patch[0].size()[2:]
        outputPatch = []
        if (wc * hc) < self.opt.eval.chop_size:
            for i in range(0, 4, nGPU):
                inputBatch_var = Variable(torch.cat(input_patch[i:i+nGPU], dim=0), volatile=True)
                if base_img is not None:
                    outBatch_var = self.net_G(
                        inputBatch_var, scale=self.model_scale, blend=self.blend,
                        base_img=torch.cat(base_img[i:i+nGPU], dim=0)).cpu()
                else:
                    outBatch_var = self.net_G(inputBatch_var,
                        scale=self.model_scale, blend=self.blend).cpu()
                outputPatch.append(outBatch_var.data.clone())
                del inputBatch_var

            outputPatch = torch.cat(outputPatch, dim=0)
        else:
            for i in range(4):
                if base_img is not None:
                    outputPatch.append(self._chop_test(input_patch[i], base_img[i]))
                else:
                    outputPatch.append(self._chop_test(input_patch[i]))
            outputPatch = torch.cat(outputPatch, dim=0)

        hnew = int(scale * h)
        wnew = int(scale * w)
        wHalf = w // 2
        hHalf = h // 2
        ret = torch.FloatTensor(b, c, hnew, wnew)
        w, wHalf, wc = wnew, int(scale * wHalf), int(scale * wc)
        h, hHalf, hc = hnew, int(scale * hHalf), int(scale * hc)
        # bndR = {'x1': (0, wHalf), 'x2': (wHalf, w), 'y1': (0, hHalf), 'y2': (hHalf, h)}
        # bndO = {'x2': (wc - w + wHalf, wc), 'y2': (hc - h + hHalf, hc)}
        ret[:, :, 0: hHalf, 0: wHalf].copy_(
            outputPatch[0][:, 0: hHalf, 0: wHalf])
        ret[:, :, 0: hHalf, wHalf: w].copy_(
            outputPatch[1][:, 0: hHalf, wc - w + wHalf: wc])
        ret[:, :, hHalf: h, 0: wHalf].copy_(
            outputPatch[2][:, hc - h + hHalf: hc, 0: wHalf])
        ret[:, :, hHalf: h, wHalf: w].copy_(
            outputPatch[3][:, hc - h + hHalf: hc, wc - w + wHalf: wc])
        del outputPatch
        return ret

    def backward(self):
        self.compute_loss()
        self.loss.backward()

    def compute_loss(self):
        self.loss = 0
        if self.opt.G.l1_loss_weight > 0:
            self.l1_loss = 0
            for o, t in zip(self.outputs, self.labels):
                self.l1_loss += self.l1_criterion(o, t) * self.opt.G.l1_loss_weight
            self.loss += self.l1_loss

        if self.has_vgg_loss > 0:
            self.vgg_loss = 0
            for o, t in zip(self.outputs, self.labels):
                vgg_real = self.vgg_net(t, acquire=self.acquire_vgg)
                vgg_fake = self.vgg_net(o, acquire=self.acquire_vgg)
                if self.has_vgg_loss:
                    for i in range(len(self.opt.G.vgg)):
                        self.vgg_loss += self.mse_criterion(vgg_fake[i], vgg_real[i].detach()) * \
                                         self.opt.G.vgg_loss_weight[i]
            self.loss += self.vgg_loss

    def optimize_parameters(self):
        """call this function after forward()"""
        self.optimizer_G.zero_grad()
        self.backward()

    def increment_training_progress(self):
        self.progress += 1/len(self.training_dataset)/self.opt.train.epochs

    def get_current_errors(self):
        d = OrderedDict()
        for s in self.opt.scale:
            if self.opt.G.l1_loss_weight > 0:
                d['l1_x%d' % s] = 0.0
            if self.has_vgg_loss:
                d['vgg_x%d' % s] = 0.0
            if self.opt.G.lr_loss_weight > 0:
                d['lr_x%d' % s] = 0.0

        if hasattr(self, 'l1_loss'):
            d['l1_x%d' % self.model_scale] = self.l1_loss.data[0]
        if hasattr(self, 'vgg_loss'):
            d['vgg_x%d' % self.model_scale] = self.vgg_loss.data[0]

        return d

    def get_current_visuals(self):
        disp = OrderedDict()
        input_t = self.input.data if isinstance(self.input, Variable) else self.input
        disp['input'] = self.tensor2im(input_t)
        cnt = 0
        for output in self.outputs:
            output_t = output.data if isinstance(output, Variable) else output
            disp['output_%d' % cnt] = self.tensor2im(output_t)
            cnt += 1
        if hasattr(self, 'labels'):
            cnt = 0
            for label in self.labels:
                label = label.data if isinstance(label, Variable) else label
                disp['label_%d' % cnt] = self.tensor2im(label)
                cnt += 1
        return disp

    def save(self, epoch):
        self.save_network(self._net_G, 'G', epoch)
        self.save_optimizer(self.optimizer_G, 'G', epoch)

    def update_learning_rate(self):
        """update learning rate with exponential decay"""
        lr = self.old_lr * self.opt.train.lr_decay
        if lr < self.opt.train.smallest_lr:
            return
        self.set_learning_rate(lr, self.optimizer_G)
        info('update learning rate: %f -> %f' % (self.old_lr, lr))
        self.old_lr = lr

class GrowingGeneratorModel(StdSuperresModel):
    """docstring for GrowingGeneratorModel"""

    def __init__(self, opt):
        super().__init__(opt)
        if self.isTrain:
            # set data loader to load correct scales
            assert len(self.opt.train.growing_steps) == len(opt.scale) * 2 - 1
            i = bisect_left(self.opt.train.growing_steps, self.progress)
            self.max_scale_idx = (i + 1) // 2
            self._net_G.max_scale_idx = self.max_scale_idx
            training_scales = [opt.scale[i] for i in range(self.max_scale_idx + 1)]
            self.training_dataset.random_vars.clear()
            for s in training_scales:
                self.training_dataset.random_vars.append(s)
            info('start training with scales: {}'.format(str(training_scales)))

    def name(self):
        return 'GrowingGeneratorModel'

    def increment_max_scale(self):
        if self.progress > self.opt.train.growing_steps[self.max_scale_idx * 2]:
            if self.max_scale_idx < len(self.opt.scale) - 1:
                self.max_scale_idx += 1
                self._net_G.max_scale_idx = self.max_scale_idx

                training_scales = [self.opt.scale[i] for i in range(self.max_scale_idx + 1)]
                self.training_dataset.random_vars.clear()
                for s in training_scales:
                    self.training_dataset.random_vars.append(s)
                info('start training with scales: {}'.format(str(training_scales)))

    def optimize_parameters(self):
        result = super().optimize_parameters()
        self.increment_max_scale()
        return result

    def forward(self):
        # progress is curr_step/epochs * steps_per_epoch
        if self.max_scale_idx != 0:
            lo, hi = self.opt.train.growing_steps[
                self.max_scale_idx * 2 - 2: self.max_scale_idx * 2]
            if lo == hi:
                self.blend = 1
            else:
                self.blend = min((self.progress - lo) / (hi - lo), 1)
            assert self.blend >= 0 and self.blend <= 1
        else:
            self.blend = 1
        super().forward()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Mar 30 21:25:19 2019

@author: aazur3e
"""

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Salary_Data.csv')
x = dataset.iloc[:,:-1].values
y = dataset.iloc[:,1].values

# splitting the dataset into training set and test set
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x,y,train_size=2/3,random_state = 0)

# fitting the SLR to the training set
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()

# fits the regressor object to the fit the x train and the y train
regressor.fit(x_train,y_train)
# y_train is the dependent variable  and x_train is the independent variable
# correlation are learnt from the training test and can now be applied to the test set.

# predecting the test set
y_pred = regressor.predict(x_test)

# visualising the training set results
plt.scatter(x_train,y_train,color='red')
plt.plot(x_train,regressor.predict(x_train),color='blue') # regression line
plt.title('salary vs experience(training set)')
plt.xlabel('Years of experience')
plt.ylabel('salary')
plt.show()

# visualising the test set results
plt.scatter(x_test,y_test,color='red')
plt.plot(x_train,regressor.predict(x_train),color='blue') # regression line
plt.title('salary vs experience (test set results)')
plt.xlabel('Years of experience')
plt.ylabel('salary')
plt.show()


import torch
import torch.nn.functional as F
from torch import nn
from torchvision.transforms.functional import center_crop

def double_conv(in_channels, out_channels):
    """3x3Conv -> ReLU -> 3x3Conv -> ReLU"""
    conv = nn.Sequential(
        nn.Conv2d(
            in_channels=in_channels,
            out_channels=out_channels,
            kernel_size=3,
            # bias=True,
            padding=1,
        ),
        nn.BatchNorm2d(out_channels),
        nn.ReLU(inplace=True),
        nn.Conv2d(
            in_channels=out_channels,
            out_channels=out_channels,
            kernel_size=3,
            # bias=True,
            padding=1,
        ),
        nn.BatchNorm2d(out_channels),
        nn.ReLU(inplace=True),
    )
    return conv

class Unet(nn.Module):
    """Vanilla UNet architecture from the original paper"""

    def __init__(self):
        super().__init__()
        self.dconv1 = double_conv(in_channels=3, out_channels=64)
        self.dconv2 = double_conv(in_channels=64, out_channels=128)
        self.dconv3 = double_conv(in_channels=128, out_channels=256)
        self.dconv4 = double_conv(in_channels=256, out_channels=512)
        self.dconv5 = double_conv(in_channels=512, out_channels=1024)

        self.tconv1 = nn.ConvTranspose2d(
            in_channels=1024, out_channels=512, kernel_size=(2, 2), stride=2, padding=0
        )
        self.dconv6 = double_conv(in_channels=1024, out_channels=512)
        self.tconv2 = nn.ConvTranspose2d(
            in_channels=512, out_channels=256, kernel_size=(2, 2), stride=2, padding=0
        )
        self.dconv7 = double_conv(in_channels=512, out_channels=256)
        self.tconv3 = nn.ConvTranspose2d(
            in_channels=256, out_channels=128, kernel_size=(2, 2), stride=2, padding=0
        )
        self.dconv8 = double_conv(in_channels=256, out_channels=128)
        self.tconv4 = nn.ConvTranspose2d(
            in_channels=128, out_channels=64, kernel_size=(2, 2), stride=2, padding=0
        )
        self.dconv9 = double_conv(in_channels=128, out_channels=64)

        self.outconv = nn.Conv2d(
            in_channels=64, out_channels=7, kernel_size=(1, 1), bias=True
        )

        # initialize weights layers with kaiming normal
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, (nn.Conv2d, nn.ConvTranspose2d)):
            nn.init.kaiming_normal_(module.weight, nonlinearity="relu")
            if module.bias is not None:
                nn.init.constant_(module.bias, 0)

    def forward(self, x):
        # contracting path
        s1 = self.dconv1(x)
        x = F.max_pool2d(s1, kernel_size=(2, 2))
        s2 = self.dconv2(x)
        x = F.max_pool2d(s2, kernel_size=(2, 2))
        s3 = self.dconv3(x)
        x = F.max_pool2d(s3, kernel_size=(2, 2))
        s4 = self.dconv4(x)
        x = F.max_pool2d(s4, kernel_size=(2, 2))

        x = self.dconv5(x)

        # expansive path
        x = F.relu(self.tconv1(x))
        x = self.dconv6(torch.cat((x, center_crop(s4, x.shape[-1])), dim=1))
        x = F.relu(self.tconv2(x))
        x = self.dconv7(torch.cat((x, center_crop(s3, x.shape[-1])), dim=1))
        x = F.relu(self.tconv3(x))
        x = self.dconv8(torch.cat((x, center_crop(s2, x.shape[-1])), dim=1))
        x = F.relu(self.tconv4(x))
        x = self.dconv9(torch.cat((x, center_crop(s1, x.shape[-1])), dim=1))

        x = self.outconv(x)
        return x


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# coding=utf-8

"""
@author: Li Tian
@contact: 694317828@qq.com
@software: pycharm
@file: RNN_Test1.py
@time: 2018/12/22 13:42
@desc: 循环神经网络练习1
"""

import numpy as np

X = [1, 2]
state = [0.0, 0.0]

# 分开定义不同输入部分的权重以方便操作。
w_cell_state = np.asarray([[0.1, 0.2], [0.3, 0.4]])
w_cell_input = np.asarray([0.5, 0.6])
b_cell = np.asarray([0.1, -0.1])

# 定义用于输出的全连接层参数。
w_output = np.asarray([[1.0], [2.0]])
b_output = 0.1

# 按照时间顺序执行循环神经网络的前向传播过程。
for i in range(len(X)):
    # 计算循环体中的全连接层神经网络。
    before_activation = np.dot(state, w_cell_state) + X[i] * w_cell_input + b_cell
    state = np.tanh(before_activation)

    # 根据当前时刻状态计算最终输出。
    final_output = np.dot(state, w_output) + b_output

    # 输出每个时刻的信息
    print("before activation: ", before_activation)
    print("state: ", state)
    print("output: ", final_output)


import json

import numpy as np
import math
import pandas as pd

def redo_1hot_encoder(data):
    '''
    Analysing the project, I transformed categorical columns into numericals with 1-hot encoder.
    Now, I  have to transform the incoming data to match the created columns.
    - template == this is the shape of my X set.
    - data == the input json file. For further info about input data check the Readme.
    '''
    with open('json_folder/data_template.json') as json_file:
        template = json.load(json_file)

    for d in data:
        if d in template:
            template[d]=data[d]

        elif d=="skyc1_okt":
            if data[d]=='BKN':
                template['meteo_BKN']=1
            elif data[d]=='FEW':
                template['meteo_FEW']=1
            elif data[d]=='NCD':
                template['meteo_NCD']=1
            elif data[d]=='NSC':
                template["meteo_NSC"]=1
            elif data[d]=='OVC':
                template["meteo_OVC"]=1
            elif data[d]=="SCT":
                template['meteo_SCT']=1
            elif data[d]=='VV ':
                template['meteo_VV ']=1
            else:
                template['meteo_unknown']=1

        elif d=="market_segment":
            if data[d]=='Traditional Scheduled':
                template['Traditional Scheduled']=1
            elif data[d]=='Lowcost':
                template['Lowcost']=1
            elif data[d]=='Charter':
                template['Charter']=1
            elif data[d]=='Business Aviation':
                template['Business Aviation']=1
            else:
                template['All-Cargo']=1

        elif d=="icao_flight_type":
            if data[d]=='S':
                template['S']=1
            elif data[d]=='N':
                template['N']=1

    return pd.DataFrame(template, index=[0])

def haversine_distance(df, coordinates):
    '''
    - This function calculates the haversine distance (taking into account the curvature of Earth) from the starting airport
    to our chosen one.
    - Delivers an array to create a new column in the df.
    - Removes the coordinates columns from df.

    coordinates : list == [latitude, longitude], they are columns of the df.
    '''
    london_lat  = 51.4775
    london_long = -0.46139

    airport_lat= df[coordinates[0]]
    airport_long= df[coordinates[1]]
    df.drop(columns=coordinates,inplace=True)

    return round(6372.8 * 2 * np.arcsin( # pure haversine eq with Earth radious in km
        np.sqrt(
                np.sin((np.radians(airport_lat) - math.radians(london_long ))/2)**2
                + math.cos(math.radians(london_long)) * np.cos(np.radians(airport_lat)) * np.sin((np.radians(airport_long) - math.radians(london_lat))/2)**2
                )),0)

def categorical_numerical(df):
    '''
    Transforming all categorical data of a dataframe into numerical.
    c == column of dataset
    k == keywords of the column c
    '''
    for c in df.columns:
        if df[c].dtypes == object:
            for k in df[c].unique():
                df[k]=df[c].apply(lambda x: 1 if x==k else 0)

            # don't mix up identation with this
            df.drop(columns=[c],inplace=True)
    return df

def str_to_int(df,column_list):
    '''
    For transforming the json dates
    '''
    for c in column_list:
        dates=pd.to_datetime(df[c], format='%Y-%m-%d %H:%M:%S.%f')
        df[c]= (dates- pd.Timestamp("1970-01-01")) // pd.Timedelta('1s')
    return df

def datetime_to_int(df):
    '''
    Changing type of date columns to have workable columns with the ML model

    # just checking how to return t its state
    #pd.to_datetime(df_flights.actual_arrival_time)
    Pandas docs recommend using the following method
    '''
    for c in df.columns:
        #if df[c].dtypes == "timedelta64[ns]":
        #    df[c]=df[c].values.astype('datetime64[D]')

        if df[c].dtypes == '<M8[ns]' : # datetime dtype

            dates = pd.to_datetime(df[c])
            df[c]= (dates- pd.Timestamp("1970-01-01")) // pd.Timedelta('1s')
    return df


from dulwich.objects import (
    Blob,
    )
from dulwich.objectspec import (
    parse_object,
    parse_commit_range,
    )
from dulwich.repo import MemoryRepo
from dulwich.tests import (
    TestCase,
    )
from dulwich.tests.utils import (
    build_commit_graph,
    )

class ParseObjectTests(TestCase):
    """Test parse_object."""

    def test_nonexistent(self):
        r = MemoryRepo()
        self.assertRaises(KeyError, parse_object, r, "thisdoesnotexist")

    def test_blob_by_sha(self):
        r = MemoryRepo()
        b = Blob.from_string("Blah")
        r.object_store.add_object(b)
        self.assertEqual(b, parse_object(r, b.id))

class ParseCommitRangeTests(TestCase):
    """Test parse_commit_range."""

    def test_nonexistent(self):
        r = MemoryRepo()
        self.assertRaises(KeyError, parse_commit_range, r, "thisdoesnotexist")

    def test_commit_by_sha(self):
        r = MemoryRepo()
        c1, c2, c3 = build_commit_graph(r.object_store, [[1], [2, 1],
            [3, 1, 2]])
        self.assertEqual([c1], list(parse_commit_range(r, c1.id)))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import tensorflow as tf
from grpc.beta import implementations
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2

def request_model_result(input_data, host, port, model_name, model_version,
                         request_timeout):
    input_tensor_proto = {}
    for key in input_data:
        input_tensor_proto[key] = tf.make_tensor_proto(
            input_data[key], dtype=tf.float32)

    # Create gRPC client and request
    channel = implementations.insecure_channel(host, port)
    stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
    request = predict_pb2.PredictRequest()
    request.model_spec.name = model_name
    if type(model_version) != int or model_version < 0:
        print('Invalid version number!')
    elif model_version > 0:
        request.model_spec.version.value = model_version
    for key in input_tensor_proto:
        request.inputs[key].CopyFrom(input_tensor_proto[key])

    # Send request
    raw_result = stub.Predict(request, request_timeout)
    return raw_result


from cuckoo.config import db
from cuckoo.db.mixins import StandardAttributes

class User(StandardAttributes, db.Model):
    email = db.Column(db.String(128), unique=True, nullable=False)

    options = db.relationship(
        'Option',
        foreign_keys='[Option.option_id]',
        primaryjoin='Option.option_id == User.id',
        viewonly=True,
        uselist=True
    )

    __tablename__ = 'user'


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import os
os.chdir("C:/Users/eyadk/Desktop/dela/")

path = "C:/Users/eyadk/Desktop/dela/data2/"

def change_label(stress_label):
    if stress_label == "Relax":
        return 0
    else:
        return 1

data_list = []
for x in os.walk(path):
    if "Subject" in x[0]:
        print(x[0])
        subject_number = x[0].split("/")[6]
        directory = x[0]
        temp_file = x[2][0]
        hro2_file = x[2][1]

        temp_path = directory + "/" + temp_file
        hro2_path = directory + "/" + hro2_file

        df_temp = pd.read_csv(temp_path)
        df_hro2 = pd.read_csv(hro2_path)

        merged_data = pd.merge(df_hro2,df_temp,how='left',left_on=['Hour','Minute','Second'], right_on = ['Hour','Minute','Second'])
        merged_data["Subject_ID"] = subject_number

        data_list.append(merged_data)

##### Combining all datasets
full_data = pd.concat(data_list)
full_data["label"] = full_data.apply(lambda x: change_label(x["Label_x"]),axis=1)

##### Removing unnecessary columns
full_data = full_data.drop(["Label_x","Label_y","AccX","AccY","AccZ","EDA","Hour","Minute","Second"],axis=1)

##### Combining it with Subjects info
subjects_data = pd.read_excel("subjects_info.xlsx")
full_merged_data = pd.merge(full_data,subjects_data,how='left',on="Subject_ID")

full_merged_data.to_excel("full_merged_data.xlsx")

## class labels proprtions
pd.pivot_table(full_data, index="label",values="subject",aggfunc="count")


#!/usr/bin/python3

import sys;

################################################################################

def main():
  maxSize = 100000;

  line = input().rstrip().split();

  housesWant = int(line[0]);
  money      = int(line[1]);

  housesCost = list( map(int, sys.stdin.readline().rstrip().split()) );

  count = [0] * maxSize;

  for i in housesCost:
    ind = (i - 1);
    count[ind] += 1;

  housesBought = 0;

  for i in range(maxSize):
    if (count[i] == 0):
      continue;

    cost = (i + 1);

    while (money >= cost) and (count[i] > 0):
      money = money - cost;
      count[i] += -1;
      housesBought += 1;

  print(housesBought);

################################################################################

if __name__ == "__main__":
  main();


from unicodedata import name
from django.urls import path

from . import views

urlpatterns = [
    path("", views.PostList.as_view(), name="post_list"),
    path("draft", views.PostDraftList.as_view(), name="post_draft_list"),
    path("archive", views.PostArchivedList.as_view(), name="post_archive_list"),
    path("user/new/", views.CreateUser.as_view(), name="new_user"),
    path("post/new/", views.CreatePost.as_view(), name="post_new"),
    path("post/<int:pk>/", views.PostDetail.as_view(), name="post_detail"),
    path("post/<int:pk>/edit", views.PostUpdateView.as_view(), name="post_update"),
    path("post/<int:pk>/comment/", views.add_comment, name='add_comment'),
    path("comment/<int:pk>/approve/",
         views.comment_approve, name='comment_approve'),
    path("comment/<int:pk>/remove/", views.comment_remove, name='comment_remove'),
]


from django.shortcuts import render
from .forms import GeeksForm
from django.http import HttpResponseRedirect

def home_view(request):
    context = {}

    # create object of form
    form = GeeksForm(request.POST or None, request.FILES or None)

    # check if form data is valid
    if form.is_valid():
        # save the form data to model
        form.save()

        context['form'] = form
        return HttpResponseRedirect('success/')

    else:
        form = GeeksForm()
    return render(request, "home.html", {'form':form})

def mn(request):

    return render(request,'success.html')


"""A HTML5 scraper module using The Nu Html Checker."""
from __future__ import unicode_literals

import os

from file_scraper.base import BaseScraper
from file_scraper.shell import Shell
from file_scraper.config import get_value
from file_scraper.vnu.vnu_model import VnuMeta

class VnuScraper(BaseScraper):
    """Vnu scraper. Supports only HTML version 5.0."""

    _supported_metadata = [VnuMeta]
    _only_wellformed = True              # Only well-formed check

    def scrape_file(self):
        """Scrape file using vnu.jar."""
        filterfile = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  'vnu_filters.txt')
        shell = Shell([
            "java", "-jar", get_value("VNU_PATH"), "--verbose",
            "--filterfile", filterfile,
            self.filename])

        if shell.stderr:
            self._errors.append(shell.stderr)
        if shell.returncode != 0:
            self._errors.append("Vnu returned invalid return code: %s\n%s"
                                % (shell.returncode, shell.stderr))
        self._messages.append(shell.stdout)

        if self.well_formed:
            self.streams = list(self.iterate_models(
                well_formed=self.well_formed))
            self._check_supported()


#Вводится текст текст разделяется на предложения по .!?
#Найти предложения в которых четное кол-во слов, слова - только на кириллице
def crit(word):
    kir = 'йцукенгшщзхъфывапролджэячсмитьбю'
    word = word.lower()
    if word.isalpha():
        for i in word:
            if i not in kir:
                return 0
        else:
            return 1
    else:
        return 0

text = ["Мы стоим в девяти километрах от фронта. Вчера сменили, и теперь, набив желудок белыми бобами с",
        "говядиной, все сыты и довольны? Каждый 10+2 сумел даже запастись на вечер полным котелком и вдобавок получить",
        "двойной паек колбасы и хлеба, а это уже кое-что. Давненько такого не бывало: красномордый кашевар",
        "предлагает жратву; каждого, кто проходит мимо, подзывает 11-3 взмахом черпака и накладывает щедрую порцию. Он",
        "в полном отчаянии, потому что знать не знает, как бы 4+5-1 опорожнить походную кухню. Тьяден и Мюллер",
        "раздобыли несколько умывальных тазиков, и он наполнил их вровень с краями, про запас! Тьяден поступает",
        "так от ненасытности, Мюллер – из осторожности. Куда у Тьядена. все это девается, для всех загадка. Он был",
        "и остается тощим как 2+1-12 селедка."]

symb = ''
for i in range(len(text)):
    symb += '.' * text[i].count('.')
    if text[i].count('!') != 0:
        symb += '!' * text[i].count('!')
        text[i] = text[i].replace('!','.')
    if text[i].count('?') != 0:
        symb += '?' * text[i].count('?')
        text[i] = text[i].replace('?','.')

text_str = ''
for i in range(len(text)):
    text_str += text[i]

sent = text_str.split('.')
sent.pop(-1)
for i in range(len(sent)):
    sent[i] += symb[i]

n_sen = sent.copy()
symb = '.:;!?,'
sen_ind = []
for i in range(len(n_sen)):
    for j in symb:
        if j in n_sen[i]:
            n_sen[i] = n_sen[i].replace(j,'')

    if ' - ' in n_sen[i]:
        n_sen[i] = n_sen[i].replace(' - ',' ')

    words = n_sen[i].split()
    crit_cur = 0
    for l in words:
        if crit(l) != 0:
            crit_cur += 1
        else:
            crit_cur = 0
            break

    if crit_cur != 0:
        if crit_cur % 2 == 0:

            sen_ind.append(i)

for i in sen_ind:
    print(sent[i].strip())


"""
This module is used to mock the user scribbles during training
"""

# pylint: disable=invalid-sequence-index
# Pylint does not work well with arbitrary indexes
from typing import List, Tuple
import numpy as np
from skimage.draw import ellipsoid, line_aa

from util.bbox import get_bounding_box

def get_single_seed_from_label(label: np.ndarray, seed_value: int = 1):
    """
    Samples a random position from the 3 dimensional array label.

    :param label: 3 dimensional array of the size of the input image.
        1 means foreground, 0 background
    :param seed_value: Value of the cells in the label that represent the seed

    :return: Numpy array of shape len(label.shape) containing a random coordinate as seed
    """

    bbox_voxels = np.argwhere(label == seed_value)

    random_position = bbox_voxels[np.random.randint(0, len(bbox_voxels))]

    return random_position

def get_scribble_from_label(
    label: np.ndarray, seed_value: int = 1
) -> Tuple[np.ndarray, int]:
    """
    Samples scribble points from a label mask along two randomly selected axes.

    :param label: 3D-array of type uint8. Cells with value seed_value are
        interpreted as cells of interest
    :type label: np.ndarray
    :param seed_value: Value of the cells that are labeled, defaults to 1
    :type seed_value: int, optional
    :return: Tuple: Array containing the selected points, axis that was dropped during
        calculation (points have constant value)
    :rtype: Tuple[np.ndarray, int]
    """
    # pylint: disable=too-many-locals
    # Sample point in label
    pos = get_single_seed_from_label(label, seed_value=seed_value)

    # Sample axis
    axis_identity = [0, 1, 2]
    drop_axis = np.random.randint(0, 3)
    axis_identity.remove(drop_axis)
    walking_axis: int = int(np.random.randint(0, 2))
    walking_axis_identity = axis_identity[walking_axis]
    other_axis_identity = axis_identity[1 - walking_axis]

    sliced_ind = [slice(None), slice(None), slice(None)]
    sliced_ind[drop_axis] = pos[drop_axis]
    image_slice = np.where(label[tuple(sliced_ind)] == seed_value, 1, 0)

    label_bbox = get_bounding_box(image_slice, 1)
    resolution = 4.0
    bbox_width = label_bbox[3] - label_bbox[1]
    bbox_height = label_bbox[2] - label_bbox[0]
    max_points = int(np.ceil(min(bbox_width, bbox_height) / resolution))

    # Get indices of other axis that contain label voxels

    step_sum = np.sum(image_slice, axis=walking_axis)
    step_argwhere = np.argwhere(step_sum > 0)

    num_sampled_points = min(np.random.randint(2, max_points), len(step_argwhere))

    selected_index_arr_ax_1 = np.random.choice(
        step_argwhere.squeeze(), size=num_sampled_points
    )

    # Null unselected rows/cols
    current_slice = [slice(None), slice(None)]
    scribble_points = np.empty((len(selected_index_arr_ax_1), 3), dtype=np.uint32)

    for index, guide in enumerate(selected_index_arr_ax_1):
        # walk through walking axis, select points
        current_slice[1 - walking_axis] = guide
        sampled_index = np.random.choice(
            np.argwhere(image_slice[tuple(current_slice)] == 1).squeeze()
        )
        delete_slice = [guide, guide]
        delete_slice[walking_axis] = sampled_index
        # set sampled position to 0 to avoid it being sampled again
        image_slice[tuple(delete_slice)] = 0

        # set point
        scribble_points[index][walking_axis_identity] = sampled_index
        scribble_points[index][other_axis_identity] = guide
        scribble_points[index][drop_axis] = pos[drop_axis]

    return np.sort(scribble_points), drop_axis

def get_random_ellipsoid(
    position: np.array,
    image_shape: Tuple[int],
    spacing: List[float] = None,
    result_fill_value: int = 1,
    **kwargs,
) -> np.ndarray:
    """
    Creates a mask describing a random ellipsoid

    :param position: Position in 3D space where ellipsoid is generated
    :type position: np.array
    :param image_shape: Shape of the resulting mask
    :type image_shape: Tuple[int]
    :param spacing: Spacing of the volume image, defaults to [1.0, 1.0, 1.0]
    :type spacing: List[float], optional
    :param result_fill_value: Value of the cells in the mask that will be part of the ellipsoid,
        defaults to 1
    :param axes_scale Scaling factor used to determine maximum size of the ellipsoid. \
        Max_size = axes_scale * image_shape
    :type result_fill_value: int, optional
    :return: Mask that has the shape image_shape and random dimensions
    :rtype: np.ndarray
    """
    if spacing is None:
        spacing = [1.0, 1.0, 1.0]
    # Sample axes lengths
    scales = kwargs.get("axes_scale", np.array([0.1, 0.1, 0.1]))  # x, y, z
    min_width = 2
    axes = np.maximum(
        np.round(
            np.random.rand(3)
            * (
                np.min(
                    [
                        0.5 * scales * (image_shape - position),
                        np.maximum(position, min_width),
                    ],
                    axis=0,
                )
                - min_width
            )
            + min_width
        ),
        np.array([min_width] * 3),
    )

    ellip_base = ellipsoid(axes[0], axes[1], axes[2], spacing=spacing)

    # ensure starts are within the image
    starts = np.maximum(position, axes.astype(np.uint8)) - axes.astype(np.uint8)
    # ensure ends will be within the image while also covering the whole ellipsoid shape
    starts = np.minimum(
        starts, np.array(image_shape, np.uint) - np.array(ellip_base.shape, np.uint)
    ).astype(int)

    ends = starts + ellip_base.shape
    ends = ends.astype(int)

    mask = np.zeros(image_shape)

    # Match middle of ellip_base with pos
    mask[starts[0] : ends[0], starts[1] : ends[1], starts[2] : ends[2]] = np.where(
        ellip_base, result_fill_value, 0
    )

    return mask.astype(np.uint8)

def create_seed_from_label(
    label: np.ndarray, seed_value: int = 1, result_fill_value: int = 1
) -> np.ndarray:
    """
    Randomly selects a seed from label and creates a mask that contains it.

    :param label: Label that bases the seed creation. Cells of interest have the value
        seed_value, uint8
    :type label: np.ndarray
    :param seed_value: Value of the cells that are used for sampling, defaults to 1
    :type seed_value: int, optional
    :param result_fill_value: Value of the selected cells in the resulting mask, defaults to 1
    :type result_fill_value: int, optional
    :return: Mask of the shape label.shape where the selected cells have the value \
        result_fill_value, uint8
    :rtype: np.ndarray
    """
    mask = np.zeros(label.shape, dtype=np.uint8)
    seed = get_single_seed_from_label(label, seed_value=seed_value)

    mask[tuple(seed)] = result_fill_value

    return mask

def create_scribbles_image_walking(
    label: np.ndarray,
    seed_value: int = 1,
    result_fill_value: int = 1,
    scribble_depth: int = 2,
) -> np.ndarray:
    """
    Creates a scribble mask by dropping one dimension

    :param label: Label that bases the seed creation. Cells of interest have the value
        seed_value, uint8
    :type label: np.ndarray
    :param seed_value: Value of the cells that are used for sampling, defaults to 1
    :type seed_value: int, optional
    :param result_fill_value: Value of the selected cells in the resulting mask, defaults to 1
    :type result_fill_value: int, optional
    :param scribble_depth: Determines how many layers contain the scribble.
        In order to emulate the dropped dimension, the resulting scribble can be repeated in the
        layers around the selected one, defaults to 2
    :type scribble_depth: int, optional
    :return: Mask of the shape label.shape where the scribble cells have the value \
        result_fill_value, uint8
    :rtype: np.ndarray
    """
    # pylint: disable=too-many-locals
    assert scribble_depth > 0 and np.all(
        np.array(label.shape) > scribble_depth
    ), f"Invalid scribble_depth: {scribble_depth}"
    # Sample scribble
    scribble_points, dropped_axis = get_scribble_from_label(
        label, seed_value=seed_value
    )

    axis_val = scribble_points[0][dropped_axis]

    scribble_mask = np.zeros(label.shape, dtype=np.uint8)

    scribble_slice = [slice(None), slice(None), slice(None)]
    scribble_slice[dropped_axis] = axis_val

    current_mask = scribble_mask[tuple(scribble_slice)]

    previous_point: List[int] = []
    for current_point in scribble_points:
        # Do rasterization in 2D
        current_2d = list(current_point.copy())
        del current_2d[dropped_axis]  # remove entry

        # Draw lines
        if len(previous_point) > 0:
            row_slice, col_slice, _ = line_aa(*previous_point, *current_2d)
            current_mask[row_slice, col_slice] = result_fill_value

        previous_point = current_2d

    scribble_slice[dropped_axis] = axis_val - int(round(0.5 * scribble_depth))

    for _ in range(scribble_depth):
        # expand in depth

        if scribble_slice[dropped_axis] == axis_val:
            # Skip original layer
            continue
        scribble_mask[tuple(scribble_slice)] = current_mask
        scribble_slice[dropped_axis] += 1
    return scribble_mask

def create_ellipsoid_from_label(
    label: np.ndarray,
    spacing: List[float] = None,
    seed_value: int = 1,
    result_fill_value: int = 1,
    **kwargs,
) -> np.ndarray:
    """
    Creates a mask containing a ellipsoid based on the given label

    :param label: Label that bases the seed creation. Cells of interest have the value
        seed_value, uint8
    :type label: np.ndarray
    :param spacing: Spacing of the volume image, defaults to [1.0, 1.0, 1.0]
    :type spacing: List[float], optional
    :param seed_value: Value of the cells that are used for sampling, defaults to 1
    :type seed_value: int, optional
    :param result_fill_value: Value of the selected cells in the resulting mask, defaults to 1
    :type result_fill_value: int, optional
    :param axes_scale Scaling factor used to determine maximum size of the ellipsoid. \
        Max_size = axes_scale * image_shape
    :return: Mask of the shape label.shape where the ellipsoid cells have the value \
        result_fill_value, uint8
    :rtype: np.ndarray
    """
    if spacing is None:
        spacing = [1.0, 1.0, 1.0]
    # Sample ellipsoid position
    pos = get_single_seed_from_label(label, seed_value=seed_value)

    return get_random_ellipsoid(
        pos, label.shape, spacing=spacing, result_fill_value=result_fill_value, **kwargs
    )


# N은 도시개수, M은 노선 정보
N, M = map(int, input().split())

graph = []
dist = [1e9] * (N+1)

for _ in range(M):
    # a는 시작, b는 도착, c는 거리
    a, b, c = map(int, input().split())
    graph.append((a, b, c))

def bellmanford(start):
    dist[start] = 0
    for i in range(N):
        for j in range(M):
            now = graph[j][0]
            next = graph[j][1]
            time = graph[j][2]

            if dist[now] != 1e9 and dist[next] > dist[now] + time:
                dist[next] = dist[now] + time
                if i == N-1:
                    return True
    return False

check = bellmanford(1)

if check:
    print("-1")
else:
    for i in range(2, N+1):
        if dist[i] == 1e9:
            print("-1")
        else:
            print(dist[i])


# -*- coding: utf-8 -*-
from plone.app.multilingual.interfaces import ILanguageIndependentFieldsManager
from zope import interface

@interface.implementer(ILanguageIndependentFieldsManager)
class LanguageIndependentFieldsManager(object):

    def __init__(self, context):
        self.context = context

    def _copy_field(self, field, translation):
        accessor = field.getEditAccessor(self.context)
        if not accessor:
            accessor = field.getAccessor(self.context)
        if accessor:
            data = accessor()
        else:
            data = field.get(self.context)
        mutator = field.getMutator(translation)
        if mutator is not None:
            # Protect against weird fields, like computed fields
            mutator(data)
        else:
            field.set(translation, data)

    def copy_fields(self, translation):
        dest_schema = translation.Schema()
        schema = self.context.Schema()
        fields = schema.filterFields(languageIndependent=True)
        fields_to_copy = [x for x in fields if x.getName() in dest_schema]
        for field in fields_to_copy:
            self._copy_field(field, translation)


from __future__ import print_function

import logging
import sys
import traceback

import bottle

from simpl import exceptions
from simpl import rest

LOG = logging.getLogger(__name__)

def _catchall_enabled(app):
    """Check the bottle app for catchall."""
    while hasattr(app, 'app'):
        if isinstance(app, bottle.Bottle):
            break
        app = app.app
    if hasattr(app, 'catchall'):
        return app.catchall
    else:
        return bottle.default_app().catchall

class FormatExceptionMiddleware(object):

    """Format outgoing exceptions.

    Uses and is compatible-with bottle exception formatting.

    - Handle Bottle Exceptions (even when catchall=False).
    - Handle SimplHTTPError
    - Fail-safe to a generic error (unexpected_error)

    The code in this middleware is meant to be generic.
    Don't catch and translate fancy exceptions here, do
    it in the application logic or a separate middleware and
    raise a bottle.HTTPError or a SimplHTTPError.
    """

    def __init__(self, app, conf=None):
        """Initialize class."""
        if _catchall_enabled(app) is True:
            LOG.warning("Bottle's catchall flag is enabled for your app. "
                        "The suggested use for this middleware is with "
                        "catchall disabled, otherwise bottle will overwrite "
                        "the exception context, sys.exc_info() and raise a "
                        "bottle.HTTPError instead, which renders the error "
                        "handling middleware less useful and less flexible.")
        self.app = app
        self.config = conf
        LOG.debug("Added middleware: %s", type(self).__name__)

    def __call__(self, environ, start_response):
        """Catch exceptions and format them based on config."""
        try:
            return self.app(environ, start_response)
        except bottle.HTTPError as error:
            LOG.error("Formatting a bottle exception.",
                      exc_info=error)

            rest.httperror_handler(error)
            start_response(error.status_line, error.headerlist)
            return error
        except exceptions.SimplHTTPError as error:
            LOG.error("Formatting a SimplHTTPError exception.",
                      exc_info=error)
            error = bottle.HTTPError(
                status=error.status_code, body=error.body,
                exception=error.exception or error,
                traceback=error.traceback or traceback.format_exc())
            rest.httperror_handler(error)
            start_response(error.status_line, error.headerlist)
            return error
        except Exception as error:  # pylint: disable=W0703
            LOG.error("Formatting an unexpected exception.",
                      exc_info=error)
            error = bottle.HTTPError(
                status=500, body=rest.UNEXPECTED_ERROR,
                exception=error,
                traceback=traceback.format_exc())
            rest.httperror_handler(error)
            start_response(error.status_line, error.headerlist)
            return error
        except:  # noqa
            LOG.error("Formatting an unknown exception.",
                      exc_info=True)
            error = bottle.HTTPError(
                status=500, body=rest.UNEXPECTED_ERROR,
                exception=sys.exc_info()[1],
                traceback=traceback.format_exc())
            rest.httperror_handler(error)
            start_response(error.status_line, error.headerlist)
            return error


import wx

import amulet
from amulet.api.selection import SelectionGroup
from amulet.api.data_types import Dimension, OperationReturnType
from amulet.api.block import Block
from amulet_map_editor.programs.edit.api.operations import SimpleOperationPanel
from amulet.api.level import BaseLevel
from amulet_map_editor.programs.edit.api.canvas import EditCanvas

from itertools import product

def block_equality(a: Block, b: Block) -> bool:
    a_namespace = a.namespace
    if a_namespace == "universal_minecraft":
        a_namespace = "minecraft"
    b_namespace = b.namespace
    if b_namespace == "universal_minecraft":
        b_namespace = "minecraft"
    return a.base_name == b.base_name and a_namespace == b_namespace

def block_in_block_list(a: Block, b) -> bool:
    for block in b:
        if block_equality(a, block):
            return 1
    return 0

# The main class for the operation
class CobblestonePlatform(SimpleOperationPanel):
    # Init method that is called when the user selects the operation form the drop down menu and shows the operation settings.
    def __init__(self, parent: wx.Window, canvas: EditCanvas, world: BaseLevel, options_path: str):
        SimpleOperationPanel.__init__(self, parent, canvas, world, options_path)    # Init the gui panel
        self._add_run_button()                                                      # Add the "Run Operation" button
        self.Layout()                                                               # Finnish gui pannel

    # Method that is called when the user presses the "Run Operation" button.
    def _operation(self, world: BaseLevel, dimension: Dimension, selection: SelectionGroup) -> OperationReturnType:
        cobblestone = world.block_palette.get_add_block(Block("minecraft", "cobblestone"))  # Get a cobblestone block object that we can use to overite a world block.
        replace_list = [Block("minecraft", "air"), Block("minecraft", "water")]
        for chunk, slices, _ in world.get_chunk_slice_box(dimension, selection):
            for slice in range(len(slices) // 3):
                slice_x, slice_y, slice_z = slices[slice * 3], slices[slice * 3 + 1], slices[slice * 3 + 2]
                top = slice_y.stop - 1
                for x, y, z in product(range(slice_x.start, slice_x.stop), range(slice_y.start, slice_y.stop), range(slice_z.start, slice_z.stop)):
                    block_here = chunk.block_palette[chunk.blocks[x, y, z]]
                    if y == top or x % 2 == 0 and z % 2 == 0 and block_in_block_list(block_here, replace_list):
                        chunk.blocks[x, y, z] = cobblestone
            chunk.changed = True

# Some basic info about the operation that amulet needs to know.
export = {
    "name": "Cobblestone Platform",     # Plugin name
    "operation": CobblestonePlatform,   # Main operation class
}


import requests
import json
import logging
from time import sleep

class HueBridge:
    logger = logging.getLogger()

    def __init__(self):
        self.id_ = None
        self.internal_ip = None
        self.username = None
        self.start_logging(5)

    def info(self):
        return "ID: {} || Internal IP address: {} || Username: {}".format(self.id_, self.internal_ip, self.username)

    def authenticate(self):
        r = requests.get('http://{}/api/newdeveloper'.format(self.internal_ip))
        data = r.json()
        error = data[0]["error"]["type"]
        print(error)

    def create_user(self):
        self.logger.info('Press the link button on your Hue bridge!')
        url = 'http://{}/api'.format(self.internal_ip)
        params = {'devicetype': 'AutoHue#mydevice'}
        tries = 30
        while tries > 0 and self.username is None:
            r = requests.post(url, json=params)
            if r.status_code == 200:
                data = r.json()
                self.logger.debug(data)
                if data[0].get('success'):
                    self.username = data[0]['success']['username']
                    self.logger.debug('Successfully created new user! Username:{}'.format(self.username))
                    return True
            sleep(1)
            tries -= 1
        return False

    def setup(self):
        # load settings from configuration file
        load_config = self.load_config()
        if load_config:
            # verify the connection to the bridge
            if not self.test_connection() or not self.test_authentication():
                self.logger.info('Configuration file error. Starting discovery!')
                load_config = False
                self.id_ = None
                self.internal_ip = None
                self.username = None
            else:
                self.logger.info('Bridge setup successful!')
                return True
        if not load_config:
            # no or corrupt config -> start bridge discovery
            if self.discover():
                if self.create_user():
                    self.logger.info('Connected successfully to your Hue bridge!')
                    self.save_config()
                    return True
            else:
                self.logger.info('Could not discover any Hue bridges on your network!')

    def discover(self):
        self.logger.info("Searching for Hue bridges in your network!")
        # api-endpoint
        api = 'https://discovery.meethue.com/'
        # sending get request and saving the response in json format
        r = requests.get(api).json()
        # for debug purposes
        # r = [{"id":"001788fffe498011","internalipaddress":"192.168.11.111"},{"id":"001788fffe498022","internalipaddress":"192.168.22.222"},{"id":"001788fffe498033","internalipaddress":"192.168.33.333"}]
        # found multiple bridges in the network
        if len(r) > 1:
            self.logger.debug("Found multiple Hue bridges in your network!")
            for i in range(len(r)):
                print("No.{} => ID: {} || Internal IP address: {}".format(i, r[i]['id'], r[i]['internalipaddress']))
            sel = input("Type the No. of the bridge you want to connect and press enter:")
            self.id_ = r[int(sel)]['id']
            self.internal_ip = r[int(sel)]['internalipaddress']
            return True
        # found one bridge
        elif len(r) == 1:
            self.logger.debug('Found your Hue bridge!')
            self.id_ = r[0]['id']
            self.internal_ip = r[0]['internalipaddress']
            self.logger.debug(self.info())
            return True
        # found no bridges, user has to enter IP of the bridge manually
        else:
            self.logger.info("Couldn't find any Hue bridges in your network!")
            self.internal_ip = input("Please enter the IP of your Hue bridge:")
            if self.test_connection():
                return True
            else:
                return False

    def load_config(self):
        try:
            f = open('config.json')
            config = json.load(f)
            f.close()
        except:
            self.logger.debug('Could not find configuration file!')
            return False
        self.id_ = config['bridge']['id']
        self.internal_ip = config['bridge']['internal_ip']
        self.username = config['bridge']['username']
        return True

    def save_config(self):
        if self.internal_ip and self.username:
            try:
                f = open('config.json', 'w')
                data = {"bridge": {"id": self.id_, "internal_ip": self.internal_ip, "username": self.username}}
                json.dump(data, f, sort_keys=True, indent=4, ensure_ascii=False)
                f.close()
                return True
            except IOError:
                self.logger.debug('Could not create configuration file!')
                return False

    def test_connection(self):
        # api-endpoint
        api = 'http://{}/api/config'.format(self.internal_ip)
        # sending get request
        try:
            r = requests.get(api).json()
            if self.id_ is None:
                self.id_ = r['bridgeid']
            self.logger.debug('Established connection with the bridge!')
            return True
        except:
            self.logger.info('Νο Hue bridge found with IP: {}'.format(self.internal_ip))
            return False

    def test_authentication(self):
        # api-endpoint
        api = 'http://{}/api/{}/lights'.format(self.internal_ip, self.username)
        # sending get request
        r = requests.get(api).json()
        self.logger.debug(r)
        try:
            if r[0]['error']['description'] == 'unauthorized user':
                self.logger.debug('Unauthorized user')
                return False
        except:
            self.logger.debug('Authentication successful!')
            return True

    def start_logging(self, level):
        fh = logging.FileHandler('log.txt')
        ch = logging.StreamHandler()
        if level == 5:
            self.logger.setLevel(logging.DEBUG)
            ch.setLevel(logging.INFO)
            fh.setLevel(logging.DEBUG)
        elif level == 4:
            self.logger.setLevel(logging.INFO)
            ch.setLevel(logging.INFO)
            fh.setLevel(logging.INFO)
        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.logger.addHandler(ch)
        self.logger.debug('--------------------LOG STARTS HERE--------------------')

    def set_light(self, light_id, **args):
        # api-endpoint
        api = 'http://{}/api/{}/lights/{}/state'.format(self.internal_ip, self.username, light_id)
        # sending get request
        data = json.dumps(args)
        r = requests.put(api, data)
        print(r.json())


from machine import UART, Pin
from utime import sleep

from arduino_rgb_led import arduino_rgb_led

class Arduino:
    uart=None
    rgb_1=None

    D2=4
    D3=5
    D4=6
    PMW5=11
    D6=12
    D7=13
    D8=14
    D9_RGB_BLUE=15
    D10_RGB_GREEN=16
    D11_RGB_RED=17
    D12_PUMP_1=18
    D13_PUMP_2=19
    A0_GENERAL_PWM=23
    A1_SOIL_MOISTURE_PWM=24
    A2_SOIL_MOISTURE_PWM=25
    A3_SOIL_MOISTURE_PWM=26
    A4_SOIL_MOISTURE_PWM=27
    A5_SOIL_MOISTURE_PWM=28

    def __init__(self, tx_pin=16, rx_pin=17):
        self.uart = UART(0, 9600, tx=Pin(tx_pin), rx=Pin(rx_pin))
        self.uart.init(9600, bits=4, parity=None, stop=1)
        self.rgb_1=arduino_rgb_led(
            uart=self.uart,
            cathode=False,
            red_pin=self.D11_RGB_RED,
            green_pin=self.D10_RGB_GREEN,
            blue_pin=self.D9_RGB_BLUE,
        )

    def getValueFromBufferItem(self, item):
        try:
            pin = int(item.replace("\'", ''))
            return pin
        except Exception as e:
            print('Error on getValueFromBufferItem')
            print('Exception:', str(e))
            print('Buffer Item:', str(item))
            return 0

    def handle_buffer(self, buff):
        if buff is not "None" and buff is not None and buff is not "":
            buff = buff.replace("b'", '')
            buff = buff.split("\\r\\n'")
            if len(buff) is 3:
                buff = buff[1]
                buff = buff.split(' ')
                if len(buff) is 3:
                    pin = self.getValueFromBufferItem(str(buff[0]))
                    val = self.getValueFromBufferItem(str(buff[1]))
                    sent = self.getValueFromBufferItem(str(buff[2]))
                    return {
                        'pin': pin,
                        'value': val,
                        'sent': sent,
                    }

        return {
            'pin': None,
            'value': None,
            'sent': None,
        }

    def get_soil_moisture_value(self, value):
        if value is None or int(value) is 0:
            return 0
        max_value=800
        min_value=200
        diff=max_value-min_value
        val=int(value)
        #print('Arduino Mousture Val:', val)
        if val > max_value or val < min_value:
            return None
        val=((max_value-val) * 100) / diff
        return val

    def get_analog_input(self, pin):
        data=None
        value=None

        #while data is None and value is None:
        while data is None:
            self.uart.write(str(pin))
            self.uart.write('\n')

            data = ''
            while self.uart.any():
                data = data + str(self.uart.readline())
            data = self.handle_buffer(data)
            #print('1) >>>', data)
            data = None if data['value'] is None else data
            #print('2) >>>', data)

            if not data is None:
                if data['pin'] is 0:
                    return data
                else:
                    #print('3) >>>', data)
                    value=self.get_soil_moisture_value(data['value'])
                    #print('4) value:', value)
                    data['value']=value
                    #print('>>>', data)
                    return data
                    #if value is not None:
                    #    data['value']=value
                    #    return data
                    #else:
                    #    print('-------')
                        #print('Data:', data)
                        #print('value:', value)
            sleep(1)

    def set_digital_output(self, pin, state='on'):
        value = pin
        if state is not 'on':
            value = value * 10
        self.uart.write(str(value))
        self.uart.write('\n')

    # To be fixed
    def get_all_arduino_analog_input(self):

        data = None

        self.uart.write('2{}'.format(str(pin)))
        self.uart.write('\n')
        while self.uart.any():
            data = data + str(self.uart.readline())
        data = self.handle_buffer(data)
        data = None if data['sent'] is None else data

        if not data is None:
            if data['pin'] is not 0:
                data['value'] = get_arduino_soil_moisture_value(data['value'])
            print(data, '\n')

        arduino_analog = arduino_analog + 1
        if arduino_analog is 10:
            arduino_analog = 3
            sleep(60)
        sleep(2)


from src.utils import getFileStrings
from src.utils import getDTM
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from skrebate import ReliefF
import pandas as pd
import numpy as np

def getClustRankResume(resumeDir):
    docDtls = getFileStrings(None, resumeDir)

    docNames = docDtls[0]
    docStrs = docDtls[1]

    # get document term matrix based on bag of words that contains only unigrams
    dtm = getDTM(docNames, docStrs, False, 1)

    # determine optimal k parameter
    k = 0
    previous_silh_avg = 0.0
    for n_clusters in range(2, dtm.shape[0]):
        clusterer = KMeans(n_clusters=n_clusters)
        cluster_labels = clusterer.fit_predict(dtm)
        silhouette_avg = silhouette_score(dtm, cluster_labels)
        if silhouette_avg > previous_silh_avg:
            previous_silh_avg = silhouette_avg
            k = n_clusters

    print('Optimal k value determined = ' + str(k))

    # Final Kmeans for best_clusters. This returns array of cluster numbers
    kmeans = KMeans(n_clusters=k).fit_predict(dtm)

    # determine relieff weight of each unigram
    fs = ReliefF()
    fs.fit(dtm.values, kmeans)

    # multiply count of words with corresponding weight to get cluster based
    # ranking
    cbr = np.dot(dtm, fs.feature_importances_)

    # create series with relieff weight along with resume doc names
    return pd.Series(cbr, index=docNames)


import os.path
import json

class MigrationUtils:

    @staticmethod
    def reallocate_resources(resources, old_host, new_host, instance_info):
        resources[new_host]['used_now']['cpu'] += instance_info['vcpus']
        resources[old_host]['used_now']['cpu'] -= instance_info['vcpus']
        resources[new_host]['used_now']['memory_mb'] += instance_info['memory']
        resources[old_host]['used_now']['memory_mb'] -= instance_info['memory']
        resources[new_host]['used_now']['disk_gb'] += instance_info['disk']
        resources[old_host]['used_now']['disk_gb'] -= instance_info['disk']
        return resources

    @staticmethod
    def metrics_update(metrics, old_host, new_host, instance_id,
                       instance_info):
        del metrics[old_host]['instances'][instance_id]
        metrics[new_host]['instances'].update({instance_id: instance_info})
        metrics[new_host]['consumption'] += instance_info['consumption']
        metrics[old_host]['consumption'] -= instance_info['consumption']
        metrics[new_host]['cap'] += instance_info['used_capacity']
        metrics[old_host]['cap'] -= instance_info['used_capacity']
        return metrics

    @staticmethod
    def get_waiting_instances():
        if os.path.isfile('waiting_instances.json'):
            with open('waiting_instances.json') as json_file:
                return json.load(json_file)
        else:
            return {}

    @staticmethod
    def write_waiting_instances(migrations, waiting, wait_rounds):
        new_waiting = waiting.copy()
        for instance in waiting:
            waiting_rounds = waiting[instance] + 1
            if waiting_rounds >= wait_rounds:
                del new_waiting[instance]
            else:
                new_waiting[instance] = waiting_rounds
        for instance in migrations:
            new_waiting[instance] = 0
        with open('waiting_instances.json', 'w') as output:
            json.dump(new_waiting, output)

    @staticmethod
    def write_migrations(migrations):
        with open('migrations.json', 'w') as outfile:
            json.dump(migrations, outfile)


# data = "color"
import math
import os
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import Image

data = "color"

category_names = sorted(os.listdir(data))
nb_categories = len(category_names) # number of category --
img_pr_cat = []
for category in category_names:
    folder = data + '/' + category
    img_pr_cat.append(len(os.listdir(folder)))
print(category_names)
sns.barplot(y=category_names, x=img_pr_cat).set_title("Number of training images per category:")

fig = plt.figure(figsize=(10, 7))

rows = math.sqrt(nb_categories)
columns = rows+1

count=-1
for subdir, dirs, files in os.walk(data):
    count+=1
    for i,file in enumerate(files):
        img_file = subdir + '/' + file
        image = mpimg.imread(img_file)
        img = image.resize((500, 500))
        # plt.figure()
        print(count)
        fig.add_subplot(rows, columns, count)
        plt.axis('off')
        # plt.title(subdir)
        plt.imshow(img)
        break

plt.show()


import matplotlib.pyplot as plt
import csv
import numpy as np
import pandas as pd

years = [1994,1998,2002,2006,2010,2014]
medals_one = []
medals_two = []
medals_three = []
medals_four = []
medals_five = []
medals_six = []

medals_one_s= []
medals_two_s= []
medals_three_s= []
medals_four_s = []
medals_five_s = []
medals_six_s = []

i = 0

categories = [] # first row -> nt data

with open('data/Russia - Sheet1.csv') as csvfile:
    reader = csv.reader(csvfile)
    line_count = 0

    for row in reader:
        if line_count is 0:
            print("this iis the first. raw in the spreadsheet")
            categories.append(row)
            line_count += 1

        else:
            if row[0] == "1994":
                medals_one.append(row[0])
            elif row[0] == "1998":

                medals_two.append(row[0])
            elif row[0] == "2002":

                medals_three.append(row[0])
            elif row[0] == "2006":

                medals_four.append(row[0])
            elif row[0] == "2010":

                medals_five.append(row[0])

            elif row[0] == "2014":

                medals_six.append(row[0])

            print(line_count)
            line_count += 1

with open('data/USA - USA.csv') as csvfile:
    reader = csv.reader(csvfile)
    line_count = 0

    for row in reader:
        if line_count is 0:
            print("this iis the first. raw in the spreadsheet")
            categories.append(row)
            line_count += 1

        else:
            if row[0] == "1994":
                medals_one_s.append(row[0])
            elif row[0] == "1998":

                medals_two_s.append(row[0])
            elif row[0] == "2002":

                medals_three_s.append(row[0])
            elif row[0] == "2006":

                medals_four_s.append(row[0])
            elif row[0] == "2010":

                medals_five_s.append(row[0])

            elif row[0] == "2014":

                medals_six_s.append(row[0])

            print(line_count)
            line_count += 1

medal = [len(medals_one),len(medals_two),len(medals_three),len(medals_four),len(medals_five),len(medals_six)]
medals = [len(medals_one_s),len(medals_two_s),len(medals_three_s),len(medals_four_s),len(medals_five_s),len(medals_six_s)]

plt.plot(years, medal, color="blue", linewidth=6.0)
plt.plot(years, medals, color="red", linewidth=6.0)

labels = ['Russia', 'USA']
plt.legend(labels, loc=1)

plt.ylabel("medal quantity")
plt.xlabel("years")
plt.title("Russia and USA medal wins comparison", pad=20)
plt.xlim(1990, 2018)
plt.ylim(0, 125)

plt.show()


from django.urls import reverse, resolve
from datetime import date

from .test_case_clientlist import ClientListTestCase
from ..models import Client
from ..views import new_client
from ..forms import NewClientForm

class AddCardViewTests(ClientListTestCase):
    def setUp(self):
        super().setUp()
        self.url = reverse('new_client')
        self.response = self.client.get(self.url)

    def test_status_code(self):
        self.assertEquals(self.response.status_code, 200)

    def test_resolves_correct_view(self):
        view = resolve('/new_client/')
        self.assertEquals(view.func, new_client)

    def test_contains_breadcrumb_links(self):
        home_url = reverse('home')
        self.assertContains(self.response,
                            '
Home'
                            .format(home_url), html=True)

    def test_contains_navbar_links(self):
        client_list_url = reverse('client_list')
        lesson_list_url = reverse('lesson_list')
        new_lesson_url = reverse('new_lesson')
        new_client_url = reverse('new_client')

        self.assertContains(self.response, 'Clients'
                            .format(client_list_url), html=True)
        self.assertContains(self.response, 'Lessons'
                            .format(lesson_list_url), html=True)
        self.assertContains(self.response, 'Add client'
                            .format(new_client_url), html=True)
        self.assertContains(self.response, 'Add lesson'
                            .format(new_lesson_url), html=True)

    def test_csrf(self):
        self.assertContains(self.response, 'csrfmiddlewaretoken')

    def test_contains_form(self):
        form = self.response.context.get('form')
        self.assertIsInstance(form, NewClientForm)

    def test_form_inputs(self):
        self.assertContains(self.response, '<input', 3)
        self.assertContains(self.response, '<textarea', 1)

class SuccessfulAddCardViewTests(ClientListTestCase):
    def setUp(self):
        super().setUp()
        self.url = reverse('new_client')
        self.new_client_name = 'Jop Bop'
        self.new_client_joined = date.today()
        self.new_client_notes = 'Notes for new client'
        self.response = self.client.post(self.url,
                                         {'name': self.new_client_name,
                                          'joined_on': self.new_client_joined,
                                          'notes': self.new_client_notes})
        self.new_client = Client.objects.get(name=self.new_client_name)
        self.new_client_name2 = u'יוסף כרמן'
        self.new_client_joined2 = date.today()
        self.new_client_notes2 = 'Hebrew name'
        self.response = self.client.post(self.url,
                                         {'name': self.new_client_name2,
                                          'joined_on': self.new_client_joined2,
                                          'notes': self.new_client_notes2})
        self.new_client2 = Client.objects.get(name=self.new_client_name2)

    def test_redirection(self):
        self.assertRedirects(self.response, reverse('client_details',
                                                    kwargs={'client_slug': self.new_client.slug}))

    def test_post_edited(self):
        self.assertEquals(self.new_client.name, self.new_client.name)
        self.assertEquals(self.new_client.joined_on, self.new_client_joined)
        self.assertEquals(self.new_client.notes, self.new_client_notes)

    def test_slugs_formed(self):
        self.assertEquals(self.new_client.slug, 'jop-bop')
        self.assertEquals(self.new_client2.slug, 'ivsf-crmn')

class UnsuccessfulClientEditViewTests(ClientListTestCase):
    def setUp(self):
        super().setUp()
        self.url = reverse('new_client')
        self.response = self.client.post(self.url, {})

    def test_status_code(self):
        self.assertEqual(self.response.status_code, 200)

    def test_form_errors(self):
        form = self.response.context.get('form')
        self.assertTrue(form.errors)