import torch
import torch.nn as nn
import torch.nn.functional as F
import math

def conv(in_planes, out_planes, kernel_size=3, stride=2, padding=1, dilation=1, bn_layer=False, bias=True):
    if bn_layer:
        return nn.Sequential(
            nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation, bias=bias),
            nn.BatchNorm2d(out_planes),
            nn.ReLU(inplace=True)
        )
    else:
        return nn.Sequential(
            nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation),
            nn.ReLU(inplace=True)
        )

def linear(in_planes, out_planes):
    return nn.Sequential(
        nn.Linear(in_planes, out_planes),
        nn.ReLU(inplace=True)
        )

class BasicBlock(nn.Module):
    expansion = 1
    def __init__(self, inplanes, planes, stride, downsample, pad, dilation):
        super(BasicBlock, self).__init__()

        self.conv1 = conv(inplanes, planes, 3, stride, pad, dilation)
        self.conv2 = nn.Conv2d(planes, planes, 3, 1, pad, dilation)

        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        out = self.conv1(x)
        out = self.conv2(out)

        if self.downsample is not None:
            x = self.downsample(x)
        out += x

        return F.relu(out, inplace=True)

class VOFlowRes(nn.Module):
    def __init__(self):
        super(VOFlowRes, self).__init__()
        inputnum = 4
        blocknums = [2,2,3,4,6,7,3]
        outputnums = [32,64,64,128,128,256,256]

        self.firstconv = nn.Sequential(conv(inputnum, 32, 3, 2, 1, 1, False),
                                       conv(32, 32, 3, 1, 1, 1),
                                       conv(32, 32, 3, 1, 1, 1))

        self.inplanes = 32

        self.layer1 = self._make_layer(BasicBlock, outputnums[2], blocknums[2], 2, 1, 1) # 40 x 28
        self.layer2 = self._make_layer(BasicBlock, outputnums[3], blocknums[3], 2, 1, 1) # 20 x 14
        self.layer3 = self._make_layer(BasicBlock, outputnums[4], blocknums[4], 2, 1, 1) # 10 x 7
        self.layer4 = self._make_layer(BasicBlock, outputnums[5], blocknums[5], 2, 1, 1) # 5 x 4
        self.layer5 = self._make_layer(BasicBlock, outputnums[6], blocknums[6], 2, 1, 1) # 3 x 2
        fcnum = outputnums[6] * 6

        fc1_trans = linear(fcnum, 128)
        fc2_trans = linear(128,32)
        fc3_trans = nn.Linear(32,3)

        fc1_rot = linear(fcnum, 128)
        fc2_rot = linear(128,32)
        fc3_rot = nn.Linear(32,3)

        self.voflow_trans = nn.Sequential(fc1_trans, fc2_trans, fc3_trans)
        self.voflow_rot = nn.Sequential(fc1_rot, fc2_rot, fc3_rot)

    def _make_layer(self, block, planes, blocks, stride, pad, dilation):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
           downsample = nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride)

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample, pad, dilation))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes,1,None,pad,dilation))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.firstconv(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.layer5(x)

        x = x.view(x.shape[0], -1)
        x_trans = self.voflow_trans(x)
        x_rot = self.voflow_rot(x)
        return torch.cat((x_trans, x_rot), dim=1)


import numpy as np
import tensorflow as tf
from config import *
import os
import random

def make_class_dict(data_root_path, train_dir_name, test_dir_name, val_dir_name):
    train_class_list = [class_name.casefold() for class_name in os.listdir(os.path.join(data_root_path, train_dir_name))]
    test_class_list = [class_name.casefold() for class_name in os.listdir(os.path.join(data_root_path, test_dir_name))]
    val_class_list = [class_name.casefold() for class_name in os.listdir(os.path.join(data_root_path, val_dir_name))]
    all_class = list(set(train_class_list) | set(test_class_list) | set(val_class_list))
    class_dict = {}
    rev_class_dict = {}
    for i, class_name in enumerate(all_class):
        class_dict[class_name] = i
        rev_class_dict[i] = class_name
    return class_dict, rev_class_dict

def get_img_path_list(data_root_path, sub_dir_name, class_dict, one_hot):
    sub_dir_class_list = [class_name for class_name in os.listdir(os.path.join(data_root_path, sub_dir_name))]
    img_path_list = []
    label_list = []

    def get_label(img_path, class_dict, one_hot):
        class_name = img_path.split('/')[-2]
        label = class_dict[class_name.casefold()]
        if one_hot:
            label = tf.one_hot(label, len(class_dict))
        return label

    for sub_dir_class_name in sub_dir_class_list:
        for img_name in os.listdir(os.path.join(data_root_path, sub_dir_name, sub_dir_class_name)):
            img_path = os.path.join(data_root_path, sub_dir_name, sub_dir_class_name, img_name)
            img_path_list.append(img_path)
            label_list.append(get_label(img_path, class_dict, one_hot))
    img_and_label = list(zip(img_path_list, label_list))
    random.shuffle(img_and_label)
    img_path_list, label_list = zip(*img_and_label)
    return list(img_path_list), list(label_list)

def read_img(img_path):
    img = tf.io.read_file(img_path)
    img = tf.image.decode_image(img, channels=3, expand_animations=False)
    img = tf.cast(img, tf.float32)
    return img

def resize(img, height, width):
    img = tf.image.resize(img, [height, width],
                          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
    return img

def random_flip(img):
    if tf.random.uniform(()) > 0.5:
        # random mirroring
        img = tf.image.flip_left_right(img)
    if tf.random.uniform(()) > 0.5:
        img = tf.image.flip_up_down(img)
    return img

def random_brightness(img):
    delta = tf.random.normal((), 0, 50)
    delta = tf.clip_by_value(delta, -5, 50)
    img = tf.image.adjust_brightness(img, delta)
    img = tf.clip_by_value(img, 0, 255)
    return img

def random_hue(img):
    delta = tf.random.normal((), 0, 0.01)
    delta = tf.clip_by_value(delta, -1, 1)
    img = tf.image.adjust_hue(img, delta)
    img = tf.clip_by_value(img, 0, 255)
    return img

def random_saturation(img):
    saturation_factor = tf.random.normal((), 1, 0.01)
    img = tf.image.adjust_saturation(img, saturation_factor)
    img = tf.clip_by_value(img, 0, 255)
    return img

def random_contrast(img):
    contrast_factor = tf.random.normal((), 1, 0.01)
    img = tf.image.adjust_contrast(img, contrast_factor)
    img = tf.clip_by_value(img, 0, 255)
    return img

def random_scale(img):
    factor = tf.clip_by_value(tf.random.normal((), 0, 10), 1, 1.1)
    height, width, _ = img.shape
    offset_h = int(tf.random.uniform((), 0, height * (factor - 1)))
    offset_w = int(tf.random.uniform((), 0, width * (factor - 1)))

    img = tf.image.resize(img, [int(height * factor), int(width * factor)],
                          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
    img = tf.image.crop_to_bounding_box(img, offset_h, offset_w, height, width)
    return img

# normalizing the images to [-1, 1]
def normalize(img):
    img = (img / 127.5) - 1
    return img

def transform_train(img_path, label, height, width):
    img = read_img(img_path)
    img = resize(img, height, width)
    img = random_flip(img)
    img = random_scale(img)
    img = random_brightness(img)
    img = random_hue(img)
    img = random_saturation(img)
    img = random_contrast(img)
    img = normalize(img)
    return img, label

def transform_test(img_path, label, height, width):
    img = read_img(img_path)
    img = resize(img, height, width)
    img = normalize(img)
    return img, label


"""
Практическое задание 5-4

Создать (не программно) текстовый файл со следующим содержимым:
One — 1
Two — 2
Three — 3
Four — 4

Напишите программу, открывающую файл на чтение и считывающую построчно данные.
При этом английские числительные должны заменяться на русские.
Новый блок строк должен записываться в новый текстовый файл.
"""

my_list = list()

sl_numbers = {'One': 'Один', 'Two': 'Два', 'Three': 'Три', 'Four': 'Четыре', 'Five': 'Пять', 'Six': 'Шесть',
             'Seven': 'Семь', 'Eight': 'Восемь', 'Nine': 'Девять', 'Zero': 'Нуль'}

my_file = open("my_fileZ4.txt", "r", encoding='cp1251')

# Чтение строк из файла и их преобразование
print(f"Начато преобразование строк из файла {my_file.name}:")
while True:
    f_str = my_file.readline()
    if len(f_str) != 0:
        str_obr = f_str.split(' ')
        #  print(f"{f_str}", end='')
        str_obr[0] = sl_numbers.get(str_obr[0])
        #  print(f"Преобразована в\n{str_obr[0] + ' ' + str_obr[1] + ' ' + str_obr[2]}")
        my_list.append(str_obr[0] + ' ' + str_obr[1] + ' ' + str_obr[2])
    else:
        break

my_file.close()

# Запись нового блока строк в файл
with open("my_fileZ4P.txt", "w", encoding='cp1251') as my_file2:
    my_file2.writelines(my_list)

print(f"Преобразование строк из файла - {my_file.name} - завершено.\n"
      f"Обработанные данные записаны в новый файл - {my_file2.name}")


import os
import sys
import time
import socket
import json
import cv2
import numpy as np

import logging as log
from imutils.video import FPS

from argparse import ArgumentParser
from inference import Network

from multiprocessing import Process, Queue
import multiprocessing
import threading
import queue

# Browser and OpenCV Window toggle
CPU_EXTENSION = "/opt/intel/openvino/deployment_tools/inference_engine/lib/intel64/libcpu_extension_sse4.so"

def build_argparser():
    """
    Parse command line arguments.
    :return: command line arguments
    """
    parser = ArgumentParser()
    parser.add_argument("-m", "--model", required=True, type=str,
                        help="Path to an xml file with a trained model.")
    parser.add_argument("-i", "--input", required=True, type=str,
                        help="Path to image or video file")
    parser.add_argument("-l", "--cpu_extension", required=False, type=str,
                        default=None,
                        help="MKLDNN (CPU)-targeted custom layers."
                             "Absolute path to a shared library with the"
                             "kernels impl.")
    parser.add_argument("-d", "--device", type=str, default="CPU",
                        help="Specify the target device to infer on: "
                             "CPU, GPU, FPGA or MYRIAD is acceptable. Sample "
                             "will look for a suitable plugin for device "
                             "specified (CPU by default)")
    parser.add_argument("-pt", "--prob_threshold", type=float, default=0.2,
                        help="Probability threshold for detections filtering"
                        "(0.5 by default)")
    return parser

def image_process_worker(cap, frame_queue, image_queue, in_n, in_c, in_h, in_w):
    # Process frames until the video ends, or process is exited
    while cap.isOpened():
        # Read the next frame
        flag, frame = cap.read()
        if not flag:
            frame_queue.put(None)
            image_queue.put(None)
            break

        # Pre-process the frame
        image_resize = cv2.resize(frame, (in_w, in_h))
        image = image_resize.transpose((2,0,1))
        image = image.reshape(in_n, in_c, in_h, in_w)

        frame_queue.put(frame)
        image_queue.put(image)

def network_inference(infer_network, frame_queue, image_queue,
                        fw, fh, prob_threshold, fps):
    current_inference, next_inference = 0, 1
    count = 0
    while True:
        image = image_queue.get()
        if image is None:
            break
        count += 1
        frame = frame_queue.get()
        # Perform inference on the frame
        infer_network.exec_net_async(image, request_id=current_inference)

        # Get the output of inference
        if infer_network.wait(next_inference) == 0:
            detection = infer_network.get_output(next_inference)
            for box in detection[0][0]: # Output shape is 1x1x100x7
                conf = box[2]
                if conf >= prob_threshold:
                    xmin = int(box[3] * fw)
                    ymin = int(box[4] * fh)
                    xmax = int(box[5] * fw)
                    ymax = int(box[6] * fh)
                    cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (0, 125, 255), 3)
        current_inference, next_inference = next_inference, current_inference

        cv2.imshow('frame', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
        fps.update()
        if count == 100:
            fps.stop()
            print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

def infer_on_stream(args):
    """
    Initialize the inference network, stream video to network,
    and output stats and video.
    :param args: Command line arguments parsed by `build_argparser()`
    :return: None
    """

    frame_queue = queue.Queue(maxsize= 4)
    image_queue = queue.Queue(maxsize= 4)

    # Initialize the Inference Engine
    infer_network = Network()

    # Set Probability threshold for detections
    prob_threshold = args.prob_threshold

    # Load the model through `infer_network`
    infer_network.load_model(args.model, args.device, CPU_EXTENSION, num_requests=2)

    # Get a Input blob shape
    in_n, in_c, in_h, in_w = infer_network.get_input_shape()

    # Get a output blob name
    _ = infer_network.get_output_name()

    # Handle the input stream
    cap = cv2.VideoCapture(args.input)
    cap.open(args.input)
    _, frame = cap.read()

    fps = FPS().start()

    _s, frame = cap.read()
    fh = frame.shape[0]
    fw = frame.shape[1]

    preprocess_thread = None

    preprocess_thread = threading.Thread(target=image_process_worker,
                        args=(cap, frame_queue, image_queue, in_n, in_c, in_h, in_w))

    preprocess_thread.start()

    network_inference(infer_network, frame_queue, image_queue, fw, fh, prob_threshold, fps)

    preprocess_thread.join()

    # Release the out writer, capture, and destroy any OpenCV windows
    cap.release()
    cv2.destroyAllWindows()
    fps.stop()
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

def main():
    """
    Load the network and parse the output.
    :return: None
    """
    # set log level
    log.basicConfig(filename='example.log',level=log.CRITICAL)

    # Grab command line args
    args = build_argparser().parse_args()

    # Inference
    infer_on_stream(args)

if __name__ == '__main__':
    main()


import json
import logging
from pathlib import Path
from textwrap import dedent

import pydantic
import pytest

from docs.build import build_schemas_from_source, dump_json_schema

def test_dump_json_schema(tmp_path):
    pth = Path(tmp_path) / "myrole"  # will be requirer/provider

    class MySchema(pydantic.BaseModel):
        foo: int = 1
        bar: str

    dump_json_schema(MySchema, pth)
    jsn = json.load(pth.with_suffix(".json").open())
    assert jsn == {
        "title": "MySchema",
        "type": "object",
        "properties": {
            "foo": {"title": "Foo", "default": 1, "type": "integer"},
            "bar": {"title": "Bar", "type": "string"},
        },
        "required": ["bar"],
    }

@pytest.mark.parametrize(
    "module_contents, schema_name",
    (
        (
            dedent(
                """import pydantic
class RequirerSchema(pydantic.BaseModel):
    foo = 1"""
            ),
            "RequirerSchema",
        ),
        (
            dedent(
                """import pydantic
class ProviderSchema(pydantic.BaseModel):
    foo = 2"""
            ),
            "ProviderSchema",
        ),
    ),
)
def test_build_schemas_from_source_one_only(tmp_path, module_contents, schema_name):
    pth = Path(tmp_path)
    schema_output_path = pth / "outputs"
    schema_path = pth / "foo" / "v42" / f"baz_one_{schema_name}.py"
    schema_path.parent.mkdir(exist_ok=True, parents=True)

    schema_path.write_text(module_contents)
    build_schemas_from_source(
        schema_path=schema_path, output_location=schema_output_path
    )

    schema_outputs = schema_output_path / "foo" / "v42"
    if schema_name == "ProviderSchema":
        assert (schema_outputs / "provider.json").exists()
        assert not (schema_outputs / "requirer.json").exists()
    if schema_name == "RequirerSchema":
        assert not (schema_outputs / "provider.json").exists()
        assert (schema_outputs / "requirer.json").exists()

def test_build_schemas_from_source_both(tmp_path):
    pth = Path(tmp_path)
    schema_path = pth / "foo" / "v42" / "baz_both.py"
    schema_path.parent.mkdir(exist_ok=True, parents=True)

    schema_path.write_text(
        dedent(
            """import pydantic
class RequirerSchema(pydantic.BaseModel):
    foo = 1
class ProviderSchema(pydantic.BaseModel):
    foo = 2"""
        )
    )
    build_schemas_from_source(schema_path=schema_path, output_location=pth)

    schema_output_path = pth / "foo" / "v42"
    assert (schema_output_path / "provider.json").exists()
    assert (schema_output_path / "requirer.json").exists()

def test_build_schemas_from_empty_source(tmp_path, caplog):
    pth = Path(tmp_path)
    schema_path = pth / "foo" / "v42" / "my_schema.py"
    schema_path.parent.mkdir(exist_ok=True, parents=True)

    # there is a module but it's empty
    schema_path.touch()

    with caplog.at_level(logging.DEBUG):
        build_schemas_from_source(
            schema_path=schema_path,
            output_location=pth,
        )

    assert (
        f"Failed to load RequirerSchema from {schema_path}: "
        f"schema not defined for role: requirer." in caplog.text
    )

    assert (
        f"Failed to load ProviderSchema from {schema_path}: "
        f"schema not defined for role: provider." in caplog.text
    )

def test_build_schemas_from_nonempty_but_bad_source(tmp_path, caplog):
    pth = Path(tmp_path)
    schema_path = pth / "foo" / "v42" / "my_schema.py"
    schema_path.parent.mkdir(exist_ok=True, parents=True)

    # there is a module, but it's full of uninteresting stuff
    schema_path.write_text(
        dedent(
            """
import pydantic
class Foo(pydantic.BaseModel):
    pass
class Bar:
    pass
"""
        )
    )
    with caplog.at_level(logging.DEBUG):
        build_schemas_from_source(
            schema_path=schema_path,
            output_location=pth,
        )

    assert (
        f"Failed to load RequirerSchema from {schema_path}: "
        f"schema not defined for role: requirer." in caplog.text
    )

    assert (
        f"Failed to load ProviderSchema from {schema_path}: "
        f"schema not defined for role: provider." in caplog.text
    )

@pytest.mark.parametrize(
    "source",
    (
        dedent(
            """import pydantic
class RequirerSchema:
    pass
        """
        ),
        dedent(
            """RequirerSchema = 42
        """
        ),
    ),
)
def test_build_schemas_broken_source(tmp_path, source, caplog):
    caplog.clear()

    pth = Path(tmp_path)
    schema_path = pth / "foo" / "v42" / "qux.py"
    schema_path.parent.mkdir(exist_ok=True, parents=True)

    # there is a module, but the name points to something unexpected
    schema_path.write_text(source)
    with caplog.at_level(logging.DEBUG):
        build_schemas_from_source(
            schema_path=schema_path,
            output_location=pth,
        )

    assert (
        f"Found object called RequirerSchema in {schema_path}; "
        f"expecting a DataBagSchema subclass, not "
    ) in caplog.text


from uplogic.nodes import ULConditionNode
from uplogic.nodes import ULOutSocket
from uplogic.utils.constants import STATUS_WAITING
from uplogic.utils import is_invalid

class ULHasProperty(ULConditionNode):
    def __init__(self):
        ULConditionNode.__init__(self)
        self.game_object = None
        self.property_name = None
        self.mode = 'GAME'
        self.OUT = ULOutSocket(self, self.get_stat)

    def get_stat(self):
        socket = self.get_output('result')
        if socket is None:
            game_object = self.get_input(self.game_object)
            property_name = self.get_input(self.property_name)
            if is_invalid(game_object, property_name):
                return STATUS_WAITING
            return self.set_output(
                'stat',
                (
                    property_name in game_object.getPropertyNames()
                )
            ) if self.mode == 'GAME' else self.set_output(
                'stat',
                (
                    property_name in [p[0] for p in game_object.blenderObject.items()]
                )
            )
        return socket

    def evaluate(self):
        self._set_ready()


#!/usr/bin/env python3

"""
@author: y00520910
@since:
"""

import argparse
import os

import numpy as np

import torch
from torch.utils.data import DataLoader
from tqdm import tqdm

import optim as optim_
from dataset import WiFiDenoiseDataset
from net import yannet

device = 'cuda' if torch.cuda.is_available() else 'cpu'

class Trainer(object):

    def __init__(self,
                 model: torch.nn.Module,
                 max_lr,
                 weight_decay,
                 num_loops):
        self._model = model
        self._parameters = list(self._model.parameters())
        self._opt = torch.optim.AdamW(
            self._parameters,
            lr=max_lr,
            weight_decay=weight_decay
        )
        self._train_scheduler = optim_.CosineWarmupAnnealingLR(self._opt, num_loops)
        self.loss_func = torch.nn.MSELoss()

    def train(self, query, key, target):
        query = query.to(device).float()
        key = key.to(device).float()
        target = target.to(device).float()

        output = self._model(query, key)  # B
        output = output * 100

        loss = self.loss_func(output, target)

        loss.backward()
        self._opt.step()
        self._opt.zero_grad()
        self._train_scheduler.step()

        return loss.cpu(), self._train_scheduler.get_last_lr()[0]

    def predict(self, query, key):
        with torch.no_grad():
            query = query.to(device).float()
            key = key.to(device).float()

            output = self._model(query, key)
            output = output * 100
            return output.cpu()

def evaluate(trainer: Trainer, test_loader: DataLoader):
    res = []

    for doc in tqdm(test_loader, leave=False, dynamic_ncols=True, desc='Evaluate'):
        output = trainer.predict(doc['query'], doc['key'])
        target = doc['target']

        for k in range(target.size(0)):
            if abs(output[k] - target[k]) <= 5:
                res.append(1)
            elif 10 >= abs(output[k] - target[k]) > 5:
                res.append(0.6)
            else:
                res.append(0)

    accuracy = np.array(res).mean()

    return accuracy

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset_dir', required=True)
    parser.add_argument('--gpu', default=[0, 1, 2, 3])
    parser.add_argument('--batch-size', type=int, default=1024)
    parser.add_argument('--max-lr', type=float, default=1e-3)
    parser.add_argument('--weight-decay', type=float, default=0.1)
    parser.add_argument('--num-epochs', type=int, default=50)
    parser.add_argument('--output_dir', type=str, default='./output_models')
    parser.add_argument('--tensorboard_dir', type=str, default='./tensorboard_')

    args = parser.parse_args()

    dataset_dir_list = args.dataset_dir.split(';')
    dataset_dir_dict = {}
    for data_dir_i in dataset_dir_list:
        content = data_dir_i.split(':')
        dataset_dir_dict[content[0]] = content[1]

    train_loader = DataLoader(
        WiFiDenoiseDataset(True,
                           dataset_dir_dict['databaseIp'],
                           dataset_dir_dict['databaseName'],
                           dataset_dir_dict['collectionName'],
                           dataset_dir_dict['username'],
                           dataset_dir_dict['password']),
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=32
    )
    test_loader = DataLoader(
        WiFiDenoiseDataset(False,
                           dataset_dir_dict['databaseIp'],
                           dataset_dir_dict['databaseName'],
                           dataset_dir_dict['collectionName'],
                           dataset_dir_dict['username'],
                           dataset_dir_dict['password']),
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=4
    )
    model = yannet()
    if device == 'cuda':
        model = torch.nn.DataParallel(model.to(device), device_ids=args.gpu)
    trainer = Trainer(
        model,
        max_lr=args.max_lr,
        weight_decay=args.weight_decay,
        num_loops=len(train_loader) * args.num_epochs
    )

    loss_g = 0.0
    accuracy_epochs = []
    loss_g_epochs = []
    loss_epochs = []
    for epoch in range(args.num_epochs):
        model.train()
        loop = tqdm(train_loader, leave=False, dynamic_ncols=True)
        this_epoch_loss = []
        for i, doc in enumerate(loop):
            loss, lr = trainer.train(doc['query'], doc['key'], doc['target'])
            loss = loss.detach().numpy()

            loss_g = 0.9 * loss_g + 0.1 * loss
            this_epoch_loss.append(loss)

            loop.set_description(f'Epoch: {epoch} L={loss_g:.06f}, LR={lr:.01e}', False)

        model.eval()
        accuracy = evaluate(trainer, test_loader)
        tqdm.write(
            f'Epoch: {epoch} L_this={np.mean(this_epoch_loss):.06f} L={loss_g:.06f} Accuracy={accuracy:.06f}'
        )
        state = {
            'state': model.state_dict(),
            'epoch': epoch,
        }
        latest_path = os.path.join(args.output_dir, 'latest.tar')
        if not os.path.exists(args.output_dir):
            os.mkdir(args.output_dir)
        torch.save(state, latest_path)

        accuracy_epochs.append(round(accuracy * 100, 2))
        loss_g_epochs.append(round(loss_g, 2))
        loss_epochs.append(round(np.array(this_epoch_loss).mean(), 2))

    # 画图
    print(f'\n"accuracy_epochs": {accuracy_epochs},')
    print(f'"loss_g_epochs": {loss_g_epochs},')
    print(f'"loss_epochs": {loss_epochs},')
    return 0

if __name__ == '__main__':
    raise SystemExit(main())


import glob, os, cv2
import numpy as np
import matplotlib.pyplot as plt
import random
from skimage.io import imread,imsave

######################################################################################
#                                                                                #
#                     The following code expects you to use the                      #
#              histopathologic-cancer-detection TRAINING datset from kaggle          #
#        Source https://www.kaggle.com/c/histopathologic-cancer-detection/data       #
#                                                                                #
######################################################################################

######################################################################################
#                                                                                #
#				BEFORE YOU BEGIN                                     #
#                                                                                #
######################################################################################

######################################################################################
#                                                                                #
#		            IMPORTANT PRE PROCESSING STEP                            #
#                                                                                #
######################################################################################

######################################################################################
#                                                                                #
#                                  RENAMING FILES                                    #
#                                                                                #
######################################################################################
#      Please make sure that the file names are named in asceding order              #
#      For example  1.tif, 2.tif, 3.tif, 4.tif, 5.tif, 6.tif, and so on              #
#      If you do not have the files named this way, please use my                    #
#      renaming_"files_ascending_order" snippet on the 'Utils' folder                #
######################################################################################

######################################################################################
#                                                                                #
#			           !!!WARNING!!                                      #
#                                                                                #
######################################################################################
#            The following code will create a training dataset with                  #
#            image pairs (clean,noise), it will be stored in your                    #
#            selected "outputfolder" below.                                          #
#            If you are going to use the TRAINING kaggle dataset                     #
#            It will create a total of 440050 images                                 #
#            220025 clean images and 220025 noise images                             #
#            Leaving your initial training folder untouched                          #
######################################################################################
#                                                                                #
#                                      REMINDER                                      #
#                                                                                #
######################################################################################
#            The following code expects that the image files are named               #
#            in asceding order                                                       #
#            For example  1.tif, 2.tif, 3.tif, 4.tif, 5.tif, 6.tif, and so on        #
#            If you do not have the files named this way, please use my              #
#            renaming_"files_ascending_order" snippet on the 'Utils' folder          #
######################################################################################

#Please state the folder path for the TRAINING dataset from kaggle
#Default name: << does not have default folder name >> USER MUST SPECIFY IT.

inputfolder   =  ""

#The shape of the input image, leave it as it is
#Because the shape of the images from the kaggle dataset are fixed at (96,96,3)
shape         = (96,96,3)

#List in which we are going to save the noise maps created
maps_of_noise = []

#List of possible standard deviation corruption, goes from 0 to 50.
noise_standard_deviation_list = np.arange(51)

#For each standard deviation, we are going to create a noise map
#and store it in our list of noise maps variable "maps_of_noise"
for noise_standard_deviation in noise_standard_deviation_list:
	#For fair comparison and reproducibility we use seed(0)
	np.random.seed(0)
	#Generating the noise map with:
	#loc   = 0                            ----> mean = 0
	#scale = noise_standard_deviation/255 ----> standard deviation and divided by 255 to normalize the map
	#size  = shape                        ----> dimensions of the image, which are (96,96,3)
	maps_of_noise.append(np.random.normal(loc=0, scale=noise_standard_deviation/255, size=shape))

#Inside the for loop below we will be increasing the standard deviation
#In order to create maps of 0 to 50 standard deviation
#Once the standard deviation hits 50 we reset it to 0
#This way we will be creating 4400 images of each standard deviation
#In other words:
#4400 for standar deviation 1
#4400 for standar deviation 2
#4400 for standar deviation 3
#...
#...
#...
#4400 for standar deviation 49
#4400 for standar deviation 50
#4400 for standar deviation 1
#4400 for standar deviation 2
#4400 for standar deviation 3
#And repeat until we reach 220025 images
standard_deviation_selector = 0

#Outputfolder for the training dataset that will be created
#Remember we need image pairs (clean,noise) to train the network
#Defaults to: "user_self_created_dataset/" folder (This is a folder name, dont get confused)
outputfolder    = "user_self_created_dataset/"
#The TRAINING dataset from kaggle has a total of 220025 images
#This is where the number in the for loop is from
#Feel free to modify it in case you are using less images
for i in range(1,220025):
	if standard_deviation_selector == 51:
		standard_deviation_selector = 0
	img   = imread(inputfolder + "{}.tif".format(i))
	img   = img / 255.
	#Clipping the image to better approximate real world noise
	#According to:  J.-S.  Lee,  “Refined  filtering  of  image  noise  using  local  statistics,”Computer  graphics  and  image  processing,  vol.  15,  no.  4,  pp.  380–389, 1981.
	noise = np.clip((img + maps_of_noise[standard_deviation_selector]),0,1)
	noise = noise.astype(np.float32)
	noise = noise * 255
	img   = img * 255
	imsave(outputfolder + "{}_noise.tif".format(i),noise.astype(np.uint8))
	imsave(outputfolder + "{}_clean.tif".format(i),img.astype(np.uint8))
	print("Current standard_deviation_selector: ",standard_deviation_selector)
	standard_deviation_selector += 1

######################################################################################
#                                                                                #
#		     Your freshly created dataset has been created                   #
#                          and ready to be used for training                         #
#                                                                                #
######################################################################################


import os
import gc

import sklearn
from sklearn.svm import SVC
from sklearn import cross_validation, grid_search
import numpy as np

from sklearn.metrics import confusion_matrix, classification_report
def train_svm_classifer(features, labels, model_output_path):

    params = [
        {
            'kernel':['linear'],
            'C':[1, 10, 100, 1000]
        },
        {
            'kernel':['rbf'],
            'C':[1, 10, 100, 1000],
            'gamma':[1e-2, 1e-3, 1e-4, 1e-5]
        }
    ]

    svm = SVC(probability=True)
    clf = grid_search.GridSearchCV(svm, params, cv=32, n_jobs=15, verbose=3)
    clf.fit(features, labels)
    if os.path.exists(model_output_path):
        sklearn.externals.joblib.dump(clf.best_estimator_, model_output_path)
    else:
        print ('Cannot save trained svm model to {0}'.format(model_output_path))
    return clf

if __name__ == '__main__':
    data_path = '../data/'
    train_true_feature = np.load(data_path+'train_true_feature.npy')
    train_fake_feature = np.load(data_path+'train_fake_feature.npy')
    train_feature = np.vstack((train_true_feature, train_fake_feature))
    train_label = np.hstack((np.ones((train_true_feature.shape[0], )), np.zeros((train_fake_feature.shape[0], ))))
    del train_true_feature
    del train_fake_feature
    gc.collect()
    model = train_svm_classifer(train_feature, train_label, 'svm_model/best_model')
    eva_true_feature = np.load(data_path+'eva_true_feature.npy')
    eva_fake_feature = np.load(data_path+'eva_fake_feature.npy')
    eva_feature = np.vstack((eva_true_feature, eva_fake_feature))
    eva_predict = model.predict(eva_feature)
    eva_labels = np.hstack((np.ones(eva_true_feature.shape[0]), np.zeros(eva_fake_feature.shape[0])))
    print (confusion_matrix(eva_labels,eva_predict,labels=[0,1]))
    '''
    eva_true_feature = np.load(data_path+'eva_true_feature.npy')
    eva_fake_feature = np.load(data_path+'eva_fake_feature.npy')
    '''


from main import bot, dp
from aiogram.types import Message
from aiogram import types
from aiogram.dispatcher import FSMContext
from states import UserState
from main import db
from image_recongition import image_recognize

from datetime import datetime

yes = ['да', 'конечно', 'ага', 'пожалуй']
no = ['нет', 'нет, конечно', 'ноуп', 'найн']

hello_message = '<b>Привет!</b>\nЯ помогу отличить кота от хлеба! Объект перед тобой квадратный?'
cat_message = 'Это кот, а не хлеб! Не ешь его!\nДля возвращения к началу диалога введите команду /start\n' \
              'Для анализа картинки введите команду /image'
bread_message = 'Это хлеб, а не кот! Ешь его!\nДля возвращения к началу диалога введите команду /start\n' \
                'Для анализа картинки введите команду /image'
error_message = 'Я не такой глупый, чтобы понять сообщение "ага", "найн" и другие, но не настолько ' \
                'умный чтобы понять ваше сообщение.\nНапишите ещё раз.'

@dp.message_handler(commands='start')
async def start(message: Message, state: FSMContext):
    if (not db.user_exists(message.from_user.id)):
        db.add_user(message.from_user.id)
    await message.answer(hello_message, parse_mode=types.ParseMode.HTML)
    db.add_message(message.from_user.id, message.chat.id, message.message_id, message.text, message.date)
    db.add_message(bot.id, message.chat.id, message.message_id + 1, hello_message,
                   datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    await state.set_state(UserState.square)

@dp.message_handler(commands='image')
async def image(message: Message, state:FSMContext):
    await message.answer('Отправь фотографию и я помогу определить кто на ней, кот или хлеб!', parse_mode=types.ParseMode.HTML)
    await state.set_state(UserState.image)

@dp.message_handler(state=UserState.image, content_types=['photo'])
async def recognize(message: Message, state: FSMContext):
    await message.photo[-1].download('images/image.jpg')
    if 'bread' in image_recognize('images/image.jpg'):
        db.update_bread(message.from_user.id)
        await message.answer(bread_message)
        await state.finish()
    elif 'cat' in image_recognize('images/image.jpg'):
        db.update_cat(message.from_user.id)
        await message.answer(cat_message)
        await state.finish()
    else:
        await message.answer('Не получилось определить объект на картинке!\n/start - начало диалога\n/image - анализ картинки')
        await state.finish()

@dp.message_handler(state=UserState.square)
async def square(message: Message, state: FSMContext):
    db.add_message(message.from_user.id, message.chat.id, message.message_id, message.text, message.date)
    if(message.text == '/start'):
        await message.answer(hello_message, parse_mode=types.ParseMode.HTML)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, hello_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    elif(message.text.lower() in yes):
        await message.answer('У него есть уши?')
        db.add_message(bot.id, message.chat.id, message.message_id + 1, 'У него есть уши?',
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        await state.set_state(UserState.ears)
    elif(message.text.lower() in no):
        db.update_cat(message.from_user.id)
        await message.answer(cat_message)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, cat_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        await state.finish()
    else:
        await message.answer(error_message)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, error_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

@dp.message_handler(state=UserState.ears)
async def ears(message: Message, state: FSMContext):
    db.add_message(message.from_user.id, message.chat.id, message.message_id, message.text, message.date)
    if(message.text == '/start'):
        await message.answer(hello_message, parse_mode=types.ParseMode.HTML)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, hello_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    elif(message.text.lower() in yes):
        db.update_cat(message.from_user.id)
        await message.answer(cat_message)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, cat_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        await state.finish()
    elif(message.text.lower() in no):
        db.update_bread(message.from_user.id)
        await message.answer(bread_message)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, bread_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        await state.finish()
    else:
        await message.answer(error_message)
        db.add_message(bot.id, message.chat.id, message.message_id + 1, error_message,
                       datetime.now().strftime('%Y-%m-%d %H:%M:%S'))


# -*- coding: utf-8 -*-
import re

import scrapy

from freelance.items import TaskItem, PRICE_PERIOD_HOURLY, PRICE_PERIOD_PROJECT

re_price_amount = re.compile(r'\D*')

class FreelansimSpider(scrapy.Spider):
    name = 'freelansim'
    allowed_domains = ['freelansim.ru']
    start_urls = ['https://freelansim.ru/']

    def parse(self, response):
        for dom_item in response.css('.content-list__item'):
            task_url = response.urljoin(dom_item.css('.task__title a::attr(href)').extract_first())
            task_title = dom_item.css('.task__title a::text').extract_first()
            task_views = self.get_views(dom_item)
            task_responses = self.get_responses(dom_item)
            task_is_safe_deal = len(dom_item.css('.icon_safe_deal')) > 0
            task_tags = dom_item.css('.tags__item_link::text').extract()
            task_price_negotiated = len(dom_item.css('.negotiated_price')) > 0
            task_price_amount = self.get_price_amount(dom_item)
            task_price_period = self.get_price_period(dom_item)

            item = TaskItem(
                url=task_url,
                title=task_title,
                views=task_views,
                responses=task_responses,
                is_safe_deal=task_is_safe_deal,
                tags=task_tags,
                price_negotiated=task_price_negotiated,
                price_amount=task_price_amount,
                price_period=task_price_period
            )

            yield scrapy.Request(url=task_url, callback=self.parse_detail, meta={"item": item})

    def parse_detail(self, response):
        item = response.meta.get('item')
        item['description'] = response.css('.task__description').xpath('text()').extract_first().strip()

        yield item

    @staticmethod
    def get_views(dom_item):
        try:
            return int(dom_item.css('.params__views > .params__count::text').extract_first())
        except (ValueError, TypeError):
            return 0

    @staticmethod
    def get_responses(dom_item):
        try:
            return int(dom_item.css('.params__responses > .params__count::text').extract_first())
        except (ValueError, TypeError):
            return 0

    @staticmethod
    def get_price_amount(dom_item):
        try:
            return int(re_price_amount.sub('', dom_item.css('.task__price > .count::text').extract_first()))
        except (ValueError,TypeError):
            return 0

    @staticmethod
    def get_price_period(dom_item):
        suffix = dom_item.css('.task__price > .count > .suffix::text').extract_first()

        if suffix == 'за час':
            return PRICE_PERIOD_HOURLY
        elif suffix == 'за проект':
            return PRICE_PERIOD_PROJECT
        else:
            return None


"""
test_suite.py
06-27-16

Contains all functionality needed to
automatically determine the yield stress
of a material, even with noisy data, given
a stress-strain curve in the form
[Strain|Stress] in each row.

Edward Alexander Nusinovich
"""

"""Basic libs"""
import numpy as np
import math
from DataModelDict import DataModelDict as dmd

"""For optimization and model training"""
from scipy.optimize import minimize #
from scipy.optimize import curve_fit
from sklearn.linear_model import LinearRegression
from sklearn.cluster import MiniBatchKMeans as mbkmeans
from sklearn.cluster import KMeans
from pybrain.optimization import GA #

# returns the model after any readings greater than 0.025, any smaller are considered noise/error-prone
def delete_noise(model,cutoff = 0.025):

    cur_index = 0

    # deleting noisy values (possible inaccuracies up to .025 by default)
    for index, num in enumerate(model[:,0]):

        if num >= cutoff:

             return model[index:]

# makes all values positive in order to be able to take logarithms
def adjust(model):

    for index, num in enumerate(model[:,1]):

        if num<=0:
            model[index,1] = 1

    return model

# gives the value that is halfway through the list, left mid if there are an odd number of points
def midpoint(lst):

    length = len(lst)
    return lst[int(length)/2]

# clusters the data into groups and returns the split data
def kmeanssplit(data, numclusters=2):

    return splitdata(data,kcluster(data,numclusters=numclusters).predict(data[:,0][:,None]))

# creates a linear model based on data and predicts its values over the domain
def predictlinear(data, step = 0.5):

    est = linfit(data)
    x_pred = np.arange(min(data[:,0]),max(data[:,0]+1), step)
    y_pred = est.predict(x_pred[:,None])

    return combine_data(x_pred,y_pred)

# given a function and an interval (two-element list) and a number of points, applies it to the function and gets some sample points
def samplepoints(function, interval, numpoints):

    x_dom = np.linspace(interval[0],interval[1],numpoints)
    y_range = np.zeros(numpoints)

    for index, point in enumerate(x_dom):
        y_range[index] = function(point)

    return combine_data(x_dom,y_range)

# fits a linear regression to the data and returns it
def linfit(data, start=None):

    return LinearRegression().fit(*format_data(data,start))

# clusters the data using mini batch kmeans
def kminicluster(data, numclusters, start=None):

    return mbkmeans(n_clusters=numclusters).fit(*format_data(data,start))

# same as above but without mini batch (runs slower, should be more accurate)
def kcluster(data,numclusters,start=None):

    return KMeans(n_clusters=numclusters).fit(*format_data(data,start))

# this method will put data in the appropriate format for regression (Scikit-Learn)
def format_data(data, start=None):

    return (expToTrain(data, start),data[start:,1])

# given two arrays, returns a combined list where each element is x_i,y_i
def combine_data(data1,data2):

    return np.array([list(a) for a in zip(data1,data2)])

# converts every non-numerical list value to zero
def regularize(data):

    for index, val in enumerate(data):
        if math.isinf(val) or math.isnan(val):
            data[index]=0

    return data

# takes predictions from kmeans clustering and split the table into two groups
def splitdata(data, predictions):

    initgroup = predictions[0]
    splitgroup = 0

    for index, val in enumerate(predictions):

        # as soon as we reach the new group, we have found our dividing point
        if val != initgroup:
            splitgroup = index
            break

    """Instead of creating tuples, we create lists"""
    elastic = combine_data(data[:splitgroup,0],data[:splitgroup,1])
    plastic = combine_data(data[splitgroup:,0],data[splitgroup:,1])

    return elastic, plastic

# takes the approximate derivative of the model by taking slopes
def get_slopes(model):

    strain = model[:,0]
    stress = model[:,1]

    slopes = []

    """Approximating the partial derivatives of stress/strain"""
    for index in xrange(len(stress)-1):

        rise = (stress[index+1]-stress[index])
        run = (strain[index+1]-strain[index])

        if run==0:
            slopes.append(0)

        else:
            slopes.append(rise/run)

    return np.array(slopes)

# converts a bunch of domain values to lists, because each domain value must be iterable for training data
def expToTrain(exp,start=None):

    x_train = []

    for data in exp[start:,0]:
        x_train.append([data, ])

    return x_train

# finds the yield stress of a dataset automatically using kmeans clustering and covariance analysis
def yield_stress(model, numpoints=1000, cutoff=0.025, startx=None, endx=None, decreasingend=False):

    """Default interval values"""
    if startx is None:
        startx=min(model[:,0])+0.1
    if endx is None:
        endx=max(model[:,0])

    """We get rid of the noise in the data, and select only positive values (so that logarithms can be taken)"""
    model = delete_noise(model,cutoff=cutoff)
    model = adjust(model)

    """a and c are parameters"""
    def fit(x, a,c):
        return a*np.log(x)+c

    strain = model[:,0]
    stress = model[:,1]

    """We are fitting a logarithmic curve as closely as possible to the dataset"""
    optimal_params, cov_matrix = curve_fit(fit,strain,stress)
    a, c = optimal_params

    """The fitted version of the dataset"""
    def bestfit(x):
        return a*np.log(x)+c

    """
    We look for the place where the slope is average over
    the domain by taking sample points of the logarithmic curve
    """
    gap_len = (endx-startx)/numpoints

    xs = np.linspace(startx,endx,numpoints)
    ys = bestfit(xs)

    pred_data = combine_data(xs,ys)
    pred_slope = get_slopes(pred_data)

    ave_slope = 0

    """If the slope decreases at the end, we cluster the data to avoid referencing the end of the dataset"""
    if decreasingend is True:

        """Defining average slope by observing clusters in data"""
        left, right = kmeanssplit(model)
        leftmid, rightmid = midpoint(left)[None,:], midpoint(right)[None,:]
        ave_slope = (rightmid[0,1]-leftmid[0,1])/(rightmid[0,0]-leftmid[0,0])

    else:
        """Otherwise, we get the slope over the whole interval to find where slope begins to decrease overall"""
        ave_slope = (stress[-1]-stress[0])/(strain[-1]-strain[0])

    """As soon as the slope at a point is less than the average slope, we stop"""
    for ind, slope in enumerate(pred_slope):
        if slope<ave_slope:
            break

    """
    We must take into account that we may not have a
    data point in the experimental set where we have found a result,
    so we find the nearest neighbor in our dataset
    """
    datapointind = ind*gap_len

    """Here we find the nearest neighbor in the dataset"""
    for ind, stra in enumerate(model[:,0]):

        if stra > datapointind:
            return model[ind][None,:]

    raise ValueError("The data does not seem to have a yield")


# -*- coding: latin1 -*-

from MCWidgets.Escape import html_escape, js_str_escape
from MCWidgets.Widgets.BaseJS import BaseJS

class SelectBox(BaseJS):

    def get_type(self):
        return 'MCWidgets.SelectBox'

    def get_value(self):
        v = BaseJS.get_value(self)
        if v is None:
            return self.get_arg('selected', [])

        if not isinstance(v, list):
            v = [v]

        return v

    def get_calc_html_attrs(self):
        calcdict = BaseJS.get_calc_html_attrs(self)

        for at in ('size', 'multiple', 'readonly'):
            a = self.get_arg(at, None)
            if a is not None:
                calcdict[at] = str(a)

        if 'readonly' in calcdict:
            calcdict['readonly'] = None

        return calcdict

    def validate_widget(self):
        errors = []
        values = [x[0] for x in self.get_arg('options', [])]
        for e in self.get_value():
            if e not in values:
                errors.append('Value "%s" is not a valid value' % e)
        return errors

    def get_args_spec(self):
        req, opt = BaseJS.get_args_spec(self)
        return (req + ['options'],
                opt + ['selected', 'sync_with', 'ajax_url',
                       'headoption', 'multiple', 'size'])

    def render(self):

        options = []
        selected = self.get_value()

        ho = self.get_arg('headoption', None)
        if ho is not None and not selected:
            options.append('    <option value="null" style="font-style: italic" disabled selected>%s</option>' % ho)

        for value, text in self.get_arg('options', []):
            s = ''
            if value in selected: s = ' selected'

            options.append('    <option value="%s"%s>%s</option>' %
                            (html_escape(value), s, html_escape(text)))

        from MCWidgets.Utils import html_dict_attrs as hda
        result = '<select %s>%s</select>' % (
            hda(self.get_calc_html_attrs()), '\n'.join(options))

        if self.get_arg('tabindex', None) == 1:
            result += '<script>document.getElementById("%s").focus()</script>' % self.get_html_id()

        return result

    def calljs_getvalue(self):
        return 'document.getElementById("%(i)s").options[document.getElementById("%(i)s").selectedIndex].value' % \
            {'i': self.get_html_id() }

    def initialize_type(self):
        form = self.get_form()
        form._add_js_file('selectbox.js')
        form._add_js_file('util.js')
        form._add_js_file('dojoio.js')

    def initialize_widget(self):

        sync_with = self.get_arg('sync_with', None)
        ajax = self.get_arg('ajax_url', None)
        if None not in (sync_with, ajax):
            import json
            from MCWidgets.Widget import Widget

            if not isinstance(sync_with, Widget):
                sync_with = self.get_form().get_widget(sync_with)

            sync_with.request_notification(
                'change',
                'mcw_sb_ajax_update("%s", "%s", %s, %s)' % (
                    self.get_html_id(),
                    ajax,
                    sync_with.calljs_getvalue(),
                    json.write(self.get_arg('headoption'))))


import unittest
import math
from talon import Talon

class TestTalon(unittest.TestCase):
    '''fonen'''
    def test_init(self):
        t = Talon()
        self.assertEqual(len(t), t.size)

    def test_pioche(self):
        t = Talon()
        t2 = Talon()
        self.assertEqual(t, t2)# on doit verifier que les listes soit bien egales
        t.pioche()
        self.assertNotEqual(len(t), len(t2))
        return True

    def test_mix(self):
        t = Talon()
        t2 = Talon()
        self.assertEqual(t, t2)# on doit verifier que les listes soit bien egales
        t2.mix()
        self.assertNotEqual(t, t2)
        return True

    def test_EstDans(self):
        t = Talon()
        self.assertTrue(not t.estDans(6))# est dans renvoi false pour true

if __name__ == "__main__":
    unittest.main()


r"""
Given a Binary Search Tree and a target number, return true if there exist two elements in the
BST such that their sum is equal to the given target.

Example 1:

    Input:

        5
       / \
      3   6
     / \   \
    2   4   7

    Target = 9

    Output: True

Example 2:

    Input:

        5
       / \
      3   6
     / \   \
    2   4   7

    Target = 28

    Output: False

"""

class TreeNode:
    """Definition for a binary tree node."""
    def __init__(self, val = 0, left = None, right = None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def findTarget1(self, root, k):    # 96ms
        def helper(node, lst = []):
            if node:
                helper(node.left)
                lst.append(node.val)
                helper(node.right)

            return lst

        vals = helper(root)

        i, j = 0, len(vals) - 1
        while i < j:
            if vals[i] + vals[j] > k:
                j -= 1
            elif vals[i] + vals[j] == k:
                return True
            else:
                i += 1

        return False

    def findTarget2(self, root, k):    # 72ms
        if not root:    return False
        d = set()
        stack = [root]
        while stack:
            node = stack.pop()
            tmp = node.val
            if tmp in d:    return True
            d.add(k - tmp)
            if node.left:    stack.append(node.left)
            if node.right:    stack.append(node.right)

        return False


import logging

from charms.signals.v0.signals import Signals, SignalEvent
from ops.charm import CharmBase
from ops.main import main
from ops.model import ActiveStatus

logger = logging.getLogger(__name__)

class MyPingCharm(CharmBase):
    def __init__(self, fw, key=None):
        super().__init__(fw, key)
        self.signals = Signals(self, 'signals')
        self.unit.status = ActiveStatus('ready')
        self.framework.observe(self.signals.on.receive,
                               self._on_signal_receive)
        self.framework.observe(self.on.ping_action,
                               self.ping)

    def ping(self, _):
        self.signals.send('ping', 'ming!')

    def _on_signal_receive(self, event: SignalEvent):
        if event.name == 'pong':
            print('got ponged!')
        else:
            raise ValueError(event.name)

if __name__ == "__main__":
    main(MyPingCharm)


import collections
import random
import json
import os

import sys
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(root_dir)
# print(f'{sys.path}')

from torch.utils.data import Dataset, DataLoader

from model.hoho_utils import *
from model.hoho_config import *

class CChessGame:

    def __init__(self, state=INIT_BOARD_STATE, restrict_count=RESTRICT_ROUND_NUM, winner=None, current_player=PLAYER_RED):
        self.state = state
        self.restrict_count = restrict_count
        self.winner = winner
        self.current_player = current_player

        print(f'{LOG_TAG_CCHESSGAME} CChessGame created!')

    def step(self, action):
        state_new = do_action_on_board(self.state, action)
        z = 0
        done = False
        if 'K' not in state_new:
            z = -1
            done = True
            self.winner = PLAYER_BLACK
        elif 'k' not in state_new:
            z = 1
            done = True
            self.winner = PLAYER_RED
        else:
            # hoho_todo: 增加restrict_count的逻辑
            pass

        self.state = state_new
        return state_new, z, done

    def reset(self):
        self.state = INIT_BOARD_STATE
        self.restrict_count = RESTRICT_ROUND_NUM
        self.winner = None
        self.current_player = PLAYER_RED

class ChessDataset(Dataset):

    def __init__(self, data_list=None):
        super(ChessDataset, self).__init__()
        # 注意：需要在这里将list转换为tensor, 否则dataloader取每个batch时，batch_size不会在第0维
        self.data_list = [(exp[0], torch.tensor(exp[1], dtype=torch.float), exp[2]) for exp in data_list]

    def __len__(self):
        return len(self.data_list)

    def __getitem__(self, idx):
        experience = self.data_list[idx]
        return experience[0], experience[1], experience[2]

    @staticmethod
    def load_from_dir(dirpath, version=None):
        if not os.path.exists(dirpath):
            return None

        all_data_list = list()
        for filename in os.listdir(dirpath):
            if filename.endswith('json') and filename.startswith('replay_buffer'):
                if version is  None:
                    with open(os.path.join(dirpath, filename), 'r') as f:
                        jsonstr = f.read()
                        data_list = json.loads(jsonstr)
                        all_data_list.extend(data_list)
                else:
                    name = filename.split('.')[0]
                    items = name.split('_')
                    if len(items) == 4:
                        check_version = int(items[3])
                        if version == check_version:
                            with open(os.path.join(dirpath, filename), 'r') as f:
                                jsonstr = f.read()
                                data_list = json.loads(jsonstr)
                                all_data_list.extend(data_list)

        dataset = ChessDataset(all_data_list)
        return dataset

class ReplayBuffer:
    ''' 经验回放池 '''
    def __init__(self, capacity=10000, data_list=None):
        self.buffer = collections.deque(maxlen=capacity)  # 队列,先进先出
        if data_list is not None:
            self.buffer.extend(data_list)

    def add(self, state, pi, z):
        """将数据加入buffer"""

        self.buffer.append((state, pi, z))

    def sample(self, batch_size):
        """从buffer中采样数据,数量为batch_size"""

        transitions = random.sample(self.buffer, batch_size)
        states, pis, zs = zip(*transitions)
        return states, pis, zs

    def size(self):
        """当前buffer中数据的数量"""

        return len(self.buffer)

    def clear(self):
        self.buffer.clear()

    def save(self, expand_data=None):
        filedir = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'output', 'data')
        if not os.path.exists(filedir):
            os.makedirs(filedir)

        model_version = 0
        if expand_data is not None:
            model_version = expand_data.get('model_version')
        filename = 'replay_buffer_{}_{}.json'.format(int(time.time()), model_version)
        filepath = os.path.join(filedir, filename)

        with open(filepath, 'w') as f:
            jsonstr = json.dumps(list(self.buffer))
            f.write(jsonstr)

    @staticmethod
    def load_from_file(filepath):
        data_list = []
        with open(filepath, 'r') as f:
            jsonstr = f.read()
            data_list = json.loads(jsonstr)
        replay_buffer = ReplayBuffer(data_list=data_list)
        return replay_buffer

    @staticmethod
    def load_from_dir(dirpath):
        if not os.path.exists(dirpath):
            return

        all_data_list = list()
        for filename in os.listdir(dirpath):
            if filename.endswith('json') and filename.startswith('replay_buffer'):
                filepath = os.path.join(dirpath, filename)
                with open(filepath, 'r') as f:
                    jsonstr = f.read()
                    data_list = json.loads(jsonstr)
                    all_data_list.extend(data_list)
        replay_buffer = ReplayBuffer(data_list=all_data_list)
        return replay_buffer

if __name__ == '__main__':
    # s = 'ERIOC<VGK1234q24ds'
    # print(('k' in s))

    # q = collections.deque(maxlen=100)
    # q.append(('23', [123, 23], 0))
    # q.append(('24', [123, 23], 0))
    # q.append(('256', [123, 23], 0))
    # print(q)
    # print(list(q))
    # result = json.dumps(list(q))
    # print(result)

    # result_load = json.loads(result)
    # print(result_load)

    # filepath = '../output/data/replay_buffer_1656040190.json'
    # rb = ReplayBuffer.load(filepath)
    # print(rb.buffer)

    dataset = ChessDataset.load_from_dir('../output/data')
    dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)
    for i, (batch_states, batch_pis, batch_zs) in enumerate(dataloader):
        print(len(batch_states), batch_pis.size(), batch_zs)
        if i == 0:
            break


"""
Ejericio 1.
Determinar si un string corresponde a un entero válido
"""

entero_str = input('Ingrese un entero válido: ')
len_str = len(entero_str)
valido = True
i = 0

while i < len_str and valido == True :
    if entero_str[i]>='0' and entero_str[i]<='9' :
        i = i + 1
    else :
        valido = False

if valido == True :
    entero = int(entero_str)
    print('el valor sí es entero ', entero)
else :
    print('dato no válido')


# piranha_test.py - Test script for the Piranha Amazon scraper

import piranha
import csv

def test():
    creds = {}
    with open("credentials/amazonrootkey") as csvfile:
        read = csv.reader(csvfile, delimiter='=')
        for line in read:
            creds[line[0]]=line[1]
    print(creds)
    p = piranha.Piranha(creds['AWSAccessKeyId'],creds['AWSSecretKey'],creds['AssociateId'])
    while True:
        result = input("What are you looking for?: ")
        print(p.ItemSearch(Keywords=result))


import os
import re
import shutil
import cv2

def atof(text):
    try:
        retval = float(text)
    except ValueError:
        retval = text
    return retval

def natural_keys(text):
    return [atof(c) for c in re.split(r'[+-]?([0-9]+(?:[.][0-9]*)?|[.][0-9]+)', text)]

# path of our input videos
videos_path = "./inference/output"
output_path = "../rectification/painting"

try:
    os.makedirs(output_path)
except:
    print('error in video frame during creation of output folder')

# list of all the videos taken from videos_path
label_list = []
videos_list = []
for file in os.listdir(videos_path):
    if file.endswith('.txt'):
        label_list.append(file)
    else:
        videos_list.append(file)

label_list.sort(key=natural_keys)
num_frame = 0
for video in videos_list:
    cap = cv2.VideoCapture(videos_path + '/' + video)

    file_path, file_extension = os.path.splitext(video)
    file_name = os.path.basename(file_path)

    frame_num = 0

    # get total number of frames
    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    # get video's fps
    fps = cap.get(cv2.CAP_PROP_FPS)

    # get total seconds of video
    total_seconds = int(total_frames / fps)

    # or set the jump of frames = fps, so that I get two frames per second
    skip_frame = int(fps) // 2

    # for every second of the video, I take one frame every skip_frame
    for i in range(1, total_frames):

        if total_frames <= skip_frame:
            ret, frame = cap.read()
            item = [l for l in label_list if l.startswith(file_name + '_' + str(i) + '.txt')]
            if item:
                cv2.imwrite(f'{output_path}/{file_name}_{i}.png', frame)
                shutil.copy(videos_path + '\\' + item[0], output_path)
            continue

        my_frame = i * skip_frame

        # check for valid frame number
        if 0 <= my_frame <= total_frames:
            # set frame position, frame to save
            cap.set(cv2.CAP_PROP_POS_FRAMES, my_frame)
        else:
            break

        ret, frame = cap.read()
        # if MOV files, then rotate 90°
        if file_extension.upper() == ".MOV":
            frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
        # save the frame in the output path created at the beginning. the image will be saved as VIRB0398-frame-0.png
        item = [l for l in label_list if l.startswith(file_name + '_' + str(my_frame) + '.txt')]

        if item:
            cv2.imwrite(f'{output_path}/{file_name}_{my_frame}.png', frame)
            shutil.copy(videos_path + '\\' + item[0], output_path)
        else:
            continue

    cv2.destroyAllWindows()


import sys

M, N = map(int, sys.stdin.readline().split())
cookies=list(map(int,sys.stdin.readline().split()))

start=0
end=max(cookies)
result=0

while(start<=end):
    ppl_cnt=0     #사람 수
    mid=(start+end)//2 #주려는 과자의 길이
    if(mid==0):     #나누어줄 수 없으면 0을 출력
        total=0
        break

    for cookie in cookies:
        if cookie>=mid:
            ppl_cnt+=(cookie//mid)

    if ppl_cnt>=M:    #최대 길이가 아닌 것, ==M 정답
        start=mid+1
        result=mid

    else:       #M명한테 못 나눠주면 end-- 종료
        end=mid-1
    # print(start)
    # print(end)
    # print("/n")
print(result)


import datetime
import typing

import celery
from mongoengine import DynamicDocument, IntField, StringField
from pymongo import UpdateOne

from .. import rds

class ItemModel(DynamicDocument):
    tid = IntField(required=True)
    name = StringField(default="")
    description = StringField(default="")
    type = StringField(default="")
    market_value = IntField(default=0)
    circulation = IntField(default=0)

    @staticmethod
    def update_items(torn_get: typing.Union[typing.Callable, celery.Task], api_key: str):
        """
        Update list of stored Torn items in the database via a Torn API call

        Parameters
        ----------
        torn_get : Callable, celery.Task
            Celery task or function calling the Torn API (e.g. `tornium_celery.tasks.tornget`)
        api_key : str
            Torn API key used for the Torn API call

        Returns
        -------
        None

        Raises
        ------
        MissingKeyError
            No API key passed
        requests.exceptions.Timeout
            API called timed out
        NetworkingError
            HTTP error code
        TornError
            Torn error code
        """

        redis_client = rds()
        last_update = redis_client.get("tornium:items:last-update")

        if last_update is not None and (  # Redis value exists for last update
            # Updated less than an hour ago
            int(datetime.datetime.utcnow().timestamp()) - int(last_update) <= 3600
            # Last update was a different day
            or datetime.datetime.fromtimestamp(int(last_update)).day != datetime.datetime.utcnow().day
        ):
            return

        items_data = torn_get(
            endpoint="torn/?selections=items",
            key=api_key,
        )
        bulk_operations = []

        item_id: int
        item: dict
        for item_id, item in items_data["items"].items():
            bulk_operations.append(
                UpdateOne(
                    {"tid": int(item_id)},
                    {
                        "$set": {
                            "tid": int(item_id),
                            "name": item.get("name", ""),
                            "description": item.get("description", ""),
                            "type": item.get("type", ""),
                            "market_value": item.get("market_value", 0),
                            "circulation": item.get("circulation", 0),
                        }
                    },
                    upsert=True,
                )
            )

        ItemModel._get_collection().bulk_write(bulk_operations, ordered=False)
        redis_client.set("tornium:items:last-update", int(datetime.datetime.utcnow().timestamp()), ex=5400)  # 1.5 hours


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn import model_selection
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor, AdaBoostRegressor
from sklearn.preprocessing import LabelEncoder, RobustScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.pipeline import make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.neighbors import KNeighborsRegressor
from sklearn.linear_model import Lasso, BayesianRidge, Ridge
from xgboost import XGBRegressor
from mlxtend.regressor import StackingRegressor

# =============================================================================

#Reading the Data
trainingPath = 'data/train.csv'
initialDF = pd.read_csv(trainingPath)

# Digging more into the target variable i.e., SalePrice
def Target(initialDF):
    target = initialDF.loc[:, "SalePrice"]
    #showing the distribution of data using seaborn
    print(target.describe())

    sns.distplot(target)

# ============================================================================
# Data cleaning - conversion, standardization

def checkCorr(initialDF):

    #corrmat = trainDF.corr()
    #correlations = corrmat["SalePrice"].sort_values(ascending=False)
    #features = correlations.index[0:10]
    #print(features)

    #sns.pairplot(trainDF[features], size = 2.5)
    #plt.show()

    # BEGIN: from https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python
    # EXPLANATION: We are visualzing the correlation matrix here using a heatmap to represent
    # different coefficients by different colors
    corrmat = initialDF.corr()
    f, ax = plt.subplots(figsize=(11,9))
    sns.heatmap(corrmat, vmax=.8, square=True)

    k = 10 #number of variables for heatmap
    cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
    cm = np.corrcoef(initialDF[cols].values.T)
    sns.set(font_scale=1.25)
    hm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
    plt.show()
    # END: from https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python

    '''
    According to the heatmap Garage Cars and the Garage Area are the most correlated attribute followed by
    YearBuilt and GarageYrBlt, GrLivArea and TotRmsAbsGr and 1stFlrSF and TotalBsmtSF

    '''

    most_corr = pd.DataFrame(cols)
    most_corr.columns = ['Most Correlated Features']
    print(most_corr)

    #We will check the collinearity between the attributes and eleminate the attribute that is not helpful in predicting sales price

    corr1 = initialDF.loc[:, "GarageArea"].corr(initialDF.loc[:, "GarageCars"])
    corr2 = initialDF.loc[:, "YearBuilt"].corr(initialDF.loc[:, "GarageYrBlt"])
    corr3 = initialDF.loc[:, "GrLivArea"].corr(initialDF.loc[:, "TotRmsAbvGrd"])
    corr4 = initialDF.loc[:, "1stFlrSF"].corr(initialDF.loc[:, "TotalBsmtSF"])

    print("Collinearity between Garage Area and Garage Cars")
    print(corr1)
    print("Collinearity between Year Built and Garage Year Built")
    print(corr2)
    print("Collinearity between Ground Living Area and Total Rooms Above Ground")
    print(corr3)
    print("Collinearity between 1st Floor SF and Total Basement SF")
    print(corr4)

def checkOutliers(initialDF):

    initialDF = featureEngineering(initialDF)

    # Plotting a scatterplot of GrivArea vs SalePrice
    # BEGIN: from https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python
    # EXPLANATION: Dataframes are concatenated and plotted using bivariate analysis
    SPvsGrLivArea = pd.concat([initialDF.loc[:, 'SalePrice'], initialDF.loc[:, 'GrLivArea']], axis=1)
    SPvsGrLivArea.plot.scatter(x='GrLivArea', y='SalePrice', ylim=(0,800000))

    # END: from https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python

    '''
    By looking at the scatterplot we can conclude that people will pay more for more living area but the two
    points on bottom right look off and so do the two points in the top right that are clustered together.
    We will be removing the outliers manually in the handleOutliers function.
    '''

    # Finding the Ids of the outliers in GrLivArea
    print(initialDF.sort_values(by = 'GrLivArea', ascending = False)[:4])

    SPvsYearsOld = pd.concat([initialDF.loc[:, 'SalePrice'], initialDF.loc[:, 'YearsOld']], axis=1)
    SPvsYearsOld.plot.scatter(x='YearsOld', y='SalePrice', ylim=(0,800000))

    '''
    The four points in the extreme right seem way too expensive for an old house. It can be dropped depending
    on the performance of the model.

    '''
    # Finding the Ids of the outliers in YearsOld
    yearsOldOutlier = (initialDF.loc[:, "YearsOld"] > 110) & (initialDF.loc[:, "SalePrice"] > 300000)
    print(initialDF.loc[yearsOldOutlier])

    SPvsLotArea = pd.concat([initialDF.loc[:, 'SalePrice'], initialDF.loc[:, 'LotArea']], axis=1)
    SPvsLotArea.plot.scatter(x='LotArea', y='SalePrice', ylim=(0,800000))

    '''
    The four points on the extreme right seem way out of the trend in the graph. We will be removing the outliers in the
    handle outliers function.
    '''
    # Finding the Ids of the outliers in vsLotArea
    print(initialDF.sort_values(by = 'LotArea', ascending = False)[:4])

    SPvsTotalBsmtSF = pd.concat([initialDF.loc[:, 'SalePrice'], initialDF.loc[:, 'TotalBsmtSF']], axis=1)
    SPvsTotalBsmtSF.plot.scatter(x='TotalBsmtSF', y='SalePrice', ylim=(0,800000))

    TotalBsmtOutlier = (initialDF.loc[:, "TotalBsmtSF"] > 6000)
    print(initialDF.loc[TotalBsmtOutlier])

    # This outlier was calculated above as well

    '''
    There are probably other outliers in the training dataset but removing outliers is not always safe. It can
    effect the model badly if there are outliers in the test dataset as well. Instead of dropping the outliers we
    will try to make some of our models more robust to tacle them.
    '''

def handleOutliers(initialDF):

    # Dropping the 4 outliers in GrLivArea vs SalePrice and 2 outliers in YearsOld vs SalePrice
    droppingOutliers = (initialDF.loc[:, "Id"] == 1299) | (initialDF.loc[:, "Id"] == 524) | (initialDF.loc[:, "Id"] == 1183) | (initialDF.loc[:, "Id"] == 692) | (initialDF.loc[:, "Id"] == 186) | (initialDF.loc[:, "Id"] == 584) | (initialDF.loc[:, "Id"] == 314) | (initialDF.loc[:, "Id"] == 336) | (initialDF.loc[:, "Id"] == 250) | (initialDF.loc[:, "Id"] == 707)
    initialDF = initialDF.drop(initialDF[droppingOutliers].index)

    return initialDF

def checkMissingValues(initialDF):

    # Check the missing values in the entire training dataset
    # Count the values and sort it in descending order
    total = initialDF.isnull().sum().sort_values(ascending=False)
    percent = (initialDF.isnull().sum()/initialDF.isnull().count()).sort_values(ascending=False)
    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    sns.heatmap(initialDF.isnull(), yticklabels=False,cbar=False)
    print("Missing Data", missing_data.head(20))

    '''There are attributes in the dataset that contain NaN values. Here, we are identifying the attributes with a lot
    of missing information
    '''
    for var in initialDF.columns:
        if initialDF[var].isnull().mean()>0.80:
            print(var, initialDF[var].unique())

    '''
    We tried to identify the variables that contain a lot of missing information (NaN) so that we can tacle that later
    when we are dropping the attributes
    '''

def handlingMissing(X):
    # We handled the missing values using the imputer
    # If the entire attribute does not have any values i.e, if the entire column consists of
    # missing values we replace the value in the column by 0
    missing_all = False
    for col in X.columns:
        if X.loc[:, col].isnull().all():
            X.loc[:, col] = 0

    # BEGIN: from https://www.kaggle.com/dansbecker/handling-missing-values
    # EXPLANATION: Imputers can fill in missing values with mean values by default. Filling
    # in missing values is usually better than dropping them entirely because
    # those that are not missing might indicate some valuable patterns.
    imputer = SimpleImputer(missing_values = np.nan , strategy = 'mean')
    imputer = imputer.fit(X)
    X = imputer.transform(X)
    # END: from https://www.kaggle.com/dansbecker/handling-missing-values
    return X

def featureEngineering(originalDF):

    # Replacing yearBuilt to yearsOld so that we can plot the attribute vs SalePrice and check if there are any outliers
    yearBuilt = originalDF.loc[:, "YearBuilt"]
    # subtracting by 2006 since the data was calculated between 2006 and 2010
    originalDF.loc[:, "YearsOld"] = 2006 - originalDF.loc[:, "YearBuilt"]

    # BEGIN: https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
    # EXPLANATION: We are replacing the categorical values such as BsmtQual with a numeric value between
    # 0 and the number of class minus 1 by the use of Label Encoder. It is being used because some categorial
    # value might contain some information in the dataset.
    EncoderList = ["KitchenQual", "GarageQual", "BsmtQual", "BsmtCond", "ExterQual", "GarageCond",
                     "OverallCond", "FireplaceQu", "ExterCond", "HeatingQC", "BsmtFinType1", "BsmtFinType2"]
	# Label Encoder
    for column in EncoderList:
        lbl = LabelEncoder()
        lbl.fit(list(initialDF[column].values))
        initialDF[column] = lbl.transform(list(initialDF[column].values))
    # END: https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard

    return originalDF

def standardize(df):

    Cols = df.select_dtypes(exclude=['object']).columns

    stds = df.loc[:, Cols].std()
    mean = df.loc[:, Cols].mean()

    df.loc[:, Cols] = (df.loc[:, Cols] - mean) / stds

    return df

# ===============================================================================

# The data is experimented with different pre-processing methods
def testResults(initialDF):

    # Build and test Grandient Boosting Regressor
    def testGBR(df, n_estimators=100, max_depth=3):
        y = df.loc[:, "SalePrice"]
        X = df.drop(["SalePrice"], axis=1)
        gbr = GradientBoostingRegressor(n_estimators=n_estimators, max_depth=max_depth)

        X = handlingMissing(X)

        cvScores = model_selection.cross_val_score(gbr, X, y, cv=10,
                                                   scoring = 'neg_mean_squared_error')
        print("\tMean Squared Error:", -1 * cvScores.mean())

    # Build and test Random Forest Regressor
    def testRF(df):
        y = df.loc[:, "SalePrice"]
        X = df.drop(["SalePrice"], axis=1)
        rf = RandomForestRegressor()

        X = handlingMissing(X)

        cvScores = model_selection.cross_val_score(rf, X, y, cv=10,
                                                   scoring = 'neg_mean_squared_error')
        print("\tMean Squared Error:", -1 * cvScores.mean())

    #Perform feature Engineering
    initialDF = featureEngineering(initialDF)

    # Dropping the Outliers
    initialDF = handleOutliers(initialDF)

    # Standardize the Data
    initialDF.loc[:, initialDF.columns != "SalePrice"] = standardize(initialDF.drop(["SalePrice"], axis=1))

    """
    One Hot Encoding and Binary encoding
    """
    # One Hot Encoding
    colNames = ['SalePrice', 'LotArea', 'Neighborhood','OverallQual','GrLivArea','FullBath','GarageCars','YearsOld']
    trainPredictors = initialDF.loc[:, colNames]
    selected_hot_DF = pd.get_dummies(trainPredictors)
    print(selected_hot_DF)
    # The attributues with more than 80% missing values are dropped
    most_hotDF = pd.get_dummies(initialDF.drop(['PoolQC', 'MiscFeature', 'Alley','Fence'], axis=1))
    # The attributes with more than 90% missing values and high correlation (GarageCars and GarageArea) values are dropped
    drop_corrDF = pd.get_dummies(initialDF.drop(['PoolQC', 'MiscFeature', 'Alley', 'GarageArea', 'YearBuilt'], axis=1))

    '''
    Displaying the Gradient Boosting Regressor and Random Forest Regressor Results
    '''

    print("Graident Boosting Regressor with 100 estimators ")
    print("Performing GBR on Selected Columns")
    testGBR(selected_hot_DF)
    print("Performing GBR on all Attributes except for ones with many missing values")
    testGBR(most_hotDF)
    print("Performing GBR on all Attributes except for ones with many missing values or high correlation")
    testGBR(drop_corrDF)

    print()

    print("Graident Boosting Regressor with 200 estimators")
    testGBR(selected_hot_DF, n_estimators=200)
    print(" Performing GBR on all Attributes except for ones that have many missing values")
    testGBR(most_hotDF, n_estimators=200)
    print("Performing GBR on all Attributes except for ones that have many missing values or Highly Correlated Attributes")
    testGBR(drop_corrDF, n_estimators=200)

    # We can see that the Mean Squared Error is redured by increasing the estimators
    print()

    print("Random Forest Regressor")
    print("Performing GBR on Selected Columns")
    testRF(selected_hot_DF)
    print("Performing GBR on all Attributes except for ones that have many missing values")
    testRF(most_hotDF)
    print("Performing GBR on all Attributes except for ones that have many missing values or Highly Correlated Attributes")
    testRF(drop_corrDF)

# The best pre processing is done by One Hot Encoder in the above trails
def preprocess(initialDF):

    # Performing feature engineering
    initialDF = featureEngineering(initialDF)
    # Dropping the outliers
    initialDF = handleOutliers(initialDF)
    # Standardizing the Data
    initialDF.loc[:, initialDF.columns != "SalePrice"] = standardize(initialDF.drop(["SalePrice"], axis=1))

    # The attributues with more than 80% missing values are dropped
    most_hotDF = pd.get_dummies(initialDF.drop(['PoolQC', 'MiscFeature', 'Alley','Fence'], axis=1))

    y = most_hotDF.loc[:, "SalePrice"]
    X = most_hotDF.drop(["SalePrice"], axis=1)
    X = handlingMissing(X)
    y = np.log1p(y)

    return X, y

# Storing X and y as Global variables
X, y = preprocess(initialDF)

# BEGIN: https://github.com/Shitao/Kaggle-House-Prices-Advanced-Regression-Techniques/blob/master/code/single_model/base_model.py
# EXPLANATION: We want the search to evaluate the model's performance using RMSE and we do that by defining a function
# that creates a scorer for GridSearchCV

def mean_squared_error_(ground_truth, predictions):
    return mean_squared_error(ground_truth, predictions) ** 0.5

RMSE = make_scorer(mean_squared_error_, greater_is_better=False)
# END: https://github.com/Shitao/Kaggle-House-Prices-Advanced-Regression-Techniques/blob/master/code/single_model/base_model.py

# Rough hyperparameter tuning is performed one at a time and is done by XSEDE allocation
# Tuning the gradient boosting regressor
def tuneGBR(df):
    X, y = preprocess(df)

    """
    param_test1 = {'min_samples_split':range(2,20)}
    gsearch1 = model_selection.GridSearchCV(
            estimator = GradientBoostingRegressor(
                learning_rate = 0.01,
                n_estimators = 1000,
                min_samples_leaf = 1,
                max_depth = 4,
                max_features = 'sqrt',
                subsample = 0.8,
                random_state = 0
                ),
            param_grid = param_test1,
            scoring = RMSE,
            iid = False,
            cv = 10
        )
    gsearch1.fit(X, y)
    print("Best param and its score", gsearch1.best_params_, -gsearch1.best_score_, sep='\n')

    param_test2 = {'min_samples_leaf':range(2,10)}
    gsearch2 = model_selection.GridSearchCV(
            estimator = GradientBoostingRegressor(
                learning_rate = 0.01,
                n_estimators = 1000,
                min_samples_split = 17,
                max_depth = 4,
                max_features = 'sqrt',
                subsample = 0.8,
                random_state = 0
                ),
            param_grid = param_test2,
            scoring = RMSE,
            iid = False,
            cv = 10
        )
    gsearch2.fit(X, y)
    print("Best param and its score", gsearch2.best_params_, -gsearch2.best_score_, sep='\n')
    """
    """
    param_test3 = {'max_depth':range(2,10)}
    gsearch3 = model_selection.GridSearchCV(
            estimator = GradientBoostingRegressor(
                learning_rate = 0.01,
                n_estimators = 1000,
                min_samples_leaf = 3,
                min_samples_split = 17,
                max_features = 'sqrt',
                subsample = 0.8,
                random_state = 0
                ),
            param_grid = param_test3,
            scoring = RMSE,
            iid = False,
            cv = 10
        )
    gsearch3.fit(X, y)
    print("Best param and its score", gsearch3.best_params_, -gsearch3.best_score_, sep='\n')
    """

    param_test4 = {'subsample': [0.76, 0.78, 0.82, 0.84]}
    gsearch4 = model_selection.GridSearchCV(
            estimator = GradientBoostingRegressor(
                learning_rate = 0.01,
                n_estimators = 1000,
                min_samples_leaf = 3,
                min_samples_split = 17,
                max_depth = 5,
                max_features = 'sqrt',
                random_state = 0
                ),
            param_grid = param_test4,
            scoring = RMSE,
            iid = False,
            cv = 10
        )
    gsearch4.fit(X, y)
    print("Best param and its score", gsearch4.best_params_, -gsearch4.best_score_, sep='\n')

# Making a dictionary of all the models so that we can calculated the RMSE later
models = {

          "gbr": GradientBoostingRegressor(n_estimators=1000,
                                           learning_rate=0.01,
                                           min_samples_split=17,
                                           min_samples_leaf=7,
                                           max_depth=5,
                                           max_features='sqrt',
                                           subsample=0.8,
                                           random_state=0),

          "xgb": XGBRegressor(n_estimators=1000,
                              learning_rate=0.01,
                              min_child_weight=3,
                              max_depth=6,
                              gamma=0,
                              subsample=0.8,
                              colsample_bytree=0.85,
                              random_state=0),
          "rf": RandomForestRegressor(n_estimators=1000,
                                      bootstrap=True,
                                      max_features='sqrt',
                                      max_depth=6,
                                      min_samples_split=3,
                                      min_samples_leaf=1,
                                      random_state=0),
          "knn": KNeighborsRegressor(n_neighbors = 10),
          "ada": AdaBoostRegressor(n_estimators=1000,
                                   learning_rate=0.01,
                                   loss='square',
                                   random_state=0),
          # BEGIN: https://www.kaggle.com/gmishrakec/multi-regression-techniques/code
          # EXPLANATION: We are using the RobustScaler to make the lasso regression more Robust
          # since it is one of the regression model that is very sensitive to outliers
          "lasso": make_pipeline(RobustScaler(), Lasso(alpha =0.0005,
                                                       random_state=0)),
          # END: https://www.kaggle.com/gmishrakec/multi-regression-techniques/code
          "bayridge": BayesianRidge(),
          "ridge": Ridge()
         }

# This function takes a model name in string
def tryModel(modelName):
    try:
        alg = models[modelName]
        X, y = preprocess(initialDF)
        cvScores = model_selection.cross_val_score(alg, X, y, cv=10, scoring = RMSE)

        print("RMSE: {:.6f} ({:.4f})".format(-cvScores.mean(), cvScores.std()))
    except:
        print("This particular model does not exist in the dictionary of models")

# BEGIN: https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard
# EXPLANATION: We are calculating the final score by taking weighted average
# of the predictions from each model. The code has been modified to calculate the weighted
# average instead of arithmetic average to get a better score
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, models, weights):
        self.models = models
        self.weights = weights
        if len(weights) != len(models):
            print("ERROR: The number of models and weights do not match.")
            return
        if sum(weights) != 1.0:
            print("The sum of weights is required to be 1")
            return

    # we define clones of the original models to fit the data in
    def fit(self, X, y):
        self.models_ = [clone(x) for x in self.models]

        # Train cloned base models
        for model in self.models_:
            model.fit(X, y)

        return self

    #Now we do the predictions for cloned models and average them
    def predict(self, X):
        predictions = np.column_stack([
            model.predict(X) for model in self.models_
        ])
        #return np.mean(predictions, axis=1)
        predictionsDF = pd.DataFrame(data=predictions)
        weightedPredictions = predictionsDF.apply(lambda prediction: prediction * self.weights, axis=1)
        return np.sum(weightedPredictions, axis=1)

# END: https://www.kaggle.com/serigne/stacked-regressions-top-4-on-leaderboard

# An array of model names in string and a meta model name in string is taken as the parameters of the function
def tryStackingModels(modelNames, metaModelName):
    X, y = preprocess(initialDF)
    regs = [models[modelName] for modelName in modelNames]
    meta = models[metaModelName]

    stack = StackingRegressor(regressors = regs,
                              meta_regressor = meta)

    cvScores = model_selection.cross_val_score(stack, X, y, cv=10, scoring = RMSE)

    print("RMSE: {:.6f} ({:.4f})".format(-cvScores.mean(), cvScores.std()))

# An array of model names in string is taken as the parameter
def tryAveragingModels(modelNames, pWeights):
    try:
        X, y = preprocess(initialDF)
        algs = tuple(models[modelName] for modelName in modelNames)
        # This gives an equal weight on every model
        # averaged_models = AveragingModels(models = algs, weights = [1/len(algs) for i in range(len(algs))])
        averaged_models = AveragingModels(models = algs, weights = pWeights)
        cvScores = model_selection.cross_val_score(averaged_models, X, y, cv=10, scoring=RMSE)

        print("RMSE: {:.6f} ({:.4f})".format(-cvScores.mean(), cvScores.std()))
    except:
        print("The specified models do not exist in the models.")

'''
Runs the algorithm on the testing set and writes the results to a csv file.
'''
def doKaggleTest(initialDF):

    print("File Name?")
    filename = input()

    # Reading the test dataset
    testDF = pd.read_csv("data/test.csv")
    testIDs = testDF.loc[:, "Id"]

    # Dropping the Outliers
    initialDF = handleOutliers(initialDF)

    y = initialDF.loc[:, "SalePrice"]
    y = y.values
    trainDF = initialDF.drop(["SalePrice"], axis=1)

    # Concatinate test and initial (=train)
    train_numRow = trainDF.shape[0]
    dataset = pd.concat(objs=[trainDF, testDF], axis=0)

    # created full dataset using One-Hot Encoding and Feature Engineering

    dataset = pd.get_dummies(dataset)
    dataset = featureEngineering(dataset)

    # The numbers of features in the testing dataset and initial dataset (trainDF) needs to match
    # So we split the dataset
    trainDF = dataset.iloc[:train_numRow, :].copy()
    testDF = dataset.iloc[train_numRow:, :].copy()

    trainDF = standardize(trainDF) #standardizing the training dataset

    """ # Weighted Average
    modelNames = ["gbr", "xgb", "lasso", "ridge"]
    algs = tuple(models[modelName] for modelName in modelNames)
    model = AveragingModels(models = algs, weights = [0.24, 0.07, 0.46, 0.09])
    """

    model = models["ridge"]

    trainDF = handlingMissing(trainDF)
    model.fit(trainDF, y)   #fitting the model with trainig data

    # Preprocessing the test dataset
    testDF = standardize(testDF)
    testDF = handlingMissing(testDF)

    predictions = model.predict(testDF)
    # Create a new dataframe with only the columns Kaggle wants from the dataset.
    submission = pd.DataFrame({
        "Id": testIDs,
        "SalePrice": predictions
    })

    # Prepare CSV
    submission.to_csv('data/testResults.csv', index=False)
    # Now, this .csv file can be uploaded to Kaggle


import ctypes, json
library = ctypes.cdll.LoadLibrary('./library.so')

hello_world = library.helloWorld
hello_world()

jsonDict = library.jsonDict
test = jsonDict()

library.jsonArray.restype = ctypes.c_char_p
test = library.jsonArray().decode('utf8')
json_string = json.dumps(test, ensure_ascii=False)
print(type(json_string))
json_dict = json.loads(test)
print(json_dict)