from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, ReduceLROnPlateau
from tensorflow.keras import layers, models
from tensorflow import lite
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
from imutils import paths
import matplotlib.pyplot as plt
import numpy as np
import os
import time
import cv2
import datetime

# КОНФИГУРАЦИЯ НЕЙРОННОЙ СЕТИ
def get_model(input_size, classes=7):
    model = models.Sequential()

    model.add(layers.Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape =input_size))
    model.add(layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(2, 2))
    model.add(layers.Dropout(0.25))

    model.add(layers.Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=regularizers.l2(0.01)))
    model.add(layers.Conv2D(256, kernel_size=(3, 3), activation='relu', kernel_regularizer=regularizers.l2(0.01)))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Dropout(0.25))

    model.add(layers.Flatten())
    model.add(layers.Dense(1024, activation='relu'))
    model.add(layers.Dropout(0.5))

    model.add(layers.Dense(classes, activation='softmax'))

    # компиляция модели
    model.compile(optimizer=Adam(learning_rate=0.0001),
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model

# ЗАГРУЗКА И ПРЕОБРАЗОВАНИЕ ИЗОБРАЖЕНИЙ
def dataset_load(im_paths, width, height, verbose):
    data = []
    labels = []
    for (i, im_path) in enumerate(im_paths):
        # загружаем изображение в переменную image
        image = cv2.imread(im_path)
        # определяем класс изображения из строки пути
        # формат пути: ../dataset/{class}/{image}.jpg
        label = im_path.split(os.path.sep)[-2]
        # изменяем размер изображения на заданный (изображение должно быть квадратным)
        image = cv2.resize(image, (width, height), interpolation=cv2.INTER_AREA)
        # переводим изображение в массив numpy
        image_array = img_to_array(image, data_format=None)
        # добавляем массив изображения в список data
        data.append(image_array)
        # добавляем в список labels метку соответствующего изображения из списка data
        labels.append(label)
        # выводим на экран количество обработанных изображений в периодичностью verbose
        if verbose > 0 and i > 0 and (i + 1) % verbose == 0:
            print("[INFO] Обработано {}/{}".format(i + 1, len(im_paths)))
    # возвращаем numpy массивы data и labels
    return (np.array(data), np.array(labels))

# 1. ПОДГОТОВКА ДАННЫХ
# указываем название каталога набора данных в папке datasets
dataset_name = "faces"
# определяем пути набора данных, сохранения графика обучения и модели нейронной сети keras
dataset_path = os.path.join("datasets", dataset_name)
name_labels = ['interested', 'uninterested']
num_classes = len(name_labels)
plot_name = "{}_output/{}_plot.png".format(dataset_name, dataset_name)
weights_name = "{}_output/{}_weights.h5".format(dataset_name, dataset_name)
tflite_name = "{}_output/{}_weights.tflite".format(dataset_name, dataset_name)
# загружаем набор данных с диска, преобразуя изображения в массив
# и масштабируя значения пикселей из диапазона [0, 255] в диапазон [0, 1]
start_time = time.time()
image_paths = list(paths.list_images(dataset_path))
print("[INFO] Загрузка изображений ...")
(data, labels) = dataset_load(image_paths, width=48, height=48, verbose=500)
data = data.astype("float") / 255.0
# разделяем данные на обучающий и тестовый наборы (75% и 25%)
(trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.25, random_state=42)
print("[INFO] Форма матрицы признаков: {}".format(data.shape))
print("[INFO] Размер матрицы признаков: {:.1f}MB".format(data.nbytes / (1024 * 1000.0)))
# преобразуем метки из целых чисел в векторы
trainY = LabelBinarizer().fit_transform(trainY)
testY = LabelBinarizer().fit_transform(testY)
print("[INFO] Время подготовки данных: {} сек".format(round(time.time() - start_time, 2)))

# 2.СБОРКА И КОМПИЛЯЦИЯ МОДЕЛИ НЕЙРОННОЙ СЕТИ
print("[INFO] Компиляция модели...")
model = get_model((48,48,1), 2)

# 3.ФОРМИРОВАНИЕ ДОПОЛНИТЕЛЬНЫХ ПАРАМЕТРОВ ОБУЧЕНИЯ
# Определение коллбэков для обучения нейронной сети
log_dir = "checkpoint/logs/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
checkpoint = ModelCheckpoint(filepath=weights_name,
                             save_best_only=True,
                             verbose=1,
                             mode='min',
                             moniter='val_loss')
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                              factor=0.2,
                              patience=6,
                              verbose=1,
                              min_delta=0.0001)
tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
csv_logger = CSVLogger('training.log')
callbacks = [checkpoint, reduce_lr, csv_logger]
# настройка метода увеличения выборки данных для обучения через модификацию существующих данных (аугментация)
aug = ImageDataGenerator(rotation_range=20, zoom_range=0.15,
                         width_shift_range=0.2, height_shift_range=0.2,
                         shear_range=0.15, horizontal_flip=True, fill_mode="nearest")

# 4. ОБУЧЕНИЕ НЕЙРОННОЙ СЕТИ
num_epochs = 30
print("[INFO] Обучение нейронной сети...")
start_time = time.time()
hist = model.fit(aug.flow(trainX, trainY, batch_size=32),
                 validation_data=(testX, testY),
                 batch_size=64,
                 epochs=num_epochs,
                 callbacks=callbacks,
                 verbose=0)
print("[INFO] Время обучения: {} сек".format(round(time.time() - start_time, 2)))

# 5.ОЦЕНКА МОДЕЛИ НЕЙРОННОЙ СЕТИ
print("[INFO] Оценка нейронной сети...")
predictions = model.predict(testX, batch_size=32)
print(classification_report(testY.argmax(axis=1),
                            predictions.argmax(axis=1),
                            target_names=name_labels))
# построение и сохранение графика потерь и точности тренировок
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, num_epochs), hist.history["loss"], label="train_loss")
plt.plot(np.arange(0, num_epochs), hist.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, num_epochs), hist.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, num_epochs), hist.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.savefig(plot_name)

# 6. СОХРАНЕНИЕ МОДЕЛИ НЕЙРОННОЙ СЕТИ
print("[INFO] Сохранение модели TFLite с квантованием...")
# конвертирование модели keras в квантованную модель tflite
converter = lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [lite.Optimize.DEFAULT]
tflite_model = converter.convert()
# сохранение модели tflite.
with open(tflite_name, 'wb') as f:
    f.write(tflite_model)


from numpy import *
import matplotlib.pyplot as plt
import functions

data = loadtxt("kplr006603043-2011145075126_slc.tab")

data_new = []
for i in data:
    if i[0] > 1691.5 and i[0] < 1693.1:
        data_new.append(i)
data = array(data_new)

mag = data[:,3]
flux = 10**((mag-median(mag))/-2.5)

o = open("lc2.dat","w")
output = [data[:,0],flux,data[:,4]]
output = transpose(output)
functions.write_table(output,o)
o.close()

plt.scatter(data[:,0],flux)
plt.show()


import copy
import numpy as np
from scipy.spatial.distance import cdist

class Server:
    def __init__(self, parameters):
        super().__init__()
        self.k = parameters['kernel_size']
        self.alpha = 1.0
        self.d_out = parameters['d_out']
        # centers, spreads, w and b can be broadcast to clients
        self.centers, self.std = None, None
        self.w = np.random.randn(self.k, self.d_out)
        self.b = np.random.randn(1, self.d_out)

    @staticmethod
    def _sort_centers(centers):
        """
        To sort the centers according to the distance from zero vector
        Please note that this fun has not consider the direction of the centers, should be change
        :param centers:
        :return: sorted centers & index
        """
        tmp_centers = copy.deepcopy(centers)
        distance = np.sum(tmp_centers ** 2, axis=1)

        sorted_index = np.argsort(distance)

        tmp_centers = tmp_centers[sorted_index, :]

        return tmp_centers, sorted_index

    @staticmethod
    def _dist(Mat1, Mat2):
        """
        rewrite euclidean distance function in Matlab: dist
        :param Mat1: matrix 1, M x N
        :param Mat2: matrix 2, N x R
        output: Mat3. M x R
        """
        Mat2 = Mat2.T

        return cdist(Mat1, Mat2)

    def average(self, selected_clients):
        stack_c, stack_w, stack_b, stack_s = [], [], [], []
        num_data = 0

        for i, client in enumerate(selected_clients):
            tmp_c, tmp_w, tmp_b, tmp_s = client.compute_update()
            nk = client.data_size()
            num_data += nk

            if i == 0:
                stack_c, stack_w, stack_b, stack_s = nk * tmp_c, nk * tmp_w, \
                                                     nk * tmp_b, nk * tmp_s
            else:
                # stack_c = np.vstack((stack_c, tmp_c))
                stack_c += nk * tmp_c
                stack_w += nk * tmp_w
                stack_b += nk * tmp_b
                stack_s += nk * tmp_s

        # k_means_c = KMeans(n_clusters=self.k).fit(stack_c)
        # self.centers = k_means_c.cluster_centers_
        self.centers = stack_c / num_data
        self.centers, tmp_index = self._sort_centers(self.centers)
        # self.w, self.b, self.std = stack_w / num_data, stack_b / num_data, stack_s / num_data
        self.w, self.b, self.std = stack_w[tmp_index] / num_data, stack_b / num_data, stack_s[tmp_index] / num_data

    def predict(self, test_x):
        N = test_x.shape[0]
        TestDistance = self._dist(self.centers, test_x.T)
        TestSpreadMat = np.tile(self.std.reshape(-1, 1), (1, N))
        TestHiddenOut = np.exp(-(TestDistance / TestSpreadMat) ** 2).T
        Test_y = np.dot(TestHiddenOut, self.w) + self.b
        return Test_y

    def broadcast(self):
        tmp_c, tmp_w, tmp_b, tmp_s = copy.deepcopy(self.centers), copy.deepcopy(self.w), \
                                     copy.deepcopy(self.b), copy.deepcopy(self.std)
        return {'centers': tmp_c, 'w': tmp_w, 'b': tmp_b, 'std': tmp_s}


"""Evaluate explanation technique on the CLEVR XAI dataset.

This module computes the saliency maps for the relation network
and evaluates how well the explanation technique matches the
ground truth heatmaps.
"""

# from lrp_relations import enable_deterministic  # noqa isort:skip

import dataclasses
import pickle
from typing import Optional, cast

import numpy as np
import pandas as pd
import savethat
import torch
from savethat import logger
from torch.utils.data import DataLoader
from tqdm import tqdm

from lrp_relations import data, lrp, train_clevr
from relation_network import model as rel_model

@dataclasses.dataclass(frozen=True)
class GroundTruthEvalArgs(savethat.Args):
    model_key: str
    dataset: str = "../data/clevr/CLEVR_v1.0/"
    question_type: str = "simple"  # "simple" or "complex"
    ground_truth: str = "single_object"  # "single_object" or "all_objects"
    n_samples: int = -1  # -1 for all samples
    device: str = "cuda" if torch.cuda.is_available() else "cpu"
    batch_size: int = 50
    checkpoint: str = "best"

@dataclasses.dataclass(frozen=True)
class GroundTruthEvalResults:
    relevance_mass: torch.Tensor
    relevance_rank_accuracy: torch.Tensor
    correct: torch.Tensor

    def as_dataframe(self) -> pd.DataFrame:
        rank = self.relevance_rank_accuracy.cpu().numpy()
        return pd.DataFrame(
            {
                "relevance_mass": self.relevance_mass.cpu().numpy(),
                "relevance_rank_accuracy": rank,
            }
        )

    def accuracy(self) -> float:
        return self.correct.float().mean().item()

def load_model(
    storage: savethat.Storage,
    key: str,
    checkpoint: str = "best",
    map_location: Optional[torch.device] = None,
) -> tuple[rel_model.RelationNetworks, train_clevr.TrainArgs]:
    """Load the model from the storage.

    Args:
        storage: Storage to load the model from.
        key: Key of the model to load.

    Returns:
        The model.
    """
    if not (storage / key).exists():
        storage.download(key)

    if checkpoint == "best":
        with open(storage / key / "results.pickle", "rb") as f:
            results = cast(train_clevr.TrainedModel, pickle.load(f))
        ckpt = results.checkpoints[-1]
        ckpt_path = storage / key / "checkpoints" / ckpt.path
        logger.debug(
            f"Loading model with accuracy {ckpt.accuracy:.4f} from {ckpt_path}"
        )
    else:
        ckpt_path = storage / key / "checkpoints" / checkpoint
        logger.debug(f"Loading model from {ckpt_path}")

    model = rel_model.RelationNetworks(data.get_n_words())
    model.load_state_dict(torch.load(ckpt_path, map_location=map_location))

    args = train_clevr.TrainArgs.from_json(storage / key / "args.json")
    return model, args

def relevance_mass(
    saliency: torch.Tensor,
    mask: torch.Tensor,
    reduce: tuple[int, ...] = (1, 2, 3),
) -> torch.Tensor:
    """Compute the relevance mass.

    Args:
        saliency: Saliency map.
        mask: Mask to apply.
        reduce: Dimensions to reduce.

    Returns:
        The relevance mass.
    """
    within = (saliency * mask).sum(dim=reduce)
    total = saliency.sum(dim=reduce)
    return within / total

def l2_norm_sq(x: torch.Tensor, dim: int = 1) -> torch.Tensor:
    """Compute the L2 norm squared.

    Args:
        x: Tensor to compute the L2 norm squared.

    Returns:
        The L2 norm squared.
    """
    return (x**2).sum(dim, keepdim=True)

def max_norm(x: torch.Tensor, dim: int = 1) -> torch.Tensor:
    """Compute the max norm.

    Args:
        x: Tensor to compute the max norm.
        dim: Dimension to compute the max norm.

    Returns:
        The max norm.
    """
    max, _ = x.abs().max(dim, keepdim=True)
    return max

# -----------------------------------------------------------------------------
# Functions for computing the relevance rank accuracy
# copied from IBA code ;)

def to_index_map(hmap: np.ndarray) -> np.ndarray:
    """Return a heatmap, in which every pixel has its value-index as value"""
    order_map = np.zeros_like(hmap, dtype=np.int64)
    for i, idx in enumerate(to_index_list(hmap)):
        order_map[idx] = -i
    return order_map

def to_index_list(
    hmap: np.ndarray, reverse: bool = False
) -> list[tuple[np.ndarray]]:
    """Return the list of indices that would sort this map.

    Sorting order is highest pixel first, lowest last
    """
    order = np.argsort((hmap if reverse else -hmap).ravel())
    indices = np.unravel_index(order, hmap.shape)  # array of two tuples
    indices_trans = np.transpose(np.stack(indices))
    return [tuple(i) for i in np.stack(indices_trans)]  # type: ignore

def get_ration_in_mask(heatmap: np.ndarray, mask: np.ndarray) -> float:
    if mask.ndim != 2:
        raise ValueError("Expected 2 dimensions")
    if heatmap.ndim != 2:
        raise ValueError("Expected 2 dimensions")
    if mask.shape != heatmap.shape:
        raise ValueError("Shapes must match")

    heatmap_idxs = to_index_map(heatmap).astype(np.int64)
    mask_np = mask > 0.5
    heatmap_bbox_idxs = heatmap_idxs.copy()
    heatmap_bbox_idxs[mask_np == 0] = heatmap_idxs.min()
    n_pixel_in_mask = mask_np.sum()
    return float(
        (heatmap_bbox_idxs > (-n_pixel_in_mask)).sum() / n_pixel_in_mask.sum()
    )

class GroundTruthEval(
    savethat.Node[GroundTruthEvalArgs, GroundTruthEvalResults]
):
    def _run(self):
        device = torch.device(self.args.device)

        model, model_args = load_model(
            self.storage,
            self.args.model_key,
            self.args.checkpoint,
            map_location=device,
        )
        model.to(device)
        lrp_model = lrp.LRPViewOfRelationNetwork(model)
        lrp_model.to(device)

        dataset = data.CLEVR_XAI(
            self.args.dataset,
            self.args.question_type,
            self.args.ground_truth,
            model_args.reverse_question,
            use_preprocessed=True,
        )

        if self.args.n_samples == -1:
            n_samples = len(dataset)
        else:
            n_samples = self.args.n_samples

        loader = DataLoader(
            dataset,
            batch_size=self.args.batch_size,
            shuffle=False,
            num_workers=1,
            pin_memory=False,
            collate_fn=dataset.collate_data,
        )

        # mse = 0
        pbar = tqdm(loader)
        rel_mass = []
        rel_rank = []

        total_samples = 0
        correct = []
        for i, (image, question, q_len, answer, gt) in enumerate(pbar):
            if i > n_samples // self.args.batch_size:
                break

            image, question, answer, gt = (
                image.to(device),
                question.to(device),
                answer.to(device),
                gt.to(device),
            )

            saliency, logits = lrp_model.get_lrp_saliency_and_logits(
                image,
                question,
                q_len,
                target=answer,
                normalize=False,
            )

            correct.append((logits.argmax(1) == answer).cpu())

            rel_mass.append(
                relevance_mass(l2_norm_sq(saliency), gt).detach().cpu()
            )
            ranks = [
                get_ration_in_mask(
                    max_norm(s, dim=0).cpu().detach().numpy()[0],
                    gt_mask.cpu().detach().numpy()[0],
                )
                for gt_mask, s in zip(gt, saliency)
            ]
            rel_rank.append(torch.tensor(ranks).cpu())
            total_samples += len(image)

        res = GroundTruthEvalResults(
            relevance_mass=torch.cat(rel_mass),
            relevance_rank_accuracy=torch.cat(rel_rank),
            correct=torch.cat(correct),
        )
        print("-" * 80)
        print(f"Statistics on {total_samples} samples:")
        print(res.as_dataframe().describe())
        print("-" * 80)
        print(f"Accuracy: {res.accuracy():.4f}")
        print("-" * 80)
        return res


from typing import Any, Dict, List, Self, Union

from attrs import define as _attrs_define
from attrs import field as _attrs_field

from ..constants.trading import (
    ConditionalCloseOrderType,
    OrderType,
    SelfTradePreventionStrategy,
    TimeInForce,
    Trigger,
    TypeOrder,
)
from ..security import get_nonce
from ..types import UNSET, Unset

@_attrs_define
class AddStandardOrderRequestBody:
    """
    Attributes:
        nonce (int): Nonce used in construction of `API-Sign` header. Default `../security.get_nonce`
        ordertype (Ordertype): Order type
        type (TypeOrder): Order direction (buy/sell)
        volume (str): Order quantity in terms of the base asset
            > Note: Volume can be specified as `0` for closing margin orders to automatically fill the requisite quantity.
        pair (str): Asset pair `id` or `altname`
        userref (Union[Unset, int]): User reference id

            `userref` is an optional user-specified integer id that can be associated with any number of orders. Many
            clients choose a `userref` corresponding to a unique integer id generated by their systems (e.g. a timestamp).
            However, because we don't enforce uniqueness on our side, it can also be used to easily group orders by pair,
            side, strategy, etc. This allows clients to more readily cancel or query information about orders in a
            particular group, with fewer API calls by using `userref` instead of our `txid`, where supported.
        displayvol (Union[Unset, str]): Used to create an iceberg order, this is the visible order quantity in terms of
            the base asset. The rest of the order will be hidden, although the full `volume` can be filled at any time by
            any order of that size or larger that matches in the order book. `displayvol` can only be used with the `limit`
            order type, must be greater than `0`, and less than `volume`.
        price (Union[Unset, str]): Price:

            * Limit price for `limit` orders
            * Trigger price for `stop-loss`, `stop-loss-limit`, `take-profit` and `take-profit-limit` orders
        price2 (Union[Unset, str]): Secondary Price:

            * Limit price for `stop-loss-limit` and `take-profit-limit` orders

            >  Note: Either `price` or `price2` can be preceded by `+`, `-`, or `#` to specify the order price as an offset
            relative to the last traded price. `+` adds the amount to, and `-` subtracts the amount from the last traded
            price. `#` will either add or subtract the amount to the last traded price, depending on the direction and order
            type used. Relative prices can be suffixed with a `%` to signify the relative amount as a percentage.
        trigger (Union[Unset, Trigger]): Price signal used to trigger `stop-loss`, `stop-
            loss-limit`, `take-profit` and `take-profit-limit` orders
            >  Note: This `trigger` type will as well be used for associated conditional close orders.
             Default: Trigger.LAST.
        leverage (Union[Unset, str]): Amount of leverage desired (default: none)
        reduce_only (Union[Unset, bool]): If `true`, order will only reduce a currently open position, not increase it
            or open a new position.
        stptype (Union[Unset, SelfTradePreventionStrategy]): Self trade prevention behavior definition:
            * cancel-newest - if self trade is triggered, arriving order will be canceled
            * cancel-oldest - if self trade is triggered, resting order will be canceled
            * cancel-both - if self trade is triggered, both arriving and resting orders will be canceled
             Default: SelfTradePreventionStrategy.CANCEL_NEWEST.
        oflags (Union[Unset, str]): Comma delimited list of order flags

              * `post` post-only order (available when ordertype = limit)
              * `fcib` prefer fee in base currency (default if selling)
              * `fciq` prefer fee in quote currency (default if buying, mutually exclusive with `fcib`)
              * `nompp` disable [market price protection](https://support.kraken.com/hc/en-us/articles/201648183-Market-
            Price-Protection) for market orders
              * `viqc`  order volume expressed in quote currency. This is supported only for market orders.
        timeinforce (Union[Unset, TimeInForce]): Time-in-force of the order to specify how
            long it should remain in the order book before being cancelled. GTC (Good-'til-cancelled) is default if the
            parameter is omitted. IOC (immediate-or-cancel) will immediately execute the amount possible and cancel any
            remaining balance rather than resting in the book. GTD (good-'til-date), if specified, must coincide with a
            desired `expiretm`.
             Default: TimeInForce.GTC.
        starttm (Union[Unset, str]): Scheduled start time, can be specified as an absolute timestamp or as a number of
            seconds in the future:
              * `0` now (default)
              * `<n>` = unix timestamp of start time
              * `+<n>` = schedule start time `<n>` seconds from now
                * Note that URL encoding of the `+` character changes it to a space, so please use `%2b` followed by the
            number of seconds instead of `+`
        expiretm (Union[Unset, str]): Expiration time, also can be specified as an absolute timestamp or as a number of
            seconds in the future:
              * `0` no expiration (default)
              * `<n>` = unix timestamp of expiration time
              * `+<n>` = expire `<n>` seconds from now, minimum 5 seconds
                * Note that URL encoding of the `+` character changes it to a space, so please use `%2b` followed by the
            number of seconds instead of `+`
        closeordertype (Union[Unset, ConditionalCloseOrderType]): Conditional close order type
            > Note: [Conditional close orders](https://support.kraken.com/hc/en-us/articles/360038640052-Conditional-Close)
            are triggered by execution of the primary order in the same quantity and opposite direction, but once triggered
            are __independent orders__ that may reduce or increase net position
        closeprice (Union[Unset, str]): Conditional close order `price`
        closeprice2 (Union[Unset, str]): Conditional close order `price2`
        deadline (Union[Unset, str]): RFC3339 timestamp (e.g. 2021-04-01T00:18:45Z) after which the matching engine
            should reject the new order request, in presence of latency or order queueing: min now() + 2 seconds, max now()
            + 60 seconds.
        validate (Union[Unset, bool]): Validate inputs only. Do not submit order.
    """

    ordertype: OrderType
    type: TypeOrder
    volume: str
    pair: str
    nonce: int = get_nonce()
    userref: Union[Unset, int] = UNSET
    displayvol: Union[Unset, str] = UNSET
    price: Union[Unset, str] = UNSET
    price2: Union[Unset, str] = UNSET
    trigger: Union[Unset, Trigger] = Trigger.LAST
    leverage: Union[Unset, str] = UNSET
    reduce_only: Union[Unset, bool] = False
    stptype: Union[
        Unset, SelfTradePreventionStrategy
    ] = SelfTradePreventionStrategy.CANCEL_NEWEST
    oflags: Union[Unset, str] = UNSET
    timeinforce: Union[Unset, TimeInForce] = TimeInForce.GTC
    starttm: Union[Unset, str] = UNSET
    expiretm: Union[Unset, str] = UNSET
    closeordertype: Union[Unset, ConditionalCloseOrderType] = UNSET
    closeprice: Union[Unset, str] = UNSET
    closeprice2: Union[Unset, str] = UNSET
    deadline: Union[Unset, str] = UNSET
    validate: Union[Unset, bool] = False
    additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        nonce = self.nonce
        ordertype = self.ordertype.value

        type = self.type.value

        volume = self.volume
        pair = self.pair
        userref = self.userref
        displayvol = self.displayvol
        price = self.price
        price2 = self.price2
        trigger: Union[Unset, str] = UNSET
        if not isinstance(self.trigger, Unset):
            trigger = self.trigger.value

        leverage = self.leverage
        reduce_only = self.reduce_only
        stptype: Union[Unset, str] = UNSET
        if not isinstance(self.stptype, Unset):
            stptype = self.stptype.value

        oflags = self.oflags
        timeinforce: Union[Unset, str] = UNSET
        if not isinstance(self.timeinforce, Unset):
            timeinforce = self.timeinforce.value

        starttm = self.starttm
        expiretm = self.expiretm
        closeordertype: Union[Unset, str] = UNSET
        if not isinstance(self.closeordertype, Unset):
            closeordertype = self.closeordertype.value

        closeprice = self.closeprice
        closeprice2 = self.closeprice2
        deadline = self.deadline
        validate = self.validate

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "nonce": nonce,
                "ordertype": ordertype,
                "type": type,
                "volume": volume,
                "pair": pair,
            }
        )
        if userref is not UNSET:
            field_dict["userref"] = userref
        if displayvol is not UNSET:
            field_dict["displayvol"] = displayvol
        if price is not UNSET:
            field_dict["price"] = price
        if price2 is not UNSET:
            field_dict["price2"] = price2
        if trigger is not UNSET:
            field_dict["trigger"] = trigger
        if leverage is not UNSET:
            field_dict["leverage"] = leverage
        if reduce_only is not UNSET:
            field_dict["reduce_only"] = reduce_only
        if stptype is not UNSET:
            field_dict["stptype"] = stptype
        if oflags is not UNSET:
            field_dict["oflags"] = oflags
        if timeinforce is not UNSET:
            field_dict["timeinforce"] = timeinforce
        if starttm is not UNSET:
            field_dict["starttm"] = starttm
        if expiretm is not UNSET:
            field_dict["expiretm"] = expiretm
        if closeordertype is not UNSET:
            field_dict["close[ordertype]"] = closeordertype
        if closeprice is not UNSET:
            field_dict["close[price]"] = closeprice
        if closeprice2 is not UNSET:
            field_dict["close[price2]"] = closeprice2
        if deadline is not UNSET:
            field_dict["deadline"] = deadline
        if validate is not UNSET:
            field_dict["validate"] = validate

        return field_dict

    @classmethod
    def from_dict(cls: Self, src_dict: Dict[str, Any]) -> Self:
        d = src_dict.copy()
        nonce = d.pop("nonce", get_nonce())

        ordertype = OrderType(d.pop("ordertype"))

        type = TypeOrder(d.pop("type"))

        volume = d.pop("volume")

        pair = d.pop("pair")

        userref = d.pop("userref", UNSET)

        displayvol = d.pop("displayvol", UNSET)

        price = d.pop("price", UNSET)

        price2 = d.pop("price2", UNSET)

        _trigger = d.pop("trigger", UNSET)
        trigger: Union[Unset, Trigger]
        trigger = UNSET if isinstance(_trigger, Unset) else Trigger(_trigger)

        leverage = d.pop("leverage", UNSET)

        reduce_only = d.pop("reduce_only", UNSET)

        _stptype = d.pop("stptype", UNSET)
        stptype: Union[Unset, SelfTradePreventionStrategy]
        if isinstance(_stptype, Unset):
            stptype = UNSET
        else:
            stptype = SelfTradePreventionStrategy(_stptype)

        oflags = d.pop("oflags", UNSET)

        _timeinforce = d.pop("timeinforce", UNSET)
        timeinforce: Union[Unset, TimeInForce]
        if isinstance(_timeinforce, Unset):
            timeinforce = UNSET
        else:
            timeinforce = TimeInForce(_timeinforce)

        starttm = d.pop("starttm", UNSET)

        expiretm = d.pop("expiretm", UNSET)

        _closeordertype = d.pop("close[ordertype]", UNSET)
        closeordertype: Union[Unset, ConditionalCloseOrderType]
        if isinstance(_closeordertype, Unset):
            closeordertype = UNSET
        else:
            closeordertype = ConditionalCloseOrderType(_closeordertype)

        closeprice = d.pop("close[price]", UNSET)

        closeprice2 = d.pop("close[price2]", UNSET)

        deadline = d.pop("deadline", UNSET)

        validate = d.pop("validate", UNSET)

        add_standard_order_request_body = cls(
            nonce=nonce,
            ordertype=ordertype,
            type=type,
            volume=volume,
            pair=pair,
            userref=userref,
            displayvol=displayvol,
            price=price,
            price2=price2,
            trigger=trigger,
            leverage=leverage,
            reduce_only=reduce_only,
            stptype=stptype,
            oflags=oflags,
            timeinforce=timeinforce,
            starttm=starttm,
            expiretm=expiretm,
            closeordertype=closeordertype,
            closeprice=closeprice,
            closeprice2=closeprice2,
            deadline=deadline,
            validate=validate,
        )

        add_standard_order_request_body.additional_properties = d
        return add_standard_order_request_body

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


import os
from pathlib import Path

import click
import numpy as np
import tensorflow as tf
from waymo_open_dataset.dataset_pb2 import Frame
from waymo_open_dataset.utils import frame_utils, transform_utils, range_image_utils
from waymo_open_dataset import dataset_pb2

from utils import save_frame, save_points
from visualization.visu_image import plot_points_on_image, save_camera_image
from visualization.visu_point_cloud import show_point_cloud

from multiprocessing import Pool

def convert_range_image_to_point_cloud(frame, range_images, camera_projections, range_image_top_pose, ri_index=(0, 1)):
    """
    Modified from the codes of Waymo Open Dataset.
    Convert range images to point cloud.
    Args:
        frame: open dataset frame
        range_images: A dict of {laser_name, [range_image_first_return, range_image_second_return]}.
        camera_projections: A dict of {laser_name,
            [camera_projection_from_first_return, camera_projection_from_second_return]}.
        range_image_top_pose: range image pixel pose for top lidar.
        ri_index: 0 for the first return, 1 for the second return.
    Returns:
        points: {[N, 3]} list of 3d lidar points of length 5 (number of lidars).
        cp_points: {[N, 6]} list of camera projections of length 5 (number of lidars).
    """
    calibrations = sorted(frame.context.laser_calibrations, key=lambda c: c.name)
    points = []
    cp_points = []
    points_NLZ = []
    points_intensity = []
    points_elongation = []

    frame_pose = tf.convert_to_tensor(np.reshape(np.array(frame.pose.transform), [4, 4]))
    # [H, W, 6]
    range_image_top_pose_tensor = tf.reshape(
        tf.convert_to_tensor(range_image_top_pose.data), range_image_top_pose.shape.dims
    )
    # [H, W, 3, 3]
    range_image_top_pose_tensor_rotation = transform_utils.get_rotation_matrix(
        range_image_top_pose_tensor[..., 0], range_image_top_pose_tensor[..., 1],
        range_image_top_pose_tensor[..., 2])
    range_image_top_pose_tensor_translation = range_image_top_pose_tensor[..., 3:]
    range_image_top_pose_tensor = transform_utils.get_transform(
        range_image_top_pose_tensor_rotation,
        range_image_top_pose_tensor_translation)

    for c in calibrations:
        points_single, cp_points_single, points_NLZ_single, points_intensity_single, points_elongation_single \
            = [], [], [], [], []
        for cur_ri_index in ri_index:
            range_image = range_images[c.name][cur_ri_index]
            if len(c.beam_inclinations) == 0:  # pylint: disable=g-explicit-length-test
                beam_inclinations = range_image_utils.compute_inclination(
                    tf.constant([c.beam_inclination_min, c.beam_inclination_max]),
                    height=range_image.shape.dims[0])
            else:
                beam_inclinations = tf.constant(c.beam_inclinations)

            beam_inclinations = tf.reverse(beam_inclinations, axis=[-1])
            extrinsic = np.reshape(np.array(c.extrinsic.transform), [4, 4])

            range_image_tensor = tf.reshape(
                tf.convert_to_tensor(range_image.data), range_image.shape.dims)
            pixel_pose_local = None
            frame_pose_local = None
            if c.name == dataset_pb2.LaserName.TOP:
                pixel_pose_local = range_image_top_pose_tensor
                pixel_pose_local = tf.expand_dims(pixel_pose_local, axis=0)
                frame_pose_local = tf.expand_dims(frame_pose, axis=0)
            range_image_mask = range_image_tensor[..., 0] > 0
            range_image_NLZ = range_image_tensor[..., 3]
            range_image_intensity = range_image_tensor[..., 1]
            range_image_elongation = range_image_tensor[..., 2]
            range_image_cartesian = range_image_utils.extract_point_cloud_from_range_image(
                tf.expand_dims(range_image_tensor[..., 0], axis=0),
                tf.expand_dims(extrinsic, axis=0),
                tf.expand_dims(tf.convert_to_tensor(beam_inclinations), axis=0),
                pixel_pose=pixel_pose_local,
                frame_pose=frame_pose_local)

            range_image_cartesian = tf.squeeze(range_image_cartesian, axis=0)
            points_tensor = tf.gather_nd(range_image_cartesian,
                                         tf.where(range_image_mask))
            points_NLZ_tensor = tf.gather_nd(range_image_NLZ, tf.compat.v1.where(range_image_mask))
            points_intensity_tensor = tf.gather_nd(range_image_intensity, tf.compat.v1.where(range_image_mask))
            points_elongation_tensor = tf.gather_nd(range_image_elongation, tf.compat.v1.where(range_image_mask))
            cp = camera_projections[c.name][0]
            cp_tensor = tf.reshape(tf.convert_to_tensor(cp.data), cp.shape.dims)
            cp_points_tensor = tf.gather_nd(cp_tensor, tf.where(range_image_mask))

            points_single.append(points_tensor.numpy())
            cp_points_single.append(cp_points_tensor.numpy())
            points_NLZ_single.append(points_NLZ_tensor.numpy())
            points_intensity_single.append(points_intensity_tensor.numpy())
            points_elongation_single.append(points_elongation_tensor.numpy())

        points.append(np.concatenate(points_single, axis=0))
        cp_points.append(np.concatenate(cp_points_single, axis=0))
        points_NLZ.append(np.concatenate(points_NLZ_single, axis=0))
        points_intensity.append(np.concatenate(points_intensity_single, axis=0))
        points_elongation.append(np.concatenate(points_elongation_single, axis=0))

    return points, cp_points, points_NLZ, points_intensity, points_elongation

def save_camera_images(idx: int, frame: Frame, output_dir: Path) -> None:
    for image in frame.images:
        save_camera_image(idx, image, frame.camera_labels, output_dir)

def save_data(frame: Frame, idx: int, points: np.ndarray,
              output_dir: Path) -> None:
    save_frame(frame, idx, output_dir)
    save_points(idx, points, output_dir)

def visualize_camera_projection(idx: int, frame: Frame, output_dir: Path,
                                pcd_return) -> None:
    points, points_cp = pcd_return
    points_all = np.concatenate(points, axis=0)
    points_cp_all = np.concatenate(points_cp, axis=0)

    images = sorted(frame.images, key=lambda i: i.name)  # type: ignore

    # distance between lidar points and vehicle frame origin
    points_tensor = tf.norm(points_all, axis=-1, keepdims=True)
    points_cp_tensor = tf.constant(points_cp_all, dtype=tf.int32)

    mask = tf.equal(points_cp_tensor[..., 0], images[0].name)

    points_cp_tensor = tf.cast(tf.gather_nd(
        points_cp_tensor, tf.where(mask)), tf.float32)
    points_tensor = tf.gather_nd(points_tensor, tf.where(mask))

    projected_points_from_raw_data = tf.concat(
        [points_cp_tensor[..., 1:3], points_tensor], -1).numpy()

    plot_points_on_image(
        idx, projected_points_from_raw_data, images[0], output_dir)

def pcd_from_range_image(frame: Frame):
    def _range_image_to_pcd(ri_index: int = 0):
        # points, points_cp = frame_utils.convert_range_image_to_point_cloud(
        #     frame, range_images, camera_projections, range_image_top_pose,
        #     ri_index=ri_index)
        points, points_cp = convert_range_image_to_point_cloud(
            frame, range_images, camera_projections, range_image_top_pose,
            ri_index=ri_index)
        return points, points_cp

    parsed_frame = frame_utils.parse_range_image_and_camera_projection(frame)
    range_images, camera_projections, _, range_image_top_pose = parsed_frame
    frame.lasers.sort(key=lambda laser: laser.name)
    return _range_image_to_pcd(), _range_image_to_pcd(1)

# def visualize_pcd_return(frame: Frame, pcd_return,
#                          visu: bool) -> None:
#     points, points_cp = pcd_return
#     points_all = np.concatenate(points, axis=0)
#     # print(f'points_all shape: {points_all.shape}')

#     # camera projection corresponding to each point
#     points_cp_all = np.concatenate(points_cp, axis=0)
#     # print(f'points_cp_all shape: {points_cp_all.shape}')

#     if visu:
#         show_point_cloud(points_all, frame.laser_labels)

def process_data(idx: int, frame: Frame, output_dir: Path, save: bool,
                 visu: bool) -> None:
    print(f'Start to process frame {idx:03}')
    # pylint: disable=no-member (E1101)
    # frame = Frame()
    # frame.ParseFromString(bytearray(data.numpy()))

    range_images, camera_projections, range_image_top_pose = frame_utils.parse_range_image_and_camera_projection(frame)
    points, cp_points, points_in_NLZ_flag, points_intensity, points_elogation = convert_range_image_to_point_cloud(
        frame, range_images, camera_projections, range_image_top_pose, ri_index=(0, )
    )

    points_all = np.concatenate(points, axis=0)
    points_in_NLZ_flag = np.concatenate(points_in_NLZ_flag, axis=0).reshape(-1, 1)
    points_intensity = np.concatenate(points_intensity, axis=0).reshape(-1, 1)
    points_elogation = np.concatenate(points_elogation, axis=0).reshape(-1, 1)

    points_all[points_in_NLZ_flag.reshape(-1) == -1]

    # pcd_return_1, pcd_return_2 = pcd_from_range_image(frame)
    # visualize_pcd_return(frame, pcd_return_1, visu)
    # visualize_pcd_return(frame, pcd_return_2, visu)

    # concatenate 1st and 2nd return
    # points, _ = concatenate_pcd_returns(pcd_return_1, pcd_return_2)

    if visu:
        save_camera_images(idx, frame, output_dir)
        show_point_cloud(points_all, frame.laser_labels, idx, output_dir)
        visualize_camera_projection(idx, frame, output_dir, (points, cp_points))

    if save:
        save_data(frame, idx, points, output_dir)

def process_segment(segment_path: str, output_dir: Path, save: bool,
                    visu: bool, parallelism: int=1) -> None:
    data_set = tf.data.TFRecordDataset(segment_path, compression_type='')
    frame_list = []
    for idx, data in enumerate(data_set):
        print(f'Loading frame: {idx}')
        frame = Frame()
        frame.ParseFromString(bytearray(data.numpy()))
        frame_list.append(frame)

    # multiprocessing?
    if parallelism > 0:
        arg_list = []
        for idx, frame in enumerate(frame_list):
            arg_list.append((idx, frame, output_dir, save, visu))

        with Pool(parallelism) as pool:
            pool.starmap(process_data, arg_list)
    else:
        for idx, frame in enumerate(frame_list):
            process_data(idx, frame, output_dir, save, visu)

@click.command(help='Point Cloud Visualization Demo')
@click.option('--save/--no-save', 'save', default=False,
              help='save frames and concatenated point clouds to disk')
@click.option('--visu/--no-visu', 'visu', default=False,
              help='visualize point clouds and save images')
@click.argument('segment_path', type=click.Path(exists=True))
@click.argument('output_dir', type=click.Path(exists=True))
def main(save: bool, visu: bool, segment_path: str, output_dir: str) -> None:
    if os.path.basename(segment_path).split('.')[-1] != 'tfrecord':
        raise ValueError(f'segment file has to be of '
                         f'{tf.data.TFRecordDataset.__name__} type')
    process_segment(segment_path, Path(output_dir), save, visu, 0)

if __name__ == '__main__':
    # pylint: disable=no-value-for-parameter
    main()


import sys
import os
import numpy as np
from numpy.linalg import svd
from numpy.linalg import eig
from skimage import io
from skimage import transform

face_folder = sys.argv[1]
if face_folder[len(face_folder)-1] != '/':
	face_folder += "/"
target = sys.argv[2]

image_data = []
for file in os.listdir(face_folder):
	filepath = os.path.join(face_folder , file)
	img = io.imread(filepath)
	img = np.array(img)
	img = img.flatten()
	image_data.append(img)

image_data = np.array(image_data)
image_data_mean = np.mean(image_data,axis=0)

x = image_data - image_data_mean

U, s, V = np.linalg.svd(x.T , full_matrices=False)

target = face_folder + target
ori_img = io.imread(target)

ori_img = np.array(ori_img)
ori_img = np.reshape(ori_img , (1,1080000))
ori_img = ori_img - image_data_mean

weights = np.dot( ori_img , U[:,:4])
recon =  image_data_mean + np.dot(weights, U[:,:4].T)

recon = np.reshape(recon,(600,600,3))
recon -= np.min(recon)
recon /= np.max(recon)
recon = (recon*255).astype(np.uint8)

io.imsave('reconstruction.png', recon)


# coding: utf-8

# In[1]:

#get_ipython().system(u'jupyter nbconvert --to script lstm_model.ipynb')
import os
import sys
import time
import pandas as pd
import datetime
#import pandas.io.data as web
from pandas_datareader import data
import matplotlib.pyplot as plt
from matplotlib import style
import glob
import numpy as np
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout
from keras.layers import Activation, LSTM
from keras.utils import plot_model
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from math import sqrt
from keras.callbacks import EarlyStopping

#import load_data

# fix random seed for reproducibility
np.random.seed(7)

# In[2]:

days_for_prediction = 30
source_dir='../data/samples'
models_dir = '../models/lstm/'
supervised_data_dir = '../data/samples2'
prediction_data_dir = '../data/prediction/samples'
rmse_csv = '../data/rsme_ltsm.csv'

# # Build train and test datasets

# In[3]:

# frame a sequence as a supervised learning problem
def to_supervised(df, lag, org_col_name='Adj Close', new_col_name='Adj Close+'):
    # new_col_name's data is created by shifting values from org_col_name
    df[new_col_name] = df.shift(-lag)[org_col_name]
    # Remove the last lag rows
    df = df.head(len(df) - lag)
    df.fillna(0, inplace=True)
    return df

def create_supervised_filename(directory, ticker):
    return os.path.join(directory, ticker + "_supervised.csv")

def create_supervised_data(source_dir, dest_dir, days_for_prediction=30, new_col_name = 'Adj Close+'):
    '''
    Input:
    - source_dir: directory where the stock price CSVs are located
    - days_for_prediction: number of days for the prediction prices. Must be at least 30 days
    Description:
    Read csv files in source_dir, load into dataframes and split into
    X_train, Y_train, X_test, Y_test
    '''
    #assert (days_for_prediction >= 30), "days_for_prediction must be >= 30"

    csv_file_pattern = os.path.join(source_dir, "*.csv")
    csv_files = glob.glob(csv_file_pattern)
    dfs = {}
    for filename in csv_files:
        arr = filename.split('/')
        ticker = arr[-1].split('.')[0]
        new_file = create_supervised_filename(dest_dir, ticker)
        #print(ticker, df.head())
        #  Date, Open, High , Low , Close, Adj Close, Volume
        #df = pd.read_csv(filename, parse_dates=[0]) #index_col='Date')
        #  Open, High , Low , Close, Adj Close, Volume
        df = pd.read_csv(filename, index_col='Date')

        #print('Before\n', df[30:40])
        #print(df.shift(2)['Adj Close'].head())

        df = to_supervised(df, days_for_prediction, new_col_name=new_col_name)
        df.to_csv(new_file)

        #print('Adding new column...\n', df[['Adj Close', new_col_name]].head(days_for_prediction+1))
        #print('After\n', df.tail())
        dfs[ticker] = df
        print(ticker, filename, new_file)

    return dfs

# # Use LSTM model for each stock

# In[4]:

dfs = create_supervised_data(source_dir=source_dir, dest_dir=supervised_data_dir, days_for_prediction=days_for_prediction)

# In[5]:

def create_lstm_model(max_features, lstm_units):
    model = Sequential()
    #model.add(LSTM(neurons, input_shape=(None, X_train.shape[1]), return_sequences=True)) #, dropout=0.2))
    #model.add(LSTM(max_features, batch_input_shape=(batch_size, None, train_X[i].shape[1]), dropout=0.2, stateful=True))
    #model.add(LSTM(1, input_shape=(max_features,1), return_sequences=True, dropout=0.2))
    #model.add(LSTM(max_features, return_sequences=False, dropout=0.2))
    #model.add(LSTM(input_dim=max_features, output_dim=300, return_sequences=True))

    model.add(LSTM(units=lstm_units[0], input_shape=(None, max_features), return_sequences=True))
    model.add(Dropout(0.2))
    model.add(LSTM(lstm_units[1], return_sequences=False))
    model.add(Dropout(0.2))

    #model.add(Dense(1)) #, activation='sigmoid'))
    model.add(Dense(1, activation='linear'))

    #model.compile(loss='mse', optimizer='rmsprop')
    #model.compile(loss='binary_crossentropy',optimizer='rmsprop', metrics=['accuracy'])
    #model.compile(loss='mean_squared_error', optimizer='adam')
    model.compile(loss='mse', optimizer='rmsprop', metrics=['accuracy'])
    return model

# In[6]:

'''
def create_train_test(data):
    X,y = data[:,0:-1], data[:, -1]
    # Transform scale
    X_scaler = MinMaxScaler(feature_range=(-1, 1))
    y_scaler = MinMaxScaler(feature_range=(-1, 1))
    scaled_X = X_scaler.fit_transform(X)
    scaled_y = y_scaler.fit_transform(y)
    print(scaled_y)
    # Now split 80/20 for train and test data
    #train_count = int(.8*len(data))

    # last test_days is for test; the rest is for train
    test_days = 90
    train_count = len(data) - test_days

    X_train, X_test = scaled_X[:train_count], scaled_X[train_count:]
    y_train, y_test = scaled_y[:train_count], scaled_y[train_count:]
    return y_scaler, X_train, y_train, X_test, y_test
'''
def create_train_test2(data):
    #X,y = data[:,0:-1], data[:, -1]
    # Transform scale
    scaler = MinMaxScaler(feature_range=(-1, 1))
    scaled_data = scaler.fit_transform(data)
    # Now split 80/20 for train and test data
    #train_count = int(.8*len(data))

    # last test_days is for test; the rest is for train
    test_days = 90
    train_count = len(data) - test_days

    train, test = scaled_data[:train_count], scaled_data[train_count:]
    X_train, y_train = train[:,0:-1], train[:, -1]
    X_test, y_test = test[:,0:-1], test[:, -1]
    return scaler, X_train, y_train, X_test, y_test

def build_models(models_dir, supervised_data_dir, lstm_units):
    # Define early stopping
    early_stopping = EarlyStopping(monitor='val_loss', patience=2) #value=0.00001

    rmse_list = list()
    models = {}
    predicted_dfs = {}
    '''
    load supervised data
    create and save models
    '''
    csv_file_pattern = os.path.join(supervised_data_dir, "*.csv")
    csv_files = glob.glob(csv_file_pattern)
    dfs = {}
    print_first_model=True
    for filename in csv_files:
        data = pd.read_csv(filename, index_col='Date')
        #print(data.head())
        arr = filename.split('/')
        ticker = arr[-1].split('.')[0].split('_')[0]
        print('Processing', ticker)
        max_features = len(data.columns) -1
        #y_scaler, X_train, y_train, X_test, y_test = create_train_test(data.values)
        scaler, X_train, y_train, X_test, y_test = create_train_test2(data.values)

        model = create_lstm_model(max_features, lstm_units)
        #plot_model(model, to_file=ticker + '.png', show_shapes=True, show_layer_names=True)
        if print_first_model:
            print(model.summary())
            print_first_model = False

        # Train data
        x1 = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1]))
        y1 = np.reshape(y_train, (y_train.shape[0], 1))
        print(x1.shape, y1.shape)
        # Test data
        x2 = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))
        y2 = np.reshape(y_test, (y_test.shape[0], 1))

        #model.fit(x, y, batch_size=100, epochs=5, shuffle=True)
        print('Training...')
        #model.fit(x1, y1, batch_size=50, epochs=20, verbose=1, validation_split=0.2, callbacks=[early_stopping])
        # Note: Early stopping seems to give worse prediction?!! We want overfitting here?
        model.fit(x1, y1, batch_size=5, epochs=20, verbose=1, validation_data=(x2, y2)) #, callbacks=[early_stopping])

        model_fname = os.path.join(models_dir, ticker + ".h5")

        print('Saving model to', model_fname)
        model.save(model_fname)

# In[ ]:

# inverse scaling for a forecasted value
def invert_scale(scaler, X, value):
    new_row = np.column_stack((X,value)) #[x for x in X] + [value]
    inverted = scaler.inverse_transform(new_row)
    return inverted[:, -1]

'''
Predict and evaluate test data
'''
def predict_evaluate(models_dir, supervised_data_dir, predicted_dir, rsme_csv):
    model_file_pattern = os.path.join(models_dir, "*.h5")
    model_files = glob.glob(model_file_pattern)
    predicted_dfs = {}
    rmse_list = list()
    print(model_file_pattern)

    for model_file in model_files:
        print('loading', model_file)
        arr = model_file.split('/')
        ticker = arr[-1].split('.')[0]
        '''
        Read supervised data and set up test data for prediction
        '''
        supervised_filename = create_supervised_filename(supervised_data_dir, ticker)
        data = pd.read_csv(supervised_filename, index_col='Date')
        scaler, X_train, y_train, X_test, y_test = create_train_test2(data.values)

        # Test data
        x2 = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))
        y2 = np.reshape(y_test, (y_test.shape[0], 1))

        print('Predicting...')
        model = load_model(model_file)
        predicted = model.predict(x2)
        predict_inversed = invert_scale(scaler, X_test, predicted)
        actual_inversed = invert_scale(scaler, X_test, y_test)

        rmse = sqrt(mean_squared_error(actual_inversed, predict_inversed))
        print('Test RMSE: %.3f' % rmse)
        rmse_list += [[ticker,rmse]]
        predicted_dfs[ticker] = pd.DataFrame({'predicted': predict_inversed.reshape(len(predict_inversed)),
                                              'actual': actual_inversed.reshape(len(actual_inversed))})
        predicted_file = os.path.join(predicted_dir, ticker + "_predicted.csv")
        print("Writing to", predicted_file)
        predicted_dfs[ticker].to_csv(predicted_file, index=False)

    rmse_df = pd.DataFrame(rmse_list, columns=['Stock Model', 'rsme'])
    rmse_df = rmse_df.sort_values(by='rsme')
    rmse_df.to_csv(rsme_csv, index=False)
    return predicted_dfs, rmse_df

# In[ ]:

build_models(models_dir, supervised_data_dir, lstm_units=[40,10])

# In[ ]:

predicted_dfs, rmse_df = predict_evaluate(models_dir,
                                          supervised_data_dir,
                                          prediction_data_dir,
                                          rmse_csv)

# In[ ]:

rmse_df

# In[ ]:

# Plot stocks based on rmse order (best -> worst)
#cnt = 0
#for index, row in rmse_df.iterrows():
#    key = row['Stock Model']
#    predicted_dfs[key].plot(title=key + ': predicted vs actual')
#    plt.show()

# In[ ]:
'''
cnt = 1
for index, row in rmse_df.iterrows():
    key = row['Stock Model']
    if (cnt % 2 != 0):
        fig, axes = plt.subplots(nrows=1, ncols=2)
        ax=axes[0]
    else:
        ax=axes[1]
    predicted_dfs[key].plot(title=key + ': price vs days', figsize=(15,4), ax=ax)
    cnt += 1
plt.show()
'''

# In[ ]:


import sys
from PyQt5.QtWidgets import QMainWindow, QApplication, QDesktopWidget, QFileDialog
from PyQt5.QtGui import QPalette, QColor
import numpy as np
from typing import *
import json

import qtmodern.styles
import qtmodern.windows

from MyModules.MyWindow import Ui_MainWindow
from MyModules.Orbits import Satellite
from MyModules.MPL3Dwidget import *

class MainWindow(QMainWindow, Ui_MainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)
        self.center()

        # Create the matplotlib 3D plot
        self.plotCanvas = MplCanvas(self.plotWidget, width=5, height=5, dpi=100)
        self.toolbar = NavigationToolbar(self.plotCanvas, self.plotWidget)
        self.plotLayout.addWidget(self.plotCanvas)
        self.plotLayout.addWidget(self.toolbar)
        self.plotWidget.setLayout(self.plotLayout)

        # connect every slider to the function that handles the ploting
        sliders = [self.slider_MA, self.slider_AOP, self.slider_ECC, self.slider_INC, self.slider_LAN, self.slider_SMA]
        for slider in sliders:
            slider.sliderReleased.connect(self.slider_released)
        self.slider_released()  # Initialize the plot

        self.actionExport_to_json.triggered.connect(lambda: self.export_to_json())
        self.actionImport_from_json.triggered.connect(lambda: self.import_from_json())
        self.planet_actions = [self.actionMercury, self.actionVenus, self.actionEarth, self.actionMars,
                               self.actionJupiter, self.actionSaturn, self.actionUranus, self.actionNeptune, self.actionPluto]
        for act in self.planet_actions:
            act.triggered.connect(lambda: self.display_planets())

    def center(self):
        """ This function centers the window at launch"""
        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def slider_released(self):
        """ Triggered when a slider is released. Computes the new positions and plots the new graph"""
        pos = self.calculate_position(self.getSliderValues())
        self.plot(pos)

    def plot(self, pos):
        """ Handles the ploting"""
        self.plotCanvas.axes.cla()
        self.plotCanvas.axes.plot(pos[:, 0], pos[:, 1], pos[:, 2], 'o', markersize=1)
        self.plotCanvas.axes.plot([0], [0], [0], 'o', color='yellow', markersize='10')
        self.plotCanvas.axes.mouse_init(rotate_btn=1, zoom_btn=3)
        set_axes_equal(self.plotCanvas.axes)
        self.plotCanvas.fig.set_facecolor(plot_background_color)
        self.plotCanvas.axes.patch.set_facecolor(plot_face_color)
        self.plotCanvas.draw()

    def getSliderValues(self) -> List[float]:
        """ Returns the current values displayed by the sliders"""
        return [float(self.slider_SMA.value()),
                float(self.slider_INC.value()),
                float(self.slider_ECC.value()) / 1e3,
                float(self.slider_LAN.value()),
                float(self.slider_AOP.value()),
                float(self.slider_MA.value())]

    def setSliderValues(self, values: Dict[str, float]):
        self.slider_SMA.setValue(int(values['SMA']))
        self.slider_INC.setValue(int(values['INC']))
        self.slider_ECC.setValue(int(values['ECC'] * 1e3))
        self.slider_LAN.setValue(int(values['LAN']))
        self.slider_AOP.setValue(int(values['AOP']))
        self.slider_MA.setValue(int(values['MA']))
        self.slider_released()

    def calculate_position(self, values: List[float]):
        obj = Satellite(*values)
        time = np.linspace(0, obj.T, 200)
        pos = obj.orbitalparam2vectorList(time)
        return pos

    def export_to_json(self):
        """Writes the current values of the sliders to a new JSON file"""
        file_name = self.FileDialog()
        with open(file_name, 'w') as f:
            keys = ["SMA", "INC", "ECC", "LAN", "AOP", "MA"]
            values = self.getSliderValues()
            json.dump(dict(zip(keys, values)), f)

    def import_from_json(self):
        file_name = self.FileDialog(save=False)
        with open(file_name, 'r') as f:
            content = json.load(f)
            self.setSliderValues(content)

    def FileDialog(self, save=True):
        options = QFileDialog.Options()
        options |= QFileDialog.DontUseNativeDialog
        if save:
            file_name, _ = QFileDialog.getSaveFileName(
                self, "Save as", "", "JSON Files (*.json)", options=options)
        else:
            file_name, _ = QFileDialog.getOpenFileName(
                self, "Open", "", "JSON Files (*.json)", options=options)
        if file_name != '':
            if not file_name.endswith('.json'):
                file_name += '.json'
            return file_name

    def display_planets(self):
        for planet in self.planet_actions:
            if planet.isChecked():
                print('hello')

if __name__ == "__main__":
    app = QApplication(sys.argv)

    app.setStyle("Fusion")
    dark_palette = QPalette()
    dark_palette.setColor(QPalette.Window, QColor(51, 54, 63))
    dark_palette.setColor(QPalette.WindowText, QColor(250, 250, 250))
    dark_palette.setColor(QPalette.Base, QColor(39, 42, 49))
    dark_palette.setColor(QPalette.AlternateBase, QColor(51, 54, 63))
    dark_palette.setColor(QPalette.ToolTipBase, QColor(250, 250, 250))
    dark_palette.setColor(QPalette.ToolTipText, QColor(250, 250, 250))
    dark_palette.setColor(QPalette.Text, QColor(250, 250, 250))
    dark_palette.setColor(QPalette.Button, QColor(51, 54, 63))
    dark_palette.setColor(QPalette.ButtonText, QColor(250, 250, 250))
    dark_palette.setColor(QPalette.BrightText, QColor(255, 0, 0))
    dark_palette.setColor(QPalette.Link, QColor(42, 130, 218))
    dark_palette.setColor(QPalette.Highlight, QColor(42, 130, 218))
    dark_palette.setColor(QPalette.HighlightedText, QColor(0, 0, 0))
    app.setPalette(dark_palette)
    plot_background_color = (51/255, 54/255, 63/255)
    plot_face_color = (39/255, 42/255, 49/255)

    win = MainWindow()
    mw = qtmodern.windows.ModernWindow(win)
    mw.show()

    sys.exit(app.exec_())


# -*- coding: utf-8 -*-

from typing import List
import math

class Solution:
    def nearestValidPoint(self, x: int, y: int, points: List[List[int]]) -> int:

        smallest_distance = math.inf
        index = -1

        for ix, c in enumerate(points):
            cx = c[0]
            cy = c[1]

            if x == cx or y == cy:
                di = abs(x - cx) + abs(y - cy)
                if di < smallest_distance:
                    smallest_distance = di
                    index = ix

        return index

# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4


#!/usr/bin/python
from pylab import *
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from pyhdf.SD import SD,SDC
import healpy as hp
import os
def modisfile(month,day=None,year=2016,datadir="/home/kawahara/exomap/sot/data/aux/"):
    import pandas as pd
    if day is None:
        dat=pd.read_csv(os.path.join(datadir,"modisM.txt"),delimiter=",")
        mask=dat["YEAR"]==2016
        mask2=dat["MONTH"]==month
        ext=(dat[mask&mask2])
        hdffile=(ext["FILE"])
        out=str(hdffile.tolist()[0])
        day=None
    else:
        dat=pd.read_csv(os.path.join(datadir,"modisE.txt"),delimiter=",")
        try:
            mask=dat["YEAR"]==2016
            mask2=dat["MONTH"]==month
            mask3=dat["DAY"]==day
            ext=(dat[mask&mask2&mask3])
            hdffile=(ext["FILE"])
            out=str(hdffile.tolist()[0])
        except:
            mask=dat["YEAR"]==2016
            mask2=dat["MONTH"]==month
            ext=(dat[mask&mask2])
            i=np.argmin(np.abs(ext["DAY"]-day))
            hdffile=(ext["FILE"])
            out=str(hdffile.tolist()[i])
            day=ext["DAY"].tolist()[i]
            print("Nearest Day is used day=",day)
    return out,month,day

def read_cloud(hdffile,N=1):
    print(hdffile)
    f = SD(hdffile,SDC.READ)
    v=f.select("Cloud_Fraction_Mean_Mean")
    vr=v.attributes()["valid_range"]
    fv=v.attributes()["_FillValue"]
    ao=v.attributes()["add_offset"]
    sf=v.attributes()["scale_factor"]

    a=np.array(v[::N,::N],dtype=float)
    a[a==fv] = None
    a=(a-ao)*sf
    return a

def to_healpix(a,nside=16):
    Nphi,Ntheta=np.shape(a)
    npix=hp.nside2npix(nside)
    hmap=np.zeros(npix)
    for ipix in range(0,npix):
        theta,phi=hp.pix2ang(nside,ipix)
        itheta=int(theta/np.pi*180)
        iphi=int(phi/np.pi*180)
        hmap[ipix]=float(a[itheta,iphi])
    return hmap

if __name__ == "__main__":
    import os
    import plotdymap
    import matplotlib
    fontsize=16
    matplotlib.rcParams.update({'font.size':fontsize})

    thetaE,phiE=plotdymap.bound_earth("/home/kawahara/exomap/sot/data/earth_boundary.npz")
    year=2016
    month=5
    day=11
    hdffile,month,day=modisfile(month,day,year=year)
    hdfdir="/home/kawahara/exomap/data/modis/MYD08"

    hdffile=os.path.join(hdfdir,hdffile)
    a=read_cloud(hdffile,N=1)
    hmap=to_healpix(a,nside=16)
    hp.mollview(hmap, title=str(year)+"-"+str(month)+"-"+str(day),flip="geo",cmap=plt.cm.pink,min=0.5,max=1.0)
    hp.projplot(thetaE, phiE,".",c="#66CC99")
    hp.projtext(-60,-25,"A",coord="G",lonlat=True,color="cyan",fontsize=26) #amazon
#    hp.projtext(-130,30,"B",coord="G",lonlat=True,color="cyan",fontsize=26) #north america

    plt.savefig("cf"+str(year)+"_"+str(month)+"_"+str(day)+".pdf")
    plt.show()


import requests, json

url = 'https://sandbox.techops.engineering/Demand/v1/Surveys/BySurveyNumber/4592039'
params = ""
headers = {'Content-type': 'application/json', 'Authorization' : 'YOUR_API_KEY_HERE', 'Accept': 'text/plain'}
response = requests.get(url, data=params, headers=headers)
print(response.content.decode())

file = open("survey_get.json","w")
file.write(response.text)
file.close()


# -*- coding: utf-8 -*-
import sys

from ccm import *
from PyQt5.QtWidgets import QMainWindow, QApplication

class CCMWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.ui = Ui_CCMTask()
        self.ui.setupUi(self)
        self.ui.retranslateUi(self)
        self.show()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    mainWindow = CCMWindow()

    mainWindow.setWindowFlags(QtCore.Qt.WindowMinimizeButtonHint)
    mainWindow.setFixedSize(mainWindow.width(), mainWindow.height())
    mainWindow.show()
    sys.exit(app.exec_())


# -*- coding: utf-8 -*-
"""User views."""
from flask import Blueprint, render_template, request, flash, redirect, url_for
from flask_login import login_required, current_user
from food_journal.user.models import User
from food_journal.user.forms import EditProfileForm

blueprint = Blueprint("user", __name__, url_prefix="/users", static_folder="../static")

@blueprint.route("/")
@login_required
def members():
    """List members."""
    return render_template("users/members.html")

@blueprint.route("/<username>")
@login_required
def profile(username):
    """Return user's profile page"""
    print("here")
    user = User.query.filter_by(username=username).first_or_404()
    return render_template("users/profile.html", user=user)

@blueprint.route('/edit_profile', methods=['GET', 'POST'])
@login_required
def edit_profile():
    form = EditProfileForm()
    if form.validate_on_submit():
        current_user.update(about_me=form.about_me.data)
        flash("Your changes have been saved.")
        return redirect(url_for('user.edit_profile'))
    elif request.method == 'GET':
        form.about_me.data = current_user.about_me
    return render_template('users/edit_profile.html', title='Edit Profile', form=form)

@blueprint.route('/follow/<username>')
@login_required
def follow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('User {} not found'.format(username))
        return redirect(url_for('index'))
    if user == current_user:
        flash('You cannot follow yourself!')
        return redirect(url_for('user', username=username))
    current_user.follow(user)
    current_user.save()
    flash('You are now following {}!'.format(username))
    return redirect(url_for('user.profile', username=username))

@blueprint.route('/unfollow/<username>')
@login_required
def unfollow(username):
    user = User.query.filter_by(username=username).first()
    if user is None:
        flash('User {} not found.'.format(username))
        return redirect(url_for('public.index'))
    if user == current_user:
        flash('You cannot unfollow yourself!')
        return redirect(url_for('user.profile', username=username))
    current_user.unfollow(user)
    current_user.save()
    flash('You are not following {}.'.format(username))
    return redirect(url_for('user.profile', username=username))


import math
import plotly
import dash_bootstrap_components as dbc
from dash import html, dcc
import dash
from django_plotly_dash import DjangoDash
from geopy.geocoders import ArcGIS
import plotly.graph_objects as go
import plotly.express as px
import multiprocessing
import re
import pandas as pd

class Mapa:
    def __init__(self, df, col):
        self.col = col
        self.df = df

    def criar_mapa(self):
        app = DjangoDash("mapa",
                add_bootstrap_links=True)

        app.layout = html.Div(
            dcc.Graph(id='mapa', figure=self.gerar_grafico()))
        return app

    def encontrar_coordenadas(self, x):
        nom = ArcGIS()
        coordenada = nom.geocode(x)
        if coordenada:
            return coordenada.latitude, coordenada.longitude

    def requisicao(self, df):
        df[['Latitude', 'Longitude']] = df[self.col].apply(lambda x: pd.Series(self.encontrar_coordenadas(x)))
        return df
    def gerar_grafico(self):
        df = self.requisicao(self.df)
        fig = go.Figure(go.Scattermapbox(
            lat=df['Latitude'],
            lon=df['Longitude'],
            mode='markers',
            marker=go.scattermapbox.Marker(
                size=15,
                color='rgb(0, 100, 58)',
                opacity=0.7
            ),
            text=df,
        ))

        # Configura o layout do mapa
        fig.update_layout(
            mapbox_style='open-street-map',
            mapbox_center_lon=0,
            margin={'r': 0, 't': 0, 'l': 0, 'b': 0}
        )
        # Obtém os valores mínimos e máximos de latitude e longitude
        lat_min, lat_max = df['Latitude'].min(), df['Latitude'].max()
        lon_min, lon_max = df['Longitude'].min(), df['Longitude'].max()

        # Calcula o centro do mapa
        center_lat = (lat_min + lat_max) / 2
        center_lon = (lon_min + lon_max) / 2
        # Calcula a extensão das coordenadas
        lat_extent = lat_max - lat_min
        lon_extent = lon_max - lon_min
        # Define o nível de zoom
        zoom_lat = math.log10(360 / lat_extent) / math.log10(2)
        zoom_lon = math.log10(360 / lon_extent) / math.log10(2)
        zoom = min(zoom_lat, zoom_lon)

        # Configura o layout do mapa com o zoom nas coordenadas marcadas
        fig.update_layout(
        mapbox={
            'center': {'lon': center_lon, 'lat': center_lat},
            'zoom': zoom
        }
    )
        return fig


"""Exceptions interface.

Exceptions allow for ignoring detected issues. This is commonly done to suppress false
positives or to ignore issues that a group has no intention of addressing.

The two types of exceptions are a list of filenames or regular expressions. If using
filename matching for the exception it is required that the reported issue contain the
absolute path to the file containing the issue to be ignored. The path for the issue is
set in the tool plugin that generates the issues.
"""
import fnmatch
import logging
import os
import re
from typing import Any, Dict, List, Match, Optional, Pattern

import yaml

from statick_tool.issue import Issue
from statick_tool.package import Package

class Exceptions:
    """Interface for applying exceptions."""

    def __init__(self, filename: Optional[str]) -> None:
        """Initialize exceptions interface."""
        if not filename:
            raise ValueError(f"{filename} is not a valid file")
        with open(filename, encoding="utf8") as fname:
            try:
                self.exceptions: Dict[Any, Any] = yaml.safe_load(fname)
            except (yaml.YAMLError, yaml.scanner.ScannerError) as ex:
                raise ValueError(f"{filename} is not a valid YAML file: {ex}") from ex

    def get_ignore_packages(self) -> List[str]:
        """Get list of packages to skip when scanning a workspace."""
        ignore: List[str] = []
        if (
            "ignore_packages" in self.exceptions
            and self.exceptions["ignore_packages"] is not None
        ):
            ignore = self.exceptions["ignore_packages"]
        return ignore

    def get_exceptions(self, package: Package) -> Dict[Any, Any]:
        """Get specific exceptions for given package."""
        exceptions: Dict[Any, Any] = {"file": [], "message_regex": []}

        if "global" in self.exceptions and "exceptions" in self.exceptions["global"]:
            global_exceptions = self.exceptions["global"]["exceptions"]
            if "file" in global_exceptions and global_exceptions["file"]:
                exceptions["file"] += global_exceptions["file"]
            if (
                "message_regex" in global_exceptions
                and global_exceptions["message_regex"]
            ):
                exceptions["message_regex"] += global_exceptions["message_regex"]

        # pylint: disable=too-many-boolean-expressions
        if (
            self.exceptions
            and "packages" in self.exceptions
            and self.exceptions["packages"]
            and package.name in self.exceptions["packages"]
            and self.exceptions["packages"][package.name]
            and "exceptions" in self.exceptions["packages"][package.name]
        ):
            package_exceptions = self.exceptions["packages"][package.name]["exceptions"]
            if "file" in package_exceptions:
                exceptions["file"] += package_exceptions["file"]
            if "message_regex" in package_exceptions:
                exceptions["message_regex"] += package_exceptions["message_regex"]
        # pylint: enable=too-many-boolean-expressions

        return exceptions

    def filter_file_exceptions_early(
        self, package: Package, file_list: List[str]
    ) -> List[str]:
        """Filter files based on file pattern exceptions list.

        Only filters files which have tools=all, intended for use after the discovery
        plugins have been run (so that Statick doesn't run the tool plugins against
        files which will be ignored anyway).
        """
        exceptions: Dict[Any, Any] = self.get_exceptions(package)
        to_remove = []
        for filename in file_list:
            removed = False
            for exception in exceptions["file"]:
                if exception["tools"] == "all":
                    for pattern in exception["globs"]:
                        # Hack to avoid exceptions for everything on Travis CI.
                        fname = filename
                        prefix = "/home/travis/build/"
                        if pattern == "*/build/*" and fname.startswith(prefix):
                            fname = fname[len(prefix) :]
                        if fnmatch.fnmatch(fname, pattern):
                            to_remove.append(filename)
                            removed = True
                            break
                    if removed:
                        break
        file_list = [filename for filename in file_list if filename not in to_remove]
        return file_list

    def filter_file_exceptions(
        self, package: Package, exceptions: List[Any], issues: Dict[str, List[Issue]]
    ) -> Dict[str, List[Issue]]:
        """Filter issues based on file pattern exceptions list."""
        for tool, tool_issues in list(  # pylint: disable=too-many-nested-blocks
            issues.items()
        ):
            warning_printed = False
            to_remove: List[Issue] = []
            for issue in tool_issues:
                if not os.path.isabs(issue.filename):
                    if not warning_printed:
                        self.print_exception_warning(tool)
                        warning_printed = True
                    continue
                rel_path: str = os.path.relpath(issue.filename, package.path)
                for exception in exceptions:
                    if exception["tools"] == "all" or tool in exception["tools"]:
                        for pattern in exception["globs"]:
                            # Hack to avoid exceptions for everything on Travis CI.
                            fname: str = issue.filename
                            prefix: str = "/home/travis/build/"
                            if pattern == "*/build/*" and fname.startswith(prefix):
                                fname = fname[len(prefix) :]
                            if fnmatch.fnmatch(fname, pattern) or fnmatch.fnmatch(
                                rel_path, pattern
                            ):
                                to_remove.append(issue)
            issues[tool] = [issue for issue in tool_issues if issue not in to_remove]

        return issues

    @classmethod
    def filter_regex_exceptions(
        cls, exceptions: List[Any], issues: Dict[str, List[Issue]]
    ) -> Dict[str, List[Issue]]:
        """Filter issues based on message regex exceptions list."""
        for exception in exceptions:  # pylint: disable=too-many-nested-blocks
            exception_re = exception["regex"]
            exception_tools = exception["tools"]
            exception_globs = []
            if "globs" in exception:
                exception_globs = exception["globs"]
            try:
                compiled_re: Pattern[str] = re.compile(exception_re)
            except re.error:
                logging.warning(
                    "Invalid regular expression in exception: %s", exception_re
                )
                continue
            for tool, tool_issues in list(issues.items()):
                to_remove = []
                if exception_tools == "all" or tool in exception_tools:
                    for issue in tool_issues:
                        if exception_globs:
                            for pattern in exception_globs:
                                if fnmatch.fnmatch(issue.filename, pattern):
                                    match: Optional[Match[str]] = compiled_re.match(
                                        issue.message
                                    )
                                    if match:
                                        to_remove.append(issue)
                        else:
                            match_re: Optional[Match[str]] = compiled_re.match(
                                issue.message
                            )
                            if match_re:
                                to_remove.append(issue)
                issues[tool] = [
                    issue for issue in tool_issues if issue not in to_remove
                ]
        return issues

    def filter_nolint(self, issues: Dict[str, List[Issue]]) -> Dict[str, List[Issue]]:
        """Filter out lines that have an explicit NOLINT on them.

        Sometimes the tools themselves don't properly filter these out if there is a
        complex macro or something.
        """
        for tool, tool_issues in list(issues.items()):
            warning_printed: bool = False
            to_remove: List[Issue] = []
            for issue in tool_issues:
                if not os.path.isabs(issue.filename):
                    if not warning_printed:
                        self.print_exception_warning(tool)
                        warning_printed = True
                    continue
                try:
                    with open(issue.filename, encoding="utf-8") as fid:
                        try:
                            lines = fid.readlines()
                        except UnicodeDecodeError as exc:
                            logging.warning(
                                "Could not read %s: %s", issue.filename, exc
                            )
                            continue
                except FileNotFoundError as exc:
                    logging.warning("Could not read %s: %s", issue.filename, exc)
                    continue
                if len(lines) <= 0:
                    continue
                line_number = int(issue.line_number) - 1
                if line_number < len(lines) and "NOLINT" in lines[line_number]:
                    to_remove.append(issue)
            issues[tool] = [issue for issue in tool_issues if issue not in to_remove]
        return issues

    def filter_issues(
        self, package: Package, issues: Dict[str, List[Issue]]
    ) -> Dict[str, List[Issue]]:
        """Filter issues based on exceptions list."""
        exceptions = self.get_exceptions(package)

        if exceptions["file"]:
            issues = self.filter_file_exceptions(package, exceptions["file"], issues)

        if exceptions["message_regex"]:
            issues = self.filter_regex_exceptions(exceptions["message_regex"], issues)

        issues = self.filter_nolint(issues)

        return issues

    @classmethod
    def print_exception_warning(cls, tool: str) -> None:
        """Print warning about exception not being applied for an issue.

        Warning will only be printed once per tool.
        """
        logging.warning(
            "[WARNING] File exceptions not available for %s tool "
            "plugin due to lack of absolute paths for issues.",
            tool,
        )


import torch
import torch.nn as nn

def domain_loss(visual_domain_logits, textual_domain_logits):
    criterion = nn.CrossEntropyLoss()
    batch_size = visual_domain_logits.shape[0]
    visual_domain_labels = torch.zeros(batch_size).long().cuda()
    textual_domain_labels = torch.ones(batch_size).long().cuda()
    loss = criterion(visual_domain_logits, visual_domain_labels) + criterion(
        textual_domain_logits, textual_domain_labels
    )
    return loss


import json
import os

from eth_hash.auto import (
    keccak,
)
from eth_utils import (
    encode_hex,
)

from eth.tools.fixtures.fillers import (
    fill_test,
)
from eth.tools.fixtures.fillers.formatters import (
    filler_formatter,
)
from eth.tools.fixtures.helpers import (
    get_test_name,
)

PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
OUTPUT_DIR = os.path.join(PARENT_DIR, "json")
FILLER_PARENT_DIR = os.path.join(OUTPUT_DIR, "fillers")
TEST_PARENT_DIR = os.path.join(OUTPUT_DIR, "tests")

DIR_STRUCTURE = {}

if __name__ == "__main__":
    for (filler_dir, test_dir), test_groups in DIR_STRUCTURE.items():
        for test_group, tests in test_groups.items():
            for filler, filler_kwargs in tests:
                test_name = get_test_name(filler)
                filename = test_name + ".json"

                filler_src_path = os.path.join(filler_dir, test_group, filename)
                filler_path = os.path.join(FILLER_PARENT_DIR, filler_src_path)
                test_path = os.path.join(
                    TEST_PARENT_DIR, test_dir, test_group, filename
                )

                for path in [filler_path, test_path]:
                    os.makedirs(os.path.dirname(path), exist_ok=True)

                formatted_filler = filler_formatter(filler)
                filler_string = json.dumps(formatted_filler, indent=4, sort_keys=True)
                with open(filler_path, "w") as filler_file:
                    filler_file.write(filler_string)

                filler_hash = keccak(filler_string.encode("ascii"))
                info = {
                    "source": filler_src_path,
                    "sourceHash": encode_hex(filler_hash),
                }

                test = fill_test(filler, info=info, **filler_kwargs or {})
                with open(test_path, "w") as test_file:
                    json.dump(test, test_file, indent=4, sort_keys=True)


#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division, print_function, unicode_literals)

from multiprocessing import cpu_count
from eventlet import monkey_patch
from eventlet.greenpool import GreenPool
from contextlib import closing

monkey_patch()

import requests
try:
    requests.packages.urllib3.disable_warnings()
except AttributeError:
    pass
monkey_patch()

def read_stock_url(symbol):
    urlname = 'http://finance.yahoo.com/q?s=' + symbol.lower() + \
              '&ql=0'
    with closing(requests.get(urlname, stream=True)) as url_:
        for line in url_.iter_lines():
            line = line.decode(errors='ignore')
            if 'yfs_l84_%s' % symbol.lower() in line:
                price = line.split('yfs_l84_%s\">' % symbol.lower())[1]
                price = float(price.split('</')[0].replace(',', ''))
                return symbol, price
    return symbol, -1

def run_stock_parser():
    stock_symbols = []
    with open('symbols.txt', 'r') as symfile:
        for n, line in enumerate(symfile):
            sym = line.strip()
            if sym:
                stock_symbols.append(sym)

    ncpu = cpu_count()

    pool = GreenPool(ncpu * 4)

    stock_prices = []
    for symbol, price in pool.imap(read_stock_url, stock_symbols):
        stock_prices.append((symbol, price))

    with open('stock_prices.csv', 'w') as outfile:
        outfile.write('Stock,Price\n')
        for symbol, price in stock_prices:
            outfile.write('%s,%s\n' % (symbol, price))

if __name__ == '__main__':
    run_stock_parser()


# only links are new should be crawled for additional links
# looks for all links that begin with /wiki/ (don't restrict to article links)
# collects the title, the 1st paragraph of content and the link to edit the page if available

from urllib.request import urlopen
from bs4 import BeautifulSoup
import re

pages = set()
def getLinks(pageUrl):
    global pages
    html = urlopen("http://en.wikipedia.org"+pageUrl)
    bsObj = BeautifulSoup(html)
    try:
        print(bsObj.h1.get_text())
        print(bsObj.find(id="mw-content-text").findAll("p")[0])
        print(bsObj.find(id="ca-edit").find("span").find("a").attrs['href'])
    except AttributeError:
        print("this page is missing something")

    for link in bsObj.findAll("a", href=re.compile("^(/wiki/)")):
        if 'href' in link.attrs:
            if link.attrs['href'] not in pages:
                # encountered a new page
                newPage = link.attrs['href']
                print("--------------------\n"+newPage)
                pages.add(newPage)
                getLinks(newPage)
getLinks("")


import os
import logging

from django.core.management.base import BaseCommand
from django.core.management.base import CommandError

from core import batch_loader
from core.management.commands import configure_logging

configure_logging('process_coordinates_logging.config',
                  'process_coordinates_%s.log' % os.getpid())

_logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = "Process word coordinates for a batch by name from a batch list file"

    def add_arguments(self, parser):
        # Positional arguments
        parser.add_argument('batch_list_filename')

    def handle(self, batch_list_filename, *args, **options):
        if len(args)!=0:
            raise CommandError('Usage is process_coordinates %s' % self.args)

        loader = batch_loader.BatchLoader()
        batch_list = open(batch_list_filename)
        _logger.info("batch_list_filename: %s" % batch_list_filename)
        for line in batch_list:
            batch_name = line.strip()
            _logger.info("batch_name: %s" % batch_name)
            parts = batch_name.split("_")
            if len(parts)==4:
                loader.process_coordinates(batch_name, strict=False)
            else:
                _logger.warning("invalid batch name '%s'" % batch_name)


import argparse
import os

import torch

from net.models import LeNet
from net.quantization import apply_weight_sharing
import util

parser = argparse.ArgumentParser(description='This program quantizes weight by using weight sharing')
parser.add_argument('model', type=str, help='path to saved pruned model')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')
parser.add_argument('--output', default='saves/model_after_weight_sharing.ptmodel', type=str,
                    help='path to model output')
args = parser.parse_args()

use_cuda = not args.no_cuda and torch.cuda.is_available()

# Define the model
model = torch.load(args.model)
print('accuracy before weight sharing')
util.test(model, use_cuda)

# Weight sharing
apply_weight_sharing(model)
print('accuacy after weight sharing')
util.test(model, use_cuda)

# Save the new model
os.makedirs('saves', exist_ok=True)
torch.save(model, args.output)


from __future__ import division
import time
import torch
import torch.nn as nn
from torch.autograd import Variable
import numpy as np
import cv2
from util import *
from darknet import Darknet

import pandas as pd
import colorsys
import random
import pickle as pkl
import argparse

def arg_parse():
    """
    Parse arguments to the detect module
    """

    parser = argparse.ArgumentParser(description='YOLO v3 Video Detection')
    parser.add_argument("--dataset", dest="dataset",
            help="Dataset on which the network has been trained", default="pascal")
    parser.add_argument("--confidence", dest="confidence",
            help="Object Confidence to filter predictions", default=0.5, type=float)
    parser.add_argument("--nms_thresh", dest="nms_thresh",
            help="NMS Threshhold", default=0.4, type=float)
    parser.add_argument("--cfg", dest="cfgfile",
            help="Config file", default="cfg/yolov3.cfg", type=str)
    parser.add_argument("--weights", dest="weightsfile",
            help="weightsfile", default="yolov3.weights", type=str)
    parser.add_argument("--reso", dest="reso",
            help="Input resolution of the network. Increase to increase accuracy. Decrease to increase learning speed",
            default=416, type=int)
    return parser.parse_args()

def main():
    args = arg_parse()
    confidence = args.confidence
    nms_thresh = args.nms_thresh
    start = 0
    CUDA = torch.cuda.is_available()

    classes = load_classes("data/coco.names")
    num_classes = len(classes)

    # Set up the neural network
    print("Loading network.....")
    model = Darknet(args.cfgfile)
    model.load_weights(args.weightsfile)
    print("Network successfully loaded")

    model.net_info["height"] = args.reso
    inp_dim = int(model.net_info["height"])
    assert inp_dim % 32 == 0
    assert inp_dim > 32

    # if there's a GPU available, put the model on GPU
    if CUDA:
        model.cuda()

    # set the model in evaluation mode
    model.eval()

    def write(x, img, color):
        c1 = tuple(x[1:3].int())
        c2 = tuple(x[3:5].int())
        cls = int(x[-1])
        label = "{0}".format(classes[cls])
        cv2.rectangle(img, c1, c2, color, 4)
        t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_PLAIN, 1, 1)[0]
        c2 = c1[0] + t_size[0] + 3, c1[1] + t_size[1] + 4
        cv2.rectangle(img, c1, c2, color, -1)
        cv2.putText(img, label, (c1[0], c1[1] + t_size[1] + 4), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (150, 150, 150), thickness=1)

    # detection phaase
    cap = cv2.VideoCapture(0)
    assert cap.isOpened(), "Cannot capture source"

    frames = 0
    start = time.time()
    hsv_tuples = [(x / num_classes, 1., 1.) for x in range(num_classes)]
    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    colors = list(map(lambda x: (int(x[0] * 200), int(x[1] * 200), int(x[2] * 200)), colors))
    np.random.seed(10000)
    np.random.shuffle(colors)
    np.random.seed(None) # reset seed to default.

    while cap.isOpened():
        ret, frame = cap.read()

        if ret:
            frame = cv2.resize(frame, dsize=(1280, 960))
            img = prep_image(frame, inp_dim)
            print(f"IMG_SHAPE: {img.shape}")
            im_dim = frame.shape[1], frame.shape[0]
            im_dim = torch.FloatTensor(im_dim).repeat(1, 2)

            if CUDA:
                im_dim = im_dim.cuda()
                img = img.cuda()

            with torch.no_grad():
                outputs = model(Variable(img, volatile=True), CUDA)
            outputs = write_results(outputs, confidence, num_classes, nms_conf=nms_thresh)

            if outputs != None:
                im_dim = im_dim.repeat(outputs.size(0), 1)
                scaling_factor = torch.min(inp_dim/im_dim, 1)[0].view(-1, 1)

                outputs[:, [1,3]] -= (inp_dim - scaling_factor * im_dim[:, 0].view(-1,1)) / 2
                outputs[:, [2,4]] -= (inp_dim - scaling_factor * im_dim[:, 1].view(-1,1)) / 2

                outputs[:, 1:5] /= scaling_factor

                for i in range(outputs.shape[0]):
                    outputs[i, [1,3]] = torch.clamp(outputs[i, [1,3]], 0.0, im_dim[i, 0])
                    outputs[i, [2,4]] = torch.clamp(outputs[i, [2,4]], 0.0, im_dim[i, 1])

                for output in outputs:
                    color = colors[int(output[-1])]
                    write(output, frame, color)

            cv2.imshow("frame", frame)
            key = cv2.waitKey(1)
            if key & 0xFF == ord('q'):
                break
            frames += 1
            print(time.time() - start)
            print("FPS of the video is {:5.2f}".format( frames / (time.time() - start) ))
        else:
            break

if __name__ == '__main__':
    main()