from src.core.interfaces.model_abc import ModelAbstract
from src.core.interfaces.repository_abc import RepositoryAbstract
from src.shared import parse_config

from sqlalchemy.orm import Session
import logging
import logging.config
import dataclasses
from typing import List
from datetime import datetime

class SqlAlchemyRepository(RepositoryAbstract):

    def __init__(self, session, object_type) -> None:
        self.session: Session = session
        self.object_type: ModelAbstract = object_type

        logging.config.dictConfig(parse_config.get_logger_config())
        self.logger = logging.getLogger("dev")
        self.logger.debug("{} repository instanciated".format(object_type.__name__))

    def get_by_filter(self, filter) -> ModelAbstract:
        self.logger.info("get_by_filter Request")
        query = self.session.query(self.object_type).filter_by(**filter)
        result = self.session.execute(query).fetchall()
        return result

    def get_by_id(self, target_id: int) -> ModelAbstract:
        """This function retrieves an object from the database.

        Return:
            The object (ModelAbstract)
        """
        self.logger.info("get_by_id Request")
        result = self.session.query(self.object_type).get(target_id)
        return result

    def get_all(self) -> List[ModelAbstract]:
        """This function retrieves all the objects in the repository.

        Returns:
            A list of the objects (List[ModelAbstract])
        """
        self.logger.info("get_all Request")
        result = self.session.query(self.object_type).all()
        return result

    def delete_by_filter(self, filter) -> int:
        """This function deletes an object using the provided filter.

        Returns:
            Affected rows provided by the database engine (int)
        """
        self.logger.info("delete_by_filter Request")
        result = self.session.query(self.object_type).filter(filter).delete()
        return result

    def delete_by_id(self, target_id: int) -> int:
        """This function deletes an object by it's id.

        Returns:
            Affected rows provided by the database engine (int)
        """
        self.logger.info("delete_by_id Request")
        result = self.session.query(self.object_type).filter(self.object_type.id == target_id).delete()
        self.session.commit()
        return result

    def insert(self, new: ModelAbstract) -> int:
        """This function inserts an object to the database.

        Returns:
            Inserted object id (int)
        """
        self.logger.info("insert Request")
        self.session.add(new)
        self.session.commit()
        return new.id

    def update(self, values_to_update, requested_object) -> ModelAbstract:
        self.logger.info("update Request")
        for key, value in values_to_update:
            if value is not None:
                setattr(requested_object, key, value)
        setattr(requested_object, "modified_at", datetime.now())
        self.session.commit()
        return requested_object


import requests
from time import sleep

timeToWait = 300 # Time to wait between callouts (in seconds)

while (True):
    # Get list of commands to run this callout
    URL = "https://slack.flemingcaleb.com:5000/api/agent/4/command/"
    r = requests.get(url=URL)

    if r.status_code == requests.codes.ok:
        # Process the list of requests
        print(r)
    elif r.status_code == requests.codes.not_found:
        # No list this time
        print("No list this time")
    else:
        #Handle Unintended Error
        print("ERROR")

    sleep(timeToWait)


import os
from pathlib import Path
from tempfile import NamedTemporaryFile

import pytest
import pytest_mock
from werkzeug.exceptions import NotFound

from iceart.models import Artist, ArtistDto, ArtistViewModel

def test_artist_vm_init():
    with pytest.raises(NotFound):
        ArtistViewModel(-1)

def test_artist_vm_search_key():
    assert ArtistViewModel(44).search_key() == {"_id": 44}

def test_artist_init():
    # Arrange
    data = {"_id": 44, "title": "t", "info": "i", "file": "f", "paintings": [5, 3, 77]}

    # Act
    model = Artist(data)

    # Assert
    assert model.identity == 44
    assert model.title == "t"
    assert model.info == "i"
    assert model.file == "f"
    assert model.paintings == [5, 3, 77]

def test_artist_dto_init():
    # Arrange
    data = {"_id": 44, "title": "t", "info": "i", "file": "f", "paintings": [5, 3, 77]}
    model = Artist(data)
    with NamedTemporaryFile("w+b", delete=False) as tmp_file:
        tmp_file.write(b"\xab\xef")
        p = Path(tmp_file.name)

    # Act
    with pytest_mock.mock.patch("pathlib.Path.joinpath", return_value=p):
        dto = ArtistDto(model, {1: "img1", 2: "img2"})

    # Assert
    assert dto.image == "q+8="
    assert dto.id == 44
    assert dto.title == "t"
    assert dto.info == "i"
    assert dto.as_json() == {
        "id": 44,
        "title": "t",
        "info": "i",
        "image": "q+8=",
        "paintings": {1: "img1", 2: "img2"},
    }
    assert dto.paintings == {1: "img1", 2: "img2"}

    # Cleanup
    os.remove(p)


import numpy as np
import time
import torch
from multiprocessing import Array, Manager

from dgl.dataloading.dataloader import GraphCollator

from collections import deque, namedtuple
from gp.utils.datasets import DatasetWithCollate

"""
Following namedtuple makes data collating nad referencing easier
"""

GraphLabelNT = namedtuple("GraphLabelNT", ["g", "labels", "index"])
ReplayBatch = namedtuple(
    "ReplayBatch",
    ["g", "plabel", "nlabel", "select_node", "move_node", "reward"],
)
GraphNMLabel = namedtuple("GraphNMLabel", ["g", "rlabel", "labels", "index"])

ReplayBatchHist = namedtuple(
    "ReplayBatchHist",
    [
        "g",
        "plabel",
        "nlabel",
        "select_node",
        "move_node",
        "reward",
        "context",
    ],
)

ReplayBatchSM = namedtuple(
    "ReplayBatchSM",
    [
        "g",
        "plabel",
        "nlabel",
        "select_node",
        "move_node",
        "reward",
    ],
)

class GraphLabelDataset(DatasetWithCollate):
    def __init__(self, graphs, labels, ind=None) -> None:
        super().__init__()

        self.graphs = graphs
        self.labels = labels
        if ind is None:
            self.ind = np.arange(len(self.graphs))
        else:
            self.ind = ind

    def __getitem__(self, index):
        return GraphLabelNT(
            self.graphs[index], np.array([self.labels[index]]), self.ind[index]
        )

    def __len__(self):
        return len(self.graphs)

    def get_collate_fn(self):
        return GraphCollator().collate

class GraphNMDataset(DatasetWithCollate):
    def __init__(self, graphs, labels, r_labels) -> None:
        super().__init__()

        self.graphs = graphs
        self.labels = labels
        self.rlabels = r_labels

    def __getitem__(self, index):
        return GraphNMLabel(
            self.graphs[index],
            np.array(self.rlabels[index]),
            np.array([self.labels[index]]),
            index,
        )

    def __len__(self):
        return len(self.graphs)

    def get_collate_fn(self):
        return GraphCollator().collate

class GraphReplayBuffer:
    def __init__(self, capacity):
        # self.buffer = deque(maxlen=capacity)
        self.capacity = capacity
        self.buffer = np.empty(capacity, dtype=object)
        self.size_count = 0
        self.pointer = 0

    def __len__(self):
        return self.size_count

    def add(self, experience):
        if isinstance(experience, list):
            ct = len(experience)
            self.size_count += ct
            next_pointer = self.pointer + ct
            if next_pointer > self.capacity:
                reminder = self.capacity - self.pointer
                self.buffer[self.pointer :] = experience[:reminder]
                self.pointer = 0
                next_pointer = next_pointer - self.capacity
                ct = next_pointer
                experience = experience[reminder:]
            self.buffer[self.pointer : next_pointer] = experience
            self.pointer = next_pointer
            self.size_count = min(self.size_count, self.capacity)
        else:
            self.buffer[self.size_count] = experience
        # if (len(self) / self.capacity) > 1.3:
        #     self.buffer = self.buffer[-self.capacity :]

    def sample(self, batch_size):
        if len(self) < batch_size:
            return None
        indices = np.random.choice(len(self), batch_size, replace=True)
        return [self.buffer[idx] for idx in indices]

    def reset(self):
        self.buffer = deque(maxlen=self.capacity)

class ReplayDataset(DatasetWithCollate):
    def __init__(self, graphs, buffer, replay_size=1, replay_type=ReplayBatch):
        super().__init__()
        self.graphs = graphs
        self.buffer = buffer
        self.replay_size = replay_size
        self.replay_type = replay_type

    def __getitem__(self, index):
        # print(len(self.buffer))
        replay = self.buffer.sample(1)[0]
        replay_graph = self.graphs[int(replay[0])]
        return self.replay_type(replay_graph, *map(np.array, replay[1:]))

    def __len__(self):
        return len(self.graphs) * self.replay_size

    def get_collate_fn(self):
        return GraphCollator().collate


"""
Plot a grid on H2
with Poincare Disk visualization.
"""

import logging
import os

import matplotlib.pyplot as plt
import numpy as np

import geomstats.visualization as visualization
from geomstats.geometry.hyperbolic import Hyperbolic

H2 = Hyperbolic(dimension=2)
METRIC = H2.metric

def main(left=-128,
         right=128,
         bottom=-128,
         top=128,
         grid_size=32,
         n_steps=512):
    starts = []
    ends = []
    for p in np.linspace(left, right, grid_size):
        starts.append(np.array([top, p]))
        ends.append(np.array([bottom, p]))
    for p in np.linspace(top, bottom, grid_size):
        starts.append(np.array([p, left]))
        ends.append(np.array([p, right]))
    starts = [H2.intrinsic_to_extrinsic_coords(s) for s in starts]
    ends = [H2.intrinsic_to_extrinsic_coords(e) for e in ends]
    ax = plt.gca()
    for start, end in zip(starts, ends):
        geodesic = METRIC.geodesic(initial_point=start,
                                   end_point=end)

        t = np.linspace(0, 1, n_steps)
        points_to_plot = geodesic(t)
        visualization.plot(
            points_to_plot, ax=ax, space='H2_poincare_disk', marker='.', s=1)
    plt.show()

if __name__ == "__main__":
    if os.environ['GEOMSTATS_BACKEND'] == 'tensorflow':
        logging.info('Examples with visualizations are only implemented '
                     'with numpy backend.\n'
                     'To change backend, write: '
                     'export GEOMSTATS_BACKEND = \'numpy\'.')
    else:
        main()


import numpy as np
from sklearn.metrics import f1_score
from sklearn.naive_bayes import BernoulliNB
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer

class NaiveBayes:
    """
    Naive Bayes implementation based on Multi-variate Bernoulli using python
    """
    def __init__(self):
        """
        self.class_probability --> Class probability of shape (output_label, ). It indicates the probability of alpha label appearing
                       without seeing the input data.
        self.phi --> Probability of alpha input feature given alpha output label. P(x|y_train). shape (output_label, input_feature)
        self.output_label --> Number of output class
        self.input_feature --> Number of input feature
        """
        self.class_probability = None
        self.phi = None
        self.output_label = None

    def fit(self, x_train, y_train):
        """
        Train the model
        :param x_train: Input training example of shape (number of training data, input feature)
        :param y_train: Output training example of shape (number of training data, )
        :return:
        """
        m = x_train.shape[0]  # Number of training example
        # Flatten the training set
        x_train = x_train.reshape(m, -1)
        input_features = x_train.shape[1]
        self.output_label = len(np.unique(y_train.reshape(-1)))
        # Initialize everything with zero
        self.class_probability = np.zeros(self.output_label)
        self.phi = np.zeros((self.output_label, input_features))
        # Calculate class probability and phi
        for label in range(self.output_label):
            # Extract the training data from an individual labels
            current_label_data = x_train[y_train == label]
            # Number of occurarances of this particular label in the training set
            current_label_occur = current_label_data.shape[0]
            # Class label of this training data
            self.class_probability[label] = (current_label_occur + 1) / (m + self.output_label)
            # Calculate phi for an individual label
            # How many times each of the input feature appeared for this label
            # One is added for laplace smoothing
            input_feature_occur = np.sum(current_label_data, axis=0) + 1
            # Fix the denominator according to the laplace smoothing
            curr_label_laplace_smoothing = current_label_occur + self.output_label
            # Calculate phi
            self.phi[label, :] = input_feature_occur / curr_label_laplace_smoothing

    def predict(self, x_test):
        """
        Make prediction
        :param x_test: data to predict of shape (number of prediction, input feature)
        :return:
        """
        # Number of prediction
        num_of_test = x_test.shape[0]
        # Probability of each of the class.
        # Initially each of the label will have zero probability
        probabilities = np.zeros((num_of_test, self.output_label))
        # Calculate for all test
        for test_index in range(num_of_test):
            # Count probabilities for each of the classes
            for label in range(self.output_label):
                # First get all the words present in this test example
                words_for_this_example = x_test[test_index] == 1
                # Get the calculated probabilities for this label and this ese words example
                words_probabilities = self.phi[label][words_for_this_example]
                # Multiply all these probability
                words_probability_multiply = np.prod(words_probabilities)
                # Multiply this with class_probability probabilities/class probabilities
                # to get the overall probability of this example
                probabilities[test_index, label] = words_probability_multiply * self.class_probability[label]
            # Normalize the probabilities
            probabilities[test_index] /= np.sum(probabilities[test_index])
        # return the maximum probability index
        return np.argmax(probabilities, axis=1)

    def get_f1_score(self, x_test, y_test):
        """
        Calculate the f1 score of our model
        :param x_test:
        :param y_test:
        :return:
        """
        return f1_score(y_test, self.predict(x_test))

if __name__ == '__main__':
    # Test our model with amazon cell review dataset.
    # Where model will predict if alpha review is positive or negetive
    # Read the dataset
    with open("amazon_cells_labelled.txt", "r") as file:
        # This will contain all the sentences
        sentences = []
        # This will contain all the output label(0, 1)
        labels = []
        for line in file.readlines():
            # The label and sentences are separated with alpha tab
            line_arr = line.strip().split("\t")
            # Remove stop words
            sentences.append(line_arr[0])
            labels.append(int(line_arr[1]))

    # Vectorize the training sentences
    vectorizer = CountVectorizer(analyzer="word", lowercase=True, stop_words="english", max_features=4500)
    data = vectorizer.fit_transform(sentences).toarray()

    # Split data
    x_train, x_test, y_train, y_test = train_test_split(data, np.array(labels))

    # Fit to our model
    naive_bayes = NaiveBayes()
    naive_bayes.fit(x_train, y_train)
    model_f1_score = naive_bayes.get_f1_score(x_test, y_test)
    print("F1 score of test set of our model is : ", str(model_f1_score))

    # Compare with scikit-learn model
    sci_naive_bayes = BernoulliNB()
    sci_naive_bayes.fit(x_train, y_train)
    sk_prediction = sci_naive_bayes.predict(x_test)
    print("F1 score of the test set of scikit learn model is : ", str(f1_score(y_test, sk_prediction)))


import pickle, ssl, logging

import numpy as np
import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

import pytorch_lightning as pl

import models, curves, utils1, utils, preresnet, resnet

DATASET = datasets.CIFAR10
TEST_ITEMS = 50_000
BATCH_SIZE = 128
WORKER_COUNT = 6
EPOCHS = 120

LEARNING_RATE=0.1
LR_GAMMA=0.1
L2_REG = 5e-4
MOMENTUM = 0.9

PARALLEL_MODELS = 1
MODEL = resnet.ResNet18 if DATASET == datasets.CIFAR10 else resnet.ResNet14MNIST

CURVE_BENDS = 3
CURVE_NUM_SAMPLES = 61

STEP_MODES = 2
STEP_BAD_MODES = 2

CODE_CHECK = False
if CODE_CHECK:
    TEST_ITEMS = 5000
    WORKER_COUNT = 0
    EPOCHS = 1
    CURVE_NUM_SAMPLES = 2

LOADER_ARGS = {"batch_size":BATCH_SIZE, "num_workers":WORKER_COUNT, "persistent_workers":WORKER_COUNT>0}
TRAINER_ARGS = {"accelerator":"gpu", "devices":"auto", "max_epochs":EPOCHS, "precision":16}

class LitModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        if PARALLEL_MODELS > 1:
            self.model = models.ParallelModel(PARALLEL_MODELS, MODEL, 10, **MODEL.kwargs)
        else:
            self.model = MODEL.base(10, **MODEL.kwargs)
        self.loss = nn.CrossEntropyLoss()
        self.mode = None

    def forward(self, x):
        return self.model(x)

    def process_batch(self, batch):
        x, y = batch
        if PARALLEL_MODELS > 1:
            y = torch.cat([y for _ in range(PARALLEL_MODELS)], dim=0)
        return self(x), y

    def training_step(self, batch, batch_idx):
        y_hat, y = self.process_batch(batch)
        loss = self.loss(y_hat, y)
        self.log("train_loss", loss.item(), on_epoch=True)
        return loss

    def validation_step(self, batch, batch_idx):
        y_hat, y = self.process_batch(batch)
        loss = self.loss(y_hat, y)
        correct = (y_hat.argmax(1) == y).type(torch.float).sum().item()
        acc = 100*correct / y.size(dim=0)
        self.log("val_loss", loss)
        self.log("val_acc", acc)

    def test_step(self, batch, batch_idx):
        y_hat, y = self.process_batch(batch)
        loss = self.loss(y_hat, y)
        correct = (y_hat.argmax(1) == y).type(torch.float).sum().item()
        acc = 100*correct / y.size(dim=0)
        self.log("test_loss", loss)
        self.log("test_acc", acc)

    def configure_optimizers(self):
        optimiser = torch.optim.SGD(self.parameters(), lr=LEARNING_RATE*PARALLEL_MODELS, momentum=MOMENTUM, weight_decay=L2_REG)
        scheduler_dict = {
            "scheduler": torch.optim.lr_scheduler.StepLR(optimiser, step_size=max(1,EPOCHS//3), gamma=LR_GAMMA),
            "interval": "epoch"
        }
        return {"optimizer":optimiser, "lr_scheduler":scheduler_dict}

class LitModelConnect(pl.LightningModule):
    def __init__(self, start_model = None, end_model = None, num_bends=CURVE_BENDS):
        super().__init__()
        self.loss = nn.CrossEntropyLoss()
        self.model = curves.CurveNet(10, curves.PolyChain, MODEL.curve, num_bends, architecture_kwargs=MODEL.kwargs)
        self.t = None
        self.update_bn = False

        # Initialise curve weights
        if start_model != None:
            self.model.import_base_parameters(start_model.model, 0)
            self.model.import_base_parameters(end_model.model, num_bends - 1)
            self.model.init_linear()

    def forward(self, x, **kwargs):
        return self.model(x, **kwargs)

    def regulariser(self):
        return 0.5 * L2_REG * self.model.l2

    def process_batch(self, batch):
        x, y = batch
        return self(x, t=self.t), y

    def set_t(self, t):
        self.t = t

    def training_step(self, batch, batch_idx):
        y_hat, y = self.process_batch(batch)
        loss = self.loss(y_hat, y) + self.regulariser()
        self.log("train_loss", loss.item(), on_epoch=True)
        return loss

    def on_test_start(self):
        if self.update_bn:
            self.model.train()

    def test_step(self, batch, batch_idx):
        y_hat, y = self.process_batch(batch)
        nll = self.loss(y_hat, y)
        loss = nll + self.regulariser()
        correct = (y_hat.argmax(1) == y).type(torch.float).sum().item()
        acc = 100*correct / y.size(dim=0)

        self.log("test_nll", nll)
        self.log("test_loss", loss)
        self.log("test_acc", acc)

    def configure_optimizers(self):
        optimiser = torch.optim.SGD(
            filter(lambda param: param.requires_grad, self.parameters()),
            lr=LEARNING_RATE,
            momentum=MOMENTUM
        )
        scheduler_dict = {
            "scheduler": torch.optim.lr_scheduler.StepLR(optimiser, step_size=max(1,EPOCHS//3), gamma=LR_GAMMA),
            "interval": "epoch"
        }
        return {"optimizer":optimiser, "lr_scheduler":scheduler_dict}

def update_bn(model, loader):
    if not utils.check_bn(model): return

    bn_trainer = pl.Trainer(logger=False,**TRAINER_ARGS)
    model.update_bn = True
    bn_trainer.test(model, loader, verbose=False)
    model.update_bn = False

def testCurve(model, trainer, test_loader, train_loader=None):
    ts = np.linspace(0.0, 1.0, CURVE_NUM_SAMPLES)
    if train_loader == None:
        train_loader = test_loader

    # BN has momentum so iter a few times to warm up
    model.set_t(0.0)
    for _ in range(3):
        update_bn(model, train_loader)

    # Test and compute max stats
    max_loss = -1
    for t in ts:
        model.set_t(t)
        update_bn(model, train_loader)

        metrics = trainer.test(model, test_loader)
        max_loss = max(max_loss, metrics[0]["test_loss"])

    return max_loss

if __name__ == "__main__":

    # Select training device
    use_cuda = torch.cuda.is_available()
    if use_cuda:
        LOADER_ARGS["pin_memory"] = True
    device = "cuda" if use_cuda else "cpu"
    print(f"Training on {device}")

    # Setup logging
    logging.getLogger("lightning").setLevel(logging.ERROR)
    def genLogger(log, path):
        return pl.loggers.CSVLogger(path, name=log)

    # Setup checkpointing
    checkpoint = pl.callbacks.ModelCheckpoint(
        #dirpath="checkpoints",
        #save_last=True,
        save_top_k=0,
        save_weights_only=True
    )

    # Setup progress bar
    progress_bar = pl.callbacks.RichProgressBar()
    TRAINER_ARGS["callbacks"] = [checkpoint, progress_bar]

    # ------------------------
    # Prepare Datasets / Loaders
    # ------------------------

    # CIFAR10 has an expired cert
    ssl._create_default_https_context = ssl._create_unverified_context

    # Too many files open with file_descriptor strategy
    torch.multiprocessing.set_sharing_strategy('file_system')

    transform = []
    # if DATASET == datasets.CIFAR10:
    #     transform += [transforms.Grayscale()]
    transform += [
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ]

    transform = transforms.Compose(transform)

    train_data = DATASET(
        root=".data",
        train=True,
        download=True,
        transform=transform
    )

    # Truncate data for quick checks
    if TEST_ITEMS < 50_000:
        train_data = torch.utils.data.Subset(train_data, range(TEST_ITEMS))

    adverse_data = utils1.NoiseDataset(train_data)

    test_data = DATASET(
        root=".data",
        train=False,
        download=True,
        transform=transform
    )

    # train_data = utils1.GPUDataset(train_data, "cuda")
    # adverse_data = utils1.GPUDataset(adverse_data, "cuda")
    # test_data = utils1.GPUDataset(test_data, "cuda")

    train_loader   = DataLoader(train_data, shuffle=True, **LOADER_ARGS)
    adverse_loader = DataLoader(adverse_data, shuffle=True, **LOADER_ARGS)
    LOADER_ARGS["batch_size"] = 1024
    test_loader    = DataLoader(test_data, **LOADER_ARGS)
    curve_loader   = DataLoader(train_data, **LOADER_ARGS)

    # ------------------------
    # Generate Data
    # ------------------------

    # Load state
    state = {"minima":[], "paths":[]}

    try:
        state_file = open('state.p', 'rb')
        state = pickle.load(state_file)
        state_file.close()
    except FileNotFoundError:
        pass

    # ------------------------
    # Generate Minima
    # ------------------------

    # Generate good minima
    good_minima = []
    for _ in range(STEP_MODES):
        model = LitModel()
        model.mode = "train"
        path=f"modes/{len(state['minima'])}"
        trainer = pl.Trainer(logger=genLogger("train", path),**TRAINER_ARGS)
        trainer.fit(model, train_loader, test_loader)
        metrics = trainer.test(model, test_loader)[0]
        trainer.save_checkpoint(path+"/model.ckpt")

        record = {"idx":len(state["minima"]), "type":"good", "path":path, "loss":metrics["test_loss"], "acc":metrics["test_acc"]}
        good_minima.append(record)
        state["minima"].append(record)

    # Generate adversarial init
    model = LitModel()
    model.mode = "adverse"
    trainer = pl.Trainer(logger=genLogger("adverse", "modes"), **TRAINER_ARGS)
    trainer.fit(model, adverse_loader)
    trainer.save_checkpoint("modes/adverse.ckpt")

    # Generate bad minima
    bad_minima = []
    for _ in range(STEP_BAD_MODES):
        model = LitModel.load_from_checkpoint("modes/adverse.ckpt")
        model.mode = "train"
        path=f"modes/{len(state['minima'])}"
        trainer = pl.Trainer(logger=genLogger("train", path), **TRAINER_ARGS)
        trainer.fit(model, train_loader, test_loader)
        metrics = trainer.test(model, test_loader)[0]
        trainer.save_checkpoint(path+"/model.ckpt")

        record = {"idx":len(state["minima"]), "type":"bad", "path":path, "loss":metrics["test_loss"], "acc":metrics["test_acc"]}
        bad_minima.append(record)
        state["minima"].append(record)

    # ------------------------
    # Generate Curves
    # ------------------------

    new_curves = []
    # Connect good minima to existing
    for mode in good_minima:
        if mode["idx"] == 0: continue

        other = np.random.randint(mode["idx"])
        while state["minima"][other]["type"] != "good":
            other = np.random.randint(mode["idx"])

        new_curves.append((mode["idx"], other))

    # Connect a bad minimum to a good minimum
    other = np.random.randint(bad_minima[0]["idx"])
    while state["minima"][other]["type"] != "good":
        other = np.random.randint(bad_minima[0]["idx"])
    new_curves.append((bad_minima[0]["idx"], other))

    # Curve directly between bad minima
    new_curves.append((bad_minima[0]["idx"], bad_minima[1]["idx"]))

    # Random new curves
    start = np.random.randint(len(state["minima"]))
    other = np.random.randint(len(state["minima"]))
    while start == other:
        other = np.random.randint(len(state["minima"]))
    new_curves.append((start, other))

    start = np.random.randint(len(state["minima"]))
    other = np.random.randint(len(state["minima"]))
    while start == other:
        other = np.random.randint(len(state["minima"]))
    new_curves.append((start, other))

    for start_idx, end_idx in new_curves:
        start = LitModel.load_from_checkpoint(state["minima"][start_idx]["path"] + "/model.ckpt")
        end = LitModel.load_from_checkpoint(state["minima"][end_idx]["path"] + "/model.ckpt")
        path=f"curves/{len(state['paths'])}"

        linear = LitModelConnect(start, end, 2)
        trainer = pl.Trainer(logger=genLogger("linear", path), **TRAINER_ARGS)
        lin_loss = testCurve(linear, trainer, curve_loader, test_loader)
        trainer.save_checkpoint(path + "/linear.ckpt")

        curve = LitModelConnect(start, end, CURVE_BENDS)
        trainer = pl.Trainer(logger=genLogger("curve", path), **TRAINER_ARGS)
        trainer.fit(curve, train_loader)
        trainer.save_checkpoint(path + "/curve.ckpt")
        curve_loss = testCurve(curve, trainer, curve_loader, test_loader)
        curve_test_loss = testCurve(curve, trainer, test_loader, test_loader)

        record = {"idx":len(state["paths"]), "path":path, "start":start_idx, "end":end_idx, "lin_loss":lin_loss, "curve_loss":curve_loss, "curve_test_loss":curve_test_loss}
        state["paths"].append(record)

    # Save state
    pickle.dump(state, open('state.p', 'wb'))


from django.urls import path

from .views import (get_user, add_user, get_categories, get_recipe,
                    add_to_favourite, get_user_favourites,
                    get_recipes_in_category, get_random_recipe,
                    add_to_dislikes)

urlpatterns = [
    path('users/<int:telegram_id>', get_user),
    path('users/add/', add_user),

    path('categories/', get_categories),
    path('category/recipes/', get_recipes_in_category),

    path('recipe/random/', get_random_recipe),

    path('favourites/recipe/', get_recipe),
    path('favourites/', get_user_favourites),
    path('favourites/add', add_to_favourite),

    path('dislikes/add', add_to_dislikes)
]


import torch
import torch.nn as nn
import torch.nn.functional as F

from .activations import ACTIVATIONS

class Embedding(nn.Module):
    '''
    Abstract base class for any module that embeds a collection of N vertices into
    N hidden states
    '''
    def __init__(self, features, hidden, **kwargs):
        super().__init__()
        self.features = features
        self.hidden = hidden

    def forward(self, x):
        pass

class Constant(Embedding):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def forward(self, x):
        return x

class OneLayer(Embedding):
    def __init__(self, features, hidden, act=None, wn=False, **kwargs):
        super().__init__(features, hidden)
        self.fc = nn.Linear(features, hidden)
        if wn:
            self.fc = nn.utils.weight_norm(self.fc, name='weight')
        self.activation = ACTIVATIONS[act]()

    def forward(self, x):
        return self.activation(self.fc(x))

class TwoLayer(Embedding):
    def __init__(self, features, hidden, act=None, wn=False, **kwargs):
        super().__init__(features, hidden)
        self.e1 = OneLayer(features, hidden, act, wn)
        self.e2 = OneLayer(hidden, hidden, act, wn)

    def forward(self, x):
        return self.e1(self.e2(x))

class NLayer(nn.Module):
    def __init__(self, dim_in=None, dim_out=None, n_layers=None, dim_hidden=None, act=None, wn=False, **kwargs):
        super().__init__()
        self.activation = ACTIVATIONS[act]()

        if dim_hidden is None:
            dim_hidden = dim_out
        dims = [dim_in] + [dim_hidden] * (n_layers-1) + [dim_out]

        self.fcs = nn.ModuleList()
        for i in range(len(dims)-1):
            fc = nn.Linear(dims[i], dims[i+1])
            if wn:
                fc = nn.utils.weight_norm(fc, name='weight')
            self.fcs.append(fc)

    def forward(self, x):
        for fc in self.fcs:
            x = fc(x)
            x = self.activation(x)
        return x

EMBEDDINGS = dict(
    n=NLayer,
    one=OneLayer,
    two=TwoLayer,
    const=Constant,
)


import tensorflow as tf
import keras.backend as K

def huber_loss(y_true, y_pred):
    return tf.losses.huber_loss(y_true, y_pred)

def adjust_binary_cross_entropy(y_true, y_pred):
    return K.binary_crossentropy(y_true, K.pow(y_pred, 2))

def MMD_Loss_func(num_source, sigmas=None):
    if sigmas is None:
        sigmas = [1, 5, 10]

    def loss(y_true, y_pred):
        cost = []

        for i in range(num_source):
            for j in range(num_source):
                domain_i = tf.where(tf.equal(y_true, i))[:, 0]
                domain_j = tf.where(tf.equal(y_true, j))[:, 0]
                single_res = mmd_two_distribution(K.gather(y_pred, domain_i),
                                                  K.gather(y_pred, domain_j),
                                                  sigmas=sigmas)
                cost.append(single_res)
        #print("wtf")
        cost = K.concatenate(cost)
        return K.mean(cost)
    return loss

def mmd_two_distribution(source, target, sigmas):

    sigmas = K.constant(sigmas)
    xy = rbf_kernel(source, target, sigmas)
    xx = rbf_kernel(source, source, sigmas)
    yy = rbf_kernel(target, target, sigmas)
    return xx + yy - 2 * xy

def rbf_kernel(x, y, sigmas):
    beta = 1. / (2. * (tf.expand_dims(sigmas, 1)))
    dist = compute_pairwise_distances(x, y)
    dot = -K.dot(beta, K.reshape(dist, (1, -1)))
    exp = K.exp(dot)
    return K.mean(exp, keepdims=True)

def compute_pairwise_distances(x, y):
    norm = lambda x: K.sum(K.square(x), axis=1)
    return norm(K.expand_dims(x, 2) - K.transpose(y))


import os
import re
import sys
import glob
import builtins
from contextlib import contextmanager

import setuptools
from setuptools import setup, Extension
from setuptools.command.build_ext import build_ext as _build_ext
from setuptools.command.install import install as _install

with open("README.md", "r") as fh:
    long_description = fh.read()

PACKAGENAME = "aliad"
VERSIONFILE = f"{PACKAGENAME}/_version.py"
verstrline = open(VERSIONFILE, "rt").read()
VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
mo = re.search(VSRE, verstrline, re.M)
if mo:
    verstr = mo.group(1)
else:
    raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,))

setup(
    name=PACKAGENAME, # Replace with your own username
    version=verstr,
    author="Alkaid Cheng",
    author_email="chi.lung.cheng@cern.ch",
    description="A library for anomaly detection.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=setuptools.find_packages(),
    package_data={PACKAGENAME: []},
    exclude_package_data={PACKAGENAME: []},
    classifiers=[
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
    ],
    install_requires=[
          'numpy',
          'pandas',
          'matplotlib',
          'click',
          'quickstats'
      ],
    scripts=[f'bin/{PACKAGENAME}'],
    python_requires='>=3.8',
)


from datetime import datetime, timezone

from config import Config
from logger import logger
from model.coordinates import Coordinates
from model.media_type import MediaType
from model.ts_source import TsSource

class MediaFile:
    original_path: str
    original_filename: str
    filename: str
    media_type: MediaType
    dir_path: list[str]

    unix_time_sec: int = None
    timestamp: datetime = None
    ts_source: TsSource = None
    mtime: datetime = None
    index: int = None

    coordinates: Coordinates = None

    md5: str = None

    target_dir = None
    target_filename = None
    target = None

    def __init__(self, original_path: str, original_filename: str, filename: str, media_type: MediaType,
                 dir_path: list[str]):
        self.original_path = original_path
        self.original_filename = original_filename
        self.filename = filename
        self.media_type = media_type
        self.dir_path = dir_path

    def __repr__(self):
        if self.ts_source == TsSource.EXIF:
            sym = Config.SYM_CHECK
        else:
            sym = Config.SYM_MULTIPLICATION
        return f'{sym} {self.ts_source} {self.dir_path}/{self.original_filename}, ts={self.timestamp}'

    def __lt__(self, other):
        return self.original_filename.lower() < other.original_filename.lower()

    def has_timestamp(self) -> bool:
        return self.timestamp is not None

    def update_time(self, ts: datetime, source: TsSource, force: bool = False):
        if ts is not None and (force or self.timestamp is None or ts < self.timestamp):
            self.timestamp = ts.astimezone(timezone.utc)
            self.unix_time_sec = int(self.timestamp.timestamp())
            self.ts_source = source
            if self.ts_source == TsSource.MTIME:
                self.mtime = ts

    def update_coordinates(self, coordinates: Coordinates):
        if coordinates is not None:
            self.coordinates = coordinates

    @staticmethod
    def create(source_dir: str, file_dir: str) -> 'MediaFile':
        original_filename, filename, media_type, split_path = MediaFile.retrieve_filename_data(source_dir, file_dir)
        return MediaFile(
            original_path=file_dir,
            original_filename=original_filename,
            filename=filename,
            media_type=media_type,
            dir_path=split_path
        )

    @staticmethod
    def retrieve_filename_data(source_dir: str, file_dir: str):
        if not source_dir.endswith('/'):
            source_dir += '/'

        split_path = file_dir.replace(source_dir, '').split('/')
        original_filename = split_path.pop()
        fixed_filename = MediaFile.replace_dots(original_filename)
        filename, extension = fixed_filename.split('.')
        media_type = MediaType.from_string(extension)

        return original_filename, filename, media_type, split_path

    @staticmethod
    def replace_dots(original_filename: str) -> str:
        result = original_filename
        while result.count('.') > 1:
            result = result.replace('.', ' ', 1)
        return result


# -*- coding: utf-8 -*
"""Graphics

.. module:: graphics
   :synopsis: Module for creating graphs

"""

# imports
import matplotlib.pyplot as plt
import numpy as np
import resoncalc.output as output
from math import ceil
from csv import DictReader

# globals
n_points = 1000 # count of points in graph
n_rows = 20.0   # count of rows in graph legend

def plot_fem_base_function(func, a, b):
    """Plot base function

    Args:
        func (func): base function
        a (float): left boundary of interval
        b (float): right boundary of interval

    """

    # coordinates init
    x = np.linspace(np.real(a), np.real(b), n_points)
    n = len(x)
    y = np.zeros(n, dtype=float)

    # calculate function values
    for i in range(n):
        y[i] = np.real(func(x[i]))

    # create graph
    plt.plot(x, y)
    plt.title('FEM base function')
    plt.tight_layout()
    plt.show()

def plot_fem_base(funcs, a, b):
    """Plot base functions

    Args:
        funcs (list): base functions
        a (float): left boundary of interval
        b (float): right boundary of interval

    """

    # coordinate init
    x = np.linspace(np.real(a), np.real(b), n_points)
    n = len(x)

    # function loop
    for i in range(len(funcs)):
        # coordinate init
        y = np.zeros(n, dtype=float)

        # bridging function
        if (type(funcs[i]) is list):
            for j in range(n):
                y[j] = np.real(funcs[i][0](x[j]) + funcs[i][1](x[j]))

        # function within element
        else:
            for j in range(n):
                y[j] = np.real(funcs[i](x[j]))

        # plot function
        plt.plot(x, y, label=i+2)

    # create graph
    plt.title('FEM base functions')
    plt.legend(bbox_to_anchor=(1, 1), borderaxespad=0.0, ncol=ceil(len(funcs)/n_rows))
    plt.tight_layout()
    plt.show()

def plot_potential(potential, a, b, *params):
    """Plot potential

    Args:
        potential (func): potential function
        a (float): left boundary of interval
        b (float): right boundary of interval
        params (args): potential specific parameters

    """

    # coordinates init
    x = np.linspace(np.real(a), np.real(b), n_points)
    n = len(x)
    y = np.zeros(n, dtype=float)

    # plot potential
    for i in range(n):
        y[i] = potential(x[i], *params)
    plt.plot(x, y)

    # create graph
    plt.title('Potential')
    plt.xlabel('x')
    plt.ylabel('E')
    plt.grid()
    plt.tight_layout()
    plt.show()

def plot_eigenstates(potential, eigenvalues, a, b, emax, fname='', *params):
    """Plot eigenstates for given potential

    Args:
        potential (func): potential function
        eigenvalues (list): bound states
        a (float): left boundary of interval
        b (float): right boundary of interval
        emax (float): maximum energy in atomic units
        fname (str): export filename
        params (args): potential specific parameters

    """

    # coordinates init
    eigenvalues = np.real(eigenvalues)
    x = np.linspace(a, b, n_points)
    n = len(x)
    y = np.zeros(n, dtype=float)

    # plot potential
    for i in range(n):
        y[i] = potential(x[i], *params)
    plt.plot(x, y)

    # plot eigenstates
    for i in range(len(eigenvalues)):
        z = np.zeros(n, dtype=float)
        val = eigenvalues[i]
        bound_state = True if (val < 0.0) else False
        cross = 0

        # line segment representing eigenstate
        for j in range(n):

            # intersection points with potential
            if (val > y[j]):
                if (cross == 0 or (cross == 2 and not bound_state)):
                    cross += 1
            elif (cross == 1):
                cross += 1

            # plot line
            if ((bound_state and cross == 1) or (not bound_state and cross == 2)):
                z[j] = val
            else:
                z[j] = np.nan

        plt.plot(x, z, 'r', label='E{0} = {1:e}'.format(i+1, val))

    # create graph
    plt.xlim(left=a, right=b)
    plt.ylim(bottom=None, top=emax)
    plt.title('Eigenstates')
    plt.xlabel('x')
    plt.ylabel('E')
    plt.grid()
    plt.legend(bbox_to_anchor=(1, 1), borderaxespad=0.0, ncol=ceil(len(eigenvalues)/n_rows))
    plt.tight_layout()

    # export graph
    if (len(fname) > 0):
        plt.savefig(fname)
        plt.close()

    # display graph
    else:
        plt.show()

def plot_complex_spectrum(eigenvalues, eigenvalues2=[], states=None, fname=''):
    """Plot complex spectrum used in ECS method

    Args:
        eigenvalues (list): eigenvalues for first angle or rotation
        eigenvalues2 (list): eigenvalues for second angle of rotation, default empty
        states (list): highlighted eigenstates
        fname (str): export filename

    """

    # plot first eigenvalues
    x = []
    y = []
    if (states is None):
        x = np.real(eigenvalues)
        y = np.imag(eigenvalues)
    else:
        limit = np.abs(states[0])
        for val in eigenvalues:
            if (np.real(val) <= limit):
                x.append(np.real(val))
                y.append(np.imag(val))

    plt.plot(x, y, 'b.')

    # plot second eigenvalues
    x = []
    y = []
    if (len(eigenvalues2) > 0):
        if (states is None):
            x = np.real(eigenvalues2)
            y = np.imag(eigenvalues2)
        else:
            limit = np.abs(states[0])
            for val in eigenvalues2:
                if (np.real(val) <= limit):
                    x.append(np.real(val))
                    y.append(np.imag(val))

    plt.plot(x, y, 'g.')

    # highlight eigenstates
    if (states is not None):
        for state in states:
            plt.plot(np.real(state), np.imag(state), 'ro', label='real={0:e}, imag={1:e}'.format(np.real(state), np.imag(state)))
        plt.legend(loc='lower left')

    # create graph
    plt.title('Complex spectrum')
    plt.xlabel('Re')
    plt.ylabel('Im')
    plt.grid()

    # export graph
    if (len(fname) > 0):
        plt.savefig(fname)
        plt.close()

    # display graph
    else:
        plt.show()

def plot_resonances_complex(infile, outfile=''):
    """Plot resonances in complex plane

    Args:
        infile (str): input filename
        outfile (str): output filename, default empty

    """

    # parse csv file
    with open(infile, 'r') as f:
        reader = DictReader(f)
        states = {}

        # get resonance states, group by param2
        for row in reader:
            if (row['type'] == 'resonance'):
                param2 = row['param2']
                if (not param2 in states):
                    states[param2] = []

                states[param2].append(float(row['real']) + 1j*float(row['imag']))

    # plot state trajectories
    for k, v in states.items():
        plt.plot(np.real(v), np.imag(v), 'b-')

    # create graph
    plt.title('Resonance states')
    plt.xlabel('Re')
    plt.ylabel('Im')
    plt.grid()

    # export graph
    if (len(outfile) > 0):
        plt.savefig(outfile)
        plt.close()

    # display graph
    else:
        plt.show()

def plot_resonances_params(infile, energy=True, outfile=''):
    """Plot resonances according to parameters

    Args:
        infile (str): input filename
        energy (bool): energy or width, default energy
        outfile (str): output filename, default empty

    """

    # parse csv file
    with open(infile, 'r') as f:
        reader = DictReader(f)
        param1, param2, value = [], [], []

        # get resonance states
        for row in reader:
            if (row['type'] == 'resonance'):
                param1.append(float(row['param1']))
                param2.append(float(row['param2']))
                value.append(float(row['real']) if (energy) else -0.5 * float(row['imag']))

    # create graph
    plt.scatter(param1, param2, c=value, cmap=plt.cm.rainbow)
    plt.colorbar()
    plt.title('Resonance states {0}'.format('energy' if (energy) else 'width'))
    plt.xlabel('param a')
    plt.ylabel('param b')
    plt.grid()

    # export graph
    if (len(outfile) > 0):
        plt.savefig(outfile)
        plt.close()

    # display graph
    else:
        plt.show()


import requests
import json
url = "https://api.telegram.org/bot5653233459:AAHWejZRnvy4luWTetBSbQY5jTzS11mA35U/sendMessage"

photo_url = "https://api.telegram.org/bot5653233459:AAHWejZRnvy4luWTetBSbQY5jTzS11mA35U/sendPhoto"

document_url = "https://api.telegram.org/bot5653233459:AAHWejZRnvy4luWTetBSbQY5jTzS11mA35U/sendDocument"

def buttons(chat_id):
  payload = {
      "photo": "https://blog.mint.com/wp-content/uploads/2013/02/1.jpg",
      #"caption": caption,
      "chat_id": chat_id,
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "Show Plans",
                      "callback_data": "Insurance Plan"
                  }
              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(photo_url, json=payload)

def button_1(chat_id):
  payload = {
      "photo": "AgACAgUAAxkBAAIGv2Mhrr21NIyhQKZHdVnObrNS0_SdAAIStTEb3U4IVSXi2GZ7s3TqAQADAgADcwADKQQ",
      "chat_id": chat_id,
    #   "text": "Select one option",
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "Term Life",
                      "callback_data": "Term Life"
                  },
                  {
                      "text": "mediclaim",
                      "callback_data": "Mediclaim"
                  },
                  {
                      "text": "Accidental",
                      "callback_data": "Accidental Insurance"
                  },

              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(photo_url, json=payload)
  print(response)

def term_life(chat_id):
  payload = {
      "chat_id": chat_id,
      'document': 'BQACAgUAAxkBAAIClGMIdmbOEsi8xsAic-Bk0UahnIl5AAJCBgACDBJBVBLbNwoHbsUhKQQ',
      #"text": "Choose Your Plan",
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "25Lac",
                      "callback_data": "25Lac"
                  },
                  {
                      "text": "50Lac",
                      "callback_data": "50Lac"
                  },
                  {
                      "text": "75Lac",
                      "callback_data": "75Lac"
                  },
                  {
                      "text": "1Cr",
                      "callback_data": "1r"
                  },

              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(document_url, json=payload)

def mediclaim(chat_id):
  payload = {
      "chat_id": chat_id,
      #"text": "Choose Your Plan",
      "document":"BQACAgUAAxkBAAICmmMIeJtMVL1v2JTsRhsdMb4uSDvCAAJEBgACDBJBVKwG4QUSEWW7KQQ",
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "2Lac",
                      "callback_data": "mediclaim up to 2Lac"
                  },
                  {
                      "text": "5Lac",
                      "callback_data": "mediclaim up to 5Lac"
                  },
                  {
                      "text": "10Lac",
                      "callback_data": "mediclaim up to 10Lac"
                  },
                  {
                      "text": "25Lac",
                      "callback_data": "mediclaim up to 25Lac"
                  },

              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(document_url, json=payload)

def accidental(chat_id):
  payload = {
      "chat_id": chat_id,
      #"text": "Choose Your Plan",
      "document":"BQACAgUAAxkBAAICmGMIeJMERoZF4atIdkk_L-cYGESUAAJDBgACDBJBVCJlGZUyjyGkKQQ",
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "5Lac",
                      "callback_data": "Plan of 5Lac"
                  },
                  {
                      "text": "10Lac",
                      "callback_data": "Plan of 10Lac"
                  },
                  {
                      "text": "25Lac",
                      "callback_data": "Plan of 25Lac"
                  },
                  {
                      "text": "50Lac",
                      "callback_data": "Plan of 50Lac"
                  },

              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(document_url, json=payload)

def existing_user(chat_id):
  payload = {
      "chat_id": chat_id,
      "text": "Seems you are an existing user! To see your details click here ",
      "reply_markup": {
          "inline_keyboard": [
              [
                  {
                      "text": "Show Details",
                      "callback_data": "show details"
                  }
              ]
          ]
      }
  }
  headers = {
      "Accept": "application/json",
      "Content-Type": "application/json"
  }

  response = requests.post(url, json=payload, headers=headers)

#print(response.text)
# import requests

# url = "https://api.telegram.org/bot5340261920:AAF2GGInKosubny7ox-CWeZyl8IMESgQg5o/sendPhoto"

# payload = {
#     "photo": "AgACAgUAAxkBAAMLYvDs4xqN8GzQGUY555yHLr5joacAAjGxMRtMw4hX56puCadfo3cBAAMCAAN5AAMpBA",
#     # "caption": "Optional",
#     # "disable_notification": False,
#     "reply_to_message_id": 0,
#     "chat_id":1091996976
# }
# headers = {
#     "Accept": "application/json",
#     "User-Agent": "Telegram Bot SDK - (https://github.com/irazasyed/telegram-bot-sdk)",
#     "Content-Type": "application/json"
# }

# response = requests.post(url, json=payload)

# print(response.text)


from selenium import webdriver
import time, re
from bs4 import BeautifulSoup
import pyautogui
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
import pyperclip
import os

# 主要功能就是访问300mium所有影片详情页，然后挨个下载封面
class Crawl_51luxu:
    def main(self, Dir='F:\\pic\\', page=1, category='300MIUM'):
        current_path = os.getcwd().replace('\\', '/') + '/'
        # custom_path = 'F:\\pic\\300MIUM\\'
        custom_path = Dir + category + "\\"
        chrome_opts = webdriver.ChromeOptions()
        chrome_opts.add_argument("--headless")
        chrome_opts.add_experimental_option(
            'excludeSwitches', ['enable-logging'])
        url = 'https://www.51luxu.com/category/sresource/' + category + '/page/' + str(page)
        def open_browser(url):
            driver = webdriver.Chrome(options=chrome_opts)
            driver.get(url)
            return driver

        def scrapy(driver):
            if not os.path.exists(Dir):
                os.mkdir(Dir)
            if not os.path.exists(custom_path):
                os.mkdir(custom_path)
            Exist = []
            if os.path.exists(custom_path + 'history.txt'):
                with open(custom_path + 'history.txt','r+') as f:
                    lines = f.readlines()
                    for line in lines:
                        Exist.append(line.replace("\n",""))
                    f.close()
            # 从history中读入历史的所下载的图片的名字，以免重复下载
            # 这一步主要是为了，当我筛选图片时，看到好看的要保留，看到不好看的要删除
            # 那么读取文件列表就乱了套了，所以把历史下载保存在txt文件中，就知道之前有没有下过这个番号了
            for page in range(1,100):
                try:
                    content = driver.page_source.encode('utf-8')
                    soup = BeautifulSoup(content, 'lxml')
                    img = soup.find_all('img')
                    src1 = re.findall(r'src=".*?"', str(img))
                    name1 = re.findall(r'alt=".*?"', str(img))
                    src2 = []
                    name2 = []
                    for i in src1:
                        src2.append(i.split('=')[1].replace("\"",""))
                    for i in name1:
                        name2.append(i.split('=')[1].replace("\"", ""))
                    if category == "Scute":
                        pattern = "S-cute"
                    else:
                        pattern = category
                    try:
                        temp = [x.replace("inggo.info", "paypp.xyz") for x in src2]
                        src3 = [x for x in temp if 'images.paypp.xyz/wp-content/uploads' in x]
                    except:
                        src3 = [x for x in src2 if 'images.paypp.xyz/wp-content/uploads' in x]
                    name3 = [x for x in name2 if pattern in x]
                    if len(name3) < 12:
                        name3 = name2
                    # 上面是name3和src3 保存了主页面的番号和相应的详情页的链接
                    # 接下来启动第二个浏览器对各个详情页的视频截图进行抓取
                    driver1 = webdriver.Chrome(options=chrome_opts)
                    for i in range(len(src3)):
                        try:
                            if '[' and '【' not in name3[i]:
                                title = name3[i]
                            else:
                                title = name3[i].split('【')[1].split('】')[0] # 简化一下番号的名字
                        except:
                            title = name3[i].split('[')[1].split(']')[0]
                        if i >= 1:
                            try:
                                if name3[i].split('[')[1].split(']')[0] == name3[i-1].split('[')[1].split(']')[0]:
                                    title = name3[i].split(']')[1].replace("[","")
                            except:
                                pass
                        if i >= 1:
                            try:
                                if name3[i].split('【')[1].split('】')[0] == name3[i-1].split('【')[1].split('】')[0]:
                                    title = name3[i].split('】')[1].replace("【","")
                            except:
                                pass
                        if title in Exist:
                            print("%s 已经下载！" % (title))
                            continue
                        # 前文提到的判断是否下过，如果是，后面就不用进行了
                        # 进入相应链接的详情页
                        driver1.get(src3[i])
                        img = driver1.find_element_by_xpath("//html/body/img")
                        img.screenshot(custom_path + title + '.jpg')
                        # wait = WebDriverWait(driver1, 10) # 等待浏览器相应，删除也可以
                        # pyautogui.rightClick(x=500, y=500) # 右击图片，位置可根据自己的屏幕调整
                        # pyautogui.typewrite(['V']) # 另存为的快捷键为 V
                        # time.sleep(2) # 等待电脑响应
                        # pyperclip.copy(custom_path + title + '.jpg')  # 复制文件名加路径到粘贴板
                        # time.sleep(1)
                        # pyautogui.hotkey('ctrlleft', 'V') # 粘贴
                        # time.sleep(1)
                        # pyautogui.press('enter') # 确认
                        # time.sleep(1)
                        while True:
                            filelist = os.listdir(custom_path)
                            if title + '.jpg' in filelist:
                                with open(custom_path + 'history.txt', 'a+') as f:
                                    f.writelines(title)
                                    f.writelines('\n')
                                    f.close()
                                print("%s 下载完成！" % (title))
                                break
                            else:
                                print("等待响应")
                                time.sleep(2)
                                # pyautogui.hotkey('ctrlleft', 'V')  # 粘贴
                                # time.sleep(1)
                                # pyautogui.press('enter')  # 确认
                                # time.sleep(1)
                        # 在txt中加入当前下载的图片名字
                        print("%s 下载完成！"%(title))
                    time.sleep(0.5)
                    driver1.quit()
                    print("第 %d 页爬完"%(page))
                    button =  "//*[@class='next page-numbers']"  #翻页按钮
                    driver.find_elements_by_xpath(button)[0].click()
                except:
                    print("第 %d 页出错！"%(page))
                    driver1.quit()
                    try:
                        button =  "//*[@class='next page-numbers']"  #翻页按钮
                        driver.find_elements_by_xpath(button)[0].click()
                    except:
                        print("爬取完毕！")
                        break
                    continue

        driver = open_browser(url)
        time.sleep(2)
        scrapy(driver)


import cv2
import numpy as np

img = cv2.imread('bookpage.jpg')

grayscaled = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#retval, threshold = cv2.threshold(grayscaled, 11, 255 , cv2.THRESH_BINARY)
threshold = cv2.adaptiveThreshold(grayscaled, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 115, 1)

median = cv2.medianBlur(threshold,3)
gaus = cv2.GaussianBlur(threshold,(5,5),0)

cv2.imshow('original',img)
cv2.imshow('gray',grayscaled)
cv2.imshow('threshold',threshold)
cv2.imshow('gaus',gaus)

cv2.waitKey(0)
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
from flask import Flask, request, jsonify, make_response, abort
import requests
from stations import stations

app = Flask(__name__)

name = [
    'station_train_code',
    'from_station_name',
    'to_station_name',
    'lishi',
    'start_time',
    'arrive_time',
    'swz_num',
    'tz_num',
    'zy_num',
    'ze_num',
    'gr_num',
    'rw_num',
    'yw_num',
    'rz_num',
    'yz_num',
    'wz_num',
    'qt_num']

@app.route('/zd')
def zd_tickets():
    tickets = []
    date = request.args.get('Date')
    from_station = request.args.get('from')
    to_station = request.args.get('to')
    if from_station in stations.keys() and to_station in stations.keys():
        from_station = stations[from_station]
        to_station = stations[to_station]
    else:
        abort(400)

    url = 'https://kyfw.12306.cn/otn/lcxxcx/query?purpose_codes=ADULT&queryDate={}&from_station={}&to_station={}'.\
        format(date, from_station, to_station)
    r = requests.get(url, verify=False)
    # if the url param 'date'was supplied incorrectly, 12306 would return -1 of int type.
    if r.json() == -1:
        abort(400)
    if 'datas' in r.json()['data']:
        contents = r.json()['data']['datas']

        # get out the information that we want in the contents dict.
        for content in contents:
            ticket = {key: content[key] for key in name}
            tickets.append(ticket)
    else:
        abort(404)

    return jsonify({'tickets': tickets})

@app.route('/hc')
def hc_tickets():
    tickets_1 = []
    tickets_2 = []
    date = request.args.get('Date')
    from_station = request.args.get('from')
    to_station = request.args.get('to')
    changed_station = request.args.get('change')
    if from_station in stations.keys() and to_station in stations.keys() and changed_station in stations.keys():
        from_station = stations[from_station]
        to_station = stations[to_station]
        changed_station = stations[changed_station]
    else:
        abort(400)

    url1 = 'https://kyfw.12306.cn/otn/lcxxcx/query?purpose_codes=ADULT&queryDate={}&from_station={}&to_station={}'. \
        format(date, from_station, changed_station)
    r1 = requests.get(url1, verify=False)

    if r1.json() == -1:
        abort(400)

    if 'datas' in r1.json()['data']:
        contents_1 = r1.json()['data']['datas']

        for content in contents_1:
            ticket = {key: content[key] for key in name}
            tickets_1.append(ticket)
    else:
        abort(404)

    for ticket in tickets_1:
        if int(ticket['lishi'][:2]) + int(ticket['start_time'][:2]) < 24:  # TODO
            url2 = 'https://kyfw.12306.cn/otn/lcxxcx/query?purpose_codes=ADULT' \
                   '&queryDate={}&from_station={}&to_station={}'. \
                format(date, changed_station, to_station)
            r2 = requests.get(url2, verify=False)

            if r2.json() == -1:
                abort(400)

            if 'datas' in r2.json()['data']:
                contents_2 = r2.json()['data']['datas']

                for content in contents_2:
                    ticket = {key: content[key] for key in name}
                    tickets_2.append(ticket)

                if tickets_2:
                    for x in tickets_1:
                        x['changed_ticket'] = [y for y in tickets_2 if
                                               1 < int(y['start_time'][:2]) - int(x['arrive_time'][:2]) < 3]
            else:
                abort(404)

        else:
            # if the first train arrived at next day, we add one day to the queryDate param.
            date2 = str(datetime.strptime(date, '%Y-%m-%d') + timedelta(days=1))[:10]

            # TODO: the following repeatedly codes should be moved to a helper method.
            url2 = 'https://kyfw.12306.cn/otn/lcxxcx/query?purpose_codes=ADULT' \
                   '&queryDate={}&from_station={}&to_station={}'. \
                format(date2, changed_station, to_station)
            r2 = requests.get(url2, verify=False)

            if r2.json() == -1:
                abort(400)

            if 'datas' in r2.json()['data']:
                contents_2 = r2.json()['data']['datas']

                for content in contents_2:
                    ticket = {key: content[key] for key in name}
                    tickets_2.append(ticket)

                if tickets_2:
                    for x in tickets_1:
                        x['changed_ticket'] = [y for y in tickets_2 if
                                           1 < int(y['start_time'][:2]) - int(x['arrive_time'][:2]) < 3]

    return jsonify({'tickets': tickets_1})

@app.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': 'Not found'}), 404)

@app.errorhandler(400)
def bad_request(error):
    return make_response(jsonify({'error': 'Please check your query param format'}), 400)

if __name__ == '__main__':
    app.run(debug=True)


from flask import g, request, current_app
import iot_api_core
import time

class InstanceVersionBaseBehavior():
  def __init__(self, widget_type, namespace, instance_id):
    self.widget_type = widget_type
    self.namespace = namespace
    self.instance_id = instance_id
    self.lumavate = iot_api_core.Lumavate()
    self.temp = {}
    self.post_version_create_handlers = []
    self.experience_info = None
    self._instance = None
    self.status_id = None
    data = request.get_json(silent=True)
    if data:
      self.status_id = data.get('statusId')

    if self.namespace is None:
      self.namespace = self.rest_get_single(self.instance_id)['namespace']

  @property
  def properties(self):
    return []

  @property
  def components(self):
    return []

  @property
  def page_security_property(self):
    page_security = {
      'always': 'Always Render',
      'user-logged-in': 'Only Render When User Logged In',
      'user-not-logged-in': 'Only Render When User NOT Logged In',
      'prod-registered': 'Only Render When Product Registered',
      'device-authorized': 'Only Render When Device Authorized',
      'prod-not-registered': 'Only Render When Product NOT Registered',
      'device-not-authorized': 'Only Render When Device NOT Authorized',
    }

    return iot_api_core.DropdownProperty('General', 'General Settings', 'pageSecurity', 'Page Security', self, default='always', options=page_security)

  @property
  def experience_id(self):
    self.load_experience_info()
    return self.experience_info.get('id')

  @property
  def model_id(self):
    self.load_experience_info()
    return self.experience_info.get('modelId')

  def load(self, version_name):
    instance = self.rest_get_single(self.instance_id)
    data = instance[version_name + 'Version']['data']

    for x in self.properties:
      if not x.name.startswith('instance__'):
        x.read(data)

  def get_general_properties(self, include_auth=False):
    return [
      self.instance_name_property(),
      self.instance_page_type_property(include_auth=include_auth),
      iot_api_core.ToggleProperty('General', 'General Settings', 'displayBackgroundImage', 'Display Background Image', self, default=False),
      iot_api_core.ImageProperty('General', 'General Settings', 'backgroundImage', 'Background Image', self),
      iot_api_core.ColorProperty('General', 'General Settings', 'backgroundColor', 'Background Color', self, default='#e2e2e2'),
      self.page_security_property
    ]

  def load_experience_info(self):
    if not self.experience_info:
      results = self.lumavate.get('/iot/v1/experiences?siteName=' + self.namespace)
      if len(results) > 0:
        self.experience_info = results[0]
        self.model_info = self.lumavate.get('/iot/v1/models/' + str(self.experience_info['modelId']))
      else:
        self.experience_info = {}
        self.model_info = {}

  def rest_get_single(self, id):
    if self._instance is None:
      self._instance  = self.lumavate.get('/iot/v1/widget-instances/' + str(id))

    return self._instance

  def get_version_id(self, instance_id, version_name):
    instance = self.rest_get_single(instance_id)
    return str(instance[version_name + 'VersionId'])

  def get_property(self, name):
    return next((p for p in self.properties if p.name == name), None)

  def get_collection_rest_uri(self):
    return '/iot/v1/widget-instances/' + str(self.instance_id) + '/versions'

  def get_single_rest_uri(self, version_id):
    return self.get_collection_rest_uri() + '/' + str(version_id)

  def rest_get_collection(self):
    return self.lumavate.get(self.get_collection_rest_uri())

  def instance_name_property(self):
    instance = self.rest_get_single(self.instance_id)
    return iot_api_core.TextProperty('General', 'General Settings', 'instance__name', 'Page Name', self, rows=0, default=instance['name'])

  def resolve_images(self, data):
    if isinstance(data, dict):
      if 'key' in data and 'versionId' in data:
        image_data = self.lumavate.post('/iot/v1/files/preview/' + data['key'] + '/' + data['versionId'], {})
        for f in ['contentType', 'url', 'mobileUrl']:
          if f in data:
            image_data[f] = data.get(f)

        return image_data

      else:
        return { k: self.resolve_images(data[k]) for k in data }

    elif isinstance(data, list):
      return [self.resolve_images(x) for x in data]

    else:
      return data

  def collapse_language(self, data):
    lang = 'en-us'

    if isinstance(data, dict):
      if lang in data:
        return data[lang]
      else:
        return { k: self.collapse_language(data[k]) for k in data }

    elif isinstance(data, list):
      return [self.collapse_language(x) for x in data]

    else:
      return data

  def instance_page_type_property(self, include_auth=False):
    page_types = {
      'home': 'Home',
      'registration': 'Registration',
      'auth': 'Auth',
      'error': 'Error',
      'normal': '<Normal>'
    }
    if include_auth == False:
      del page_types['auth']

    return iot_api_core.DropdownProperty('General', 'General Settings', 'pageType', 'Page Type', self, default='normal', options=page_types)

  def rest_create(self):
    instance = self.rest_get_single(self.instance_id)
    if instance:
      payload = {'data': self.validate_data(request.get_json())}
      results = self.lumavate.post(self.get_collection_rest_uri() + '-direct', payload)
      instance['futureVersionId'] = results['id']
      for vch in self.post_version_create_handlers:
        vch()
      self.post_version_create_handlers = []

      if instance.get('futureVersion') is not None:
        results['delta'] = self.get_delta_document(instance.get('futureVersion').get('data'), results['data'])
      else:
        results['delta'] = results.get('data')

      return results

  def background(self, function, args=[], kwargs={}):
    data = {
      'namespace': self.namespace,
      'widgetInstanceId': self.instance_id,
      'widgetType': current_app.config['WIDGET_ID'],
      'method': function.__name__,
      'args': args,
      'kwargs': kwargs,
      'statusId': self.status_id
    }
    self.lumavate.post('/iot/v1/background', data)

  def run_background(self):
    self.lumavate.put('/iot/v1/statuses/' + self.status_id, {'percent': 100})

    return {'a': 4}
    data = request.get_json()
    method = data.get('method')
    args = data.get('args')
    kwargs = data.get('kwargs')
    getattr(self, method)(*args, **kwargs)

  def publish(self, version_name):
    instance = self.rest_get_single(self.instance_id)
    if not instance:
      return

    if version_name not in ['future', 'draft', 'production', 'current']:
      return

    version = instance.get(version_name + 'Version')
    #for x in self.properties:
    #  x.read(version['data'])
    #  x.publish()
    if self.status_id:
      self.lumavate.put('/iot/v1/statuses/' + self.status_id, {'percent': 100})
    else:
      print('NOPE', flush=True)

    return {'Status': 'Ok'}

  def store_data(self, category, record_id, data, latitude=None, longitude=None, version='future'):
    version_id = self.get_version_id(self.instance_id, version)
    payload = {
      'recordId': str(record_id),
      'data': data,
      'latitude': latitude,
      'longitude': longitude
    }
    return self.lumavate.post(self.get_single_rest_uri(version_id) + '/data/' + category, payload)

  def clear_data(self, category, version='future'):
    version_id = self.get_version_id(self.instance_id, version)
    return self.lumavate.delete(self.get_single_rest_uri(version_id) + '/data/' + category)

  def get_current_version(self):
    if hasattr(g, 'iot_context'):
      return g.iot_context['token_data']['version']
    else:
      return 'future'

  def get_data(self, category, default=[], qs=''):
    version_id = self.get_version_id(self.instance_id, self.get_current_version())

    res = self.lumavate.get('/iot/v1/widget-instances/' + str(self.instance_id) + '/versions/' + version_id + '/data/' + category + '?' + qs)
    for x in res:

      if 'distance' in x:
        x['data']['distance'] = x['distance']

    res = [x['data'] for x in res]

    if len(res) == 0:
      return default
    else:
      return res

  def load_activation_info(self):
    api_result = {}
    try:
      api_result = self.lumavate.get('/iot/v1/labels/' + str(g.iot_context['token_data'].get('activationId', 0)))
    except Exception as e:
      pass

    return {
      'key': api_result.get('key'),
      'serialNumber': api_result.get('serialNumber')
    }

  def get_config_data(self):
    # Check if there is a valid version for the current context
    instance = self.rest_get_single(self.instance_id)
    if instance[self.get_current_version() + 'Version'] is None:
      raise Exception('Version ' + self.get_current_version() + ' does not exist for instance ' + str(self.instance_id))

    result = instance[self.get_current_version() + 'Version']['data']

    # Is there any activation to report?
    result['activation'] = self.load_activation_info()

    # Are there any 'special' pages that the UI should know about?
    result['authCheck'] = None
    if g.iot_context['token_data'].get('authUrl') is not None:
      root, part, instance = g.iot_context['token_data']['authUrl'].rpartition('/')
      result['authCheck'] = '{}/api/instances/{}/check-login-status'.format(root, instance)

    return result

  def default_image_data(self, data, prop):
    return {
      'preview': data.get(prop, {}).get('preview', '/icons/iot/page/api/instances/icons/no_image_available.png'),
      'previewLarge': data.get(prop, {}).get('previewLarge', '/icons/iot/page/api/instances/icons/no_image_available.png'),
      'previewMedium': data.get(prop, {}).get('previewMedium', '/icons/iot/page/api/instances/icons/no_image_available.png'),
      'previewSmall': data.get(prop, {}).get('previewSmall', '/icons/iot/page/api/instances/icons/no_image_available.png')
    }

  def get_delta_document(self, original, current):
    if original is None:
      return current
    else:
      result = {}

      for x in self.properties:
        dd = x.delta_doc(original.get(x.name), current.get(x.name))
        if dd is not None:
          result[x.name] = dd

      return result

  def validate_data(self, data):
    result = {}
    instance_payload = {}

    for x in self.properties:
      if x.name.startswith('instance__'):
        field = x.name.split('__')[-1]
        new_val = x.read(data)
        if self._instance[field] != new_val:
          self._instance[field] = new_val
          instance_payload[x.name.split('__')[-1]] = x.read(data)
      else:
        result[x.name] = x.read(data)

    if len(instance_payload.keys()) > 0:
      self.lumavate.put('/iot/v1/widget-instances/' + str(self.instance_id), instance_payload)

    return result

  def get_all_components(self):
    return [
      {
        'label': x.instantiate().label,
        'type': x.instantiate().component_type,
        'icon': x.instantiate().icon_url,
        'section': x.instantiate().section,
        'category': x.instantiate().category
      } for x in self.components
    ]

  def get_component_properties(self, component_type):
    comp = next((x.instantiate() for x in self.components if x.instantiate().component_type == component_type), None)
    if comp:
      return comp.get_properties()

  def get_component_property(self, component_type, property_name):
    comp = next((x.instantiate() for x in self.components if x.instantiate().component_type == component_type), None)
    if comp:
      return comp.get_property(property_name)

  def get_widget_properties(self):
    return [x.to_json() for x in self.properties]

  def handle_language_fields(self, data):
    lang = 'en-us'

    if isinstance(data, dict):
      if lang in data:
        return self.handle_language_fields(data[lang])
      else:
        return { k: self.handle_language_fields(data[k]) for k in data }
    if isinstance(data, list):
      return [ self.handle_language_fields(x) for x in data ]
    else:
      return data


__authors__ = ["T. Vincent"]
__license__ = "MIT"
__date__ = "07/01/2019"

import os
import logging
import weakref
from . import qt
import silx.resources
from silx.utils import weakref as silxweakref

_logger = logging.getLogger(__name__)
"""Module logger"""

_cached_icons = None
"""Cache loaded icons in a weak structure"""

def getIconCache():
    """Get access to all cached icons

    :rtype: dict
    """
    global _cached_icons
    if _cached_icons is None:
        _cached_icons = weakref.WeakValueDictionary()
        # Clean up the cache before leaving the application
        # See https://github.com/silx-kit/silx/issues/1771
        qt.QApplication.instance().aboutToQuit.connect(cleanIconCache)
    return _cached_icons

def cleanIconCache():
    """Clean up the icon cache"""
    _logger.debug("Clean up icon cache")
    _cached_icons.clear()

_supported_formats = None
"""Order of file format extension to check"""

class AbstractAnimatedIcon(qt.QObject):
    """Store an animated icon.

    It provides an event containing the new icon everytime it is updated."""

    def __init__(self, parent=None):
        """Constructor

        :param qt.QObject parent: Parent of the QObject
        :raises: ValueError when name is not known
        """
        qt.QObject.__init__(self, parent)

        self.__targets = silxweakref.WeakList()
        self.__currentIcon = None

    iconChanged = qt.Signal(qt.QIcon)
    """Signal sent with a QIcon everytime the animation changed."""

    def register(self, obj):
        """Register an object to the AbstractAnimatedIcon.
        If no object are registred, the animation is paused.
        Object are stored in a weaked list.

        :param object obj: An object
        """
        if obj not in self.__targets:
            self.__targets.append(obj)
        self._updateState()

    def unregister(self, obj):
        """Remove the object from the registration.
        If no object are registred the animation is paused.

        :param object obj: A registered object
        """
        if obj in self.__targets:
            self.__targets.remove(obj)
        self._updateState()

    def hasRegistredObjects(self):
        """Returns true if any object is registred.

        :rtype: bool
        """
        return len(self.__targets)

    def isRegistered(self, obj):
        """Returns true if the object is registred in the AbstractAnimatedIcon.

        :param object obj: An object
        :rtype: bool
        """
        return obj in self.__targets

    def currentIcon(self):
        """Returns the icon of the current frame.

        :rtype: qt.QIcon
        """
        return self.__currentIcon

    def _updateState(self):
        """Update the object according to the connected objects."""
        pass

    def _setCurrentIcon(self, icon):
        """Store the current icon and emit a `iconChanged` event.

        :param qt.QIcon icon: The current icon
        """
        self.__currentIcon = icon
        self.iconChanged.emit(self.__currentIcon)

class MovieAnimatedIcon(AbstractAnimatedIcon):
    """Store a looping QMovie to provide icons for each frames.
    Provides an event with the new icon everytime the movie frame
    is updated."""

    def __init__(self, filename, parent=None):
        """Constructor

        :param str filename: An icon name to an animated format
        :param qt.QObject parent: Parent of the QObject
        :raises: ValueError when name is not known
        """
        AbstractAnimatedIcon.__init__(self, parent)

        qfile = getQFile(filename)
        self.__movie = qt.QMovie(qfile.fileName(), qt.QByteArray(), parent)
        self.__movie.setCacheMode(qt.QMovie.CacheAll)
        self.__movie.frameChanged.connect(self.__frameChanged)
        self.__cacheIcons = {}

        self.__movie.jumpToFrame(0)
        self.__updateIconAtFrame(0)

    def __frameChanged(self, frameId):
        """Callback everytime the QMovie frame change
        :param int frameId: Current frame id
        """
        self.__updateIconAtFrame(frameId)

    def __updateIconAtFrame(self, frameId):
        """
        Update the current stored QIcon

        :param int frameId: Current frame id
        """
        if frameId in self.__cacheIcons:
            icon = self.__cacheIcons[frameId]
        else:
            icon = qt.QIcon(self.__movie.currentPixmap())
            self.__cacheIcons[frameId] = icon
        self._setCurrentIcon(icon)

    def _updateState(self):
        """Update the movie play according to internal stat of the
        MovieAnimatedIcon."""
        self.__movie.setPaused(not self.hasRegistredObjects())

class MultiImageAnimatedIcon(AbstractAnimatedIcon):
    """Store a looping QMovie to provide icons for each frames.
    Provides an event with the new icon everytime the movie frame
    is updated."""

    def __init__(self, filename, parent=None):
        """Constructor

        :param str filename: An icon name to an animated format
        :param qt.QObject parent: Parent of the QObject
        :raises: ValueError when name is not known
        """
        AbstractAnimatedIcon.__init__(self, parent)

        self.__frames = []
        for i in range(100):
            try:
                frame_filename = os.sep.join((filename, ("%02d" %i)))
                frame_file = getQFile(frame_filename)
            except ValueError:
                break
            try:
                icon = qt.QIcon(frame_file.fileName())
            except ValueError:
                break
            self.__frames.append(icon)

        if len(self.__frames) == 0:
            raise ValueError("Animated icon '%s' do not exists" % filename)

        self.__frameId = -1
        self.__timer = qt.QTimer(self)
        self.__timer.timeout.connect(self.__increaseFrame)
        self.__updateIconAtFrame(0)

    def __increaseFrame(self):
        """Callback called every timer timeout to change the current frame of
        the animation
        """
        frameId = (self.__frameId + 1) % len(self.__frames)
        self.__updateIconAtFrame(frameId)

    def __updateIconAtFrame(self, frameId):
        """
        Update the current stored QIcon

        :param int frameId: Current frame id
        """
        self.__frameId = frameId
        icon = self.__frames[frameId]
        self._setCurrentIcon(icon)

    def _updateState(self):
        """Update the object to wake up or sleep it according to its use."""
        if self.hasRegistredObjects():
            if not self.__timer.isActive():
                self.__timer.start(100)
        else:
            if self.__timer.isActive():
                self.__timer.stop()

def getWaitIcon():
    """Returns a cached version of the waiting AbstractAnimatedIcon.

    :rtype: AbstractAnimatedIcon
    """
    return getAnimatedIcon("process-working")

def getAnimatedIcon(name):
    """Create an AbstractAnimatedIcon from a resource name.

    The resource name can be prefixed by the name of a resource directory. For
    example "silx:foo.png" identify the resource "foo.png" from the resource
    directory "silx".

    If no prefix are specified, the file with be returned from the silx
    resource directory with a specific path "gui/icons".

    See also :func:`silx.resources.register_resource_directory`.

    Try to load a mng or a gif file, then try to load a multi-image animated
    icon.

    In Qt5 mng or gif are not used, because the transparency is not very well
    managed.

    :param str name: Name of the icon, in one of the defined icons
                     in this module.
    :return: Corresponding AbstractAnimatedIcon
    :raises: ValueError when name is not known
    """
    key = name + "__anim"
    cached_icons = getIconCache()
    if key not in cached_icons:

        qtMajorVersion = int(qt.qVersion().split(".")[0])
        icon = None

        # ignore mng and gif in Qt5
        if qtMajorVersion != 5:
            try:
                icon = MovieAnimatedIcon(name)
            except ValueError:
                icon = None

        if icon is None:
            try:
                icon = MultiImageAnimatedIcon(name)
            except ValueError:
                icon = None

        if icon is None:
            raise ValueError("Not an animated icon name: %s", name)

        cached_icons[key] = icon
    else:
        icon = cached_icons[key]
    return icon

def getQIcon(name):
    """Create a QIcon from its name.

    The resource name can be prefixed by the name of a resource directory. For
    example "silx:foo.png" identify the resource "foo.png" from the resource
    directory "silx".

    If no prefix are specified, the file with be returned from the silx
    resource directory with a specific path "gui/icons".

    See also :func:`silx.resources.register_resource_directory`.

    :param str name: Name of the icon, in one of the defined icons
                     in this module.
    :return: Corresponding QIcon
    :raises: ValueError when name is not known
    """
    cached_icons = getIconCache()
    if name not in cached_icons:
        qfile = getQFile(name)
        icon = qt.QIcon(qfile.fileName())
        cached_icons[name] = icon
    else:
        icon = cached_icons[name]
    return icon

def getQPixmap(name):
    """Create a QPixmap from its name.

    The resource name can be prefixed by the name of a resource directory. For
    example "silx:foo.png" identify the resource "foo.png" from the resource
    directory "silx".

    If no prefix are specified, the file with be returned from the silx
    resource directory with a specific path "gui/icons".

    See also :func:`silx.resources.register_resource_directory`.

    :param str name: Name of the icon, in one of the defined icons
                     in this module.
    :return: Corresponding QPixmap
    :raises: ValueError when name is not known
    """
    qfile = getQFile(name)
    return qt.QPixmap(qfile.fileName())

def getQFile(name):
    """Create a QFile from an icon name. Filename is found
    according to supported Qt formats.

    The resource name can be prefixed by the name of a resource directory. For
    example "silx:foo.png" identify the resource "foo.png" from the resource
    directory "silx".

    If no prefix are specified, the file with be returned from the silx
    resource directory with a specific path "gui/icons".

    See also :func:`silx.resources.register_resource_directory`.

    :param str name: Name of the icon, in one of the defined icons
                     in this module.
    :return: Corresponding QFile
    :rtype: qt.QFile
    :raises: ValueError when name is not known
    """
    global _supported_formats
    if _supported_formats is None:
        _supported_formats = []
        supported_formats = qt.supportedImageFormats()
        order = ["mng", "gif", "svg", "png", "jpg"]
        for format_ in order:
            if format_ in supported_formats:
                _supported_formats.append(format_)
        if len(_supported_formats) == 0:
            _logger.error("No format supported for icons")
        else:
            _logger.debug("Format %s supported", ", ".join(_supported_formats))

    for format_ in _supported_formats:
        format_ = str(format_)
        filename = silx.resources._resource_filename('%s.%s' % (name, format_),
                                                     default_directory='gui/icons')
        qfile = qt.QFile(filename)
        if qfile.exists():
            return qfile
        _logger.debug("File '%s' not found.", filename)
    raise ValueError('Not an icon name: %s' % name)


import requests
import pandas
from bs4 import BeautifulSoup
import json

url = 'https://www.imdb.com/chart/top/'

response = requests.get(url).content

soup = BeautifulSoup(response,'html.parser')

title = soup.find_all('td', class_='titleColumn')

rating = soup.find_all('strong')

images = soup.find_all('img')

movie_name = []
movie_year =[]
movie_href =[]
movie_image = []
movie_rating = []

for t in title:
    imdb_title_num = t.a.get('href').split('/')[2]
    href = 'https://www.imdb.com/title/'+imdb_title_num
    movie_href.append(href)
    imdb_title = t.a.text
    movie_name.append(imdb_title)
    year = t.span.text
    movie_year.append(year)

for rate in rating:
    r = rate.text
    movie_rating.append(r)

for img in images:
    i = img.get('src')
    movie_image.append(i)

model = pandas.DataFrame({'title': movie_name, 'year': movie_year, 'rating': movie_rating, 'image': movie_image, 'href': movie_href})
model.to_json('movies_data.json', orient="records")


# -*- coding: utf-8 -*-
import scrapy
from LaGou.items import LagouItem
import LaGou.settings as settings

class LagouSpider(scrapy.Spider):
    name = 'lagou'
    allowed_domains = ['https://www.lagou.com']
    start_urls = ['http://https://www.lagou.com/']

    def parse(self, response):
        if response.status==200:
            items=response.css('ul .con_list_item')
            data=LagouItem()
            for item in items:
                data['position']=item.css('.p_top .position_link h3::text').extract_first()
                data['company']=item.css('.company_name a::text').extract_first()
                data['sadd']=item.css('.p_top .add em::text').extract_first()
                data['salary']=item.css('.li_b_l .money::text').extract_first()
                data['claim']=item.css('.p_bot .li_b_l::text').extract()[-1].strip()
                # data['tags']=item.css('.list_item_bot .li_b_l span::text').extract()
                data['joburl']=item.css('.p_top .position_link::attr(href)').extract_first()
                yield data

    def start_requests(self):
        for page in range(3,settings.MAX_PAGE+1):
            url='https://www.lagou.com/jobs/list_%s?city=%s&cl=false&fromSearch=true'%(settings.KEY,settings.CITY)   #输入uid
            yield scrapy.Request(url=url,callback=self.parse,meta={'page':page},dont_filter=True)


import math
import random

import numba as nb
import numpy as np

@nb.jit(nopython=True)
def dimension_selector_uniform(n_dimensions):
    return random.randrange(n_dimensions)

def get_dimension_selector_expovariate(
    lambd=None,
    rel_lambd=None,
):
    if lambd is not None and rel_lambd is not None:
        raise ValueError("Cannot set both lambd and rel_lambd")

    if lambd is None and rel_lambd is None:
        # the default, using a rel_lambd of 4.0, placing the pseudo-mean
        # 1/4 of the way into the dimensions
        rel_lambd = 4.

    @nb.jit(nopython=True)
    def dimension_selector_expovariate(n_dimensions):
        nonlocal lambd, rel_lambd

        value = math.inf
        while value >= n_dimensions:
            value = random.expovariate(
                lambd if lambd is not None else rel_lambd/n_dimensions
            )
        return int(value)

    return dimension_selector_expovariate

def get_finder_for_cluster_obeying(
    check_func,
    min_count=1,
    max_count=-1,
    max_depth=-1,
    dimension_selector=dimension_selector_uniform,
    fixed_dimensional_parameters=-1,
    fixed_non_dimensional_parameters=-1,
    fixed_n=-1,
    verbose=False,
    jit_kwargs={},
):
    @nb.jit(nopython=True, **jit_kwargs)
    def _find_cluster_obeying(
        dimensional_parameters,
        non_dimensional_parameters,
        random_seed=None,
        iterations=-1,
    ):
        if dimensional_parameters.shape[1] != non_dimensional_parameters.shape[0]:
            raise ValueError(
                "Minor dimension of dimensional_parameters must match "
                "major dimension of non_dimensional_parameters"
            )

        if (
            fixed_dimensional_parameters != -1
            and fixed_dimensional_parameters != dimensional_parameters.shape[0]
        ):
            raise ValueError("Number of dimensional parameters not expected value")

        if (
            fixed_non_dimensional_parameters != -1
            and fixed_non_dimensional_parameters != non_dimensional_parameters.shape[1]
        ):
            raise ValueError("Number of non-dimensional parameters not expected value")

        if (
            fixed_n != -1
            and fixed_n != non_dimensional_parameters.shape[0]
        ):
            raise ValueError("Number of candidates not expected value")

        final_max_depth = max_depth if max_depth != -1 else (1 + int(
            math.floor(math.log(dimensional_parameters.shape[-1]) / math.log(2))
        ))

        if final_max_depth < 2:
            raise ValueError("max_depth < 2 makes no sense")

        if random_seed is not None:
            random.seed(random_seed)

        bitmap_stack = np.zeros(
            (final_max_depth, dimensional_parameters.shape[-1]),
            dtype=np.bool_,
        )
        bitmap_stack[0,:] = True
        right_branch_stack = np.zeros((final_max_depth,), dtype=np.int8)

        current_level = 1
        iteration = 0

        while True:
            if right_branch_stack[current_level] == 0:
                chosen_dimension = dimension_selector(dimensional_parameters.shape[0])
                vals_count = 0
                # initialize these to any value of the correct type
                vals_min = vals_max = dimensional_parameters[0,0]
                # scan for range of remaining values in this dimension
                for i in range(dimensional_parameters.shape[1]):
                    if bitmap_stack[current_level-1,i]:
                        v = dimensional_parameters[chosen_dimension,i]

                        if vals_count == 0:
                            vals_min = vals_max = v
                        else:
                            vals_min = min(vals_min, v)
                            vals_max = max(vals_max, v)

                        vals_count+=1

                chosen_split_point = random.uniform(vals_min, vals_max)

                # mark values greater than threshold
                remaining_count = 0
                for i in range(dimensional_parameters.shape[1]):
                    if bitmap_stack[current_level-1,i]:
                        is_chosen = (
                            dimensional_parameters[chosen_dimension,i] >= chosen_split_point
                        )
                        bitmap_stack[current_level,i] = is_chosen
                        if is_chosen:
                            remaining_count+=1
            elif right_branch_stack[current_level] == 1:
                # invert current_level's bitmap, masked by the previous level's
                remaining_count = 0
                for i in range(bitmap_stack.shape[1]):
                    if bitmap_stack[current_level-1,i]:
                        is_chosen = not bitmap_stack[current_level,i]
                        bitmap_stack[current_level,i] = is_chosen
                        if is_chosen:
                            remaining_count+=1
            else:
                # tidy up then unwind
                right_branch_stack[current_level] = 0
                bitmap_stack[current_level,:] = False
                if current_level > 1:
                    current_level-=1
                    # advance branch at underlying level
                    right_branch_stack[current_level]+=1
                else:
                    # we're at the root
                    iteration+=1
                    if iterations != -1 and iteration >= iterations:
                        return None

                    # start again by continuing at the
                    # same current_level

                continue

            if verbose:
                print("current_level = ", current_level, " remaining_count = ", remaining_count)

            if remaining_count < min_count:
                right_branch_stack[current_level]+=1
                continue

            if max_count == -1 or remaining_count <= max_count:
                ndp_subset = np.empty(
                    (remaining_count, non_dimensional_parameters.shape[1],),
                    dtype=non_dimensional_parameters.dtype,
                )

                j = 0
                for i in range(bitmap_stack.shape[1]):
                    if j < remaining_count and bitmap_stack[current_level,i]:
                        for k in range(ndp_subset.shape[1]):
                            ndp_subset[j,k] = non_dimensional_parameters[i,k]
                        j+=1

                check_result = check_func(ndp_subset)
                if check_result:
                    if check_result > 0:
                        return bitmap_stack[current_level,:]
                    else:
                        # negative result signals to stop checking this branch
                        right_branch_stack[current_level]+=1
                        continue

            if remaining_count <= 1:
                # dividing any more makes no sense
                right_branch_stack[current_level]+=1
                continue

            if current_level+1 >= final_max_depth:
                # can't descend any deeper
                right_branch_stack[current_level]+=1
                continue

            current_level+=1

    return _find_cluster_obeying