import functools
import os
from contextlib import nullcontext
from os.path import join, expanduser

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
from sacred import Experiment
from sacred import SETTINGS
from sacred.observers import FileStorageObserver
from scipy.optimize import linear_sum_assignment
from sklearn.metrics import pairwise_distances
from torch import nn
from torch.optim import Adam
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import DataLoader

from gsoftmax.lagrangian.models import CNNPos, Sum, EpsilonLR
from gsoftmax.lagrangian.datasets import SyntheticSMLMDataset, SMLMDataset
from gsoftmax.lagrangian.postprocessing import cluster_and_trim
from gsoftmax.lagrangian.sinkhorn import MeasureDistance

SETTINGS.HOST_INFO.INCLUDE_GPU_INFO = False

base_dir = expanduser('~/output/g-softmax/lagrangian')
exp = Experiment('vae')
exp.observers.append(FileStorageObserver.create(base_dir))

@exp.config
def system():
    device = 0
    seed = 200
    checkpoint = None
    log_interval = 100

    n_jobs = 8

    test_only = False
    train_only = False

@exp.config
def base():
    test_source = 'MT0.N1.LD'
    modality = 'AS'
    dimension = 3

    measure = 'sinkhorn'
    p = 2
    q = 2
    sigma = [1]
    epsilon = 1e-2
    rho = 1

    beads = 256

    zero = 1e-16

    architecture = 'resnet'
    batch_norm = False

    lifted = False
    batch_size = 512
    train_size = int(1024 * 1024)
    eval_size = 2048
    test_size = None

    mass_norm = True

    lr = 1e-4
    gamma = 1

    n_epochs = 100

    repeat = False

    epsilon_gamma = 1.

@exp.named_config
def sinkhorn():
    """ 0.93 eval accuracy"""
    measure = 'sinkhorn'
    sigma = [0.01, 0.02, 0.04, 0.12]
    epsilon = 1
    p = 1
    q = 1
    rho = 1
    epsilon_gamma = 1

    batch_size = 512
    train_size = int(1024 * 512)

    lr = 1e-4
    epoch = 100

    architecture = 'deep_loco'
    mass_norm = True

@exp.named_config
def sinkhorn2():
    measure = 'sinkhorn'
    sigma = [1]
    epsilon = 1e-2
    rho = 1
    epsilon_gamma = 1

    batch_size = 512
    train_size = int(1024 * 512)

    lr = 1e-4
    epoch = 100

    architecture = 'deep_loco'
    mass_norm = True

@exp.named_config
def right_hausdorff():
    measure = 'right_hausdorff'
    epsilon = 1e-1
    epsilon_gamma = .5
    gamma = 1
    rho = None

    batch_size = 512
    train_size = int(1024 * 128)
    test_size = 512

    lr = 1e-4

    architecture = 'resnet'
    mass_norm = True
    lifted = False

@exp.named_config
def lifted():
    measure = 'right_hausdorff'
    epsilon = 1
    epsilon_gamma = .5
    gamma = 1
    rho = None

    batch_size = 512
    train_size = int(1024 * 128)
    test_size = 512

    lr = 1e-4

    seed = 98

    architecture = 'deep_loco'
    mass_norm = False
    lifted = False
    zero = 1e-16

@exp.named_config
def mmd():
    measure = 'mmd'
    p = 1
    q = 1
    sigma = [0.01, 0.02, 0.04, 0.12]
    gamma = .5
    zero = 0
    batch_size = 1024
    train_size = 1024 * 1024
    n_epochs = 100
    lr = 1e-4
    epoch = 10

    architecture = 'deep_loco'
    batch_norm = False
    mass_norm = False

@exp.named_config
def single_batch():
    device = 0
    batch_size = 4
    train_size = int(4)
    eval_size = 4
    test_size = 4
    seed = 17
    repeat = True
    train_only = True

@exp.named_config
def test_only():
    device = 'cpu'
    test_only = True
    checkpoint = None

class ClampedModelLoss(nn.Module):
    def __init__(self, model, loss_fn, zero=1e-7):
        super().__init__()
        self.model = model
        self.loss_fn = loss_fn
        self.zero = zero

    def forward(self, imgs, positions, weights):
        pred_positions, pred_weights = self.model(imgs)
        weights = torch.clamp(weights, min=self.zero)
        return self.loss_fn(pred_positions, pred_weights,
                            positions, weights), pred_positions, pred_weights

def plot_example(datasets, output_dir):
    fig, axes = plt.subplots(1, 3, figsize=(7, 3))

    for ax, (fold, dataset) in zip(axes, datasets.items()):
        img, positions, weights = dataset[np.random.randint(0, len(dataset))]
        c, m, n = img.shape
        ax.imshow(img[0])
        ax.scatter(positions[:, 0] * m,
                   positions[:, 1] * n,
                   s=weights * 10, color='red')
        ax.set_xlim([0, m])
        ax.set_ylim([0, n])
        ax.set_title(f'{fold} example')
        ax.axis('off')
    plt.savefig(join(output_dir, 'examples.png'))
    plt.close(fig)

@exp.capture
def plot_pred_ground_truth(pred_positions, pred_weights,
                           positions, weights,
                           img, filename, zero):
    fig, axes = plt.subplots(1, 2, figsize=(5, 5))
    c, m, n = img.shape
    for ax, pos, w in zip(axes, (pred_positions, positions),
                          (pred_weights, weights)):
        pos = pos[w > zero]
        w = w[w > zero]
        ax.imshow(img[0])
        ax.scatter(pos[:, 0] * m, pos[:, 1] * n,
                   s=w * 10, color='red')
        ax.set_xlim([0, 64])
        ax.set_ylim([0, 64])
        ax.axis('off')

    plt.savefig(filename)
    plt.close(fig)

@exp.capture
def metrics(pred_positions, pred_weights, positions, weights,
            reduction='mean', threshold=100):
    dim = pred_positions.shape[2]

    jaccards = []
    rmses_xy = []
    rmses_z = []

    for ppos, pweight, pos, weight in zip(pred_positions, pred_weights,
                                          positions, weights):
        ppos = ppos[pweight > 0]

        pos = pos[weight > 0]

        if len(pos) == 0 or len(ppos) == 0:
            rmse_xy = 0.
            rmse_z = 0.
            if len(pos) == 0 and len(ppos) > 0:
                tp = 0
                fp = len(ppos)
                fn = 0
                jaccard = 0.
            elif len(ppos) == 0 and len(pos) > 0:
                fn = len(ppos)
                tp = 0
                fp = 0
                jaccard = 0.
            else:
                fp = fn = tp = 0
                jaccard = 1.
        else:
            cost_matrix = pairwise_distances(pos, ppos, metric='euclidean')
            row_ind, col_ind = linear_sum_assignment(cost_matrix)

            matched_cost = cost_matrix[row_ind, col_ind]
            mask = matched_cost < threshold
            tp = mask.sum()
            fn = len(mask) - tp
            non_matched_mask = np.ones(cost_matrix.shape[1], dtype=np.bool)
            non_matched_mask[col_ind] = 0
            non_matched_cost = cost_matrix[:, non_matched_mask]
            fp = np.all(non_matched_cost > threshold, axis=0).sum()
            jaccard = float(tp) / (fn + fp + tp)

            if dim == 2:
                if tp > 0:
                    rmse_xy = np.sqrt((matched_cost[mask] ** 2).mean())
                else:
                    rmse_xy = 0.
                rmse_z = 0.
            else:
                cost_matrix_xy = pairwise_distances(pos[:, :2],
                                                    ppos[:, :2],
                                                    metric='euclidean')
                cost_matrix_z = pairwise_distances(pos[:, 2:3],
                                                   ppos[:, 2:3],
                                                   metric='euclidean')
                if tp > 0:
                    rmse_xy = np.sqrt(
                        (cost_matrix_xy[row_ind, col_ind][mask]
                         ** 2).mean())
                    rmse_z = np.sqrt((cost_matrix_z[row_ind, col_ind][mask]
                                      ** 2).mean())
                else:
                    rmse_xy = 0.
                    rmse_z = 0.
        jaccards.append(jaccard)
        # print(tp, fn, fp)
        rmses_xy.append(rmse_xy)
        rmses_z.append(rmse_z)

    jaccard = torch.tensor(jaccards)
    rmse_xy = torch.tensor(rmses_xy)
    rmse_z = torch.tensor(rmses_z)
    if reduction == 'mean':
        jaccard = jaccard.mean()
        rmse_xy = rmse_xy.mean()
        rmse_z = rmse_z.mean()
    elif reduction == 'sum':
        jaccard = jaccard.sum()
        rmse_xy = rmse_xy.sum()
        rmse_z = rmse_z.sum()
    return jaccard, rmse_xy, rmse_z

def worker_init_fn(worker_id, offset):
    np.random.seed((torch.initial_seed() + worker_id + offset) % (2 ** 31))

def save_checkpoint(model, optimizer, scheduler, filename):
    state_dict = {'model': model.state_dict(),
                  'optimizer': optimizer.state_dict(),
                  'scheduler': scheduler.state_dict()}
    torch.save(state_dict, filename)

def load_checkpoint(filename, model, optimizer=None, scheduler=None):
    state_dict = torch.load(filename)
    model.load_state_dict(state_dict['model'])
    if optimizer is not None:
        optimizer.load_state_dict(state_dict['optimizer'])
    if scheduler is not None:
        scheduler.load_state_dict(state_dict['scheduler'])

@exp.capture
def train_eval_loop(model_loss, loader, fold, epoch, output_dir, zero,
                    device, log_interval, _run, optimizer=None,
                    ):
    records = dict(loss=0.)
    records['jaccard'] = 0.
    records['rmse_xy'] = 0.
    records['rmse_z'] = 0.
    n_samples = 0
    if fold == 'train':
        model_loss.train()
    else:
        model_loss.eval()
    offset, scale, _, _, _ = loader.dataset.get_geometry()
    offset = offset.to(device)
    scale = scale.to(device)
    if fold == 'test':
        all_preds = []
    with torch.no_grad() if not fold == 'train' else nullcontext():
        for batch_idx, (imgs, positions, weights) in enumerate(loader):
            batch_size = imgs.shape[0]

            imgs = imgs.to(device)
            positions = positions.to(device)
            weights = weights.to(device)
            if fold == 'train':
                optimizer.zero_grad()
            loss, pred_positions, pred_weights = model_loss(imgs, positions,
                                                            weights)
            if fold == 'train':
                loss.backward()

            if fold != 'train' or batch_idx == 0:
                # For plot
                if batch_idx == 0:
                    plot_pred_ground_truth(pred_positions[0].detach().cpu().numpy(),
                                           pred_weights[0].detach().cpu().numpy(),
                                           positions[0].detach().cpu().numpy(),
                                           weights[0].detach().cpu().numpy(),
                                           imgs[0].detach().cpu().numpy(), join(output_dir,
                                                f'img_{fold}_e_{epoch}.png'))

                pred_positions = pred_positions * scale
                positions = positions * scale
                pred_positions = pred_positions + offset
                positions = positions + offset
                # Postprocess
                pred_positions, pred_weights = cluster_and_trim(pred_positions,
                                                                pred_weights,
                                                                zero, 100,
                                                                1e-1)
                pred_positions = pred_positions.detach().cpu().numpy()
                pred_weights = pred_weights.detach().cpu().numpy()
                positions = positions.detach().cpu().numpy()
                weights = weights.detach().cpu().numpy()

                if fold != 'train' or batch_idx == 0:
                    size = len(loader.dataset) if fold == 'train' else batch_size
                    jaccard, rmse_xy, rmse_z = metrics(
                        pred_positions, pred_weights, positions, weights)
                    records['jaccard'] += jaccard * size
                    records['rmse_xy'] += rmse_xy * size
                    records['rmse_z'] += rmse_z * size

                if fold == 'test':
                    frames = np.repeat(np.arange(n_samples + 1,
                                                 batch_size + n_samples + 1,
                                                 dtype=np.float32)[:, None, None],
                                       pred_positions.shape[1], axis=1)
                    preds = np.concatenate([frames, pred_positions], axis=2)
                    preds = preds[pred_weights != 0]
                    all_preds.append(preds)

            records['loss'] += loss.item() * batch_size
            n_samples += batch_size
            if fold == 'train':
                optimizer.step()
                if batch_idx % log_interval == 0:
                    print(f'{fold} epoch: {epoch} '
                          f'[{batch_idx * batch_size}/{len(loader.dataset)} '
                          f'({100. * batch_idx / len(loader):.0f}%)]\t'
                          f'loss: {loss.item():.6f}')
    if fold == 'test':
        all_preds = np.concatenate(all_preds, axis=0)
        if all_preds.shape[1] == 4:
            columns = ['frame', 'xnano', 'ynano', 'znano']
        else:
            columns = ['frame', 'xnano', 'ynano']
        df = pd.DataFrame(data=all_preds, columns=columns)
        df['frame'] = df['frame'].astype('int32')
        df['intensity'] = 1
        df.to_csv(join(output_dir, 'pred_activations.csv'))
    print(f"=================> {fold} epoch: {epoch}", flush=False)
    for name, record in records.items():
        print(f'{fold}.{name} : {record / n_samples} ', flush=False)
        _run.log_scalar(f'{fold}.{name}', record / n_samples, epoch)
    print('')

@exp.main
def main(test_source, train_size, n_jobs,
         eval_size, batch_size, n_epochs, checkpoint,
         test_only, dimension, test_size, architecture,
         measure, sigma, epsilon, rho, lr, zero, p, q, gamma,
         epsilon_gamma, lifted,
         batch_norm, train_only, beads, repeat,
         modality, device, _seed, _run):
    output_dir = join(base_dir, str(_run._id), 'artifacts')
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    torch.manual_seed(_seed)
    device = torch.device(f"cuda:{device}" if isinstance(device, int)
                          else "cpu")

    lengths = {'train': train_size, 'eval': eval_size}
    datasets = {}
    loaders = {}

    datasets['test'] = SMLMDataset(name=test_source, modality=modality,
                                   dimension=dimension,
                                   return_activation=False, length=test_size)
    loaders['test'] = DataLoader(datasets['test'], batch_size=batch_size,
                                 num_workers=1)
    # Use geometrical information to construct synthetics datasets
    offset, scale, shape, max_beads, w_range = datasets['test'].get_geometry()
    print(f'Dataset info: offset {offset}, scale {scale}, shape {shape},'
          f' max_beads {max_beads}, w_range {w_range}')
    for fold in ['train', 'eval']:
        datasets[fold] = SyntheticSMLMDataset(max_beads=max_beads, noise=100,
                                              length=lengths[fold],
                                              offset=offset,
                                              scale=scale,
                                              shape=shape,
                                              psf_radius=2700,
                                              w_range=w_range,
                                              batch_size=1,
                                              dimension=dimension,
                                              return_activation=False,
                                              modality=modality)
        loaders[fold] = DataLoader(datasets[fold],
                                   batch_size=batch_size,
                                   worker_init_fn=functools.partial(
                                       worker_init_fn,
                                       offset=1000 * (fold == 'eval')),
                                   num_workers=n_jobs,
                                   pin_memory=torch.device.type == 'cuda',
                                   shuffle=False)

    model = CNNPos(beads=beads, dimension=dimension, batch_norm=batch_norm,
                   architecture=architecture, zero=zero)

    if len(sigma) > 1:
        loss_fn = Sum(MeasureDistance(measure=measure, p=p, q=q,
                                      max_iter=100,
                                      sigma=this_sigma,
                                      epsilon=epsilon, rho=rho,
                                      lifted=lifted,
                                      reduction='mean')
                      for this_sigma in sigma)
        epsilon_scheduler = None
        # Desactivated for now
    else:
        if hasattr(sigma, '__iter__'):
            sigma = sigma[0]
        loss_fn = MeasureDistance(measure=measure, p=p, q=q,
                                  max_iter=100,
                                  sigma=sigma, lifted=lifted,
                                  epsilon=epsilon, rho=rho,
                                  reduction='mean')
        epsilon_scheduler = EpsilonLR(loss_fn, step_size=10,
                                      gamma=epsilon_gamma, min_epsilon=1e-4)
    loss_model = ClampedModelLoss(model, loss_fn, zero=zero)

    if not test_only:
        optimizer = Adam(loss_model.parameters(), lr=lr, amsgrad=True)
        scheduler = StepLR(optimizer, step_size=1, gamma=gamma)
    else:
        n_epochs = 1
        optimizer = None
        scheduler = None

    if checkpoint is not None:
        load_checkpoint(checkpoint, model, optimizer=optimizer,
                        scheduler=scheduler)

    loss_model.to(device)

    for epoch in range(n_epochs):
        if repeat:
            torch.manual_seed(_seed)
        else:
            torch.manual_seed(_seed + epoch)
        if not train_only:
            train_eval_loop(loss_model, loaders['eval'], 'eval', epoch,
                            output_dir=output_dir)
            train_eval_loop(loss_model, loaders['test'], 'test', epoch,
                            output_dir=output_dir)
        if not test_only:
            scheduler.step(epoch)
            if epsilon_scheduler is not None:
                epsilon_scheduler.step(epoch)
            train_eval_loop(loss_model, loaders['train'], 'train', epoch,
                            optimizer=optimizer,
                            output_dir=output_dir)
            if epoch % 10 == 0:
                save_checkpoint(model, optimizer, scheduler,
                                join(output_dir, f'checkpoint.pkl'))

if __name__ == '__main__':
    exp.run_commandline()


"""Pullers Collection"""
# -*- coding: utf-8 -*-
from __future__ import annotations
import os
import asyncio
from asyncio import Future
from typing import Iterable, Mapping, Any, Callable, Awaitable, Sequence, TypeVar, TYPE_CHECKING
from abc import abstractmethod
from httpx import Request, Response, Timeout
from utils import DummyAioFileStream as Dummyf, EventHint as Hint, LoggerLike, to_ordinal
from utils import ActionChain, AsyncActionChain, EventHook, StrChain
from utils import ensure_async
from httpx._types import HeaderTypes, ProxiesTypes, CookieTypes, QueryParamTypes
import aiofiles as aiof
import httpx

if TYPE_CHECKING:
    import rich.progress

__all__ = [
    "Request",
    "Response",
    "HeaderTypes",
    "ProxiesTypes",
    "CookieTypes",
    "QueryParamTypes",
    "HttpActionType",
    "AsyncPuller",
    "AsyncPullerEventHook",
    "AsyncActionChain",
    "AsyncHttpActionChain",
    "MaxRetryReached",
    "Modifier",
]

class MaxRetryReached(Exception):
    """Raised when the maximum number of retries is reached."""

class BasePuller:
    """Base class for pullers."""
    @property
    def event_hooks(self) -> EventHook:
        """Event hook for this puller."""
        raise NotImplementedError()

    @abstractmethod
    def pull(self, url: str, path: str) -> Any:
        """Pull the file from the url to the path."""

class BaseWorker:
    """Base class for workers."""

class BaseMaster:
    """Base class for masters."""

HttpActionType = Callable[[Request | Response], Awaitable]
ActionVar = TypeVar('ActionVar', bound=Callable)

class AsyncHttpActionChain(ActionChain[HttpActionType]):
    """Async Http Action Chain"""

    def __init__(self, actions: Iterable[HttpActionType] = None) -> None:
        super().__init__(actions=actions)

class AsyncPullerEventHook(EventHook):
    """Async Puller Event Hook"""

    def __init__(self, chain: Mapping[str, Iterable[Callable[..., Any]]] | None = None):
        self.hook("request", AsyncHttpActionChain())
        self.hook("response", AsyncHttpActionChain())
        self.hook("worker.spawn", AsyncActionChain())
        self.hook("worker.destroy", AsyncActionChain())
        self.hook("worker.start", AsyncActionChain())
        self.hook("worker.response_get", AsyncActionChain())
        self.hook("worker.bytes_get", AsyncActionChain())
        self.hook("worker.retry", AsyncActionChain())
        self.hook("worker.success", AsyncActionChain())
        self.hook("worker.fail", AsyncActionChain())
        self.hook("puller.spawn", AsyncActionChain())
        self.hook("puller.destroy", AsyncActionChain())
        self.hook("puller.join", AsyncActionChain())
        EventHook.__init__(self, chain=chain)

class AsyncWorker(BaseWorker):
    def __init__(
        self,
        puller: AsyncPuller,
        url: str,
        path: str | None,
        retry: int,
        overwrite: bool,
        *,
        future: Future | None = None,
        timeout: Timeout | float | None,
        extra_headers: HeaderTypes | None,
        extra_params: QueryParamTypes | None,
        extra_cookies: CookieTypes | None,
        **kw
    ):
        self.puller = puller
        self.url = url
        self.path = path
        self.future = future
        self.timeout = timeout
        self.max_retry = retry
        self.overwrite = overwrite
        self.extra_headers = extra_headers
        self.extra_params = extra_params
        self.extra_cookies = extra_cookies
        self.kw = kw
        self.event_hooks = AsyncPullerEventHook(
            self.puller.event_hooks)  # snapshot

    async def run(self):
        """Run the worker."""
        event_hooks = self.event_hooks
        try:
            await event_hooks.aemit("worker.start", self)
            if self.path and os.path.exists(self.path) and not self.overwrite:
                raise FileExistsError(f"{self.path} already exists")
            client = self.puller.client
            retry = 0
            while True:
                try:
                    # Open file in async mode, if path is None, write to void
                    async with aiof.open(self.path, "wb") if self.path else Dummyf() as f:
                        # Establish connection
                        async with client.stream(
                            method=self.kw.pop("method", "GET"),
                            url=self.url,
                            params=self.extra_params,
                            headers=self.extra_headers,
                            cookies=self.extra_cookies,
                            timeout=self.timeout or self.puller.client.timeout,
                            **self.kw
                        ) as r:
                            await event_hooks.aemit("worker.response_get", self, r)
                            # Iterate over response chunks
                            # 256KB
                            async for chunk in r.aiter_bytes(chunk_size=2**18):
                                await event_hooks.aemit("worker.bytes_get", self, r, chunk)
                                await f.write(chunk)
                            await f.flush()
                            await event_hooks.aemit("worker.success", self, r)
                    # set result for placeholder
                    self.future.set_result(self.path)
                    break  # Quit successfully
                # Retry on network IO error
                except (httpx.HTTPError, httpx.StreamError) as e:
                    retry += 1
                    if retry > self.max_retry:
                        raise MaxRetryReached(
                            f"Max retry reached: {retry} times") from e
                    await event_hooks.aemit("worker.retry", self, e, retry)
                    await asyncio.sleep(min(30, 1.7 ** retry))
        # Fatal Errors
        except (Exception, asyncio.CancelledError) as e:
            handled = await event_hooks.aemit("worker.fail", self, e)
            if True not in handled:
                self.future.cancel()
                raise e
        finally:
            await event_hooks.aemit("worker.destroy", self)
            self.puller._workers.get_nowait()
            self.puller._workers.task_done()  # workers count - 1

    def __repr__(self) -> str:
        return f"AsynWorker({self.url}, {self.path})"

class AsyncMaster(BaseMaster):
    def __init__(self, puller: AsyncPuller):
        self.puller = puller

    async def run(self):
        """Run the master."""
        try:
            buffer = self.puller._buffer
            workers = self.puller._workers
            while True:
                worker = await buffer.get()
                if worker is None:  # Kill signal
                    buffer.task_done()
                    break
                await workers.put(worker)  # blocks when max_workers reached
                self.puller._loop.create_task(worker.run())
                buffer.task_done()
                await asyncio.sleep(self.puller.interval)  # sleep a while
        finally:
            self.puller._master = None  # Reset master

class AsyncPuller(BasePuller):
    """### Async Puller"""

    def __init__(
        self,
        *,
        headers: HeaderTypes | None = None,
        params: QueryParamTypes | None = None,
        proxies: ProxiesTypes | None = None,
        cookies: CookieTypes | None = None,
        event_hooks: Mapping[str, Sequence[Callable]] = None,
        interval: float = 0.0,
        max_workers: int = 8,
        timeout: Timeout | float | None = 10,
        retry: int = 3,
        overwrite: bool = False,
        loop: asyncio.AbstractEventLoop | None = None,
        **kw
    ):
        """### Init Puller
        * `headers`: Default headers for all requests
        * `params`: Default query params for all requests
        * `proxies`: Default proxies for all requests
        * `cookies`: Default cookies for all requests
        * `event_hooks`: dict[str, Iterable[Callable]]
        * `interval`: Interval between each request
        * `max_workers`: Max downloading threads count
        * `timeout`: Timeout for each request
        * `retry`: Max retry times for each request
        * `overwrite`: Overwrite existing files
        * `loop`: Event loop
        * `**kw`: Other keyword arguments for httpx.Client
        """
        self._loop = loop
        self.interval = interval
        self.max_retry = max(retry, 0)
        self.overwrite = overwrite

        self._proxies = proxies
        self._master: AsyncMaster | None = None
        self._buffer: asyncio.Queue = asyncio.Queue()  # Pending workers
        self._workers: asyncio.Queue = asyncio.Queue(
            maxsize=max_workers)  # Running workers

        limits = httpx.Limits(
            max_connections=None,
            max_keepalive_connections=None,
            keepalive_expiry=10)

        self._client = httpx.AsyncClient(
            headers=headers,
            params=params,
            proxies=proxies,
            cookies=cookies,
            timeout=timeout,
            limits=limits,
            follow_redirects=True,
            http2=True,
            **kw
        )
        self.event_hooks = event_hooks or AsyncPullerEventHook()

        # Set up event hook
        def subscribe(event: StrChain, action: ActionVar) -> ActionVar:
            ev = str(event)
            self._event_hooks[ev].append(action)
            if ev in ["request", "response"]:
                self._client.event_hooks = self._event_hooks  # type: ignore[assignment]
            return action

        self._on = PullerEventHint(
            strchain=StrChain(joint='.', callback=subscribe))

    @property
    def client(self):
        return self._client

    @property
    def loop(self):
        return self._loop

    @property
    def headers(self):
        """Get headers."""
        return self._client.headers

    @headers.setter
    def headers(self, value: HeaderTypes):
        """Set headers for all requests."""
        self._client.headers = value  # type: ignore[assignment]

    @property
    def cookies(self):
        return self._client.cookies

    @cookies.setter
    def cookies(self, value: CookieTypes):
        """Set cookies for all requests."""
        self._client.cookies = value  # type: ignore[assignment]

    @property
    def params(self):
        return self._client.params

    @params.setter
    def params(self, value: QueryParamTypes):
        """Set params for all requests."""
        self._client.params = value  # type: ignore[assignment]

    @property
    def event_hooks(self):
        return self._event_hooks

    @event_hooks.setter
    def event_hooks(self, hooks: Mapping[str, Sequence[Callable]]):
        """Set event hooks for all requests."""
        self._event_hooks = AsyncPullerEventHook(hooks)  # deepcopy
        self._client.event_hooks = self._event_hooks  # type: ignore[assignment]

    @property
    def on(self):
        """Shortcut for `Puller.event_hooks.on`"""
        return self._on

    @property
    def proxies(self):
        return self._proxies

    async def pull(
        self,
        url: str,
        path: str | None,
        *,
        extra_headers: HeaderTypes | None = None,
        extra_params: QueryParamTypes | None = None,
        extra_cookies: CookieTypes | None = None,
        timeout: Timeout | float | None = 0,
        retry: int | None = None,
        overwrite: bool | None = None,
        **kw
    ) -> Future:
        """
        ### Asyncronously pull a file from a url.
        File will be decompressed and saved in binary mode.

        Returns a `Future` instance refer to the path of the downloaded file.
        * `url`: url to pull from
        * `path`: path to save to, set to `None` or `""` will not save file

        Optional Parameters:
        * `extra_headers`: extra headers to add to request
        * `extra_params`: extra params to add to request
        * `extra_cookies`: extra cookies to add to request
        * `timeout`: timeout for this request, set to None will be no limit, 0 for default
        * `retry`: retry times for this request, set to None will use default
        * `overwrite`: overwrite file if exists, set to None will use default
        * `**kw`: extra keyword arguments for httpx.stream
        """
        timeout = self.client.timeout if timeout == 0 else timeout
        self._loop = self.loop or asyncio.get_running_loop()
        if self._master is None:
            self._master = AsyncMaster(self)
            self._loop.create_task(self._master.run())
        future: Future = Future()  # A placeholder for the worker.run() task
        worker = AsyncWorker(
            self,
            url=url,
            path=path,
            extra_headers=extra_headers,
            extra_params=extra_params,
            extra_cookies=extra_cookies,
            future=future,
            timeout=timeout,
            retry=retry or self.max_retry,
            overwrite=overwrite or self.overwrite,
            **kw
        )
        await self._event_hooks.aemit("worker.spawn", worker)
        await self._buffer.put(worker)
        return future

    async def join(self) -> None:
        """### Wait for all workers to finish."""
        await self._event_hooks.aemit("puller.join", self)
        await self._buffer.join()  # Make sure no pending tasks
        await self._workers.join()  # Make sure all tasks are done

    async def aclose(self):
        await self.join()  # Make sure all tasks are done
        await self._buffer.put(None)  # Kill master
        await self._event_hooks.aemit("puller.destroy", self)
        await self._client.aclose()

    async def __aenter__(self) -> AsyncPuller:
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb) -> bool:
        await self.aclose()
        return False

    def __repr__(self):
        return f"{self.__class__.__name__}:\n" +\
            f"  max_retry: {self.max_retry}\n" +\
            f"  overwrite: {self.overwrite}\n" +\
            f"  headers: {self.headers}\n" +\
            f"  params: {self.params}\n" +\
            f"  cookies: {self.cookies}\n" +\
            f"  proxies: {self.proxies}\n" +\
            f"  event_hooks: {self.event_hooks}\n"

################### Useful Modifiers #####################

class Modifier:
    progress = None

    @classmethod
    def show_progress(
        cls,
        puller: BasePuller,
        desc_len: int = os.get_terminal_size().columns//3,
        delay_on_finish: float | None = 0.5,
        progress: rich.progress.Progress | None = None,
    ) -> None:
        """
        ### Show progress of all tasks.
        * `puller`: Puller instance
        * `desc_len`: max length of url description
        * `delay_on_finish`: time to remove finished progress bar, set to `None` to keep it
        * `progress`: rich.progress.Progress instance, set to `None` to use default one
        """
        from rich.progress import Progress
        if cls.progress is None:
            cls.progress = Progress(refresh_per_second=30)
        _progress = progress or cls.progress

        if isinstance(puller, AsyncPuller):
            @puller.on.worker.start
            async def create_task(event: str, worker: AsyncWorker):
                nonlocal puller
                progress = _progress  # In case outer progress is changed
                desc = worker.url.split("://")[-1]
                if len(desc) > desc_len:
                    desc = desc[:(desc_len-3)//2] + "..." + \
                        desc[-(desc_len-3)//2:]
                progress.start()  # start progress bar
                task = progress.add_task(desc)
                total = 0
                failed = False

                @worker.event_hooks.on.worker.response_get
                async def get_total(event: str, worker: AsyncWorker, r: Response):
                    nonlocal task
                    nonlocal total
                    total = int(r.headers.get("Content-Length", 0))
                    # type: ignore[arg-type]
                    progress.update(task, total=total or 100)

                @worker.event_hooks.on.worker.bytes_get
                async def update_task(ev: str, w: AsyncWorker, r: Response, b: bytes):
                    nonlocal task
                    nonlocal total
                    if total:
                        # type: ignore[arg-type]
                        progress.update(task, completed=r.num_bytes_downloaded)

                @worker.event_hooks.on.worker.fail
                async def fail_task(ev: str, w: AsyncWorker, e: Exception):
                    nonlocal failed
                    nonlocal task
                    failed = True
                    if isinstance(e, FileExistsError):
                        progress.remove_task(task)

                @worker.event_hooks.on.worker.destroy
                async def remove_task(event: str, worker: AsyncWorker):
                    nonlocal task
                    nonlocal total
                    if not failed:
                        if not total:  # Show cpmpleted for unknown size
                            progress.update(task, total=1, completed=1)
                        progress.refresh()
                        if delay_on_finish is not None:
                            await asyncio.sleep(delay_on_finish)
                            progress.update(task, visible=False, refresh=True)
                            progress.remove_task(task)
                    if progress.finished:
                        progress.stop()

    @staticmethod
    def ignore_failure(puller: BasePuller) -> None:
        """
        ### Ignore all exceptions.
        * `puller`: Puller instance
        """
        if isinstance(puller, AsyncPuller):
            @puller.on.worker.fail
            async def ignore_failure(event: str, worker: AsyncWorker, e: Exception):
                return True

    @staticmethod
    def ignore_file_exists(puller: BasePuller) -> None:
        """
        ### Ignore `FileExistsError`.
        * `puller`: Puller instance
        """
        if isinstance(puller, AsyncPuller):
            @puller.on.worker.fail
            async def ignore_file_exists(event: str, worker: AsyncWorker, e: Exception):
                if isinstance(e, FileExistsError):
                    return True

    @staticmethod
    def raise_for_status(puller: BasePuller) -> None:
        """
        ### Raise exception on non-200 status.
        * `puller`: Puller instance
        """
        if isinstance(puller, AsyncPuller):
            @puller.on.worker.response_get
            async def raise_for_status(event: str, worker: AsyncWorker, r: Response):
                r.raise_for_status()

    @staticmethod
    def add_logging(puller: BasePuller, logger: LoggerLike) -> None:
        """
        ### Add logging for events.
        * `puller`: Puller instance
        * `logger`: logger-like instance
        """
        if isinstance(puller, AsyncPuller):
            @puller.on.puller.spawn
            async def p_log_spawn(event: str, puller: AsyncPuller):
                logger.debug(f"Puller spawned. {puller}")

            @puller.on.puller.join
            async def p_log_join(event: str, puller: AsyncPuller):
                logger.debug(f"Puller joined. {puller}")

            @puller.on.puller.destroy
            async def p_log_destroy(event: str, puller: AsyncPuller):
                logger.debug(f"Puller destroyed. {puller}")

            @puller.on.worker.spawn
            async def log_spawn(event: str, worker: AsyncWorker):
                logger.debug(f"Spawned worker {worker}")

            @puller.on.worker.start
            async def log_start(event: str, worker: AsyncWorker):
                logger.debug(f"Start: {worker}")

            @puller.on.worker.response_get
            async def log_response(event: str, worker: AsyncWorker, r: Response):
                logger.debug(f"Response: {r.status_code}  {r.url}")

            @puller.on.worker.bytes_get
            async def log_bytes(event: str, worker: AsyncWorker, r: Response, b: bytes):
                logger.debug(f"Bytes size: {len(b)} from {r.url}")

            @puller.on.worker.retry
            async def log_retry(event: str, worker: AsyncWorker, e: Exception, retry: int):
                logger.debug(f"Retry: {worker} at {to_ordinal(retry)} time")

            @puller.on.worker.destroy
            async def log_destroy(event: str, worker: AsyncWorker):
                logger.debug(f"Destroy: {worker}")

            @puller.on.worker.fail
            async def log_fail(event: str, worker: AsyncWorker, e: Exception):
                logger.error(f"Fail: {worker} due to {e}")
                logger.exception(e)

    @staticmethod
    def on_every_event(puller: BasePuller, func: Callable) -> None:
        """
        ### Call function on every event.
        * `puller`: Puller instance
        * `func`: function to call
        """
        if isinstance(puller, AsyncPuller):
            @puller.on.puller.spawn
            @puller.on.puller.join
            @puller.on.puller.destroy
            @puller.on.worker.spawn
            @puller.on.worker.start
            @puller.on.worker.response_get
            @puller.on.worker.bytes_get
            @puller.on.worker.retry
            @puller.on.worker.destroy
            @puller.on.worker.fail
            async def on_every_event(*args, **kw):
                await ensure_async(func)(*args, **kw)

    @staticmethod
    def clear_hooks(puller: BasePuller) -> None:
        """
        ### Clear all hooks.
        * `puller`: Puller instance
        """
        puller.event_hooks.clear()

#### Start Hinting: Add hints for event names ####

class on_puller_hint(Hint):
    @property
    def spawn(self):
        """Callback Type: (event_name: str, AsyncPuller) -> None"""
        return self._chain.spawn

    @property
    def destroy(self):
        """Callback Type: (event_name: str, AsyncPuller) -> None"""
        return self._chain.destroy

    @property
    def join(self):
        """Callback Type: (event_name: str, AsyncPuller) -> None"""
        return self._chain.join

class on_worker_hint(Hint):
    @property
    def spawn(self):
        """Callback Type: (event_name: str, AsyncWorker) -> None"""
    @property
    def destroy(self):
        """Callback Type: (event_name: str, AsyncWorker) -> None"""
    @property
    def start(self):
        """Callback Type: (event_name: str, AsyncWorker) -> None"""
    @property
    def response_get(self):
        """Callback Type: (event_name: str, AsyncWorker, Response) -> None"""
    @property
    def bytes_get(self):
        """Callback Type: (event_name: str, AsyncWorker, Response, bytes) -> None"""
    @property
    def retry(self):
        """Callback Type: (event_name: str, AsyncWorker, Exception, retry: int) -> None"""
    @property
    def success(self):
        """Callback Type: (event_name: str, AsyncWorker) -> None"""
    @property
    def fail(self):
        """Callback Type: (event_name: str, AsyncWorker, Exception) -> bool:
        whether to ignore the exception"""

class PullerEventHint(Hint):
    """Puller Event Hint."""
    @property
    def puller(self) -> on_puller_hint:
        return self._chain.puller  # type: ignore

    @property
    def worker(self) -> on_worker_hint:
        return self._chain.worker  # type: ignore

    @property
    def request(self):
        """Callback Type: (Request) -> None"""
        return self._chain.request

    @property
    def response(self):
        """Callback Type: (Response) -> None"""
        return self._chain.response

### End Hinting ###


import socket
import time

TARGET_IP = "192.168.137.146"
UDP_PORT = 5005
FREQUENCY = 20

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

i = 0

while(True):
    message = bytes(str("message:" + str(i) + "$$$$$"), 'utf-8')

    sock.sendto(message,(TARGET_IP,UDP_PORT))
    time.sleep(1/FREQUENCY)

    i += 1


SUMMERNOTE_CONFIG = {
    # Using SummernoteWidget - iframe mode, default
    'iframe': True,

    # Or, you can set it as False to use SummernoteInplaceWidget by default - no iframe mode
    # In this case, you have to load Bootstrap/jQuery stuff by manually.
    # Use this when you're already using Bootstraip/jQuery based themes.
    # 'iframe': False,

    # You can put custom Summernote settings
    'summernote': {
        # As an example, using Summernote Air-mode
        # 'airMode': True,

        # Change editor size
        'width': '90%',
        'height': '820',

        # Use proper language setting automatically (default)
        'lang': "es-ES",

        # Or, set editor language/locale forcely
        # 'lang': 'ko-KR',

        # You can also add custom settings for external plugins
        # 'print': {
        #     'stylesheetUrl': '/some_static_folder/printable.css',
        # },
        # 'codemirror': {
        #     'mode': 'htmlmixed',
        #     'lineNumbers': 'true',
        #     # You have to include theme file in 'css' or 'css_for_inplace' before using it.
        #     'theme': 'monokai',
        # },
    },

    # Need authentication while uploading attachments.
    # 'attachment_require_authentication': False,

    # Set `upload_to` function for attachments.
    # 'attachment_upload_to': my_custom_upload_to_func(),

    # Set custom storage class for attachments.
    # 'attachment_storage_class': 'my.custom.storage.class.name',

    # Set custom model for attachments (default: 'django_summernote.Attachment')
    # must inherit 'django_summernote.AbstractAttachment'
    # 'attachment_model': 'my.custom.attachment.model',

    # You can disable attachment feature.
    # 'disable_attachment': False,

    # Set `True` to return attachment paths in absolute URIs.
    # 'attachment_absolute_uri': False,

    # test_func in summernote upload view. (Allow upload images only when user passes the test)
    # https://docs.djangoproject.com/en/2.2/topics/auth/default/#django.contrib.auth.mixins.UserPassesTestMixin

    # def example_test_func(request):
    #     return request.user.groups.filter(name='group_name').exists()

    # 'test_func_upload_view': example_test_func,

    # You can add custom css/js for SummernoteWidget.
    # 'css': (
    # ),
    # 'js': (
    # ),

    # You can also add custom css/js for SummernoteInplaceWidget.
    # !!! Be sure to put {{ form.media }} in template before initiate summernote.
    # 'css_for_inplace': (
    # ),
    # 'js_for_inplace': (
    # ),

    # Codemirror as codeview
    # If any codemirror settings are defined, it will include codemirror files automatically.
    # 'css': (
    #     '//cdnjs.cloudflare.com/ajax/libs/codemirror/5.29.0/theme/monokai.min.css',
    # ),

    # Lazy initialize
    # If you want to initialize summernote at the bottom of page, set this as True
    # and call `initSummernote()` on your page.
    # 'lazy': True,

    # To use external plugins,
    # Include them within `css` and `js`.
    # 'js': {
    #     '/some_static_folder/summernote-ext-print.js',
    #     '//somewhere_in_internet/summernote-plugin-name.js',
    # },
}
# Django summernote default theme for text editor
SUMMERNOTE_THEME = 'bs4'


# Peaks and Valleys: In an array of integers, a "peak" is an element which is greater than or equal
# to the adjacent integers and a "valley" is an element which is less than or equal to the adjacent integers.
# For example, in the array {5, 8, 6, 2, 3, 4, 6}, {8, 6} are peaks and {5, 2} are valleys.
# Given an array of integers, sort the array into an alternating sequence of peaks and valleys.

# partition around pivot
def partition(arr, start, end):
    pivot = arr[start]
    j = 0
    for i in range(start + 1, end + 1):
        if arr[i] <= pivot:
            j += 1
            arr[i], arr[j] = arr[j], arr[i]
    arr[start], arr[j] = arr[j], arr[start]
    return j + 1

# quick sort
def quick_sort(arr, start, end):
    if start < end:
        ind = partition(arr, start, end)
        quick_sort(arr, start, ind - 1)
        quick_sort(arr, ind + 1, end)

def peaks_and_valleys(arr):
    quick_sort(arr, 0, len(arr) - 1)
    print('Sorted Array : ', arr)
    half = len(arr) // 2
    # swap every couple
    for i in range(0, half, 2):
        arr[i], arr[i + 1] = arr[i + 1], arr[i]
    return arr

if __name__ == "__main__":
    arr = [5, 3, 1, 2, 3]
    print('Initial array:', arr)
    print('Array with peaks and valleys', peaks_and_valleys(arr))

Cracking the Coding Interview/10_Sorting and Searching/10_11.py


# Given a non-empty string s and a dictionary wordDict containing a list of non-empty words,
# add spaces in s to construct a sentence where each word is a valid dictionary word.
# Return all such possible sentences.
# Note:
# The same word in the dictionary may be reused multiple times in the segmentation.
# You may assume the dictionary does not contain duplicate words.

# Example 1:
# Input:
# s = "catsanddog"
# wordDict = ["cat", "cats", "and", "sand", "dog"]
# Output:
# [
#   "cats and dog",
#   "cat sand dog"
# ]

# Example 2:
# Input:
# s = "pineapplepenapple"
# wordDict = ["apple", "pen", "applepen", "pine", "pineapple"]
# Output:
# [
#   "pine apple pen apple",
#   "pineapple pen apple",
#   "pine applepen apple"
# ]
# Explanation: Note that you are allowed to reuse a dictionary word.

# Example 3:
# Input:
# s = "catsandog"
# wordDict = ["cats", "dog", "sand", "and", "cat"]
# Output:
# []

class Solution(object):

    # M1. DFS
    # 定义一个函数check - DP判断子字符串是否可以由wordDict中的词汇组成；
    # def wordBreak(self, s, wordDict):
    #     """
    #     :type s: str
    #     :type wordDict: List[str]
    #     :rtype: List[str]
    #     """
    #     wordDict = set(wordDict)
    #     res = []
    #     self.dfs(s, wordDict, 0, '', res)
    #     return res

    # def check(self, s, wordDict):
    #     n = len(s)
    #     dp = [False] * (n + 1)    # s[:i]
    #     dp[0] = True
    #     for i in range(1, n+1):
    #         for j in range(i-1, -1, -1):
    #             if dp[j] and s[j:i] in wordDict:
    #                 dp[i] = True
    #                 break
    #     return dp[-1]

    # def dfs(self, s, wordDict, i, temp, res):
    #     if i == len(s):
    #         # 越过结尾空格
    #         res.append(temp[:-1])
    #     if self.check(s[i:], wordDict):
    #         for j in range(i+1, len(s)+1):
    #             if s[i:j] in wordDict:
    #                 self.dfs(s, wordDict, j, temp + s[i:j] + ' ', res)

    # M1改进. dp已经统计了截止当前位置的字符串是否可以由wordDict中的词汇组成。
    # def wordBreak(self, s, wordDict):
    #     """
    #     :type s: str
    #     :type wordDict: List[str]
    #     :rtype: List[str]
    #     """
    #     wordDict = set(wordDict)
    #     dp = self.check(s, wordDict)
    #     res = []
    #     self.dfs(s, wordDict, dp, '', res)
    #     return res

    # def check(self, s, wordDict):
    #     n = len(s)
    #     dp = [False] * (n+1)
    #     dp[0] = True
    #     for i in range(1, n+1):
    #         for j in range(i-1, -1, -1):
    #             if dp[j] and s[j:i] in wordDict:
    #                 dp[i] = True
    #                 break
    #     return dp

    # def dfs(self, s, wordDict, dp, temp, res):
    #     if not s:
    #         res.append(temp[:-1])
    #     for i in range(len(s)):
    #         if dp[i] and s[i:] in wordDict:
    #             self.dfs(s[:i], wordDict, dp, s[i:] + ' ' + temp, res)

    # M2. 带记忆DP 存储截止s[:i], 满足题意的所有可能组合
    def wordBreak(self, s, wordDict):
        """
        :type s: str
        :type wordDict: List[str]
        :rtype: List[str]
        """
        wordDict = set(wordDict)
        n = len(s)
        dp = [[] for _ in range(n+1)]    # 截止s[:i], 满足题意的所有可能组合
        dp[0] = ['']

        if self.check(s, wordDict):
            for i in range(1, n+1):
                temp = []
                for j in range(i):
                    if dp[j] and s[j:i] in wordDict:
                        for prev in dp[j]:
                            space = ' ' if prev else ''
                            temp.append(prev + space + s[j:i])
                dp[i] = temp
        return dp[-1]

    def check(self, s, wordDict):
        n = len(s)
        dp = [False] * (n+1)
        dp[0] = True
        for i in range(1, n+1):
            for j in range(i-1, -1, -1):
                if dp[j] and s[j:i] in wordDict:
                    dp[i] = True
                    break
        return dp[-1]

LeetCode/Algorithms/DynamicProgramming/140. Word Break II.py


from django.conf.urls import *
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.conf.urls.i18n import i18n_patterns
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from apps.ecidadania.voting.views.polls import ViewPoll, DeletePoll, \
    ListPolls, ViewPollResults
from apps.ecidadania.voting.views.voting import ViewVoting, ListVotings, \
    AddVoting, EditVoting, DeleteVoting
from apps.ecidadania.voting.url_names import *

urlpatterns = patterns('apps.ecidadania.voting.views',

    url(r'^$', ListVotings.as_view(), name=LIST_VOTING),

    url(r'^poll/$', ListPolls.as_view(), name=LIST_POLL),

    url(r'^add/$', AddVoting.as_view(), name=ADD_VOTING),

    url(r'^add/poll/$', 'polls.add_poll', name=ADD_POLL),

    url(r'^poll/(?P<poll_id>\d+)/edit/$', 'polls.edit_poll', name=EDIT_POLL),

    url(r'^(?P<voting_id>\d+)/edit/$', EditVoting.as_view(),
        name=EDIT_VOTING),

    url(r'^poll/(?P<poll_id>\d+)/delete/$', DeletePoll.as_view(),
        name=DELETE_POLL),

    url(r'^(?P<voting_id>\d+)/delete/$', DeleteVoting.as_view(),
        name=DELETE_VOTING),

    url(_(r'^poll/(?P<pk>\d+)/$'), ViewPoll.as_view(), name=VIEW_POLL),

    url(_(r'^poll/(?P<pk>\d+)/results/$'), ViewPollResults.as_view(),
        name=VIEW_RESULT),

    url(r'^(?P<voting_id>\d+)/$', ViewVoting.as_view(), name=VIEW_VOTING),

    url(r'^vote/poll/(?P<poll_id>\d+)/$', 'polls.vote_poll', name=VOTE_POLL),

    url(r'^vote/voting/$', 'voting.vote_voting', name=VOTE_VOTING),

    url(r'^vote/validate/(?P<token>\w+)/$', 'voting.validate_voting',
        name=VALIDATE_VOTE),
)


from rest_framework.response import Response
from rest_framework.decorators import api_view
from .serializers import RegisterSerializer

@api_view(['POST'])
def register(request):
    try:
        serializer = RegisterSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response({'msg':'operation successful'})
        else:
            return Response(serializer.errors, 400)

    except Exception as ex:
        print('internal server error -> ', ex)
        return Response({'msg':'internal server error'}, 500)


import nextcord
from nextcord import ui, ButtonStyle

from Commands.CivFR.Database import db
from Commands.CivFR.Ranked.RankedFunc import update_player_ranks
from Commands.CivFR.constant import RANKED_ADMIN_ROLES, RANKED_ADMIN_USERS
from util.exception import Forbidden

def member_is_authorised(member) -> bool:
    if member.id in RANKED_ADMIN_USERS:
        return True
    try:
        member_roles = [i.id for i in member.roles]
        for i in RANKED_ADMIN_ROLES:
            if i in member_roles:
                return True
    except AttributeError:
        return False
    return False

class ModButton(ui.Button):
    async def callback(self, interaction: nextcord.Interaction):
        if not member_is_authorised(interaction.user):
            await interaction.send("You are not an authorised Administrator", ephemeral=True)
            raise Forbidden("Member is not a ranked Administrator")

class ModSelect(ui.Select):
    async def callback(self, interaction: nextcord.Interaction):
        if not member_is_authorised(interaction.user):
            await interaction.send("You are not an authorised Administrator", ephemeral=True)
            raise Forbidden("Member is not a ranked Administrator")

class ValidButton(ModButton):
    def __init__(self, view):
        super().__init__(label="Valider", style=ButtonStyle.green, row=3, disabled=not view.report.report_status.is_valid)

    async def callback(self, interaction: nextcord.Interaction):
        await super().callback(interaction)
        view = self.view
        if self.view.report.validated:
            return await interaction.send("Error: Match already Validated", ephemeral=True)
        db.valid_s1_match(view.report)
        await view.report.update_embed(client=view.client)
        await update_player_ranks(view.report, client=view.client)

class ScrapButton(ModButton):
    def __init__(self, view):
        super().__init__(label="Scrap", style=ButtonStyle.red, row=4, disabled=False)

    async def callback(self, interaction: nextcord.Interaction):
        await super().callback(interaction)
        view = self.view
        if self.view.report.validated:
            return await interaction.send("Error: Match already Scrapped", ephemeral=True)
        db.scrap_s1_match(view.report)
        await update_player_ranks(view.report, client=view.client)
        await view.report.update_embed(client=view.client)

class DeleteButton(ModButton):
    def __init__(self, view):
        super().__init__(label="Delete", style=ButtonStyle.red, row=4, disabled=False)

    async def callback(self, interaction: nextcord.Interaction):
        await super().callback(interaction)
        view = self.view
        db.delete_s1_match(view.report)
        await view.report.delete(client=view.client)

class EditButton(ModButton):
    def __init__(self):
        super().__init__(label="Assigner la position", style=ButtonStyle.blurple, row=3)

    async def callback(self, interaction: nextcord.Interaction):
        await super().callback(interaction)
        view : RankedView = self.view
        if not view.player_select.values:
            await interaction.send("Error: No player selected", ephemeral=True)
            return
        player_id = int(view.player_select.values[0])
        if not view.position_select.values:
            await interaction.send("Error: No position selected", ephemeral=True)
            return
        if self.view.report.validated:
            return await interaction.send("Error: Match already Validated", ephemeral=True)
        position = int(view.position_select.values[0])
        view.report.set_player_position(player_id, position)
        await view.report.update_embed(view.client)
        db.update_s1_match(view.report)

class ReportQuitButton(ModButton):
    def __init__(self):
        super().__init__(label="Bannir pour Quit", style=ButtonStyle.red, row=3, disabled=True)

class AutoFillButton(ModButton):
    def __init__(self):
        super().__init__(label="AutoFill players", style=ButtonStyle.red, row=4)

    async def callback(self, interaction: nextcord.Interaction):
        await super().callback(interaction)
        view: RankedView = self.view
        view.report.fill_unreported_players(interaction.user)
        db.update_s1_match(view.report)
        await view.report.update_embed(client=view.client)

class PlayerSelect(ModSelect):
    def __init__(self, view):
        player_names = [self.get_player_name(view.client.get_user(i), i) for i in view.report.players]
        super().__init__(placeholder="Select a player to Administrate", row=1,
                         options=[nextcord.SelectOption(label=name, value=str(pl))
                                  for pl, name in zip(view.report.players, player_names)])

    @staticmethod
    def get_player_name(user : nextcord.User, discord_id : int):
        if user:
            return user.name
        return f"<{discord_id}>"

class PositionSelect(ModSelect):
    def __init__(self, view):
        super().__init__(placeholder="Select a position to force assign", row=2,
                         options=[nextcord.SelectOption(label=str(i+1), value=str(i+1))
                                  for i, _ in enumerate(view.report.players)])

class RankedView(ui.View):
    def __init__(self, report, parent : nextcord.PartialMessage, client : nextcord.Client):
        super().__init__(timeout=30)
        self.report = report
        self.parent : nextcord.PartialMessage = parent
        self.client : nextcord.Client = client

        self.player_select = PlayerSelect(self)
        self.position_select = PositionSelect(self)
        self.valid_button = ValidButton(self)

        self.add_item(self.valid_button)
        self.add_item(self.player_select)
        self.add_item(self.position_select)
        self.add_item(EditButton())
        self.add_item(ReportQuitButton())
        self.add_item(ScrapButton(self))
        self.add_item(DeleteButton(self))
        self.add_item(AutoFillButton())

    async def on_timeout(self) -> None:
        await self.parent.edit(view=None)


"""
NN model with glove embeddings
layers:
    1. embedding layer (glove)
    2. SpatialDropout1D (0.24)
    3. bidirectional lstm & gru
    4. [global_max_pooling1d, attention, features]
    5. dense 128 & 16
    6. output (sigmoid)
"""
import os
import gc
import numpy as np
import pandas as pd
from sklearn.preprocessing import minmax_scale
from keras.layers import (Input, Embedding, SpatialDropout1D, Bidirectional,
                          LSTM, GRU, GlobalMaxPool1D, Concatenate, Dropout,
                          Dense)
from keras.models import Model

from neural_networks import Attention, DropConnect
from neural_networks import NeuralNetworkClassifier

from tqdm import tqdm
tqdm.pandas()

# model configs
MAX_FEATURES = int(2.5e5)  # total word count = 227,538; clean word count = 186,551   # noqa
MAX_LEN = 75    # mean_len = 12; Q99_len = 40; max_len = 189;
RNN_UNITS = 40
F_DROPOUT = 0.05
DENSE_UNITS_1 = 128
DENSE_UNITS_2 = 16

# file configs
MODEL_FILEPATH = os.path.join(
    os.environ['DATA_PATH'],
    'models',
    'model_v50.hdf5'
)
EMBED_FILEPATH = os.path.join(
    os.environ['DATA_PATH'],
    'embeddings',
    'glove.840B.300d',
    'glove.pkl'
)

def get_network(embed_filepath):
    # features network
    input_features = Input(shape=(136, ), name='input_features')
    dense_features = DropConnect(
        Dense(16, activation="relu"),
        prob=F_DROPOUT)(input_features)
    # tokens network
    input_tokens = Input(shape=(MAX_LEN, ), name='input_tokens')
    # 1. embedding layer
    # get embedding weights
    print('load pre-trained embedding weights ......')
    embed_weights = pd.read_pickle(embed_filepath)
    input_dim = embed_weights.shape[0]
    output_dim = embed_weights.shape[1]
    x = Embedding(
        input_dim=input_dim,
        output_dim=output_dim,
        weights=[embed_weights],
        trainable=False,
        name='embedding'
    )(input_tokens)
    # clean up
    del embed_weights, input_dim, output_dim
    gc.collect()
    # 2. dropout
    x = SpatialDropout1D(rate=0.24)(x)
    # 3. bidirectional lstm & gru
    x = Bidirectional(
        layer=LSTM(RNN_UNITS, return_sequences=True),
        name='bidirectional_lstm'
    )(x)
    x = Bidirectional(
        layer=GRU(RNN_UNITS, return_sequences=True),
        name='bidirectional_gru'
    )(x)
    # 4. concat global_max_pooling1d and attention
    max_pool = GlobalMaxPool1D(name='global_max_pooling1d')(x)
    atten = Attention(step_dim=MAX_LEN, name='attention')(x)
    x = Concatenate(axis=-1)([max_pool, atten, dense_features])
    # 5. dense
    x = Dense(units=DENSE_UNITS_1, activation='relu', name='dense_1')(x)
    x = Dropout(rate=0.1)(x)
    x = Dense(units=DENSE_UNITS_2, activation='relu', name='dense_2')(x)
    # 6. output (sigmoid)
    output_layer = Dense(units=1, activation='sigmoid', name='output')(x)
    return Model(inputs=[input_tokens, input_features], outputs=output_layer)

def get_model():
    print('build network ......')
    model = get_network(embed_filepath=EMBED_FILEPATH)
    print(model.summary())
    return NeuralNetworkClassifier(
        model,
        balancing_class_weight=True,
        filepath=MODEL_FILEPATH)

"""
text cleaning
"""

def token_transformer(df_text):
    from model_v30 import transform
    return transform(df_text)

def features_transformer(df_text):
    from nlp import meta_features_transformer
    from nlp import topic_features_transformer
    # get features
    meta_features = meta_features_transformer(df_text).values
    topic_features = topic_features_transformer(df_text).values
    # concat
    joined_features = np.hstack([meta_features, topic_features])
    return minmax_scale(joined_features)

def transform(df_text):
    X1 = token_transformer(df_text)
    X2 = features_transformer(df_text)
    return [X1, X2]


from eyed3 import mp3
import os
path = os.getcwd()
for music_file in os.listdir(path) :
    cursor = "{}/{}".format(path, music_file)
    if music_file[-4:] == ".mp3" :
        trackinfo = mp3.Mp3AudioFile(cursor)
        dir_album = "{}/{} - {}".format(path, trackinfo.tag.artist, trackinfo.tag.album)
        # ensure album dir exists
        if not os.path.isdir(dir_album) :
            os.mkdir(dir_album)
        os.rename(path+"/"+music_file, dir_album+"/"+music_file)


# 2023/8/8 by DKZ https://github.com/davidkingzyb/BlenderSlot

import bpy
import os
from .ui import MainPanel

root_path=os.path.dirname(__file__)

class DeleteCode(bpy.types.Operator):
    bl_idname='bs.deletecode'
    bl_label='delete'
    bl_options = {'REGISTER', 'UNDO'}

    file_name:bpy.props.StringProperty(
        name="file_name",
        description="file name",
        default="",
    )

    def execute(self, context):
        try:
            os.remove(os.path.join(root_path,'slot\\'+self.file_name))
        except Exception as e:
            pass

        bpy.utils.unregister_class(MainPanel)
        bpy.utils.register_class(MainPanel)
        return {'FINISHED'}

class SaveCode(bpy.types.Operator):
    bl_idname='bs.savecode'
    bl_label='save'
    bl_options = {'REGISTER', 'UNDO'}

    file_name:bpy.props.StringProperty(
        name="file_name",
        description="file name",
        default="",
    )

    code:bpy.props.StringProperty(
        name="code",
        description="code",
        default="",
    )

    def execute(self, context):
        text = bpy.data.texts.get(self.file_name)
        self.code=text.as_string()
        with open(os.path.join(root_path,'slot\\'+self.file_name),'w') as f:
            f.write(self.code)
        bpy.utils.unregister_class(MainPanel)
        bpy.utils.register_class(MainPanel)
        return {'FINISHED'}

class ExecCode(bpy.types.Operator):
    bl_idname='bs.execcode'
    bl_label='exec'
    bl_options = {'REGISTER', 'UNDO'}

    file_name:bpy.props.StringProperty(
        name="file_name",
        description="file name",
        default="",
    )

    code:bpy.props.StringProperty(
        name="code",
        description="code",
        default="",
    )

    def execute(self, context):
        text = bpy.data.texts.get(self.file_name)
        if text is not None:
            self.code=text.as_string()
        else:
            try:
                with open(os.path.join(root_path,'slot\\'+self.file_name),'r') as f:
                    self.code=f.read()
            except Exception as e:
                self.report({'ERROR'}, f"Error executing generated code: {e}")
                return {'CANCELLED'}
        global_namespace = globals().copy()
        try:
            exec(self.code,global_namespace)
        except Exception as e:
            self.report({'ERROR'}, f"Error executing generated code: {e}")
            return {'CANCELLED'}
        return {'FINISHED'}

class ShowCode(bpy.types.Operator):
    bl_idname='bs.showcode'
    bl_label='show'
    bl_options = {'REGISTER', 'UNDO'}

    file_name:bpy.props.StringProperty(
        name="file_name",
        description="file name",
        default="",
    )

    code:bpy.props.StringProperty(
        name="code",
        description="code",
        default="",
    )

    def execute(self, context):
        print(self.file_name)
        text = bpy.data.texts.get(self.file_name)
        if text is None:
            text = bpy.data.texts.new(self.file_name)
            try:
                with open(os.path.join(root_path,'slot\\'+self.file_name),'r') as f:
                    self.code=f.read()
            except Exception as e:
                # self.report({'ERROR'}, f"Error executing generated code: {e}")
                pass
            text.clear()
            text.write(self.code)
        else:
            self.code=text.as_string()

        text_editor_area = None
        for area in context.screen.areas:
            if area.type == 'TEXT_EDITOR':
                text_editor_area = area
                break

        if text_editor_area is None:
            text_editor_area = split_area_to_text_editor(context)

        text_editor_area.spaces.active.text = text

        return {'FINISHED'}

def split_area_to_text_editor(context):
    area = context.area
    for region in area.regions:
        if region.type == 'WINDOW':
            override = {'area': area, 'region': region}
            bpy.ops.screen.area_split(override, direction='VERTICAL', factor=0.5)
            break

    new_area = context.screen.areas[-1]
    new_area.type = 'TEXT_EDITOR'
    return new_area


from django.conf import settings
from django.http import Http404
from django.db.models import Q
from accounts.models import CustomUser
from django.core.exceptions import FieldDoesNotExist
import datetime

import logging
logger = logging.getLogger()

def search(keyword):
    """ 検索機能 """
    if keyword:
        queryset = CustomUser.objects.filter(
            Q(deleted_at=None)
            & (Q(name__icontains=keyword)
                | Q(description__icontains=keyword)
            )
        )
    else:
        queryset = CustomUser.objects.all().filter(
            Q(is_active=True)
            & Q(partner__isnull=False)
        )[:100]
    return queryset

def getEngineer(user_id):
    engineer = CustomUser.objects.filter(Q(id=user_id) & Q(is_active=True)).first()
    if not engineer:
        raise Http404("Project Id={} Not Found".format(user_id))
    return engineer

# """ 案件作成/更新 """
def save_user(form, partner_id):
    # 保存する前に一旦取り出す
    user = form.save(commit=False)
    # パスワードをハッシュ化してセット
    user.set_password(form.cleaned_data['password'])
    # ユーザーオブジェクトを保存
    user.partner_id = partner_id
    user.save()
    form.save_m2m()

# """ 案件削除 """
# def delete_project(project, user_id):
#     now = datetime.datetime.now()
#     project.deleted_at = now
#     project.deleted_id = user_id
#     project.save()


def same_strings(string1, string2):
    lista1 = list(string1)
    lista2 = list(string2)
    if lista1 == lista2:
        print("stringovi su jednaki")
    else:
        print("stringovi nisu jednaki")

def stepByStep(string1, string2):
    lista1 = list(string1)
    lista2 = list(string2)
    if len(lista1) != len(lista2):
        print("nisu")
    else:
        for i in range(len(lista1)):
            if lista1[i] != lista2[i]:
                print("nisu")
                break
            else:
                if i == len(lista1) - 1 and lista1[i] == lista2[i]:
                    print("jesu")

int = "prvi string"
int2 = "drugi string"
int3 = "drugi string"
same_strings(int, int2)
same_strings(int2, int3)
stepByStep(int, int2)
stepByStep(int2, int3)


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
import gym
from gym import spaces
import torch
from itertools import combinations
from scipy.stats import multivariate_normal

def z_func(x, y, cos_mag_scale=3., cos_freq_scale=5., norm_scale=1.):
    return cos_mag_scale * np.cos(cos_freq_scale * x) * np.cos(
        cos_freq_scale * y) - norm_scale * (np.abs(x) + np.abs(y))

class SineRewardEnv(gym.Env):
    """
    This guy is a bit different. What's different? |a|_2 (\sum_i sin(a_i)). But maybe more interestingly,
    there will be state AND action within those things. So, the state is given to you, and the
    action you need to choose yourself. So it's not a bandit anymore. That's actually a much better test-bed. We can do
    standard RBFDQN, latent RBFDQN, and DDPG!
    """
    def get_reward(self, s, a):
        assert s.shape == (self.s_dim, ), f"{s.shape} {self.s_dim}"
        assert a.shape == (self.a_dim, ), f"{a.shape} {self.a_dim}"
        magnitude = np.linalg.norm(np.concatenate([s, a]))
        assert magnitude.shape == (), "Expects a single number"
        sines_1 = np.sin(s*self.FREQ_SCALE)
        sines_2 = np.sin(a*self.FREQ_SCALE)
        sines_sum = (np.sum(sines_1) + np.sum(sines_2)) / (self.s_dim + self.a_dim)
        return magnitude * sines_sum

    def __init__(self, s_dim=1, a_dim=1, episode_length=float('inf')):
        """
        Action is what makes reward
        Observation is always the same.
        For now, episode_length is determined through the registration thing in init
        We'll have a special case where s_dim being zero makes the observation space constant and zero.
        """
        was_zero = False
        if s_dim <= 0:
            s_dim = 1
            was_zero = True

        self.s_dim = s_dim
        self.a_dim = a_dim
        print(f"sdim: {s_dim}\t adim: {a_dim}")
        self.episode_length = episode_length
        self.FREQ_SCALE = 4.

        if was_zero:
            self.observation_space = spaces.Box(0.0, 0.0, shape=(self.s_dim, ))
        else:
            self.observation_space = spaces.Box(-1.0, 1.0, shape=(self.s_dim, ))

        self.action_space = spaces.Box(-1.0, 1.0, shape=(self.a_dim, ))

        self._last_state = None
        self._t = 0

        self.COS_MAG_SCALE = 3.
        self.COS_FREQ_SCALE = 5.
        self.NORM_SCALE = 1.

    def reset(self):
        self._last_state = self.observation_space.sample()
        self._t = 0
        return self._last_state.copy()

    def step(self, action):
        reward = self.get_reward(self._last_state, action)
        assert self.action_space.contains(action), action
        self._last_state = self.observation_space.sample()
        self._t += 1
        done = (self._t >= self.episode_length)

        return self._last_state.copy(), reward, done, {}

    def plot(self, savefile):
        assert self.a_dim == 2
        x = np.arange(-1.0, 1.0, 0.05)
        y = np.arange(-1.0, 1.0, 0.05)
        X, Y = np.meshgrid(x, y)  # grid of point
        actions = np.concatenate([X.reshape(-1, 1), Y.reshape(-1, 1)], axis=1)
        states = np.zeros((actions.shape[0], self.s_dim))
        Z = np.array([self.get_reward(states[i], actions[i]) for i in range(len(actions))])
        Z = Z.reshape(X.shape)

        fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
        surf = ax.plot_surface(X,
                               Y,
                               Z,
                               cmap=cm.coolwarm,
                               linewidth=0,
                               antialiased=False)
        fig.colorbar(surf, shrink=0.5, aspect=5)
        plt.savefig(savefile)
        plt.close()

    def plot_agent(self, Q_object, savefile):
        assert self.a_dim == 2
        x = np.arange(-1.0, 1.0, 0.05)
        y = np.arange(-1.0, 1.0, 0.05)
        X, Y = np.meshgrid(x, y)  # grid of point
        actions = np.concatenate([X.reshape(-1, 1), Y.reshape(-1, 1)], axis=1)
        actions_torch = torch.Tensor(actions).to(Q_object.device)

        states_torch = torch.zeros(actions.shape[0], self.s_dim).to(Q_object.device)
        Z_torch = Q_object.forward(states_torch, actions_torch)
        Z = Z_torch.detach().cpu().numpy()
        Z = Z.reshape(X.shape)

        fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
        surf = ax.plot_surface(X,
                               Y,
                               Z,
                               cmap=cm.coolwarm,
                               linewidth=0,
                               antialiased=False)
        fig.colorbar(surf, shrink=0.5, aspect=5)
        plt.savefig(savefile)
        plt.close()

class MultimodalEnv(gym.Env):
    def __init__(self, s_dim = 1, a_dim = 1, std = 1, scale=1):
        self.observation_space = spaces.Box(-1.0, 1.0, shape=(s_dim, ))
        self.action_space = spaces.Box(-1.0, 1.0, shape=(a_dim, ))
        self.s_dim = s_dim
        self.a_dim = a_dim
        self.std = std
        self.scale = scale

        self.distributionCenters = []
        for count in range(0, self.a_dim + 1):
            Combs = combinations(range(0, self.a_dim), count)
            for indices in Combs:
                center = np.ones((self.a_dim)) * 0.5
                for index in indices:
                    center[index] = -0.5
                self.distributionCenters.append(center)
        self.distributionCenters = np.array(self.distributionCenters)
        self.distributions = []
        for center in self.distributionCenters:
            var = multivariate_normal(mean=center, cov=self.std)
            self.distributions.append(var)

    def reset(self):
        # Since it's a bandit, we should have a constant state
        return np.ones((self.s_dim,))

    def step(self, action):
        reward = 0
        for distribution in self.distributions:
            reward += distribution.pdf(action) * self.scale
        reward = int(reward * 1000) / 1000 # only keep 3 digits after the decimal point
        done = True
        return np.zeros((self.s_dim,)), reward, done, {}

if __name__ == "__main__":
    env = MultimodalEnv(s_dim=1, a_dim=2, std=0.1, scale = 0.5)
    breakpoint()


from pypdf import PdfReader
import os
import openai
import json
from PIL import Image
from io import BytesIO
import urllib

openai.api_key = ""

'''

Get byte stream of image

Input:
- img_path (str): path to image to generate
- image_size (tuple_int): image size

Output:

- byte_array(byte stream): output byte stream of image file

'''

def image_to_byte_stream(img_path, image_size):

    image = Image.open(img_path)
    image = image.resize(image_size)

    # Convert the image to a BytesIO object
    byte_stream = BytesIO()
    image.save(byte_stream, format='PNG')
    byte_array = byte_stream.getvalue()
    return byte_array

'''

Get variation generation of image

Input:

- img_stream(byte stream): output byte stream of image file

- img_out_path (str): output path of image

'''

def generate_variation(img_stream, img_out_path):
    # Generate augmentation
    response = openai.Image.create_variation(
      image= img_stream,
      n=1,
      size="512x512"
    )

    image_url = response['data'][0]['url']

    urllib.request.urlretrieve(image_url, img_out_path)

# Quick using
if __name__ == "__main__":

    img_path = "test.png"
    path_out = "out.png"

    img_byte = image_to_byte_stream(img_path, 256)
    generate_variation(img_byte, path_out)


"""
 Analyzer
 API for analyzing user activity in relation with hobbies
 API will receive signals from home such as create , view, love, share, comment or follow
 API can manage hobby_map and primary hobby of user for personalized experience
"""
import math
from insight.models import *
from insight.workers.interface import AnalyzerInterface
from celery import shared_task
from insight.utils import next_sunday
from insight.competitions.main import RankReportManager
from rest_framework.authtoken.models import Token
from django.db.models import Q, QuerySet, ExpressionWrapper, F, DecimalField
from math import log

class Analyzer(AnalyzerInterface):
    counts = None

    @staticmethod
    def calculate_freshness_score(post: Post) -> float:
        diff = get_ist() - post.created_at
        return (2 * math.exp(-diff.days / 4)) + 0.8

    @staticmethod
    def audit_post_counts(post: Post, after=None) -> dict:
        audits = {'love': post.loves.count(), 'view': post.views.count(), 'comment': post.comments.count(),
                  'share': post.shares.count(), 'up_vote': post.up_votes.count(), 'down_vote': post.down_votes.count()}
        return audits

    def manage_hobby_report(self, hobby: str, **reports) -> HobbyReport:
        hobbies: QuerySet = Hobby.objects.filter(code_name=hobby)
        if not hobbies.exists():
            return None
        hobby: Hobby = hobbies.first()
        hobby_reports: QuerySet = HobbyReport.objects.filter(Q(account=self.user) & Q(hobby=hobby))
        hobby_report: HobbyReport = None
        if hobby_reports.exists():
            hobby_report: HobbyReport = hobby_reports.first()
        else:
            hobby_report: HobbyReport = HobbyReport.objects.create(
                account=self.user, hobby=hobby
            )
        for key, value in reports.items():
            # only view,love, share supported
            hobby_report.__dict__[f'{key}s'] += value
        hobby_report.save()
        return hobby_report

    def score_post(self, counts: dict, for_comp=False) -> float:
        score: float = (self.WEIGHT_LOVE * counts['love']) + (self.WEIGHT_VIEW * counts['view']) + \
                       (self.WEIGHT_SHARE * counts['share'])
        score += (self.WEIGHT_UP_VOTE * counts['up_vote']) - (self.WEIGHT_DOWN_VOTE * counts['down_vote'])

        return score * 0.01

    def manage_score_post(self, post: Post, is_new: bool = False, after: datetime = None, view_action: bool = False):
        post.freshness_score = self.calculate_freshness_score(post)
        post.score = self.score_post(self.audit_post_counts(post))
        if view_action:
            post.score -= self.WEIGHT_LOVE/2
        post.net_score = post.freshness_score + post.score
        post.last_modified = get_ist()
        post.save()
        return post

    def manage_scoreboard(self, post: Post, **actions):
        scoreboards: QuerySet = Scoreboard.objects.filter(Q(account=post.account) & Q(expires_on__gte=get_ist()) &
                                                          Q(created_at__lte=get_ist()))
        if not scoreboards.exists():
            scoreboard: Scoreboard = Scoreboard.objects.create(account=post.account, original_creation=get_ist(),
                                                               created_at=get_ist(),
                                                               expires_on=next_sunday(get_ist()))
        else:
            scoreboard: Scoreboard = scoreboards.first()
        if not scoreboard.posts.filter(post_id=post.post_id).exists():
            scoreboard.posts.add(post)
        for key, value in actions.items():
            scoreboard.__dict__[f'{key}s'] += value
        scoreboard.save()
        return scoreboard

    def user_activity(self, scoreboard: Scoreboard):
        posts: QuerySet = scoreboard.posts.all()
        if posts.exists():
            number: int = posts.count()
            avg_number_in_week: float = number / 4  # minimum required posts are 4 in a week
            if avg_number_in_week > 1:
                scoreboard.retention = log(avg_number_in_week)
                scoreboard.save()
        reports: QuerySet = HobbyReport.objects.select_related('hobby').filter(account=self.user) \
            .annotate(focus=ExpressionWrapper(
            F('views') + F('shares') + F('loves') + F('comments'), output_field=DecimalField()
        )).order_by('-focus')
        if reports.exists():
            report: HobbyReport = reports.first()
            self.user.primary_hobby = report.hobby.code_name
            self.user.save()

    @staticmethod
    @shared_task
    def log_data(path: str, header: dict, body):
        if 'HTTP_AUTHORIZATION' not in header:
            return None
        tokens: QuerySet = Token.objects.filter(key=header['HTTP_AUTHORIZATION'].replace("Token ", ""))
        if not tokens.exists():
            return None
        user: Account = tokens.first().user
        data_logs: QuerySet = DataLog.objects.filter(user_id=user.account_id)
        data_log: DataLog = None
        if data_logs.exists():
            data_log = data_logs.first()
        else:
            data_log = DataLog(user_id=user.account_id, created_at=get_ist(),)
        data_log.logs.append({"header": {}, "body": body, "process": path})
        data_log.save()
        return None

    @staticmethod
    @shared_task
    def background_task(user_id: str, post_id: str, created: bool = False, *count, **kwargs) -> None:
        users: QuerySet = Account.objects.filter(account_id=user_id)
        posts: QuerySet = Post.objects.filter(post_id=post_id)
        if users.count() == 0 or posts.count() == 0:
            return None
        user = users.first()
        post = posts.first()
        analyzer = Analyzer(user)
        if 'hobby' in kwargs and 'report' in kwargs:
            analyzer.manage_hobby_report(kwargs['hobby'], **kwargs['report'])
        if 'post' in kwargs:
            del kwargs['post']
        analyzer.user_activity(analyzer.manage_scoreboard(post, **kwargs))
        return None

    def analyzer_create_post(self, post: Post):
        self.manage_hobby_report(post.hobby.code_name, post=1)
        self.manage_score_post(post, is_new=True)
        self.background_task.delay(self.user.account_id, post.post_id, post=1),

    def analyze_post_action(self, post: Post, for_test: bool = False, **actions):
        self.manage_score_post(post, view_action='view' in actions)
        RankReportManager.update_score(post)
        self.background_task.delay(self.user.account_id, post.post_id, **actions)


import soundfile as sf
import os

# def compute_similar(au1, au2):
#     audiofolder = '/home/hieuld/workspace/ASVmodel/input/test'
#     audiolist = os.listdir(audiofolder)
#     audiolist.sort()
#     print(audiolist)
#     audiodata = []
#     for audio in audiolist:
#         audio = os.path.join(audiofolder, audio)
#         data, fs = sf.read(audio)
#         audiodata.append(data)

#     import numpy as np
#     corr1 = np.correlate(a=audiodata[3], v=audiodata[4])
#     corr2 = np.correlate(a=audiodata[0], v=audiodata[1])

#     print(corr1)
#     print(corr2)
ROOTDIR = '/home/hieuld/workspace/AntiSpoof/ASVapp/ASVmodel/input/tmp'
import numpy as np
def compute_similar(au1, au2):
    audio1 = os.path.join(ROOTDIR, au1)
    audio2 = os.path.join(ROOTDIR, au2)
    au1, _ = sf.read(audio1)
    au2, _ = sf.read(audio2)
    if len(au1) > len(au2):
        au2 = np.pad(au2, (0, len(au1) - len(au2)), 'constant', constant_values=(0))
    else :
        au1 = np.pad(au1, (0, len(au2) - len(au1)), 'constant', constant_values=(0))
    corr = np.correlate(a=au1, v=au2)
    return corr
if __name__ == '__main__':
    au1 = 'atachfromaMinh.flac'
    au2 = 'atachfromaMinh_cutnoise.flac'
    au1 = os.path.join(ROOTDIR, au1)
    au2 = os.path.join(ROOTDIR, au2)
    print(compute_similar(au1,au2))


from datetime import datetime,timedelta
import trio
import semver
from fastapi import APIRouter
import io
import pprint
from json import loads
import pandas as pd
from app.services.search import ElasticService
router = APIRouter()

@router.get('/api/v1/graph/{uuid}')
async def graph(uuid: str):
    index = ""
    meta = await getMetadata(uuid)
    print(meta)
    metrics = []
    if meta["benchmark"] == "k8s-netperf" :
        uuids = await getMatchRuns(meta,False)
        print(uuids)
        index = "k8s-netperf"
        oData = await getResults(uuid,uuids,index)
        cData = await getResults(uuid,[uuid],index)
        oMetrics = await processNetperf(oData)
        oMetrics = oMetrics.reset_index()
        nMetrics = await processNetperf(cData)
        nMetrics = nMetrics.reset_index()
        x=[]
        y=[]
        for index, row in oMetrics.iterrows():
            test = "{}-{}".format(row['profile'], row['messageSize'])
            value = "{}".format(row['throughput'])
            x.append(value)
            y.append(test)
        old = {'y' : x,
            'x' : y,
            'name' : 'Previous results average',
            'type' : 'bar',
            'orientation' : 'v'}
        x=[]
        y=[]
        for index, row in nMetrics.iterrows():
            test = "{}-{}".format(row['profile'], row['messageSize'])
            value = "{}".format(row['throughput'])
            x.append(value)
            y.append(test)
        new = {'y' : x,
            'x' : y,
            'name' : 'Current results average',
            'type' : 'bar',
            'orientation' : 'v'}
        metrics.append(old)
        metrics.append(new)

    elif meta["benchmark"] == "ingress-perf":
        uuids = await getMatchRuns(meta, False)
        index = "ingress-performance"
        data = await getResults(uuid, uuids, index)
    else:
        index = "ripsaw-kube-burner*"
        uuids = await getMatchRuns(meta,True)

        # We need to look at the jobSummary to ensure all UUIDs have similar iteration count.
        job = await jobSummary([uuid])
        jobs = await jobSummary(uuids)
        ids = jobFilter(job,jobs)

        oData = await getBurnerResults(uuid,ids,index)

        oMetrics = await processBurner(oData)
        oMetrics = oMetrics.reset_index()
        cData = await getBurnerResults(uuid,[uuid],index)
        nMetrics = await processBurner(cData)
        nMetrics = nMetrics.reset_index()
        x=[]
        y=[]
        for index, row in oMetrics.iterrows():
            test = "PodLatency-p99"
            value = row['P99']
            x.append(int(value)/1000)
            y.append(test)
        old = {'y' : x,
            'x' : y,
            'name' : 'Previous results p99',
            'type' : 'bar',
            'orientation' : 'v'}
        x=[]
        y=[]
        for index, row in nMetrics.iterrows():
            test = "PodLatency-p99"
            value = row['P99']
            x.append(int(value)/1000)
            y.append(test)
        new = {'y' : x,
            'x' : y,
            'name' : 'Current results P99',
            'type' : 'bar',
            'orientation' : 'v'}
        metrics.append(old)
        metrics.append(new)
    return metrics

async def jobSummary(uuids: list):
    index = "ripsaw-kube-burner*"
    ids = "\" OR uuid: \"".join(uuids)
    query = {
        "query": {
            "query_string": {
                "query": (
                    f'( uuid: \"{ids}\" )'
                    f' AND metricName: "jobSummary"'
                    )
            }
        }
    }
    print(query)
    es = ElasticService(airflow=False,index=index)
    response = await es.post(query)
    await es.close()
    runs = [item['_source'] for item in response["hits"]["hits"]]
    return runs

async def processBurner(data: dict) :
    pprint.pprint(data)
    df = pd.json_normalize(data)
    filterDF = burnerFilter(df)
    ptile = filterDF.groupby(['quantileName'])['P99'].quantile([.99])
    return ptile

async def processNetperf(data: dict) :
    pprint.pprint(data)
    df = pd.json_normalize(data)
    filterDF = netperfFilter(df)
    tput = filterDF.groupby(['profile','messageSize'])['throughput'].mean()
    return tput

def jobFilter(pdata: dict, data: dict):
    columns = ['uuid','jobConfig.jobIterations']
    pdf = pd.json_normalize(pdata)
    pick_df = pd.DataFrame(pdf, columns=columns)
    iterations = pick_df.iloc[0]['jobConfig.jobIterations']
    df = pd.json_normalize(data)
    ndf = pd.DataFrame(df, columns=columns)
    ids_df = ndf.loc[df['jobConfig.jobIterations'] == iterations ]
    return ids_df['uuid'].to_list()

def burnerFilter(data: dict) :
    #
    # Filter out aspects of the test to norm results
    #
    pprint.pprint(data)
    columns = ['quantileName','metricName', 'P99']
    ndf = pd.DataFrame(data, columns=columns)
    print(ndf)
    return ndf

def netperfFilter(df):
    #
    # Filter out aspects of the test to norm results
    #
    columns = ['profile','hostNetwork','parallelism','service','acrossAZ','samples',
               'messageSize','throughput','test']
    ndf = pd.DataFrame(df, columns=columns)
    hnfilter = df[ (ndf.hostNetwork == True) ].index
    hnd = ndf.drop(hnfilter)
    sfilter = hnd[ (hnd.service == True)].index
    sdf = hnd.drop(sfilter)
    azfilter = sdf[ (sdf.acrossAZ == True)].index
    adf = sdf.drop(azfilter)
    d = adf[ (adf.parallelism == 1) ]
    d = d[d.profile.str.contains('TCP_STREAM')]
    return d

async def getBurnerResults(uuid: str, uuids: list, index: str ):

    if len(uuids) > 1 :
        uuids.remove(uuid)
    ids = "\" OR uuid: \"".join(uuids)
    print(ids)
    query = {
        "query": {
            "query_string": {
                "query": (
                    f'( uuid: \"{ids}\" )'
                    f' AND metricName: "podLatencyQuantilesMeasurement"'
                    f' AND quantileName: "Ready"'
                    )
            }
        }
    }
    print(query)
    es = ElasticService(airflow=False,index=index)
    response = await es.post(query)
    await es.close()
    runs = [item['_source'] for item in response["hits"]["hits"]]
    return runs

async def getResults(uuid: str, uuids: list, index: str ):
    if len(uuids) > 1 :
        uuids.remove(uuid)
    ids = "\" OR uuid: \"".join(uuids)
    print(ids)
    query = {
        "query": {
            "query_string": {
                "query": (
                    f'(uuid: \"{ids}\")')
            }
        }
    }
    print(query)
    es = ElasticService(airflow=False,index=index)
    response = await es.post(query)
    await es.close()
    runs = [item['_source'] for item in response["hits"]["hits"]]
    return runs

async def getMatchRuns(meta: dict, workerCount: False):
    index = "perf_scale_ci"
    version = meta["ocpVersion"][:4]
    query = {
        "query": {
            "query_string": {
                "query": (
                    f'benchmark: "{meta["benchmark"]}"'
                    f' AND workerNodesType: "{meta["workerNodesType"]}"'
                    f' AND masterNodesType: "{meta["masterNodesType"]}"'
                    f' AND platform: "{meta["platform"]}"'
                    f' AND ocpVersion: {version}*'
                    f' AND jobStatus: success'
                    )
            }
        }
    }
    if workerCount :
        query = {
            "query": {
                "query_string": {
                    "query": (
                        f'benchmark: "{meta["benchmark"]}"'
                        f' AND workerNodesType: "{meta["workerNodesType"]}"'
                        f' AND masterNodesType: "{meta["masterNodesType"]}"'
                        f' AND masterNodesCount: "{meta["masterNodesCount"]}"'
                        f' AND workerNodesCount: "{meta["workerNodesCount"]}"'
                        f' AND platform: "{meta["platform"]}"'
                        f' AND ocpVersion: {version}*'
                        f' AND jobStatus: success'
                        )
                }
            }
        }
    print(query)
    es = ElasticService(airflow=False)
    response = await es.post(query)
    await es.close()
    runs = [item['_source'] for item in response["hits"]["hits"]]
    uuids = []
    for run in runs :
        uuids.append(run["uuid"])
    return uuids

async def getMetadata(uuid: str) :
    index = "perf_scale_ci"
    query = {
        "query": {
            "query_string": {
                "query": (
                    f'uuid: "{uuid}"')
            }
        }
    }
    print(query)
    es = ElasticService(airflow=False)
    response = await es.post(query)
    await es.close()
    meta = [item['_source'] for item in response["hits"]["hits"]]
    return meta[0]

"""
    [ {
        'y' : ["4,13","4.14"],
        'x' : [100,120],
        'type' : 'bar',
        'orientation' : 'v'
    }]
"""


import math

def jc(n):
    res = 1
    while n > 0:
        res *= n
        n -= 1
    return res

l = [1, 2, 3, 4, 5, 6, 7, 8, 9]
n = int(input())
k = int(input())
res = ""
while n > 0:
    temp = jc(n - 1)
    index = int(math.ceil(k / temp))-1
    if index<0:
        index = 0
    res += str(l.pop(index))
    k %= temp
    n -= 1
print(res)


from mitmproxy import ctx, http
import logging
import time
import os
import html
import json
import pdf_printer

# Global variables for the request template and article list
articles = []
template_flow = None

def response(flow: http.HTTPFlow) -> None:
    if "https://mp.weixin.qq.com/mp/profile_ext?action=getmsg" in flow.request.pretty_url:
        global articles

        response_json = json.loads(flow.response.text)
        has_next = response_json['can_msg_continue']
        needs_replay = has_next or not flow.is_replay

        # Modify the query parameters for the next request
        query_params = dict(flow.request.query.fields)
        updated_params = query_params.copy()

        # template, no need to handle the response yet
        if not flow.is_replay:
            template_flow = flow.copy()
            updated_params['offset'] = 0

        # start adding new articles; modify parameters for next replay
        else:
            santinised_json_raw = response_json['general_msg_list'].replace("\/", "/")
            new_articles = json.loads(santinised_json_raw)['list']
            articles += flat_records_from(new_articles)

            offset = int(query_params.get('offset', '0'))
            count = int(query_params.get('count', '10'))
            updated_params['offset'] = str(offset + count)

        # Replay the request every 2 seconds if needed
        if needs_replay:
            replay_flow = flow.copy()
            replay_flow.request.query.fields = list(updated_params.items())
            time.sleep(2)
            ctx.master.commands.call("replay.client", [replay_flow])

        # Stop intercepting traffic once no longer needed
        else:
            with open('articles.json', 'w', encoding='utf8') as f:
                json.dump(articles, f, ensure_ascii=False)

            logging.info('Finished capturing articles. Saved to articles.json')
            ctx.master.shutdown()

def done():
    logging.info('Done')

def replace_uncommon_characters(input_str):
    for s in ['(',')','|',' ','/','*','\\','.','<','>','&','?',' ']:
        output_str = input_str.replace(s, '-')
    return output_str

def sub_dict(data, keys):
    return {k: html.unescape(data[k]) for k in data if k in keys}

def sanitised_record(msg_info, p_date):
    date_pretty = time.strftime('%Y%m%d',time.localtime(p_date))

    keys = ['title', 'author', 'content_url', 'digest', 'cover', 'source_url']
    data = sub_dict(msg_info, keys)
    title = replace_uncommon_characters(data['title'])
    record_item = {}
    record_item['title'] = title
    record_item['date'] = date_pretty
    record_item['url'] = data['content_url']

    # record = "{}_{} : {}\n".format(date_pretty, title, data['content_url'])
    return record_item

def flat_records_from(msg_list):
    records = []
    for msg in msg_list:
        p_date = msg.get('comm_msg_info').get('datetime')

        msg_info = msg.get("app_msg_ext_info")

        if msg_info:
            multi_msg_info = msg_info.get("multi_app_msg_item_list")
            if multi_msg_info:
                for multi_msg_item in multi_msg_info:
                    records.append(sanitised_record(multi_msg_item, p_date))
            else:
                records.append(sanitised_record(msg_info, p_date))
        else:
            ctx.log.info(u"Non articles message，data=%s" % json.dumps(msg.get("comm_msg_info")))
    return records


class GPUInfo:
    def __init__(self, countGpu, driverVersion, indexGpu, nameGpu,memoryTotal,
                memoryUsed,memoryFree,enforcedPowerLimit,powerUsage,
                fanSpeed, utilizationRate,temperature):
        self.countGpu = countGpu
        self.driverVersion=driverVersion
        self.indexGpu = indexGpu
        self.nameGpu = nameGpu
        self.memoryTotal = memoryTotal
        self.memoryUsed = memoryUsed
        self.memoryFree = memoryFree
        self.enforcedPowerLimit = enforcedPowerLimit
        self.powerUsage = powerUsage
        self.fanSpeed = fanSpeed
        self.utilizationRate = utilizationRate
        self.temperature = temperature

# 经过 getGPUInfo 实际得到的数据类型
# count:  2       <class 'int'>
# driver_version: 470.141.03      <class 'str'>
# index:  1       <class 'int'>
# name:   NVIDIA GeForce RTX 2080 Ti      <class 'str'>
# memory_toal:    11019   <class 'int'>
# memory_used:    7831    <class 'int'>
# memory_free:    3187    <class 'int'>
# enforced_power_limit:   250     <class 'int'>
# power_usage:    184     <class 'int'>
# fan_speed:      90      <class 'int'>
# utilization_rate:       48      <class 'int'>
# temperature:    78      <class 'int'>