from contextlib import nullcontext
from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union
from uuid import uuid4

import torch
from fairscale.nn.data_parallel import FullyShardedDataParallel as FSDP
from fairscale.nn.data_parallel import ShardedDataParallel as SDDP
from torch.nn.parallel import DataParallel as DP
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader as DL
from torch.utils.data import Dataset
from torch.utils.data.distributed import DistributedSampler, Sampler

from stoke.configs import (
    AMPConfig,
    ApexConfig,
    ClipGradConfig,
    ClipGradNormConfig,
    DDPConfig,
    DeepspeedConfig,
    FairscaleFSDPConfig,
    FairscaleOSSConfig,
    FairscaleSDDPConfig,
    HorovodConfig,
    StokeOptimizer,
)
from stoke.data import StokeDataLoader
from stoke.distributed import RunnerDistEnum
from stoke.extensions import RunnerOptimizerEnum
from stoke.fp16 import RunnerFP16Enum
from stoke.io_ops import RunnerIOEnum
from stoke.status import DistributedOptions, FP16Options, StokeStatus
from stoke.utils import (
    ParamNormalize,
    T_co,
    _collate_fn_t,
    _worker_init_fn_t,
    zero_optimizer_grads,
)

class Stoke:
    """High level stoke object that manages all necessary configs and provides a unified interface to ops

    This is the main class within Stoke. Functionally it manages all interfaces to the necessary wrapped ops (model,
    loss, backward, step), provides helper functions, and dynamically constructs the runtime that handles the
    combinatorics problem of underlying frameworks (DDP, Horovod, Deepspeed, Fairscale),
    mixed-precision (AMP or APEX) and devices (CPU or GPU)

    Attributes
    ----------
    amp_config
    apex_config
    batch_size
    cuda
    ddp_config
    deepspeed_config
    distributed
    effective_batch_size
    ema_loss
    fp16
    fsdp_config
    fully_sharded
    gpu
    grad_accum
    grad_clip
    horovod_config
    is_amp
    is_apex
    is_ddp
    is_deepspeed
    is_horovod
    loss_access
    model_access
    nccl
    num_model_parameters
    optimizer
    oss
    oss_config
    rank
    scaler
    sddp_config
    sharded
    status
    world_size
    _agg_loss: Union[float, List[float], Tuple[float]]
        aggregated loss for grad accumulation (single or multiple losses)
    _backward_steps: int
        Number of times gradients have been calculated on a batch of samples (calls to backward)
    _grad_accum_counter: int
        counter for grad accumulation steps
    _loss: Union[Callable, List[Callable], Tuple[Callable]]
        callable function that calculates a loss from the model outputs
    _last_step_loss: list, tuple, or float
        last loss step calculation aggregated over device(s)
    _model: torch.nn.Module
        instance of torch.nn.Module for Stoke to handle
    _optimizer: StokeOptimizer
        StokeOptimizer config object that describes the torch.optim.Optimizer and it's kwargs
    _optimizer_steps: int
        Number of times step has been called on the optimizer
    _runner: StokeRunner
        the dynamically created runtime object that handles all ops
    _status: StokeStatus
        StokeStatus object that sets and maintains the current configuration
    _verbose: bool
        print verbosity
    _rolling_loss_steps: int
        number of steps that have been called for the rolling loss
    _rolling_mean_loss: list, tuple, or float
        current ema loss
    _ema_weight: float
        weight used for any ema calculation on metrics

    """

    def __init__(
        self,
        model: torch.nn.Module,
        optimizer: StokeOptimizer,
        loss: Union[Callable, List[Callable], Tuple[Callable]],
        batch_size_per_device: int,
        grad_accum_steps: Optional[int] = 1,
        grad_clip: Optional[Union[ClipGradConfig, ClipGradNormConfig]] = None,
        gpu: bool = False,
        fp16: Optional[FP16Options] = None,
        distributed: Optional[DistributedOptions] = None,
        fairscale_oss: bool = False,
        fairscale_sddp: bool = False,
        fairscale_fsdp: bool = False,
        configs: Optional[
            List[
                Union[
                    AMPConfig,
                    ApexConfig,
                    DDPConfig,
                    DeepspeedConfig,
                    FairscaleOSSConfig,
                    FairscaleSDDPConfig,
                    FairscaleFSDPConfig,
                    HorovodConfig,
                ]
            ]
        ] = None,
        info_rank: Optional[Union[int, List[int]]] = 0,
        verbose: bool = True,
        ema_weight: float = 0.1,
    ):
        """Init for Stoke class object

        Parameters
        ----------
        model: torch.nn.Module
            PyTorch model
        optimizer: StokeOptimizer
            Optimizer configuration
        loss: Union[Callable, List[Callable], Tuple[Callable]]
            Callable loss function or functions
        batch_size_per_device: int
            Batch size at the single device level
        grad_accum_steps: Optional[int], default: 1
            Number of gradient accumulation steps
        grad_clip: Optional[Union[ClipGradConfig, ClipGradNormConfig]], default: None
            Gradient clipping configuration
        gpu: bool, default: False
            flag to use GPU device(s)
        fp16: Optional[FP16Options], default: None
            Choice of mixed-precision backend
        distributed: Optional[DistributedOptions], default: None
            Choice of distributed backend
        fairscale_oss: bool, default: False
            Flag to activate optimizer state sharding using Fairscale
        fairscale_sddp: bool, default: False
            Flag to activate sharded DDP using Fairscale
        fairscale_fsdp: bool, default: False
            Flag to activate fully sharded DDP using Fairscale
        configs: Optional[List[Union[AMPConfig, ApexConfig, DDPConfig, DeepspeedConfig, FairscaleOSSConfig, FairscaleSDDPConfig, FairscaleFSDPConfig, HorovodConfig]], default: None
            Configuration objects for runtimes
        info_rank: Optional[Union[int, List[int]]], default = 0
            Constrain prints to specific devices
        verbose: bool, default: True
            Flag for verbosity
        ema_weight: float, default: 0.5
            weight used for any ema calculation on metrics

        """
        # Verbosity
        self._verbose = verbose
        # Info rank
        self._info_rank = info_rank
        # EMA
        self._ema_weight = ema_weight
        # Setup the StokeState
        self._status = StokeStatus(
            batch_size_per_device=batch_size_per_device,
            grad_accum=grad_accum_steps,
            grad_clip=grad_clip,
            gpu=gpu,
            fp16=fp16,
            distributed=distributed,
            fairscale_oss=fairscale_oss,
            fairscale_sddp=fairscale_sddp,
            fairscale_fsdp=fairscale_fsdp,
            configs=configs,
        )
        # Run some checks
        self._model = self._check_model(model)
        self._optimizer = self._check_optimizer(optimizer)
        self._loss = self._check_loss(loss)
        # Dynamically construct the StokeRunner from the StokeStatus
        self._runner, class_info = self._build_runner()
        # Setup distributed backend
        self._runner.setup_distributed()
        # Post here the runner will have the print_device function that is mapped to the self.print here
        # as it needs rank to be accessible before working
        if self._verbose:
            dev_id = (
                self.rank
                if (self.rank == "cpu" or self.rank == "gpu")
                else self._info_rank
            )
            self.print(f"Printing verbose information on rank(s): {dev_id}")
            # Print the runner class info from the mixins
            self.print(class_info)
        # Possibly place model on GPU depending on StokeStatus -- before wrap calls
        self._place_model_on_gpu()
        # Handle the wrap ops in the correct order
        self._handle_ordered_wrap_ops(optimizer=optimizer)
        # Create some tracking vars
        self._grad_accum_counter = 0
        self._optimizer_steps = 0
        self._backward_steps = 0
        self._last_step_loss = self._set_loss_to_zero()
        self._agg_loss = self._set_loss_to_zero()
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0
        # Set post-init status variables
        self._status.set_post_init_values(world_size=self.world_size)
        # Print the final configuration
        if self._verbose:
            self.print(msg=self._status)

    def _wrap_optimizer_then_model(self, optimizer: StokeOptimizer):
        """Handles wrapping of optimizer then the model

        This holds only for SDDP, Horovod, and APEX as these need to use an instantiated optimizer before wrapped
        methods are called

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # Build the optimizer
        self._optimizer = self._runner.build_optimizer(
            optimizer=optimizer["optimizer"],
            optimizer_kwargs=optimizer["optimizer_kwargs"],
            model=self._model,
        )
        # Setup/Initialize FP16 backend -- in this case the optimizer is passed through
        self._runner.wrap_fp16(model=self._model, optimizer=self._optimizer)
        # Wrap with distributed backend -- in this case the optimizer is passed through
        self._model, self._optimizer = self._runner.wrap_distributed(
            model=self._model, grad_accum=self.grad_accum, optimizer=self._optimizer
        )

    def _wrap_model_then_optimizer(self, optimizer: StokeOptimizer):
        """Handles wrapping of model then optimizer

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # Wrap with distributed backend -- in this case the optimizer is passed as None since it doesn't exist yet
        # don't use the return for the optimizer in this case
        self._model, _ = self._runner.wrap_distributed(
            model=self._model, grad_accum=self.grad_accum, optimizer=None
        )
        # Setup/Initialize FP16 backend -- in this case the optimizer is passed as None since it doesn't exist yet
        self._runner.wrap_fp16(model=self._model, optimizer=None)
        # Build the optimizer
        self._optimizer = self._runner.build_optimizer(
            optimizer=optimizer["optimizer"],
            optimizer_kwargs=optimizer["optimizer_kwargs"],
            model=self._model,
        )

    def _handle_ordered_wrap_ops(self, optimizer: StokeOptimizer):
        """Handles wrapping model, using FP16, and wrapping optimizer in the correct order depending on Stoke Status

        Parameters
        ----------
        optimizer: StokeOptimizer
            Optimizer configuration

        Returns
        -------
        None

        """
        # if SDDP + OSS, Horovod, and APEX then we need to make sure that the optimizer gets wrapped before the model
        # gets wrapped, all other models follow standard DDP paradigm (or their own DeepSpeed)
        if (self.sharded and self.oss) or self.is_apex or self.is_horovod:
            self._wrap_optimizer_then_model(optimizer=optimizer)
        else:
            self._wrap_model_then_optimizer(optimizer=optimizer)

    def _check_accum(self):
        """Checks if the current step is the last accumulation step

        Returns
        -------
        bool

        """
        return (self._grad_accum_counter + 1) % (self.grad_accum + 1) == 0

    def _check_pre_accum(self):
        """Checks if we are at the pre-accumulate step

        Returns
        -------
        bool

        """
        return (self._grad_accum_counter + 1) % (self.grad_accum + 1) == self.grad_accum

    def _set_loss_to_zero(self):
        """Used to set a loss tracker to zero depending on the type

        Returns
        -------
        float or list or tuple of reset loss

        """
        return (
            type(self._loss)([0.0] * len(self._loss))
            if isinstance(self._loss, (list, tuple))
            else 0.0
        )

    def reset_ema(self):
        """Used to reset the current state of the rolling mean loss

        Returns
        -------
        None

        """
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0

    def print_ema_loss(
        self, prepend_msg: str = "Current EMA Loss", single_line: bool = False
    ):
        """Prints the current ema loss synced across all devices

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        prepend_msg: str, default: "Current EMA Loss"
            message prepend to print
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        if isinstance(self._rolling_mean_loss, (list, tuple)):
            print_vals = [
                f"{prepend_msg} {idx}: {val:.3f}"
                for idx, val in enumerate(self._rolling_mean_loss)
            ]
            self.print(print_vals, single_line=single_line)
        else:
            self.print(f"{prepend_msg}: {self._rolling_mean_loss:.3f}")

    def print_mean_accumulated_synced_loss(
        self,
        prepend_msg: str = "Mean Accumulated & Synced Loss",
        pre_backwards: bool = True,
        single_line: bool = False,
    ):
        """Prints the mean accumulated and device synced loss only after the grad accumulation step

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        prepend_msg: str, default: "Mean Accumulated & Synced Loss"
            message prepend to print
        pre_backwards: bool, default: True
            if being called pre backward step
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        check_fn = self._check_pre_accum if pre_backwards else self._check_accum
        if check_fn():
            if isinstance(self._agg_loss, (list, tuple)):
                print_vals = self._scale_agg_loss()
                self.print(print_vals, single_line=single_line)
            else:
                self.print(f"{prepend_msg}: {self._scale_agg_loss():.3f}")

    def _scale_agg_loss(self):
        """Scales the mean aggregated loss by  grad accum

        Returns
        -------
        scale_vals: list or float of mean aggregated loss

        """
        if isinstance(self._agg_loss, (list, tuple)):
            scale_vals = [
                val / self.grad_accum for idx, val in enumerate(self._agg_loss)
            ]
        else:
            scale_vals = self._agg_loss / self.grad_accum
        return scale_vals

    def print_synced_loss(
        self,
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]],
        prepend_msg: str = "Step Synced Loss",
        device=None,
        single_line: bool = False,
    ):
        """Prints a device synced loss at a single step

        Handles single or multiple losses. Prints only on devices specified by self._info_rank

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            current loss(es) on the device
        prepend_msg: str, default: "Step Synced Loss"
            message prepend to print
        device: default: None
            specify the device to place the synced loss on (defaults to same device)
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        printable_loss = self.detach_and_sync_loss(loss, device)
        if isinstance(printable_loss, (list, tuple)):
            print_vals = [
                f"{prepend_msg} {idx}: {val * self.grad_accum:.3f}"
                for idx, val in enumerate(printable_loss)
            ]
            self.print(print_vals, single_line=single_line)
        else:
            self.print(msg=f"{prepend_msg}: {printable_loss * self.grad_accum:.3f}")

    def print_on_devices(
        self, msg: Union[str, List[str]], rank: Optional[Union[int, List[int]]] = 0
    ):
        """Wraps runner print interface for shorter semantics

        Parameters
        ----------
        msg: str
            message to print
        rank: Union[int, List[int]], default: 0
            which ranks to print on

        Returns
        -------
        None

        """
        self._runner.print_device(msg=msg, rank=rank)

    def print(self, msg: Union[str, List[str]], single_line: bool = False):
        """Wraps the runners print device and forces print on the _info_rank attribute(s)

        Parameters
        ----------
        msg: str
            message to print
        single_line: bool, default: False
            if iterable print all on one line space and comma separated

        Returns
        -------
        None

        """
        self._runner.print_device(
            msg=msg, rank=self._info_rank, single_line=single_line
        )

    @staticmethod
    def _check_model(model: torch.nn.Module):
        """Verifies the type of the model

        Parameters
        ----------
        model: torch.nn.Module
            current torch model

        Returns
        -------
        None

        """
        # Check if the model is an nn.Module such that it has a forward method
        if not isinstance(model, torch.nn.Module):
            raise TypeError(
                f"Stoke -- Model is not of type torch.nn.Module, currently {type(model)}"
            )
        return model

    @staticmethod
    def _check_optimizer(optimizer: StokeOptimizer):
        """Verifies the type of the optimizer

        Parameters
        ----------
        optimizer: StokeOptimizer
            Current optimizer configuration TypedDict (aka dict)

        Returns
        -------
        None

        """
        if not isinstance(optimizer, dict):
            raise TypeError(
                f"Stoke -- Optimizer is not of type torch.optim.Optimizer, currently {type(optimizer)}"
            )
        return optimizer

    def _check_loss(self, loss: Union[Callable, List[Callable], Tuple[Callable]]):
        """Checks to make sure the loss function(s) is/are callable

        Parameters
        ----------
        loss: Union[Callable, List[Callable], Tuple[Callable]]
            Current callable loss(es)

        Returns
        -------
        None

        """
        if isinstance(loss, (list, tuple)):
            loss = [self._check_loss(val) for val in loss]
            return loss
        elif isinstance(loss, Callable):
            return loss
        else:
            raise TypeError(
                f"Stoke -- Loss is not of type Callable, currently {type(loss)}"
            )

    def _place_model_on_gpu(self):
        """Automatically moves the model to GPU device(s)

        Returns
        -------
        None

        """
        if self.gpu and not self.is_deepspeed:
            if self._verbose:
                self.print(f"Automatically handling moving model to GPU(s)...")
            self._model.cuda()

    def _build_runner(self):
        """Builds the runtime object from the mixin style classes

        Mixes the distributed class, fp16 class, and optimizer class into a single object such that all can be called
        from the same interface. Prevents verbose calls to multiple objects and unifies all functionality under a
        a single interface. Might prevent some IDE type-hinting as it's dynamic

        Returns
        -------
        StokeRunner
            runtime runner object

        """
        # Get the classes
        dist_class = self._get_distributed_mixin()
        fp16_class = self._get_fp16_mixin()
        optimizer_class = self._get_optimizer_mixin()
        io_class = self._get_io_mixin()

        # Python MRO hack to make sure the inits of all the Mixin classes get called
        def __multiple_mixin_init__(*args, **kwargs):
            dist_class.__init__(*args, **kwargs)
            fp16_class.__init__(*args, **kwargs)
            optimizer_class.__init__(*args, **kwargs)
            io_class.__init__(*args, **kwargs)

        # Configs pass through
        kwargs_dict = {
            "amp_config": self.amp_config,
            "apex_config": self.apex_config,
            "ddp_config": self.ddp_config,
            "deepspeed_config": self.deepspeed_config,
            "horovod_config": self.horovod_config,
            "oss_config": self.oss_config,
            "sharded_config": self.sddp_config,
            "fully_sharded_config": self.fsdp_config,
        }
        # Generate the runner class from the mixins based on the StokeStatus
        runner_class = type(
            "StokeRunner",
            (dist_class, fp16_class, optimizer_class, io_class),
            {"__init__": __multiple_mixin_init__},
        )(
            verbose=self._verbose,
            batch_size_per_device=self.batch_size,
            grad_accum_steps=self.grad_accum,
            grad_clip=self.grad_clip,
            info_rank=self._info_rank,
            loss=self._loss,
            **kwargs_dict,
        )
        # Make a list of class info for print later
        class_info = [
            f"Distributed Mixin: {dist_class.__name__}",
            f"Optimizer Mixin: {dist_class.__name__}",
            f"FP16 Mixin: {fp16_class.__name__}",
            f"IO Mixin: {io_class.__name__}",
        ]
        return runner_class, class_info

    def _get_io_mixin(self):
        """Determines which IO class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated ioclass

        """
        if self.is_deepspeed:
            return_class = RunnerIOEnum.deepspeed.value
        elif self.is_horovod:
            return_class = RunnerIOEnum.horovod.value
        elif self.is_ddp:
            return_class = RunnerIOEnum.ddp.value
        else:
            return_class = RunnerIOEnum.base.value
        return return_class

    def _get_optimizer_mixin(self):
        """Determines which optimizer class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated optimizer class

        """
        if self.oss:
            return_class = RunnerOptimizerEnum.oss.value
        else:
            return_class = RunnerOptimizerEnum.base.value
        return return_class

    def _get_distributed_mixin(self):
        """Determines which distributed class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated distributed class

        """
        # if not gpu then fall to cpu single
        if not self.gpu:
            return_class = RunnerDistEnum.cpu.value
        # if gpu but no distributed then fall to single gpu
        elif self.gpu and (self.distributed is None):
            return_class = RunnerDistEnum.gpu.value
        elif self.gpu and (self.distributed is not None):
            return_class = RunnerDistEnum[self.distributed].value
        else:
            raise ValueError("Stoke -- Cannot map to a valid distributed class")
        return return_class

    def _get_fp16_mixin(self):
        """Determines which fp16 class to use

        Embedded logic based on the enum class

        Returns
        -------
        ABCMeta
            un-instantiated fp16 class

        """
        if self.fp16 is not None:
            return_class = RunnerFP16Enum[self.fp16].value
        else:
            return_class = RunnerFP16Enum.full.value
        return return_class

    def DataLoader(
        self,
        dataset: Dataset[T_co],
        shuffle: bool = False,
        sampler: Optional[Sampler[int]] = None,
        batch_sampler: Optional[Sampler[Sequence[int]]] = None,
        num_workers: int = 0,
        collate_fn: _collate_fn_t = None,
        pin_memory: bool = False,
        drop_last: bool = False,
        timeout: float = 0,
        worker_init_fn: Optional[_worker_init_fn_t] = None,
        multiprocessing_context=None,
        generator=None,
        *,
        prefetch_factor: int = 2,
        persistent_workers: bool = False,
    ):
        """Provides a shim interface to torch.utils.data.DataLoader with mapped kwargs.

        Shim is necessary for two reasons... to inject some horovod runtime configs (make sure forkserver is called)
        and to automatically handle device placement since the gpu/fp16 flags can't be determined until the StokeStatus
        object is available which is post init. This could be disconnected from this class but it would require the
        user to forward on device or fp16 configs which breaks the paradigm that the flags only need to be set and
        never handled

        Parameters
        ----------
        dataset: Dataset
            dataset from which to load the data.
        shuffle: bool, default: False
            set to ``True`` to have the data reshuffled at every epoch.
        sampler: Sampler or Iterable, default: None
            defines the strategy to draw samples from the dataset. Can be any ``Iterable`` with ``__len__``
            implemented. If specified, :attr:`shuffle` must not be specified.
        batch_sampler: Sampler or Iterable, default: None:
            like :attr:`sampler`, but returns a batch of indices at a time. Mutually exclusive with
            :attr:`batch_size`, :attr:`shuffle`, :attr:`sampler`, and :attr:`drop_last`.
        num_workers: int, default: 0
            how many subprocesses to use for data loading. ``0`` means that the data will be loaded in the main process.
        collate_fn: callable, optional:
            merges a list of samples to form a mini-batch of Tensor(s).  Used when using batched loading from a
            map-style dataset.
        pin_memory: bool, default: False:
            If ``True``, the data loader will copy Tensors into CUDA pinned memory before returning them. If your
            data elements are a custom type, or your :attr:`collate_fn` returns a batch that is a custom type,
            see the example below.
        drop_last: bool, default: False
            set to ``True`` to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
            If ``False`` and the size of dataset is not divisible by the batch size, then the last batch
            will be smaller.
        timeout: numeric, default: 0
            if positive, the timeout value for collecting a batch from workers. Should always be non-negative.
        worker_init_fn: callable, default: None
            If not ``None``, this will be called on each worker subprocess with the worker id
            (an int in ``[0, num_workers - 1]``) as input, after seeding and before data loading.
        generator: torch.Generator: None
            If not ``None``, this RNG will be used by RandomSampler to generate random indexes and multiprocessing
            to generate `base_seed` for workers.
        prefetch_factor: int, default: 2
            Number of samples loaded in advance by each worker. ``2`` means there will be a total of 2 * num_workers
            samples prefetched across all workers.
        persistent_workers: bool, default: False
            If ``True``, the data loader will not shutdown the worker processes after a dataset has been
            consumed once. This allows to maintain the workers `Dataset` instances alive.

        Returns
        -------
        StokeDataLoader
            wrapped torch.utils.data.DataLoader object

        """
        # Check if forkserver is available for horovod and use
        if (
            num_workers > 0
            and "forkserver" in torch.multiprocessing.get_all_start_methods()
            and self.is_horovod
            and self.horovod_config.use_fork_server
        ):
            torch.multiprocessing.set_start_method("forkserver")
            if self._verbose:
                print(
                    f"Stoke -- Attempting to use forkserver as multiprocessing_context"
                )

        if self.distributed is not None and not isinstance(sampler, DistributedSampler):
            raise TypeError(
                f"Stoke -- Using a distributed backend requires passing an instance of a "
                f"DistributedSampler to the sampler argument"
            )
        if self._verbose and self.gpu:
            self.print(
                f"Stoke -- Automatically handling moving model input data to GPU(s)..."
            )
        # Assemble a kwargs dict as the super call with direct named args can cause un-traceable behavior (#23)
        # Insight from PyTorch Lightning that has to handle their DataLoader shims with all kwargs
        # https://github.com/PyTorchLightning/pytorch-lightning/blob/6609b2e46f5eb2cde6c42aedf5b843d050a4bb8d/pytorch_lightning/trainer/data_loading.py#L215
        kwargs = {
            "batch_size": self.batch_size,
            "shuffle": shuffle,
            "sampler": sampler,
            "batch_sampler": batch_sampler,
            "num_workers": num_workers,
            "collate_fn": collate_fn,
            "pin_memory": pin_memory,
            "drop_last": drop_last,
            "timeout": timeout,
            "worker_init_fn": worker_init_fn,
            "multiprocessing_context": multiprocessing_context,
            "generator": generator,
            "prefetch_factor": prefetch_factor,
            "persistent_workers": persistent_workers,
        }
        # Forward the already known options from the Stoke status
        return StokeDataLoader(dataset, gpu=self.gpu, fp16=self.fp16, **kwargs)

    def model(self, *args, **kwargs):
        """Wrapped model forward call

        Parameters
        ----------
        *args: list or tuple
            Additional arguments should be passed as keyword arguments
        **kwargs: dict, optional
            Extra arguments passed to the model forward call

        Returns
        -------
        model forward output

        """
        with self._runner.model_context:
            return self._model(*args, **kwargs)
            # return self.model_access(*args, **kwargs)

    def loss(self, *args, **kwargs):
        """Wrapped callable loss function call

        Handles internal logic of aggregating up the losses for single and multiple losses

        Parameters
        ----------
        *args: list or tuple
            Additional arguments should be passed as keyword arguments
        **kwargs: dict, optional
            Extra arguments passed to the loss function call(s)

        Returns
        -------
        outputs of callable loss function(s)

        """
        # TODO: WIP Handle multiple losses. Should support list/tuple of losses. Check non base PyTorch
        with self._runner.loss_context:
            if isinstance(self._loss, (list, tuple)):
                loss = type(self._loss)(val(*args, **kwargs) for val in self._loss)
                sync_loss = [self.detach_and_sync_loss(val) for val in loss]
                self._last_step_loss = type(self._loss)(
                    val for idx, val in enumerate(sync_loss)
                )
                self._agg_loss = type(self._loss)(
                    self._agg_loss[idx] + val for idx, val in enumerate(sync_loss)
                )
                self._handle_ema_loss(loss=sync_loss)
                if self.grad_accum > 1 and self.model_access.training:
                    loss = type(loss)(val / self.grad_accum for val in loss)
            else:
                loss = self._loss(*args, **kwargs)
                sync_loss = self.detach_and_sync_loss(loss)
                self._last_step_loss = sync_loss
                self._agg_loss += sync_loss
                self._handle_ema_loss(loss=sync_loss)
                # Handle grad accumulation by dividing by the accumulation steps
                if self.grad_accum > 1 and self.model_access.training:
                    loss = loss / self.grad_accum
            return loss

    def _handle_ema_loss(self, loss: Union[float, List[float], Tuple[float]]):
        """Handles calculating the ema loss

        Parameters
        ----------
        loss: Union[float, List[float], Tuple[float]]
            current calculated loss list, tuple or float

        Returns
        -------
        None

        """
        self._rolling_loss_steps += 1
        if isinstance(loss, (list, tuple)):
            self._rolling_mean_loss = type(self._rolling_mean_loss)(
                self._ema_loss(value=val, current_mean=self._rolling_mean_loss[idx])
                for idx, val in enumerate(loss)
            )
        else:
            self._rolling_mean_loss = self._ema_loss(
                value=loss, current_mean=self._rolling_mean_loss
            )

    def _ema_loss(self, value: float, current_mean: float):
        """Calculate the ema of the loss

        Parameters
        ----------
        value: float
            current loss value
        current_mean: float
            current mean value

        Returns
        -------
        current ema value: float

        """
        if self._rolling_loss_steps == 1:
            return value
        else:
            return (self._ema_weight * value) + (
                (1.0 - self._ema_weight) * current_mean
            )

    def backward(
        self, loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
    ):
        """Wrapped backwards call

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            Callable loss function(s)

        Returns
        -------
        None

        """
        # Increment the grad counter
        self._grad_accum_counter += 1
        # Set the context based on the counter
        dist_cm = (
            nullcontext()
            if self._check_accum()
            else self._runner.grad_accum_context(self._model)
        )
        with dist_cm:
            self._runner.backward_call(
                loss=loss, model=self.model_access, optimizer=self._optimizer
            )
        # Increment the number of total calls to backward (each backward to a loss is only considered 1)
        self._backward_steps += 1

    def step(self):
        """Wrapped step call

        Handles grad clipping internally

        Returns
        -------
        None

        """
        # Step the optimizer only if the modulo is zero
        if self._check_accum():
            if self._verbose and self.grad_accum > 0:
                self.print(f"Gradient Accumulation Steps: {self.grad_accum}")
            # Clip if needed
            if self.grad_clip is not None:
                self._runner.clip_grad(
                    self.grad_clip,
                    self._model if self.fully_sharded else self.model_access,
                    self._optimizer,
                    oss=self.oss,
                    horovod=self.is_horovod,
                    deepspeed=self.is_deepspeed,
                    fsdp=self.fully_sharded,
                )
            # Handle the optimizer step
            step_cm = (
                self._runner.step_context(self._optimizer)
                if self.grad_clip is not None
                else nullcontext()
            )
            with step_cm:
                self._runner.step_call(
                    model=self.model_access, optimizer=self._optimizer
                )
            # Reset for the accumulated step
            self._reset()
            # Increment the number of step calls to the optimizer
            self._optimizer_steps += 1
        # if deepspeed we need to step everytime as it handles the grad accumulation internally
        elif self.is_deepspeed:
            # Handle the optimizer step
            step_cm = (
                self._runner.step_context(self._optimizer)
                if self.grad_clip is not None
                else nullcontext()
            )
            with step_cm:
                self._runner.step_call(
                    model=self.model_access, optimizer=self._optimizer
                )

    def _reset(self):
        """Resets the state post optimizer step call

        Returns
        -------
        None

        """
        if self._verbose:
            self.print("Resetting all grad/variables for next optimizer step")
        # Zero the grads if not deepspeed
        if not self.is_deepspeed:
            self.zero_grads()
        # Reset counter
        self._grad_accum_counter = 0
        # Reset agg loss -- single or mutiple losses
        self._agg_loss = self._set_loss_to_zero()

    def save(
        self,
        path: str,
        name: str = uuid4(),
        extension: str = "pt",
        create_directory: bool = True,
        extras: Optional[dict] = None,
    ):
        """Saves a model checkpoint using the correct backend interface

        Parameters
        ----------
        path: str
            path to directory to save the model checkpoint (prefer absolute paths over relative paths)
        name: str, default: uuid4()
            name used to save checkpoint file
        extension: str, default: '.pt'
            extension used to save PyTorch model checkpoint
        create_directory: bool, default: True
            flag to create the directory path if it doesn't exist
        extras: dict, default: None
            a dictionary of any extra things to save

        Returns
        -------
        path: str
            path to directory that the model checkpoint was saved
        tag: str
            full tag name the model checkpoint was saved as

        """
        out_path, tag = self._runner.save(
            model=self._model if self.fully_sharded else self.model_access,
            optimizer=self.optimizer,
            path=path,
            backward_step=self._backward_steps,
            grad_accum_step=self._grad_accum_counter,
            optimizer_step=self._optimizer_steps,
            name=name,
            scaler_dict=self.fp16_state_dict,
            extension=extension,
            create_directory=create_directory,
            extras=extras,
            status=self.status.status,
        )
        self.print(f"Successfully saved model checkpoint to {out_path}/{tag}")
        return out_path, tag

    def load(self, path: str, tag: str, strict: bool = True):
        """Loads a model checkpoint using the correct backend interface

        Parameters
        ----------
        path: str
            path to directory that the model checkpoint was saved (prefer absolute paths over relative paths)
        tag: str
            full tag name the model checkpoint was saved as
        strict: bool
            ignore non-matching keys

        Returns
        -------
        extras: dict, default: None
            a dictionary of any custom fields the user passed to the save function

        """
        # TODO: How to deal with mapping between backends? e.g. FP16 model back to FP32? Or multi-gpu to CPU?
        backward_step, grad_accum_step, optimizer_step, extras = self._runner.load(
            model=self._model if self.fully_sharded else self.model_access,
            optimizer=self.optimizer,
            gpu=self.gpu,
            path=path,
            tag=tag,
            scaler_dict_fn=self._load_fp16_state_dict_fn(),
            strict=strict,
        )
        # Reset values based on what was in the load dict
        self._backward_steps = backward_step
        self._grad_accum_counter = grad_accum_step
        self._optimizer_steps = optimizer_step
        self.print(f"Successfully loaded model checkpoint from {path}/{tag}")
        # Return the extras dict
        return extras

    def print_num_model_parameters(
        self, normalize: ParamNormalize = ParamNormalize.MILLION
    ):
        """

        Parameters
        ----------
        normalize: ParamNormalize, default: ParamNormalize.MILLION
            ParamNormalize choice for pretty print normalizing

        Returns
        -------
        None

        """
        self.print(
            f"Total Trainable Model Parameters: "
            f"{(self.num_model_parameters / normalize.value):.3f} {normalize.name}"
        )

    def detach_and_sync_loss(
        self,
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]],
        device=None,
    ):
        """Shorthand method to detach and sync loss

        Maps to the runner function of the same name

        Parameters
        ----------
        loss: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
            current loss(es)
        device: default: None
            device to sync across

        Returns
        -------
        loss that is synced across devices and all_reduced w/ SUM

        """
        return self._runner.detach_and_sync_loss(loss=loss, device=device)

    def zero_grads(self):
        """Zeros the optimizer grads depending on the optimizer type

        Returns
        -------
        None

        """
        zero_optimizer_grads(
            optimizer=self._optimizer, apex=self.is_apex, horovod=self.is_horovod
        )

    def reset(self):
        """Public method for resetting the underlying stoke state

        Returns
        -------
        None

        """
        self._reset()

    def reset_tracking(self):
        """Public method for resetting all underlying stoke tracked variables

        Returns
        -------
        None

        """
        # Create some tracking vars
        self._grad_accum_counter = 0
        self._optimizer_steps = 0
        self._backward_steps = 0
        self._last_step_loss = self._set_loss_to_zero()
        self._agg_loss = self._set_loss_to_zero()
        self._rolling_mean_loss = self._set_loss_to_zero()
        self._rolling_loss_steps = 0

    def dump_model_parameter_info(self):
        """Dumps all parameter information for named parameters (shape, device, dtype)

        Returns
        -------
        None

        """
        self.print("Dumping all model parameter information to stdout....")
        for name, param in self.model_access.named_parameters():
            if param.requires_grad:
                self.print(
                    f"Name: {name}, Shape: {param.shape}, "
                    f"Device: {param.device}, dtype: {param.dtype}"
                )

    def _load_fp16_state_dict_fn(self):
        """Returns the function to load the sacler state dict

        Returns
        -------
        mp_state_dict_fn: Callable, default: None
            callable function to load the scaler state dict

        """
        mp_state_dict_fn = None
        if self.scaler is not None:
            if self.is_apex:
                try:
                    from apex import amp

                    mp_state_dict_fn = amp.load_state_dict
                except ImportError as e:
                    print(
                        e,
                        ": Stoke -- apex cannot be imported -- please install (https://github.com/NVIDIA/apex)",
                    )
            else:
                mp_state_dict_fn = self.scaler.load_state_dict
        return mp_state_dict_fn

    def barrier(self):
        """Calls the underlying distributed barrier if available"""
        self._runner.barrier()

    @property
    def step_loss(self):
        """Gets the last step loss synced across device(s) (unscaled)"""
        return self._last_step_loss

    @property
    def model_access(self):
        """Interface for model access due to the different types between the DP, DDP, and SDDP implementations"""
        if isinstance(self._model, (DDP, DP, SDDP, FSDP)):
            return self._model.module
        else:
            return self._model

    @property
    def loss_access(self):
        """Gets loss tensor(s)"""
        return self._loss

    @property
    def optimizer(self):
        """Gets the optimizer"""
        return self._optimizer

    @property
    def scaler(self):
        """Gets the current scaler object"""
        return self._runner.scaler

    @property
    def fp16_state_dict(self):
        """Gets the fp16 state dict from various methods"""
        mp_state_dict = None
        if self.scaler is not None:
            if self.is_apex:
                try:
                    from apex import amp

                    mp_state_dict = amp.state_dict()
                except ImportError as e:
                    print(
                        e,
                        ": Stoke -- apex cannot be imported -- please install (https://github.com/NVIDIA/apex)",
                    )
            elif self.is_amp:
                mp_state_dict = self.scaler.state_dict()
        return mp_state_dict

    @property
    def status(self):
        """Gets the StokeStatus object"""
        return self._status

    @property
    def batch_size(self):
        """Shortcut to batch size"""
        return self._status.batch_size

    @property
    def effective_batch_size(self):
        """Shortcut to effective batch size"""
        return self._status.effective_batch_size

    @property
    def grad_clip(self):
        """Shortcut to get grad clip"""
        return self._status.grad_clip

    @property
    def grad_accum(self):
        """Shortcut to get grad accumulation"""
        return self._status.grad_accum

    @property
    def gpu(self):
        """Shortcut to get GPU status"""
        return self._status.gpu

    @property
    def cuda(self):
        """Shortcut to get cuda status"""
        return self._status.cuda

    @property
    def nccl(self):
        """Shortcut to get nccl status"""
        return self._status.nccl

    @property
    def fp16(self):
        """Shortcut to get FP16 status"""
        return self._status.fp16

    @property
    def is_apex(self):
        """Returns if APEX is activated"""
        return self._status.is_fp16_apex

    @property
    def is_amp(self):
        """Returns if AMP is activated"""
        return self._status.is_fp16_amp

    @property
    def distributed(self):
        """Shortcut to distributed status"""
        return self._status.distributed

    @property
    def is_ddp(self):
        """Returns if DDP is activated"""
        return self._status.is_distributed_ddp

    @property
    def is_horovod(self):
        """Returns if Horovod is activated"""
        return self._status.is_distributed_horovod

    @property
    def is_deepspeed(self):
        """Returns if Deepspeed is acticated"""
        return self._status.is_distributed_deepspeed

    @property
    def oss(self):
        """Returns if Fairscale optimizer state sharding status"""
        return self._status.oss

    @property
    def sharded(self):
        """Returns if Fairscale sharded DDP status"""
        return self._status.sharded

    @property
    def fully_sharded(self):
        """Returns if Fairscale fully sharded DDP status"""
        return self._status.fully_sharded

    @property
    def world_size(self):
        """Shortcut to get world size"""
        return self._runner.world_size

    @property
    def rank(self):
        """Shortcut to get rank"""
        return self._runner.rank

    @property
    def amp_config(self):
        """Returns amp config or None based on amp state"""
        return self._status.amp_config if self.is_amp else None

    @property
    def apex_config(self):
        """Returns apex config or None based on apex state"""
        return self._status.apex_config if self.is_apex else None

    @property
    def ddp_config(self):
        """Returns ddp config or None based on ddp state"""
        return self._status.ddp_config if self.is_ddp else None

    @property
    def deepspeed_config(self):
        """Returns deepspeed config or None based on deepspeed state"""
        return self._status.deepspeed_config if self.is_deepspeed else None

    @property
    def oss_config(self):
        """Returns oss config or None based on ossstate"""
        return self._status.oss_config if self.oss else None

    @property
    def sddp_config(self):
        """Returns sddp config or None based on sddp state"""
        return self._status.sddp_config if self.sharded else None

    @property
    def fsdp_config(self):
        """Returns fsdp config or None based on fsdp state"""
        return self._status.fsdp_config if self.fully_sharded else None

    @property
    def horovod_config(self):
        """Returns horovod config or None based on horovod state"""
        return self._status.horovod_config if self.is_horovod else None

    @property
    def num_model_parameters(self):
        """Returns number of parameters that require gradients"""
        return sum(p.numel() for p in self.model_access.parameters() if p.requires_grad)

    @property
    def ema_loss(self):
        """Returns the current rolling mean loss"""
        return self._rolling_mean_loss


import matplotlib.pyplot as plt
import numpy as np

dict_races = {}
dict_race_count_mr_sr = {}#list for each race - 0idx - count of SR, 1 idx - count of MR

count_mr = 0
# with open('pat_don_mr_high_geno_races.csv') as in_f:
with open('new_data_geno.txt') as in_f:
    for line in in_f:
        line = line.strip().split(',')
        race1 = line[2]
        race2 = line[3]

        races = [race1, race2]
        if race1 != race2:
            if race1 in dict_race_count_mr_sr:
                dict_race_count_mr_sr[race1][1] +=1
            else:
                dict_race_count_mr_sr[race1] = [0,1]

            if race2 in dict_race_count_mr_sr:
                dict_race_count_mr_sr[race2][1] += 1
            else:
                dict_race_count_mr_sr[race2] = [0, 1]

            count_mr +=1
        else:
            if race1 in dict_race_count_mr_sr:
                dict_race_count_mr_sr[race1][0] += 1
                #dict_race_count_mr_sr[race1] += 1
            else:
                dict_race_count_mr_sr[race1] = [1, 0]
                #dict_race_count_mr_sr[race1] = 1
        races = ('_').join(sorted(races))

        dict_races[races] = dict_races.get(races, 0) + 1

print(count_mr)
print(len(dict_races))

sorted_dict_races = sorted(dict_races.items(), key=lambda kv: kv[1], reverse=True)
sorted_dict = sorted(dict_race_count_mr_sr.items(), key=lambda kv: kv[1], reverse=True)
sum = 0

x, y = [], []
v1, v2 = [],[]

for key, val in sorted_dict:
    value = val[0]+val[1]
    x.append(key)
    y.append(value)
    v1.append(val[0])
    v2.append(val[1])

    print(key + '-  ' + str(val))
    sum += val[0]

# #mr_sr_plot
# plt.figure()
# plt.bar(x, v1, color='royalblue', label='sr')
# plt.bar(x, v2, color='silver',  label='mr', bottom=v1)
# plt.xticks(np.arange(len(sorted_dict)), x, rotation='vertical',fontsize=7)
# plt.yticks(fontsize=7)
# plt.ylabel('Number in category')
# plt.xlabel('Races')
# plt.title('Number of individuals per race')
# plt.legend()
# plt.tight_layout()
# plt.show()
# print(sum)

plt.figure()
plt.bar(x, y,color='royalblue', label='sr')
plt.xticks(np.arange(len(sorted_dict)), x, rotation='vertical',fontsize=7)
plt.yticks(fontsize=7)
plt.ylabel('Number in category')
plt.xlabel('Races')
plt.title('Number of individuals per race')
plt.legend()
plt.tight_layout()
plt.show()
print(sum)


# -*- coding: utf-8 -*-
from somali.utils import roles
from somali import config, log
from somali.apis import Twitch
from somali.bots import TwitchBot
import sys, os , sqlite3
from somali.database.models.channel import Channel
from somali.cogs.basics.routines.new_channels import routine

# aliases = [""]
description = "Comando para mandar o boto para canais apos reiniciar, precisa de 'nick'"
extra_checks = [roles.owner]

async def command(ctx, name: str):
    bot: TwitchBot = TwitchBot(config)
    cu = "{}"
    name = name.lower()
    id = await Twitch.account_id(name)
    # try:
    #     print("Tentando entrar")
    #     ctx.bot.add_channel(name, id)
    #     # await ctx.bot.adiciona()
    # except:
    #     print("Falhou")

    try:
        sqliteConnection = sqlite3.connect('/home/pi/Desktop/Bots/Oboto/db.sqlite3')
        cursor = sqliteConnection.cursor()
        log.info("Successfully Connected to SQLite")

        sql1 = f'''INSERT INTO "main"."user"("id", "name")VALUES ({id}, '{name}');'''
        sql2 = f'''INSERT INTO "main"."channel"("banwords", "disabled", "prefix", "user_id")VALUES ('{cu}', '{cu}', '+', '{id}');'''

        try:
            try:
                count1 = cursor.execute(sql1)
            except:
                log.info("Failed to insert data into sqlite table")

        finally:
            try:
                count1 = cursor.execute(sql2)
            except:
                log.info("Failed to insert data into sqlite table")

        sqliteConnection.commit()
        log.info("Record inserted successfully into SqliteDb_developers table ")
        cursor.close()

    except sqlite3.Error as error:
        log.info("Failed to insert data into sqlite table")
    finally:
        if sqliteConnection:
            sqliteConnection.close()
            ctx.response = f"Entrei em {name} com sucesso, agora é so reinicar."
            log.info("The SQLite connection is closed")
            log.info("Tentando reinicar")
            await ctx.bot.adiciona()


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 23 16:42:35 2022

@author: jonpr
"""

# Regional social care use: England
import os
os.chdir(r"C:\Users\jonpr\Documents\Data projects\Python\FrontDoor")
path = os.getcwd()

import pandas as pd
import plotly.express as px
from plotly.offline import plot
import json

# Load SALT/ASCFR dataset 2016-2021
df = pd.read_csv(os.path.join(path, 'LA_dataset.csv'))

# Specify age range: either 18-64 or over 65
Age = '65' # '18_64' or '65'

# Specify variables of interest
AccessOptions = ['DischargefromHospital_'+Age,'Diversion_hosp_serv_'+Age,'self_deplet_funds_'+Age,'depl_fund_DP_'+Age, 'CommunityOtherRoute_'+Age,'Prison_'+Age]
DestinationOptions = ['ShortTermCaretomaximiseind_'+Age,'LongTermCareNursing_'+Age,'LongTermCareResidential_'+Age,'LongTermCareCommunity_'+Age,'LongTermCarePrison_'+Age,
                      'NHSFundedCare_'+Age,'EndofLife_'+Age,'OngoingLowLevelSupport_'+Age,'ShortTermCareothershortter_'+Age,'UniversalServicesSignpostedt_'+Age,
                      'NoServicesProvidedDeceased_'+Age,'NoServicesProvided_'+Age]
CareVs = ['ShortTermCaretomaximiseind_'+Age,'ShortTermCareothershortter_'+Age,'EndofLife_'+Age,'LongTermCareNursing_'+Age,'LongTermCareResidential_'+Age,'LongTermCareCommunity_'+Age,'LongTermCarePrison_'+Age,'NHSFundedCare_'+Age,'OngoingLowLevelSupport_'+Age]
NoCareVs = ['NoServicesProvided_'+Age,'NoServicesProvidedDeceased_'+Age,'UniversalServicesSignpostedt_'+Age]
STCareVs = ['ShortTermCaretomaximiseind_'+Age,'ShortTermCareothershortter_'+Age,'EndofLife_'+Age]
LTCareVs = ['LongTermCareNursing_'+Age,'LongTermCareResidential_'+Age,'LongTermCareCommunity_'+Age,'LongTermCarePrison_'+Age,'NHSFundedCare_'+Age,'OngoingLowLevelSupport_'+Age]
RegionalID = ['Geography_code','LAname']

# Convert social care access and destination variables to % of population
acc_df = pd.DataFrame()
acc_df[AccessOptions] = df[AccessOptions].apply(lambda x: x/df['Population_'+Age]*100)
acc_df = pd.concat([df[RegionalID],df['year'].astype(int), acc_df[AccessOptions]], axis=1)

dest_df = pd.DataFrame()
dest_df[DestinationOptions] = df[DestinationOptions].apply(lambda x: x/df['Population_'+Age]*100)
dest_df = pd.concat([df[RegionalID],df['year'].astype(int), dest_df[DestinationOptions]], axis=1)

# Drop 2016 due to missing population data
acc_df, dest_df = acc_df[acc_df['year'] != 2016], dest_df[dest_df['year'] != 2016]

# Create dataframe of totals
ScTotals = pd.concat([acc_df.iloc[:,:3],acc_df[AccessOptions].sum(axis=1, skipna=True), dest_df[DestinationOptions].sum(axis=1, skipna=True)], axis=1)
ScTotals.columns = ['AreaCD','AreaNM','Year','Access_rate','Destination_rate']
ScTotals['Care_rate'] = dest_df[CareVs].sum(axis=1, skipna=True)
ScTotals['NoCare_rate'] = dest_df[NoCareVs].sum(axis=1, skipna=True)
ScTotals['STCare_rate'] = dest_df[STCareVs].sum(axis=1, skipna=True)
ScTotals['LTCare_rate'] = dest_df[LTCareVs].sum(axis=1, skipna=True)
ScTotals['AC_Disparity'] = ScTotals['Care_rate'] - ScTotals['Access_rate']

# Take the mean across time within each region
ScTotals_by_region = ScTotals.groupby(['AreaCD','AreaNM']).mean().reset_index().drop('Year', axis=1)

# Load geoJSON file from ONS
with open(os.path.join(path, 'Counties_and_Unitary_Authorities_2015.geojson')) as fp:
    GeoData = json.load(fp)

# Plot social care data as map
def ukPlot(dataset, variable, label):
    fig = px.choropleth_mapbox(dataset, locations="AreaNM", featureidkey="properties.ctyua15nm", geojson=GeoData,
                               color=variable, hover_name="AreaNM", mapbox_style="carto-positron", zoom=4,
                               center = {"lat": 55, "lon": 0}, labels={variable:label})
    return fig

disparity_map = ukPlot(ScTotals_by_region,'AC_Disparity','Disparity (%)')
plot(disparity_map)


#!/usr/bin/env python
'''
Module for in silico digestion of WT and Variant proteins and writing to fasta files.
'''

import re

from pyteomics import fasta

from PoolSeqProGen import dna_seq

__author__ = "Rigbe G. Weldatsadik"
__copyright__ = "Copyright (c) 2020 Rigbe G. Weldatsadik"
__license__ = "Apache 2.0"
__version__ = "0.0.1"

def writewtpro_returnwtpep(k, seq_entry,gbk_file,fastaOutput,poolID):
    '''
    Writes the sequences of the 'wild type' proteins to the fasta output and
    returns the tryptic peptides of the 'wild type' protein.
    '''

    entries = []

    '''protein_id_cds_index=dna_seq.index_genbank_features(gbk_file,"CDS",
                                                "protein_id")'''

    protein_id_cds_index = dna_seq.index_genbank_features(gbk_file, "CDS",
                                                  "locus_tag")

    index = protein_id_cds_index[k]
    cds_feature = gbk_file.features[index]
    try:  # there are cds with no translation and protein_id
        immu_tran = cds_feature.qualifiers['translation'][0]
        protein_id = cds_feature.qualifiers['protein_id'][0]
    except KeyError:
        pass
    else:
        seq_entry += 1
        if poolID:
            header = protein_id + '|0_WT|' + \
                str(seq_entry) + '_' + str(poolID)
        else:
            header = protein_id + '|0_WT|' + str(seq_entry)

        entries.append((header, immu_tran))
        fasta.write(entries, fastaOutput)

        # split by K or R when they are not followd by P
        peptide_list_org = re.compile(".(?:(?<![KR](?!P)).)*").findall(str(immu_tran))

        return (seq_entry, peptide_list_org)

def return_varpep(k,v,gbk_file,gbk_fileSeq,geneticCodeID):
    '''
    Returns the variant peptides to be written to the fasta output
    '''

    '''protein_id_cds_index=dna_seq.index_genbank_features(gbk_file,"CDS",\
                                                "protein_id")'''

    protein_id_cds_index = dna_seq.index_genbank_features(gbk_file, "CDS",
                                                  "locus_tag")

    index = protein_id_cds_index[k]
    cds_feature = gbk_file.features[index]
    try:
        immu_tran = cds_feature.qualifiers['translation'][0]
        protein_id = cds_feature.qualifiers['protein_id'][0]
    except KeyError:
        pass
    else:
        loc = cds_feature.location
        strand = cds_feature.location.strand  # +1 or -1
        feature_seq = cds_feature.extract(gbk_file.seq)
        pr_name = cds_feature.qualifiers['product'][0]

        # this wont return the reverse complement for protein on the complement strand
        # unlike extract method
        mut_cds_seq = gbk_file[loc.start:loc.end].seq
        mut_cds_len = len(mut_cds_seq)
        del_range_list = []
        ins_range_list = []
        modified = False

        # subset is in the form of (var_pos,alt,ref,effect,snp) for snp
        # (var_pos,alt,ref,effect,ins) for insertion
        # (var_pos,nt_after_del,alt,ref,del,effect) for deletion
        for subset in v:
            var_pos = subset[0]
            alt_allele = subset[1]
            ref = subset[2]
            effect = subset[5]

            # since var_pos is 1-based position from SnpEff and relative to
            # the whole genome length
            zero_based_pos = int(var_pos) - 1
            pos_relativeto_cds = abs(loc.start - zero_based_pos)

            diff = len(alt_allele) - len(ref)

            if effect == "START_LOST":
                modified = False
                break

            if diff >= 0:  # ins and snp
                # checking deletion range list first since if that particular
                # position has been deleted, it doesn't make sense to
                # insert there
                in_del = dna_seq.check_if_in_del(pos_relativeto_cds, del_range_list)

                if in_del == "True":
                    modified = False
                    break
                elif in_del == "False":
                    pos_relativeto_cds = pos_relativeto_cds
                else:
                    pos_relativeto_cds = in_del

                pos_relativeto_cds = dna_seq.check_if_in_ins(
                    pos_relativeto_cds, ins_range_list)

                # adding len(ref) to the last indices in both the insertion
                # and deletion cases insures we skip and delete as many as
                # the length of the ref so we avoid repeating the nt in the
                # insertion case and we delete the right amount of nts in
                # the deletion case. the len(alt) in the deletion case ensures
                # we dont delete the trailing nt that are reported in the
                # vcf but shouldnt be deleted.

                if(diff > 0):
                    ins_range_list.append(range(pos_relativeto_cds +
                                                 len(ref), pos_relativeto_cds +
                                                 len(alt_allele)))

                try:
                    # this will create a copy of cds_seq so we dont need to
                    # changs cds_seq to mutable explicitly to modify it
                    # this way
                    mut_cds_seq = mut_cds_seq[:pos_relativeto_cds] + \
                        alt_allele + \
                        mut_cds_seq[pos_relativeto_cds +
                                    len(ref):]
                except IndexError:
                    modified = False
                    break
                else:

                    modified = True

            else:  # deletion
                in_del = dna_seq.check_if_in_del(pos_relativeto_cds, del_range_list)

                if in_del == "True":
                    modified = False
                    break
                elif in_del == "False":

                    pos_relativeto_cds = pos_relativeto_cds
                else:

                    pos_relativeto_cds = in_del

                pos_relativeto_cds = dna_seq.check_if_in_ins(
                    pos_relativeto_cds, ins_range_list)

                del_range_list.append(range(pos_relativeto_cds + len(alt_allele),
                                             pos_relativeto_cds + len(ref)))

                try:
                    mut_cds_seq = mut_cds_seq[:pos_relativeto_cds + len(alt_allele)] + \
                        mut_cds_seq[pos_relativeto_cds + len(ref):]
                except IndexError:
                    modified = False
                    break
                else:

                    modified = True

        if modified:
            if strand == -1:
                cur_pos = len(mut_cds_seq) % 3
                if cur_pos == 0:
                    initial = loc.start - 1
                elif cur_pos == 1:
                    initial = loc.start  # since there is a nt at loc.start of the cds
                else:
                    initial = loc.start + 1
                start_index, last_index = dna_seq.check_stopcodon_index_backward(
                    initial,gbk_fileSeq,geneticCodeID)

                # the +1 since from the backward the indices are 1-based rather
                # than 0-based
                lengthmodified_cds_seq = gbk_fileSeq[last_index +
                                                     1:loc.start] + mut_cds_seq

                lengthmodified_cds_seq = lengthmodified_cds_seq.reverse_complement()

            else:
                cur_pos = len(mut_cds_seq) % 3
                initial = loc.end - cur_pos
                start_index, last_index = dna_seq.check_stopcodon_index_forward(
                    initial,gbk_fileSeq,geneticCodeID)

                lengthmodified_cds_seq = mut_cds_seq + gbk_fileSeq[loc.end:last_index]

            mut_tran = str(lengthmodified_cds_seq.translate(
                table=11, to_stop=True))
            peptide_list = re.compile(".(?:(?<![KR](?!P)).)*").findall(mut_tran)

            return (protein_id, peptide_list)

        else:
            return None


from setuptools import setup

package_name = 'trajectory_gen_mpc'

setup(
    name=package_name,
    version='0.0.0',
    packages=[package_name],
    data_files=[
        ('share/ament_index/resource_index/packages',
            ['resource/' + package_name]),
        ('share/' + package_name, ['package.xml']),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='krishna',
    maintainer_email='krishna-satish.deshkulkarni-shankaranarayana@altran.com',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
        'console_scripts': [
            "trajectory_gen_mpc = trajectory_gen_mpc.main:main",
            "TrajectoryServer = trajectory_gen_mpc.TrajectoryGenServer:main",
            "CarlaClient = trajectory_gen_mpc.CarlaClient:main",
            "CarlaNode =trajectory_gen_mpc.CarlaNode:main",
            "TrajectorNode = trajectory_gen_mpc.TrajectorGenMpcNode:main"
        ],
    },
)


#!/home/ckdqja0225/anaconda3/bin/python3

import pandas as pd
from matplotlib import pyplot as plt
from random import shuffle
import sys
import time

class Sort() :

    def __init__(self, rangeToSort=range(1000), desc=False) :
        listToSort = [i for i in rangeToSort]
        shuffle(listToSort)
        self.originalDf = pd.DataFrame(listToSort)
        self.arrDf = self.originalDf.copy()
        self.fig = None
        self.rects = None

        if desc == True :
            self.compare = self._descending_compare
        else :
            self.compare = self._ascending_compare
        self.plot_init()

    def initialize(self) :
        self.arrDf = self.originalDf.copy()

    def _ascending_compare(self, a, b) :
        #self.plot_update()
        return a < b

    def _descending_compare(self, a, b) :
        #self.plot_update()
        return a > b

    def plot_init(self) :
        x = self.arrDf.index.values.tolist()
        y = [i[0] for i in self.arrDf.values.tolist()]
        if self.rects == None :
            self.fig = plt.figure(figsize=(24, 8))
            self.rects = plt.bar(x, y)
            plt.show(block=False)

    def plot_update(self, idx_list) :
        y = [self.arrDf.values[i][0] for i in idx_list]
        rects = [self.rects[i] for i in idx_list]
        for rect, height in zip(rects, y) :
            rect.set_height(height)
        self.fig.canvas.update()
        plt.pause(0.000001)

    def swap(self, a_idx, b_idx) :
        if a_idx == b_idx :
            return
        t_a = self.arrDf.iloc[a_idx].values[0]
        self.arrDf.iloc[a_idx] = self.arrDf.iloc[b_idx].values[0]
        self.arrDf.iloc[b_idx] = t_a
        self.plot_update([a_idx, b_idx])

class MergeSort(Sort) :

    def sort(self) :
        self._topDownMerge(0, len(self.arrDf.index.values))

    def _topDownMerge(self, start, end) :
        if end-start < 2 :
            return
        else :
            df = self.arrDf
            divider = (end-start+1)//2
            self._topDownMerge(start, start+divider)
            self._topDownMerge(start+divider, end)
            self._merge(start, start+divider, end)
            print(start, end)

    def _merge(self, start, mid, end) :
        df = self.arrDf
        tdf = self.arrDf.copy()

        a_idx = start
        b_idx = mid
        for idx in range(start, end) :
            if (not b_idx < end) or ((a_idx < mid) and self.compare(tdf.iloc[a_idx].values[0], tdf.iloc[b_idx].values[0])) :
                df.iloc[idx] = tdf.iloc[a_idx].values[0]
                a_idx += 1
            else :
                df.iloc[idx] = tdf.iloc[b_idx].values[0]
                b_idx += 1
            self.plot_update([idx])

class FuckingSlowSort(Sort) :

    def sort(self) :
        idx = 0
        size = len(self.arrDf)
        while idx+1 != size :
            t_a = self.arrDf.iloc[idx].values[0]
            t_b = self.arrDf.iloc[idx+1].values[0]
            if not self.compare(t_a, t_b) :
                self.swap(idx, idx+1)
                idx = 0
                continue
            idx += 1

class QuickSort(Sort) :

    def sort(self) :
        self._topDownQuickSort(0, len(self.arrDf.index.values))

    def _topDownQuickSort(self, start, end) :
        if end-start < 2 :
            return
        else :
            pivot_idx = end-1
            pivot_val = self.arrDf.iloc[pivot_idx].values[0]
            leftmost_idx = start
            rightmost_idx = end - 2
            while True :
                leftmost_val = self.arrDf.iloc[leftmost_idx].values[0]
                rightmost_val = self.arrDf.iloc[rightmost_idx].values[0]
                while self.compare(leftmost_val, pivot_val) :
                    leftmost_idx += 1
                    leftmost_val = self.arrDf.iloc[leftmost_idx].values[0]
                while self.compare(pivot_val, rightmost_val) and leftmost_idx < rightmost_idx:
                    rightmost_idx -= 1
                    rightmost_val = self.arrDf.iloc[rightmost_idx].values[0]
                if leftmost_idx < rightmost_idx :
                    self.swap(leftmost_idx, rightmost_idx)
                    leftmost_idx += 1
                    rightmost_idx -= 1
                    continue
                else :
                    break
            self.swap(leftmost_idx, pivot_idx)
            self._topDownQuickSort(start, leftmost_idx)
            self._topDownQuickSort(leftmost_idx+1, end)

class RadixSort(Sort) :

    def __init__(self, rangeToSort=range(1000), desc=False, base=10) :
        super(RadixSort, self).__init__(rangeToSort, desc)
        self.base = base

    def sort(self) :
        maximum_exponent = self.getMaximumExponent()
        self._radixSort(0, len(self.arrDf.index.values), maximum_exponent, current_exponent=0)

    def getMaximumExponent(self) :
        max_val = max([i[0] for i in self.arrDf.values.tolist()])
        exponent = 0
        while not max_val//(self.base**exponent) < self.base :
            exponent += 1
        return exponent

    def _radixSort(self, start, end, maximum_exponent, current_exponent) :
        base = self.base
        divider = base**current_exponent
        if current_exponent > maximum_exponent :
            return
        else :
            countList = [0]*base
            tmpList = [0]*(end-start)
            for i in range(len(tmpList)) :
                remainder = (self.arrDf.iloc[start+i].values[0] // divider) % base
                print(remainder)
                countList[remainder] += 1

            for i in range(1, base) :
                countList[i] = countList[i] + countList[i-1];

            for idx in range(end-start-1, -1, -1) :
                remainder = (self.arrDf.iloc[start+idx].values[0] // divider) % base
                tmpList[countList[remainder]-1] = self.arrDf.iloc[start+idx].values[0]
                countList[remainder] -= 1
                idx -= 1

            for i in range(len(tmpList)) :
                self.arrDf.iloc[start+i] = tmpList[i]
                self.plot_update([i])

            self._radixSort(start, end, maximum_exponent, current_exponent+1)

class HeapSort(Sort) :

    def __init__(self, rangeToSort=range(1000), desc=False) :
        super(HeapSort, self).__init__(rangeToSort, desc)
        self.heapLength = len(self.arrDf.index)

    def sort(self) :
        self.build_heap()
        for idx in range(len(self.arrDf.index)-1, -1, -1) :
            self.arrDf.iloc[idx] = self.pop()
            self.plot_update([idx])

    def build_heap(self) :
        start_idx = self.get_parent_index(self.heapLength-1)
        end_idx = -1
        for node_idx in range(start_idx, end_idx, -1) :
            self.heapify(node_idx)

    def isOutOfSize(self, node_idx) :
        return node_idx >= self.heapLength or node_idx < 0

    def heapify(self, node_idx) :
        print("heapify : " + str(node_idx))
        #parent_index = self.get_parent_index(node_idx)
        left_child_index = self.get_left_child_index(node_idx)
        right_child_index = self.get_right_child_index(node_idx)

        if self.isOutOfSize(left_child_index) or self.isOutOfSize(node_idx) :
            return
        else :
            node_val = self.get_value_with_node_idx(node_idx)
            left_child_val = self.get_value_with_node_idx(left_child_index)

            if self.isOutOfSize(right_child_index) :
                if self.compare(left_child_val, node_val) :
                    return
                else :
                    self.swap(left_child_index, node_idx)
                    self.heapify(left_child_index)
            else :
                right_child_val = self.get_value_with_node_idx(right_child_index)

                if self.compare(left_child_val, node_val) and self.compare(right_child_val, node_val) :
                    return
                else :
                    if self.compare(left_child_val, right_child_val) :
                        self.swap(right_child_index, node_idx)
                        self.heapify(right_child_index)
                    else :
                        self.swap(left_child_index, node_idx)
                        self.heapify(left_child_index)

    def push(self, value) :
        self.arrDf.iloc[self.heapLength][0] = -1
        self.increaseKey(self.heapLength, value)
        self.heapLength += 1

    def increaseKey(self, index, key) :
        if self.heapLength == 0 :
            return
        elif self.get_value_with_node_idx(index) > key :
            return
        else :
            self.arrDf.iloc[index][0] = key
            while( (not isOutOfSize(index))
                    and (not self.compare(
                        self.get_value_with_node_idx(index),
                        self.get_value_with_node_idx(self.get_parent_index(index))))) :
                self.swap(index, self.get_parent_index(index))
                index = self.get_parent_index(index)

    def pop(self) :
        retToVal = self.get_value_with_node_idx(0)
        self.arrDf.iloc[0][0] = self.arrDf.iloc[self.heapLength-1][0]
        self.heapify(0)
        self.heapLength -= 1
        return retToVal

    def get_left_child_index(self, idx) :
        return (idx+1)*2-1

    def get_right_child_index(self, idx) :
        return (idx+1)*2

    def get_parent_index(self, idx) :
        return (idx+1)//2-1

    def get_value_with_node_idx(self, node_idx) :
        return self.arrDf.values[node_idx][0]

if __name__ == "__main__" :
    sys.setrecursionlimit(100000)
    sorter = RadixSort(range(100), desc=False, base=7)
    #sorter = MergeSort(range(300), desc=False)
    #sorter = QuickSort(range(300), desc=False)
    #sorter = HeapSort(range(20), desc=False)
    sorter.sort()


from django.urls import path
from django.views.decorators.csrf import csrf_exempt
from .views import (
                    VideoLectureView,
                    VideoLectureNew,
                    VideoLectureUserView,
                    VideoLectureEdit
)

urlpatterns = [
    path('video/create/', VideoLectureNew.as_view(), name="create-video"),
    path('video/<int:vid_lec_id>/', VideoLectureView.as_view(), name="video-lecture"),
    path('videos/user', VideoLectureUserView.as_view(), name="user-videos"),
    path('video/update/<int:pk>/', VideoLectureEdit.as_view(), name="edit-video"),

]


import numpy as np, pyemma as py
# from msmbuilder.decomposition.tica import tICA
from sklearn.kernel_approximation import Nystroem

"""modified from https://github.com/msmbuilder/msmbuilder/blob/master/msmbuilder/decomposition/ktica.py"""
"""reference: [1] Schwantes, Christian R., and Vijay S. Pande. J. Chem Theory Comput. 11.2 (2015): 600--608."""

class Kernel_tica(object):
    def __init__(self, n_components, lag_time,
                 gamma,             # gamma value for rbf kernel
                 n_components_nystroem=100,  # number of components for Nystroem kernel approximation
                 landmarks = None,
                 shrinkage = None,
                 weights='empirical'    # if 'koopman', use Koopman reweighting for tICA (see Wu, Hao, et al. "Variational Koopman models: slow collective variables and molecular kinetics from short off-equilibrium simulations." The Journal of Chemical Physics 146.15 (2017): 154104.)
                 ):
        self._n_components = n_components
        self._lag_time = lag_time
        self._n_components_nystroem = n_components_nystroem
        self._landmarks = landmarks
        self._gamma = gamma
        self._nystroem = Nystroem(gamma=gamma, n_components=n_components_nystroem)
        self._weights = weights
        # self._tica = tICA(n_components=n_components, lag_time=lag_time, shrinkage=shrinkage)
        self._shrinkage = shrinkage
        return

    def fit(self, sequence_list):
        if self._landmarks is None:
            self._nystroem.fit(np.concatenate(sequence_list))
        else:
            print("using landmarks")
            self._nystroem.fit(self._landmarks)
        sequence_transformed = [self._nystroem.transform(item) for item in sequence_list]
        # define tica object at fit() with sequence_list supplied for initialization, as it is required by
        # Koopman reweighting
        self._tica = py.coordinates.tica(sequence_transformed, lag=self._lag_time,
                                         dim=self._n_components, kinetic_map=True,
                                         weights=self._weights)
        return

    def transform(self, sequence_list):
        return self._tica.transform(
            [self._nystroem.transform(item) for item in sequence_list])

    def fit_transform(self, sequence_list):
        self.fit(sequence_list)
        return self.transform(sequence_list)

    def score(self, sequence_list):
        model = self.__class__(n_components = self._n_components, lag_time=self._lag_time, gamma=self._gamma,
                               n_components_nystroem=self._n_components_nystroem, landmarks=self._landmarks,
                               shrinkage=self._shrinkage)
        model.fit(sequence_list)
        return np.sum(model._tica.eigenvalues)


import bge
from collections import OrderedDict
import 载具物理性质
import MathLib
import math

if not hasattr(bge, "__component__"):
    scene = bge.logic.getCurrentScene()

class MyVehicle(bge.types.KX_PythonComponent):
    args = OrderedDict([
    ])

    def start(self, args):
        self.载具 = 载具物理性质.载具物理(self.object)
        print(self.object.childrenRecursive)
        self.摄影机 = [obj for obj in self.载具.childrenRecursive if "Camera" in obj][0]

        self.摄影机跟随 = False

        # scene = bge.logic.getCurrentScene()
        # cam1 = scene.objects["Camera.002"]
        # cam2 = scene.objects["Camera.001"]

        # cam1.useViewport = True
        # cam2.useViewport = True
        # cam2.setOnTop()

        # width = bge.render.getWindowWidth()
        # height = bge.render.getWindowHeight()
        # print(width,height,'height----------------------------')

        # bottom = 0
        # top = height

        # # here are the boundaries for the right viewport
        # left_1 = 0
        # right_1 = width

        # # here are the boundaries for the left viewport
        # left_2 = 0
        # right_2 = int(width/2)

        # # set player viewports with player1 at the right, player 2 at the left
        # cam1.setViewport( right_2, int(height/2), width, height)
        # cam2.setViewport( 0, 0, right_2, int(height/2))

    def update(self):

        if self.摄影机跟随:
            self.载入摄像机()

    def 载入摄像机(self):

        车辆局部旋转 = self.载具.局部方向.to_euler()
        车辆局部旋转.rotate_axis('X', math.radians(-5.0))
        车辆局部旋转[1] = 0.0

        目标位置 = 车辆局部旋转.to_matrix()

        速度 = self.载具.DeltaTime*5
        self.摄影机.localOrientation = self.摄影机.localOrientation.lerp(目标位置, 速度)


from os.path import join

from pythonforandroid.recipe import Recipe
from pythonforandroid.util import current_directory
from pythonforandroid.logger import shprint
import sh

class OpenSSLRecipe(Recipe):
    '''
    The OpenSSL libraries for python-for-android. This recipe will generate the
    following libraries as shared libraries (*.so):

        - crypto
        - ssl

    The generated openssl libraries are versioned, where the version is the
    recipe attribute :attr:`version` e.g.: ``libcrypto1.1.so``,
    ``libssl1.1.so``...so...to link your recipe with the openssl libs,
    remember to add the version at the end, e.g.:
    ``-lcrypto1.1 -lssl1.1``. Or better, you could do it dynamically
    using the methods: :meth:`include_flags`, :meth:`link_dirs_flags` and
    :meth:`link_libs_flags`.

    .. warning:: This recipe is very sensitive because is used for our core
        recipes, the python recipes. The used API should match with the one
        used in our python build, otherwise we will be unable to build the
        _ssl.so python module.

    .. versionchanged:: 0.6.0

        - The gcc compiler has been deprecated in favour of clang and libraries
          updated to version 1.1.1 (LTS - supported until 11th September 2023)
        - Added two new methods to make easier to link with openssl:
          :meth:`include_flags` and :meth:`link_flags`
        - subclassed versioned_url
        - Adapted method :meth:`select_build_arch` to API 21+
        - Add ability to build a legacy version of the openssl libs when using
          python2legacy or python3crystax.

    .. versionchanged:: 2019.06.06.1.dev0

        - Removed legacy version of openssl libraries

    '''

    version = '1.1'
    '''the major minor version used to link our recipes'''

    url_version = '1.1.1m'
    '''the version used to download our libraries'''

    url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz'

    built_libraries = {
        'libcrypto{version}.so'.format(version=version): '.',
        'libssl{version}.so'.format(version=version): '.',
    }

    @property
    def versioned_url(self):
        if self.url is None:
            return None
        return self.url.format(url_version=self.url_version)

    def get_build_dir(self, arch):
        return join(
            self.get_build_container_dir(arch), self.name + self.version
        )

    def include_flags(self, arch):
        '''Returns a string with the include folders'''
        openssl_includes = join(self.get_build_dir(arch.arch), 'include')
        return (' -I' + openssl_includes +
                ' -I' + join(openssl_includes, 'internal') +
                ' -I' + join(openssl_includes, 'openssl'))

    def link_dirs_flags(self, arch):
        '''Returns a string with the appropriate `-L<lib directory>` to link
        with the openssl libs. This string is usually added to the environment
        variable `LDFLAGS`'''
        return ' -L' + self.get_build_dir(arch.arch)

    def link_libs_flags(self):
        '''Returns a string with the appropriate `-l<lib>` flags to link with
        the openssl libs. This string is usually added to the environment
        variable `LIBS`'''
        return ' -lcrypto{version} -lssl{version}'.format(version=self.version)

    def link_flags(self, arch):
        '''Returns a string with the flags to link with the openssl libraries
        in the format: `-L<lib directory> -l<lib>`'''
        return self.link_dirs_flags(arch) + self.link_libs_flags()

    def get_recipe_env(self, arch=None):
        env = super().get_recipe_env(arch)
        env['OPENSSL_VERSION'] = self.version
        env['MAKE'] = 'make'  # This removes the '-j5', which isn't safe
        env['CC'] = 'clang'
        env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir
        return env

    def select_build_arch(self, arch):
        aname = arch.arch
        if 'arm64' in aname:
            return 'android-arm64'
        if 'v7a' in aname:
            return 'android-arm'
        if 'arm' in aname:
            return 'android'
        if 'x86_64' in aname:
            return 'android-x86_64'
        if 'x86' in aname:
            return 'android-x86'
        return 'linux-armv4'

    def build_arch(self, arch):
        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            # sh fails with code 255 trying to execute ./Configure
            # so instead we manually run perl passing in Configure
            perl = sh.Command('perl')
            buildarch = self.select_build_arch(arch)
            config_args = [
                'shared',
                'no-dso',
                'no-asm',
                buildarch,
                '-D__ANDROID_API__={}'.format(self.ctx.ndk_api),
            ]
            shprint(perl, 'Configure', *config_args, _env=env)
            self.apply_patch('disable-sover.patch', arch.arch)

            shprint(sh.make, 'build_libs', _env=env)

recipe = OpenSSLRecipe()


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

matplotlib.interactive(True)
params = np.load('./mnist_gen_params.npy')
accs = np.load('./mnist_gen_accs.npy')
losss = np.load('./mnist_gen_losss.npy')

hist = []
new_accs = [[],[]]
new_losss = [[],[]]
for i,v in enumerate(params):
	if v not in hist:
		hist.append(v)
		new_accs[0].append(accs[0][i])
		new_accs[1].append(accs[1][i])
		new_losss[0].append(losss[0][i])
		new_losss[1].append(losss[1][i])

print("Acutual Model Num: {}".format(len(hist)))

plt.plot(hist, new_accs[0], 'o', label="train_acc")
plt.plot(hist, new_accs[1], 'o', label="test_acc")
plt.xlabel('parameters')
plt.ylabel('acc')
plt.title('Model Accuracy by Patameters')
plt.legend()
plt.savefig('./accs.png')
plt.close()

plt.plot(hist, new_losss[0], 'o', label="train_loss")
plt.plot(hist, new_losss[1], 'o', label="test_loss")
plt.xlabel('parameters')
plt.ylabel('loss')
plt.title('Model loss by Patameters')
plt.legend()
plt.savefig('./losss.png')
plt.close()


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

'''

ptr2inject
by JMK

GitHub: https://github.com/jmk-developer/ptr2inject

'''

import wx
import shutil
import os
import time
import math
import json

import modmanager

CurrentImage        = None
CurrentModFile      = None

CurrentImagePackage = None
CurrentModPackage   = None

ImageReady          = False
ModFileReady        = False

temp_directory      = None
temp_directory_name = ".temporary"
temp_directory_path = "\\" + temp_directory_name + "\\"
temp_directory_fullpath = os.getcwd() + temp_directory_path

if not os.path.isdir(temp_directory_fullpath):
    os.makedirs(temp_directory_fullpath)

modmanager.set_temporary_directory(temp_directory_fullpath)

def check_start_enabled(thisobject):
    print("Image Ready: " + str(ImageReady))
    print("ModFile Ready: " + str(ModFileReady))

    startbutton = thisobject.GetParent().Start
    if ImageReady == True and ModFileReady == True:
        startbutton.Enable()
    else:
        startbutton.Disable()

def not_yet_implemented(self):
    mod_data_dialog = wx.MessageDialog(self,
                                       "This feature is not yet implemented.",
                                       "Info",
                                       wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
    mod_data_dialog.ShowModal()
    mod_data_dialog.Destroy()

def show_package_data(self, mod_package):
    mod_data_dialog = wx.MessageDialog(self,
                                       "Mod Name: " + mod_package['mod_data_json']['name'] +
                                       "\nMod Description: " + mod_package['mod_data_json']['description'] +
                                       "\nMod Version: " + str(mod_package['mod_data_json']['version']),

                                       "Mod Info",
                                       wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
    mod_data_dialog.ShowModal()
    mod_data_dialog.Destroy()

class MyFrame(wx.Frame):
    def __init__(self, *args, **kwds):
        # begin wxGlade: MyFrame.__init__
        kwds["style"] = kwds.get("style", 0) | wx.CAPTION | wx.CLIP_CHILDREN | wx.CLOSE_BOX | wx.MINIMIZE_BOX | wx.SYSTEM_MENU
        wx.Frame.__init__(self, *args, **kwds)
        self.SetSize((400, 180))

        # Menu Bar
        self.MenuBar = wx.MenuBar()
        wxglade_tmp_menu = wx.Menu()

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "Create Mod File from INT", "Create a mod file from INT files.")
        self.Bind(wx.EVT_MENU, self.create_mod_file, id=item.GetId())

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "View Mod Info", "View a mod file's info.")
        self.Bind(wx.EVT_MENU, self.view_mod_info, id=item.GetId())

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "View ISO Info", "View an ISO file's info.")
        self.Bind(wx.EVT_MENU, self.view_iso_info, id=item.GetId())

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "Clear Temporary Files", "Clear all temporary files.")
        self.Bind(wx.EVT_MENU, self.clear_temporary_files, id=item.GetId())

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "About", "Info about this program")
        self.Bind(wx.EVT_MENU, self.about_menu, id=item.GetId())

        self.MenuBar.Append(wxglade_tmp_menu, "File")
        wxglade_tmp_menu = wx.Menu()

        item = wxglade_tmp_menu.Append(wx.ID_ANY, "Ignore ISO/Mod Region", "Automatically, this program detects when mods are incompatible with the ISO you give them, based on region.", wx.ITEM_CHECK)
        self.Bind(wx.EVT_MENU, self.ignore_mod_region, id=item.GetId())
        self.MenuBar.Append(wxglade_tmp_menu, "Settings")
        self.SetMenuBar(self.MenuBar)
        # Menu Bar end
        self.ChooseImage = wx.Button(self, wx.ID_ANY, "Choose ISO")
        self.ChooseFile = wx.Button(self, wx.ID_ANY, "Choose File")
        self.Start = wx.Button(self, wx.ID_ANY, "Start")

        self.__set_properties()
        self.__do_layout()

        self.Bind(wx.EVT_BUTTON, self.choose_image_event, self.ChooseImage)
        self.Bind(wx.EVT_BUTTON, self.choose_file_event, self.ChooseFile)
        self.Bind(wx.EVT_BUTTON, self.start_event, self.Start)
        # end wxGlade
        self.Start.Disable()

    def __set_properties(self):
        # begin wxGlade: MyFrame.__set_properties
        self.SetTitle("ptr2inject")
        self.ChooseImage.SetFont(wx.Font(12, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, 0, "Segoe UI"))
        self.ChooseImage.SetToolTip("This button chooses your Parappa The Rapper 2 ISO image.")
        self.ChooseFile.SetFont(wx.Font(12, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, 0, "Segoe UI"))
        self.ChooseFile.SetToolTip("This button selects the .p2m file you want to inject.")
        self.Start.SetFont(wx.Font(12, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, 0, "Segoe UI"))
        self.Start.SetToolTip("This button begins the ISO injection. Once you click this, you will be asked where to save the newly created ISO.")
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: MyFrame.__do_layout
        RootSizer = wx.BoxSizer(wx.HORIZONTAL)
        RootSizer.Add(self.ChooseImage, 1, wx.ALL | wx.EXPAND, 8)
        RootSizer.Add(self.ChooseFile, 1, wx.ALL | wx.EXPAND, 8)
        RootSizer.Add(self.Start, 2, wx.ALL | wx.EXPAND, 8)
        self.SetSizer(RootSizer)
        self.Layout()
        # end wxGlade

    def create_mod_file(self, event):  # wxGlade: MyFrame.<event_handler>
        not_yet_implemented(self)

    def view_mod_info(self, event):  # wxGlade: MyFrame.<event_handler>
        if ModFileReady == False:
            view_mod_error_dialog = wx.MessageDialog(self,
                                                        "Please select a mod file before using this option.",
                                                        "Info",
                                                        wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
            view_mod_error_dialog.ShowModal()
            view_mod_error_dialog.Destroy()
        else:
            show_package_data(self, CurrentModPackage)

    def view_iso_info(self, event):
        not_yet_implemented(self)
        '''
        if ImageReady == False:
            view_iso_error_dialog = wx.MessageDialog(self,
                                                     "Please select an ISO file before using this option.",
                                                     "Info",
                                                     wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
            view_iso_error_dialog.ShowModal()
            view_iso_error_dialog.Destroy()
        else:
            show_image_data(self, CurrentModPackage)
        '''

    def clear_temporary_files(self, event):
        begin_time = time.perf_counter()
        modmanager.clear_temporary_directory()
        end_time = time.perf_counter()
        clear_temporary_files_dialog = wx.MessageDialog(self,
                                           "Finished in " + str(math.ceil(end_time - begin_time)) +
                                           " seconds. Thank you!",
                                           "Done",
                                           wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
        clear_temporary_files_dialog.ShowModal()
        clear_temporary_files_dialog.Destroy()

    def about_menu(self, event):  # wxGlade: MyFrame.<event_handler>
        not_yet_implemented(self)

    def ignore_mod_region(self, event):  # wxGlade: MyFrame.<event_handler>
        not_yet_implemented(self)

    def choose_image_event(self, event):  # wxGlade: MyFrame.<event_handler>
        global CurrentImage
        global ImageReady

        iso_dialog = wx.FileDialog(self, "Open", "", "", "PS2 ISO images (*.iso)|*.iso", wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)
        iso_dialog.ShowModal()
        CurrentImage = iso_dialog.GetPath()
        if not CurrentImage or CurrentImage == "":
            CurrentImage = None
            return
        myobject = event.GetEventObject()
        myobject.Disable()
        myobject.SetLabel("Completed")
        ImageReady = True
        check_start_enabled(myobject)

    def choose_file_event(self, event):  # wxGlade: MyFrame.<event_handler>
        global CurrentModFile
        global CurrentModPackage
        global ModFileReady

        file_dialog = wx.FileDialog(self, "Open", "", "", "PTR2 mod files (*.p2m)|*.p2m", wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)
        file_dialog.ShowModal()
        CurrentModFile = file_dialog.GetPath()

        if not CurrentModFile or CurrentModFile == "":
            CurrentModFile = None
            return
        CurrentModPackage = modmanager.unpackage_mod_file(CurrentModFile, "ptr2mod")

        myobject = event.GetEventObject()
        myobject.Disable()
        myobject.SetLabel("Completed")
        ModFileReady = True
        check_start_enabled(myobject)

    def start_event(self, event):  # wxGlade: MyFrame.<event_handler>
        global CurrentModFile
        global CurrentModPackage
        global CurrentImage

        start_button = event.GetEventObject()
        start_button.Disable()
        start_button.SetLabel("Starting...")
        wx.BeginBusyCursor()
        show_package_data(self, CurrentModPackage)
        begin_time = time.perf_counter()
        start_button.SetLabel("Cleaning up...")
        modmanager.clear_temporary_directory()

        start_button.SetLabel("Extracting ISO...")
        unpackaged_iso = modmanager.unpackage_iso(CurrentImage, "ptr2iso")
        start_button.SetLabel("Moving files...")
        start_button.SetLabel("Patching ISO...")
        modmanager.package_iso(unpackaged_iso, CurrentImage)

        end_time = time.perf_counter()
        wx.EndBusyCursor()
        finished_dialog = wx.MessageDialog(self,
                                            "Finished in " + str(math.ceil(end_time - begin_time)) +
                                            " seconds. Thank you!",
                                            "Done",
                                            wx.OK | wx.STAY_ON_TOP | wx.CENTRE)
        finished_dialog.ShowModal()
        finished_dialog.Destroy()
# end of class MyFrame

class AboutDialog(wx.Dialog):
    def __init__(self, *args, **kwds):
        # begin wxGlade: MyDialog.__init__
        kwds["style"] = kwds.get("style", 0) | wx.CAPTION | wx.CLOSE_BOX
        wx.Dialog.__init__(self, *args, **kwds)
        self.SetSize((320, 220))

        self.__set_properties()
        self.__do_layout()
        # end wxGlade

    def __set_properties(self):
        # begin wxGlade: MyDialog.__set_properties
        self.SetTitle("About ptr2inject")
        self.SetSize((320, 220))
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: MyDialog.__do_layout
        AboutRootSizer = wx.BoxSizer(wx.VERTICAL)
        AboutText = wx.StaticText(self, wx.ID_ANY, "Program created by JMK\n\nThis program is free.\nHow to use: \n1. Choose your Parappa The Rapper 2 ISO\n2. Choose your mod file\n3. Click \"Start\" \n4. Profit", style=wx.ALIGN_CENTER)
        AboutText.SetFont(wx.Font(10, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, 0, "Segoe UI"))
        AboutRootSizer.Add(AboutText, 1, wx.ALL | wx.EXPAND, 20)
        self.SetSizer(AboutRootSizer)
        self.Layout()
        # end wxGlade

# end of class MyDialog

class MyApp(wx.App):
    def OnInit(self):
        self.RootFrame = MyFrame(None, wx.ID_ANY, "")
        self.SetTopWindow(self.RootFrame)
        self.RootFrame.Show()
        return True

# end of class MyApp

if __name__ == "__main__":
    app = MyApp(0)
    app.MainLoop()