from typing import List

import fvcore.nn.weight_init as weight_init
import torch
from torch import nn
from torch.nn import functional as F
from einops import rearrange
from detectron2.config import configurable
from detectron2.layers import Conv2d, ShapeSpec, cat, get_norm
from detectron2.modeling import ROI_MASK_HEAD_REGISTRY
from detectron2.modeling.roi_heads.mask_head import BaseMaskRCNNHead
from detectron2.structures import Instances
# from .mask_encoding_V2 import DctMaskEncodingV2
from .mask_encoding import DctMaskEncoding

@ROI_MASK_HEAD_REGISTRY.register()
class MaskRCNNPatchDCTHead_NSTAGE(BaseMaskRCNNHead):
    """
    A mask head with several conv layers, plus an upsample layer (with `ConvTranspose2d`).
    Predictions are made with a final 1x1 conv layer.
    """

    @configurable
    def __init__(self, input_shape: ShapeSpec, *, num_classes, conv_dims, conv_norm="",
                 dct_vector_dim, mask_size, pooler_resolution, hidden_features, fine_features_resolution,
                 mask_size_assemble, patch_size, patch_dct_vector_dim, mask_loss_para, dct_loss_type,
                 num_stage, mask_loss_para_each_stage, patch_threshold, eval_gt,
                 **kwargs):
        """
        NOTE: this interface is experimental.

        Args:
            input_shape (ShapeSpec): shape of the input feature
            num_classes (int): the number of classes. 1 if using class agnostic prediction.For COCO,num_classes=80
            conv_dims (list[int]): a list of N>0 integers representing the output dimensions
                of N-1 conv layers and the last upsample layer.
            conv_norm (str or callable): normalization for the conv layers.
                See :func:`detectron2.layers.get_norm` for supported types.
            dct_vector_dim: dct vector dim in DCT_MASK(default=300)
            mask_size: resolution of mask to be refined(default=112)
            hidden_features: feature dim of linear layer(default=1024)
            fine_features_resolution: feature map in PatchDCT(default=42)
            mask_size_assemble: mask size in PatchDCT(default=112)
            patch_size: patch size(default=8)
            patch_dct_vector_dim: DCT vector dim for each patch
            mask_loss_para: coefficient of total loss(default=1)
            dct_loss_type: loss type of DCT vector regressor(default=l1, option=[l1, sl1,l2])
            num_stage: number of segmentation stage, equals to number of PatchDCT blocks+1(default=2)
            mask_loss_para_each_stage: coefficient of loss for each segmentation stage
            patch_threshold: threshold used for classifier(default=0.3),
            eval_gt: use for calculate the upper bound of the model

        """
        super().__init__(**kwargs)
        assert len(conv_dims) >= 1, "conv_dims have to be non-empty!"
        assert num_stage == len(mask_loss_para_each_stage)
        self.patch_dct_vector_dim = patch_dct_vector_dim
        self.mask_size_assemble = mask_size_assemble
        self.patch_size = patch_size
        self.num_classes = num_classes
        self.hidden_features = hidden_features
        self.dct_vector_dim = dct_vector_dim
        self.mask_size = mask_size
        self.dct_loss_type = dct_loss_type
        self.mask_loss_para = mask_loss_para
        self.scale = self.mask_size // self.patch_size
        self.ratio = fine_features_resolution // self.scale
        self.num_stage = num_stage - 1  # num_stage below is the number of PatchDCT block
        self.loss_para = mask_loss_para_each_stage
        self.patch_threshold = patch_threshold
        self.eval_gt = eval_gt
        self.dct_encoding = DctMaskEncoding(vec_dim=self.dct_vector_dim, mask_size=self.mask_size)
        self.patch_dct_encoding = DctMaskEncoding(vec_dim=self.patch_dct_vector_dim, mask_size=self.patch_size)

        self.conv_norm_relus = []

        cur_channels = input_shape.channels
        for k, conv_dim in enumerate(conv_dims[:-1]):
            conv = Conv2d(
                cur_channels,
                conv_dim,
                kernel_size=3,
                stride=1,
                padding=1,
                bias=not conv_norm,
                norm=get_norm(conv_norm, conv_dim),
                activation=F.relu,
            )
            self.add_module("mask_fcn{}".format(k + 1), conv)
            self.conv_norm_relus.append(conv)
            cur_channels = conv_dim

        self.predictor = nn.Sequential(
            nn.Linear(pooler_resolution ** 2 * conv_dim, self.hidden_features),
            nn.ReLU(),
            nn.Linear(self.hidden_features, self.hidden_features),
            nn.ReLU(),
            nn.Linear(self.hidden_features, self.dct_vector_dim)
        )
        self.reshape = Conv2d(
            1,
            conv_dim,
            kernel_size=3,
            stride=1,
            padding=1,
            activation=F.relu
        )
        self.fusion = nn.Sequential(
            Conv2d(cur_channels,
                   conv_dim,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu),
            Conv2d(cur_channels,
                   conv_dim,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu)
        )

        self.downsample = nn.Sequential(
            Conv2d(
                cur_channels,
                self.hidden_features,
                kernel_size=self.ratio,
                stride=self.ratio,
                padding=0,
                bias=not conv_norm,
                norm=get_norm(conv_norm, conv_dim),
                activation=F.relu, ),
            Conv2d(self.hidden_features,
                   self.hidden_features,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu),
        )

        self.predictor1 = Conv2d(self.hidden_features,
                                 self.patch_dct_vector_dim * self.num_classes,
                                 kernel_size=1,
                                 stride=1,
                                 padding=0,
                                 )
        self.predictor_bfg = Conv2d(self.hidden_features,
                                    3 * self.num_classes,
                                    kernel_size=1,
                                    stride=1,
                                    padding=0,
                                    )

        for layer in self.conv_norm_relus:
            weight_init.c2_msra_fill(layer)

    @classmethod
    def from_config(cls, cfg, input_shape):
        ret = super().from_config(cfg, input_shape)
        conv_dim = cfg.MODEL.ROI_MASK_HEAD.CONV_DIM
        num_conv = cfg.MODEL.ROI_MASK_HEAD.NUM_CONV
        ret.update(
            conv_dims=[conv_dim] * (num_conv + 1),  # +1 for ConvTranspose
            conv_norm=cfg.MODEL.ROI_MASK_HEAD.NORM,
            input_shape=input_shape,
            hidden_features=cfg.MODEL.ROI_MASK_HEAD.HIDDEN_FEATURES,
            pooler_resolution=cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION,
            dct_vector_dim=cfg.MODEL.ROI_MASK_HEAD.DCT_VECTOR_DIM,
            mask_loss_para=cfg.MODEL.ROI_MASK_HEAD.MASK_LOSS_PARA,
            mask_size=cfg.MODEL.ROI_MASK_HEAD.MASK_SIZE,
            dct_loss_type=cfg.MODEL.ROI_MASK_HEAD.DCT_LOSS_TYPE,
            fine_features_resolution=cfg.MODEL.ROI_MASK_HEAD.FINE_FEATURES_RESOLUTION,
            mask_size_assemble=cfg.MODEL.ROI_MASK_HEAD.MASK_SIZE_ASSEMBLE,
            patch_size=cfg.MODEL.ROI_MASK_HEAD.PATCH_SIZE,
            patch_dct_vector_dim=cfg.MODEL.ROI_MASK_HEAD.PATCH_DCT_VECTOR_DIM,
            num_stage=cfg.MODEL.ROI_MASK_HEAD.NUM_STAGE,
            mask_loss_para_each_stage=cfg.MODEL.ROI_MASK_HEAD.MASK_LOSS_PARA_EACH_STAGE,
            patch_threshold=cfg.MODEL.ROI_MASK_HEAD.PATCH_THRESHOLD,
            eval_gt=cfg.MODEL.ROI_MASK_HEAD.EVAL_GT,
        )

        if cfg.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK:
            ret["num_classes"] = 1
        else:
            ret["num_classes"] = cfg.MODEL.ROI_HEADS.NUM_CLASSES
        return ret

    def layers(self, x, fine_mask_features, instances):
        """

        Args:
            x: feature map used in DCT-Mask
            fine_mask_features: feature map used in PatchDCT

        Returns:
            x (Tensor): [B, D]. D is dct-dim. [B, D]. DCT_Vector in DCT-Mask.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
        """
        for layer in self.conv_norm_relus:
            x = layer(x)
        # DCT_Mask
        x = self.predictor(x.flatten(start_dim=1))
        if not self.training:
            num_masks = x.shape[0]
            if num_masks == 0:
                return x, {}, {}

        # reverse transform to obtain high-resolution masks
        # if use DctMaskEncodingV2, plese use: masks = self.dct_encoding.decode(x)
        masks = self.dct_encoding.decode(x).real
        masks = masks[:, None, :, :]

        # PatchDCT
        if self.training:
            classes = self.get_gt_classes(instances)
        else:
            classes = instances[0].pred_classes
        num_instance = classes.size()[0]
        indices = torch.arange(num_instance)

        bfg, patch_vector = self.patchdct(fine_mask_features, masks, classes, indices)

        bfg_dict = {}
        patch_dict = {}

        bfg_dict[0] = bfg
        patch_dict[0] = patch_vector

        for i in range(1, self.num_stage):
            masks = self.patch2mask_nstage(bfg, patch_vector)
            bfg, patch_vector = self.patchdct(fine_mask_features, masks, classes, indices)
            bfg_dict[i] = bfg
            patch_dict[i] = patch_vector

        return x, bfg_dict, patch_dict

    def patchdct(self, fine_mask_features, masks, classes, indices):
        """
        PatchDCT block
        Args:
            fine_mask_features: feature map cropped from FPN P2
            masks: masks to be refined
            classes: a tensor of shape [B], classes of each instance
            indices: a tensor of shape [B]

        Returns:
            bfg and patch_vector of each PatchDCT block
        """
        masks = F.interpolate(masks, size=(self.scale * self.ratio, self.scale * self.ratio))
        masks = self.reshape(masks)
        fine_mask_features = masks + fine_mask_features
        fine_mask_features = self.fusion(fine_mask_features)
        fine_mask_features = self.downsample(fine_mask_features)
        patch_vector = self.predictor1(fine_mask_features)
        bfg = self.predictor_bfg(fine_mask_features)

        bfg = rearrange(bfg, "b (n c) h w -> b n c h w", n=self.num_classes)
        patch_vector = rearrange(patch_vector, "b (n c) h w -> b n c h w", n=self.num_classes)

        bfg = rearrange(bfg[indices, classes], "b c h w -> (b h w) c")
        patch_vector = rearrange(patch_vector[indices, classes], "b c h w -> (b h w) c")
        return bfg, patch_vector

    def patch2mask_nstage(self, bfg, patch_vector):
        """

        Args:
            bfg: A tensor of shape [B*scale*scale,3]
            patch_vector: A tensor of shape [B*scale*scale,patch_dct_vector_dim]

        Returns:
            masks: A tensor of shape [B,1,mask_size,mask_size]
        """
        index = torch.argmax(bfg, dim=1)
        bg = torch.zeros_like(patch_vector, device=bfg.device)
        bg[index == 1] = 1
        fg = torch.zeros_like(patch_vector, device=bfg.device)
        fg[index == 2, 0] = self.patch_size
        patch_vector = patch_vector * bg + fg
        # if use DctMaskEncodingV2, plese use: patch_masks = self.patch_dct_encoding.decode(patch_vector)
        patch_masks = self.patch_dct_encoding.decode(patch_vector).real
        masks = rearrange(patch_masks, "(b s1 s2) p1 p2 -> b (s1 p1) (s2 p2)", s1=self.scale, s2=self.scale)
        return masks[:, None, :, :]

    def forward(self, x, fine_mask_features, instances: List[Instances]):
        """
        Args:
            x: input region feature(s) provided by :class:`ROIHeads`.
            fine_mask_features: features cropped from FPN-P2
            instances (list[Instances]): contains the boxes & labels corresponding
                to the input features.
                Exact format is up to its caller to decide.
                Typically, this is the foreground instances in training, with
                "proposal_boxes" field and other gt annotations.
                In inference, it contains boxes that are already predicted.

        Returns:
            A dict of losses in training. The predicted "instances" in inference.
        """
        x, bfg_dict, patch_dict = self.layers(x, fine_mask_features, instances)
        if self.training:
            return {"loss_mask": self.mask_rcnn_dct_loss(x, bfg_dict, patch_dict, instances, self.vis_period)}
        else:
            pred_instances = self.mask_rcnn_dct_inference(x, bfg_dict, patch_dict, instances)
            return pred_instances

    def mask_rcnn_dct_loss(self, pred_mask_logits, bfg_dict, patch_dict, instances, vis_period=0):
        """
        Compute the mask prediction loss defined in the PatchDCT paper.

        Args:
            pred_mask_logits (Tensor): [B, D]. D is dct-dim. [B, D]. DCT_Vector in DCT-Mask.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
            instances (list[Instances]): A list of N Instances, where N is the number of images
                in the batch. These instances are in 1:1
                correspondence with the pred_mask_logits. The ground-truth labels (class, box, mask,
                ...) associated with each instance are stored in fields.
            vis_period (int): the period (in steps) to dump visualization.

        Returns:
            mask_loss (Tensor): A scalar tensor containing the loss.
        """
        if self.dct_loss_type == "l1":
            loss_func = F.l1_loss
        elif self.dct_loss_type == "sl1":
            loss_func = F.smooth_l1_loss
        elif self.dct_loss_type == "l2":
            loss_func = F.mse_loss
        else:
            raise ValueError("Loss Type Only Support : l1, l2; yours: {}".format(self.dct_loss_type))

        gt_masks, gt_masks_coarse, gt_bfg = self.get_gt_mask(instances, pred_mask_logits)

        mask_loss = loss_func(pred_mask_logits, gt_masks_coarse)
        mask_loss = mask_loss * self.loss_para[0]
        for i in range(self.num_stage):
            bfg = bfg_dict[i]
            patch_vector = patch_dict[i]
            patch_vector = patch_vector[gt_bfg == 1]
            mask_loss += self.loss_para[i + 1] * (F.cross_entropy(bfg, gt_bfg) + loss_func(patch_vector, gt_masks))

        mask_loss = self.mask_loss_para * mask_loss
        return mask_loss

    def mask_rcnn_dct_inference(self, pred_mask_logits, bfg_dict, patch_dict, pred_instances):
        """
        Convert pred_mask_logits to estimated foreground probability masks while also
        extracting only the masks for the predicted classes in pred_instances. For each
        predicted box, the mask of the same class is attached to the instance by adding a
        new "pred_masks" field to pred_instances.

        Args:
            pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask)
                for class-specific or class-agnostic, where B is the total number of predicted masks
                in all images, C is the number of foreground classes, and Hmask, Wmask are the height
                and width of the mask predictions. The values are logits.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
            pred_instances (list[Instances]): A list of N Instances, where N is the number of images
                in the batch. Each Instances must have field "pred_classes".

        Returns:
            None. pred_instances will contain an extra "pred_masks" field storing a mask of size (Hmask,
                Wmask) for predicted class. Note that the masks are returned as a soft (non-quantized)
                masks the resolution predicted by the network; post-processing steps, such as resizing
                the predicted masks to the original image resolution and/or binarizing them, is left
                to the caller.
        """

        num_patch = pred_mask_logits.shape[0]
        device = pred_mask_logits.device
        if num_patch == 0:
            pred_instances[0].pred_masks = torch.empty([0, 1, self.mask_size, self.mask_size]).to(device)
            return pred_instances
        else:
            with torch.no_grad():
                last_block = self.num_stage - 1
                bfg = bfg_dict[last_block]
                patch_vector = patch_dict[last_block]
                bfg = F.softmax(bfg, dim=1)
                bfg[bfg[:, 0] > self.patch_threshold, 0] = bfg[bfg[:, 0] > self.patch_threshold, 0] + 1
                bfg[bfg[:, 2] > self.patch_threshold, 2] = bfg[bfg[:, 2] > self.patch_threshold, 2] + 1
                index = torch.argmax(bfg, dim=1)

                # --------------only for upper bound evaluation------------
                if self.eval_gt:
                    gt_masks, gt_bfg = self.get_gt_mask_inference(pred_instances, pred_mask_logits)
                    index = gt_bfg
                    patch_vector[gt_bfg == 1] = gt_masks
                # ---------------------------------------------------------

                patch_vector[index == 0, ::] = 0
                patch_vector[index == 2, ::] = 0
                patch_vector[index == 2, 0] = self.patch_size

                pred_mask_rc = self.patch_dct_encoding.decode(patch_vector)
                # assemble patches to obtain an entire mask
                pred_mask_rc = rearrange(pred_mask_rc,
                                         "(b s1 s2) p1 p2 -> b (s1 p1) (s2 p2)", s1=self.scale, s2=self.scale)

            pred_mask_rc = pred_mask_rc[:, None, :, :]
            pred_instances[0].pred_masks = pred_mask_rc
            return pred_instances

    def get_gt_classes(self, instances):
        gt_classes = []
        for instances_per_image in instances:
            if len(instances_per_image) == 0:
                continue
            gt_classes_per_image = instances_per_image.gt_classes.to(dtype=torch.int64)
            gt_classes.append(gt_classes_per_image)
        gt_classes = cat(gt_classes, dim=0)  # [N_instance]
        return gt_classes

    def get_gt_mask(self, instances, pred_mask_logits):
        gt_masks = []
        gt_masks_coarse = []
        for instances_per_image in instances:

            if len(instances_per_image) == 0:
                continue

            gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                instances_per_image.proposal_boxes.tensor, self.mask_size)
            gt_masks_coarse.append(gt_masks_per_image)

            gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                instances_per_image.proposal_boxes.tensor, self.mask_size_assemble)
            # divided masks into scale x scale patch,patch size=8
            gt_masks_per_image = rearrange(gt_masks_per_image,
                                           "b (s1 p1) (s2 p2) -> (b s1 s2) p1 p2", s1=self.scale, s2=self.scale)

            gt_masks.append(gt_masks_per_image)

        if len(gt_masks) == 0:
            return pred_mask_logits.sum() * 0
        gt_masks = cat(gt_masks, dim=0)
        gt_masks = self.patch_dct_encoding.encode(gt_masks)  # [N, dct_v_dim]
        gt_masks = gt_masks.to(dtype=torch.float32)  # [N_instance,patchdct_vector_dim]

        gt_masks_coarse = cat(gt_masks_coarse, dim=0)
        gt_masks_coarse = self.dct_encoding.encode(gt_masks_coarse).to(dtype=torch.float32)

        gt_masks, gt_bfg = self.get_gt_bfg(gt_masks)
        return gt_masks, gt_masks_coarse, gt_bfg

    def get_gt_mask_inference(self,instances,pred_mask_logits):
        gt_masks = []

        for instances_per_image in instances:
            if len(instances_per_image) == 0:
                continue
            if instances_per_image.has("gt_masks"):
                gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                    instances_per_image.pred_boxes.tensor, self.mask_size_assemble)
            else:
                # print("gt_mask is empty")
                shape = instances_per_image.pred_boxes.tensor.shape[0]
                device = instances_per_image.pred_boxes.tensor.device
                gt_masks_per_image = torch.zeros((shape, self.mask_size_assemble, self.mask_size_assemble),
                                                 dtype=torch.bool).to(device)

            gt_masks_per_image = rearrange(gt_masks_per_image,
                                           "b (s1 p1) (s2 p2) -> (b s1 s2) p1 p2", s1=self.scale, s2=self.scale)

            gt_masks.append(gt_masks_per_image)

        if len(gt_masks) == 0:
            return pred_mask_logits.sum() * 0

        gt_masks = cat(gt_masks, dim=0)
        gt_masks = self.patch_dct_encoding.encode(gt_masks)
        gt_masks = gt_masks.to(dtype=torch.float32)

        gt_masks, gt_bfg = self.get_gt_bfg(gt_masks)
        return gt_masks, gt_bfg

    def get_gt_bfg(self, gt_masks):
        gt_bfg = gt_masks[:, 0].clone()
        gt_bfg[(gt_bfg > 0) & (gt_bfg < self.patch_size)] = 1.
        gt_bfg[gt_bfg == self.patch_size] = 2
        gt_bfg = gt_bfg.to(dtype=torch.int64)
        gt_masks = gt_masks[gt_bfg == 1, :]
        return gt_masks, gt_bfg


import billboard
import pandas as pd
import json

def all_sundays(year):
    """Return list of Sunday`s dates on particular year"""
    return pd.date_range(start=str(year), end=str(year+1), freq='W-SUN').strftime('%Y-%m-%d').tolist()

def get_chart(date):
    """Return Billboard chart object on particular date."""
    return billboard.ChartData('hot-100', date=date)

charts = {}
"""Write all Billboard charts for past 60 years to json file"""
for year in range(1958, 2019):
    for date in all_sundays(year):
        try:
            chart = get_chart(date)
            li = [
                {
                    "title": ch.title,
                    "artist": ch.artist,
                    "rank": ch.rank
                } for ch in chart
            ]
            charts[date] = li
        except:
            pass
with open("us_charts.json", "w+") as file:
    file.write(json.dumps(charts))


from sklearn.feature_selection import f_regression
from sklearn.linear_model import Lasso, LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import numpy as np

def rank_scaled_dict(rank, names, order=1):
    """ Scale the rankings from different models and add
    them all to a dictionary for comparison
    """
    # scale the scores between 0 and 1
    scaled_rank = MinMaxScaler().fit_transform(order*np.array([rank]).T).T[0]
    final_ranks = map(lambda x: round(x, 3), scaled_rank)
    adict = dict(zip(names, final_ranks))
    print([(k, v) for k, v in sorted(adict.items(), key=lambda x: x[1], reverse=True) if abs(v) > 0.0])

    return dict(zip(names, final_ranks))

def rank_features(X, y, collinear):
    """ Returns most important features based on different techniques
    """
    rank = {}

    col_names = X.columns

    print("\n\nStandarized variables before using different models for ranking")
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    y_scaled = scaler.fit_transform(y).ravel()

    print("\n\nRanking from univariate regression based on F-scores")  # only identifies linear relationship
    f, pval = f_regression(X_scaled, y_scaled)
    rank["Corr"] = rank_scaled_dict(f, col_names)
    print("\nAll very significant features with p-val < 0.01 from univariate regression")
    pvals_dict = dict(zip(col_names, pval))
    print([(k, v) for k, v in sorted(pvals_dict.items(), key=lambda x: x[1]) if v < 0.01])

    if not collinear:
        lr = LinearRegression().fit(X_scaled, y_scaled)
        print("\n\nRanking based on linear regression")
        rank["Linear Regression"] = rank_scaled_dict(np.abs(lr.coef_), col_names)

    print("\n\nRanking based on lasso ( L1 penalty) based on very low regularization value alpha - 0.001")
    lasso = Lasso(alpha=0.001).fit(X_scaled, y_scaled)
    rank['Lasso'] = rank_scaled_dict(lasso.coef_, col_names)

    print("\n\nRanking based on Random forests variable importance graph; using mutual information")
    rf = RandomForestRegressor().fit(X_scaled, y_scaled)
    rank["Random Forest"] = rank_scaled_dict(rf.feature_importances_, col_names)

    # Get the mean score based on all the algorithms
    temp = {}
    for col in col_names:
        temp[col] = round(np.mean([rank[algo][col] for algo in rank.keys()]), 3)

    rank['avg_scores'] = temp
    # print(rank)
    methods = sorted(rank.keys())

    print("\n\nRanking based on different metrics")
    sorted_colnames = sorted(rank['avg_scores'], key=rank['avg_scores'].__getitem__, reverse=True)
    print("\t%s" % "\t".join(methods))
    for name in sorted_colnames:
        print("%s\t%s" % (name, "\t\t".join(map(str, [rank[method][name] for method in methods]))))

    select_features = [k for k, v in rank['avg_scores'].items() if v > 0]
    print("\n\nImportant predictors are based on average score:", select_features)

    print("\n\n One can also use Recursive feature elimination for feature engineering,"
          "but given so many features, this would take long time.")

    return select_features


# Import necessary libraries
import pandas as pd
import matplotlib.pyplot as plt

# Define the base Report class
class Report:
    def __init__(self, dataframe, title, **kwargs):
        self.dataframe = dataframe
        self.title = title
        self.properties = kwargs

    def generate_table(self):
        """This method generates a table using Matplotlib."""
        fig, ax =plt.subplots(1,1)
        ax.axis('tight')
        ax.axis('off')
        self.table = ax.table(cellText=self.dataframe.values,
                              colLabels=self.dataframe.columns,
                              cellLoc = 'center',
                              loc='center')
        self.fig = fig

    def apply_style(self):
        """This method applies common styling to the table."""
        self.table.auto_set_font_size(False)
        self.table.set_fontsize(self.properties.get('font_size', 10))

    def generate(self):
        """This method generates the report by creating and styling the table."""
        self.generate_table()
        self.apply_style()
        self.fig.suptitle(self.title)
        plt.show()

class DataFrameReport(Report):
    """This subclass of Report represents a report with a dataframe."""
    def __init__(self, dataframe, title, **kwargs):
        super().__init__(dataframe, title, **kwargs)

    def apply_style(self):
        """This method applies styling specific to DataFrameReport."""
        super().apply_style()
        # Apply additional or modified styling here

class MatplotlibReport(Report):
    """This subclass of Report represents a report with a matplotlib plot."""
    def __init__(self, dataframe, title, **kwargs):
        super().__init__(dataframe, title, **kwargs)

    def apply_style(self):
        """This method applies styling specific to MatplotlibReport."""
        super().apply_style()
        # Apply additional or modified styling here

class DataFramePlotReport(Report):
    """This subclass of Report represents a report with a dataframe and a plot."""
    def __init__(self, dataframe, title, **kwargs):
        super().__init__(dataframe, title, **kwargs)

    # If no specific styling is needed, no need to override apply_style method.
    # It will use the common styling from the base class.

class ReportFactory:
    """This class generates a report based on a given type."""
    @staticmethod
    def create_report(report_type, dataframe, title, **kwargs):
        if report_type == "dataframe":
            return DataFrameReport(dataframe, title, **kwargs)
        elif report_type == "matplotlib":
            return MatplotlibReport(dataframe, title, **kwargs)
        elif report_type == "dataframe_plot":
            return DataFramePlotReport(dataframe, title, **kwargs)
        else:
            raise ValueError(f"Invalid report_type: {report_type}")

# Create settings dictionary with report parameters
settings = {
    "rai": {
        "report_type": "dataframe",
        "dataframe": df,  # pandas dataframe
        "title": "Your RAI Report Title",
        "font_size": 10,
        # other properties to customize the look and feel of the report
    },
    # More settings...
}

# Generate report for each set of settings
for report_id, report_settings in settings.items():
    report = ReportFactory.create_report(report_settings['report_type'],
                                         report_settings['dataframe'],
                                         report_settings['title'],
                                         **report_settings)
    report.generate()  # generate the report


import cntk as C
import numpy as np

import ast

from .agent import AgentBaseClass
from .shared.cntk_utils import negative_of_entropy_with_softmax
from .shared.models import Models
from .shared.policy_gradient_parameters import PolicyGradientParameters

class ActorCritic(AgentBaseClass):
    """
    Actor-Critic Policy Gradient.

    See https://arxiv.org/pdf/1602.01783.pdf for a description of algorithm.
    """

    def __init__(self, config_filename, o_space, a_space):
        """
        Constructor for policy gradient.

        Args:
            config_filename: configure file specifying training details.
            o_space: observation space, gym.spaces.tuple_space.Tuple is not
                supported.
            a_space: action space, limits to gym.spaces.discrete.Discrete.
        """
        super(ActorCritic, self).__init__(o_space, a_space)

        self._parameters = PolicyGradientParameters(config_filename)

        # Create preprocessor.
        if self._parameters.preprocessing:
            preproc = self._import_method(self._parameters.preprocessing)
            self._preprocessor = preproc(
                self._shape_of_inputs,
                *ast.literal_eval(self._parameters.preprocessing_args))

        self._set_up_policy_network_and_value_network()

        self._trajectory_states = []
        self._trajectory_actions = []
        self._trajectory_rewards = []

        # Training data for the policy and value networks. Note they share the
        # same input.
        self._input_buffer = []
        self._value_network_output_buffer = []
        self._policy_network_output_buffer = []
        self._policy_network_weight_buffer = []

        self.episode_count = 0
        self.step_count = 0

    def start(self, state):
        """
        Start a new episode.

        Args:
            state (object): observation provided by the environment.

        Returns:
            action (int): action choosen by agent.
            debug_info (dict): auxiliary diagnostic information.
        """
        # Call _process_accumulated_trajectory() to process unused trajectory
        # data from previous episode.
        self._process_accumulated_trajectory(False)

        # Reset preprocessor.
        if self._preprocessor is not None:
            self._preprocessor.reset()

        # Append new state and action
        o = self._preprocess_state(state)
        action, _ = self._choose_action(o)
        self._trajectory_states.append(o)
        self._trajectory_actions.append(action)

        self.episode_count += 1

        return action, {}

    def step(self, reward, next_state):
        """
        Observe one transition and choose an action.

        Args:
            reward (float) : amount of reward returned after previous action.
            next_state (object): observation provided by the environment.

        Returns:
            action (int): action choosen by agent.
            debug_info (dict): auxiliary diagnostic information.
        """
        o = self._preprocess_state(next_state)
        self._trajectory_rewards.append(reward)
        self._trajectory_states.append(o)
        self.step_count += 1

        # Update every self._parameters.update_frequency
        if self.step_count % self._parameters.update_frequency == 0:
            self._process_accumulated_trajectory(True)
            self._update_networks()

        action, _ = self._choose_action(o)
        self._trajectory_actions.append(action)
        return action, {}

    def end(self, reward, next_state):
        """
        Last observed reward/state of the episode (which then terminates).

        Args:
            reward (float) : amount of reward returned after previous action.
            next_state (object): observation provided by the environment.
        """
        self._trajectory_rewards.append(reward)
        self.step_count += 1

        # Update every self._parameters.update_frequency
        if self.step_count % self._parameters.update_frequency == 0:
            self._process_accumulated_trajectory(False)
            self._update_networks()

    def set_as_best_model(self):
        """Copy current model to best model."""
        self._best_model = self._policy_network.clone('clone')

    def _set_up_policy_network_and_value_network(self):
        shape_of_inputs = self._shape_of_inputs if self._preprocessor is None \
            else self._preprocessor.output_shape()
        self._input_variables = \
            C.ops.input_variable(shape=shape_of_inputs, dtype=np.float32)

        # Set up policy network.
        if self._parameters.policy_representation == 'nn':
            model = Models.feedforward_network(
                shape_of_inputs,
                self._num_actions,
                self._parameters.policy_network_hidden_layers,
                C.losses.cross_entropy_with_softmax,
                use_placeholder_for_input=True)
        else:
            try:
                model_definition_function = self._import_method(
                    self._parameters.policy_representation)
                model = model_definition_function(
                    shape_of_inputs,
                    self._num_actions,
                    C.losses.cross_entropy_with_softmax,
                    use_placeholder_for_input=True)
            except ValueError:
                raise ValueError(
                    'Unknown representation for policy: "{0}"'
                    '\n'.format(self._parameters.policy_representation))

        self._policy_network = model['f']
        self._policy_network.replace_placeholder(self._input_variables)
        self._policy_network_output_variables = model['outputs']
        # The weight is computed as part of the Actor-Critic algorithm.
        self._policy_network_weight_variables = \
            C.ops.input_variable(shape=(1,), dtype=np.float32)
        self._policy_network_loss = \
            model['loss'] * self._policy_network_weight_variables

        # Initialized from a saved model.
        if self._parameters.initial_policy_network:
            self._policy_network.restore(
                self._parameters.initial_policy_network)

        print("Parameterized the agent's policy using neural networks "
              '"{0}" with {1} actions.\n'
              ''.format(self._parameters.policy_representation,
                        self._num_actions))

        # Set up value network.
        if self._parameters.shared_representation:
            # For shared representation, policy pi and value function V share
            # all non-output layers. To use cross_entropy_with_softmax loss
            # from cntk, _policy_network defined here doesn't include softmax
            # output layer. Therefore _value_network becomes _policy_network
            # plus one additional linear output layer.
            self._value_network = C.layers.Dense(1, activation=None)(
                self._policy_network)
            self._value_network_output_variables = C.ops.input_variable(
                shape=(1,), dtype=np.float32)
            self._value_network_loss = C.losses.squared_error(
                self._value_network, self._value_network_output_variables)
        else:
            if self._parameters.value_function_representation == 'nn':
                model = Models.feedforward_network(
                    shape_of_inputs,
                    1,  # value network outputs a scalar
                    self._parameters.value_network_hidden_layers,
                    use_placeholder_for_input=True)
            else:
                try:
                    model_definition_function = self._import_method(
                        self._parameters.value_function_representation)
                    model = model_definition_function(
                        shape_of_inputs,
                        1,  # value network outputs a scalar
                        use_placeholder_for_input=True)
                except ValueError:
                    raise ValueError(
                        'Unknown representation for value function: "{0}"'
                        '\n'.format(self._parameters.value_function_representation))

            self._value_network = model['f']
            self._value_network.replace_placeholder(self._input_variables)
            self._value_network_output_variables = model['outputs']
            self._value_network_loss = model['loss']  # squared_error by default

        combined_networks = C.ops.combine(
            [self._policy_network, self._value_network])
        combined_loss = self._policy_network_loss + \
            self._parameters.regularization_weight * \
            negative_of_entropy_with_softmax(self._policy_network) + \
            self._parameters.relative_step_size * self._value_network_loss

        # The learning rate will be updated later before each minibatch
        # training.
        # TODO: allow user to specify learner through config file.
        self._trainer = C.train.trainer.Trainer(
            combined_networks,
            (combined_loss, None),
            C.learners.adam(
                combined_networks.parameters,
                C.learners.learning_parameter_schedule_per_sample(
                    self._parameters.initial_eta),
                momentum=C.learners.momentum_schedule(self._parameters.momentum),
                variance_momentum=C.learners.momentum_schedule(0.999),
                minibatch_size=C.learners.IGNORE))

        print("Parameterized the agent's value function using neural network "
              '"{0}".\n'.format(
                self._parameters.policy_representation
                if self._parameters.shared_representation
                else self._parameters.value_function_representation))

    def _adjust_learning_rate(self):
        if self._parameters.initial_eta != self._parameters.eta_minimum:
            eta = self._parameters.eta_minimum + max(
                    0,
                    (self._parameters.initial_eta - self._parameters.eta_minimum) *
                    (1 - float(self.step_count)/self._parameters.eta_decay_step_count))
            self._trainer.parameter_learners[0].reset_learning_rate(
                C.learners.learning_parameter_schedule_per_sample(eta))

    def _choose_action(self, state):
        """
        Choose an action according to policy.

        Args:
            state (object): observation seen by agent, which can be different
                from what is provided by the environment. The difference comes
                from preprcessing.

        Returns:
            action (int): action choosen by agent.
            debug_info (object): probability vector the action is sampled from.
        """
        action_probs = \
            C.ops.softmax(self._evaluate_model(self._policy_network, state)).eval()
        return np.random.choice(self._num_actions, p=action_probs), action_probs

    def save(self, filename):
        """Save model to file."""
        self._best_model.save(filename)

    def save_parameter_settings(self, filename):
        """Save parameter settings to file."""
        self._parameters.save(filename)

    def _evaluate_model(self, model, state):
        r"""Evaluate log of pi(\cdot|state) or v(state)."""
        return np.squeeze(model.eval({model.arguments[0]: [state]}))

    def _process_accumulated_trajectory(self, keep_last):
        """Process accumulated trajectory to generate training data.

        Args:
            keep_last (bool): last state without action and reward will be kept
                if True.
        """
        if not self._trajectory_states:
            return

        # If trajectory hasn't terminated, we have _trajectory_states
        # and sometimes _trajectory_actions having one more item than
        # _trajectory_rewards. Same length is expected if called from
        # start() or end(), where the trajectory has terminiated.
        if len(self._trajectory_states) == len(self._trajectory_rewards):
            bootstrap_r = 0
        else:
            # Bootstrap from last state
            bootstrap_r = np.asscalar(self._evaluate_model(
                self._value_network, self._trajectory_states[-1]))
            last_state = self._trajectory_states.pop()
            if len(self._trajectory_actions) != len(self._trajectory_rewards):
                # This will only happen when agent calls start() to begin
                # a new episode without calling end() before to terminate the
                # prevous episode. The last action thus can be discarded.
                self._trajectory_actions.pop()

        if len(self._trajectory_states) != len(self._trajectory_rewards) or \
           len(self._trajectory_actions) != len(self._trajectory_rewards):
            raise RuntimeError("Can't pair (state, action, reward). "
                               "state/action can only be one more step ahead "
                               "of rewrad in trajectory.")

        for transition in zip(
                self._trajectory_states,
                self._trajectory_actions,
                self._discount_rewards(bootstrap_r)):
            self._input_buffer.append(transition[0])
            self._value_network_output_buffer.append([transition[2]])
            # TODO: consider using cntk.ops.one_hot instead of _index_to_vector
            self._policy_network_output_buffer.append(
                self._index_to_vector(transition[1], self._num_actions))
            self._policy_network_weight_buffer.append([transition[2]
                - self._evaluate_model(self._value_network, transition[0])])

        # Clear the trajectory history.
        self._trajectory_states = []
        self._trajectory_actions = []
        self._trajectory_rewards = []
        if keep_last:
            self._trajectory_states.append(last_state)

    def _update_networks(self):
        self._adjust_learning_rate()

        # Train the policy network on one minibatch.
        self._trainer.train_minibatch(
            {
                self._input_variables: np.array(self._input_buffer).astype(
                    np.float32),
                self._policy_network_output_variables:
                    np.array(self._policy_network_output_buffer).astype(
                        np.float32),
                self._policy_network_weight_variables:
                    np.array(self._policy_network_weight_buffer).astype(
                        np.float32),
                self._value_network_output_variables:
                    np.array(self._value_network_output_buffer).astype(
                        np.float32)
            })

        # Clear training data.
        self._input_buffer = []
        self._value_network_output_buffer = []
        self._policy_network_output_buffer = []
        self._policy_network_weight_buffer = []

    def _discount_rewards(self, bootstrap_r):
        discounted_rewards = [0] * len(self._trajectory_rewards)
        r = bootstrap_r
        for t in reversed(range(len(self._trajectory_rewards))):
            r = r * self._parameters.gamma + self._trajectory_rewards[t]
            discounted_rewards[t] = r
        return discounted_rewards


import numpy as np
from pathlib import Path

PRED_TXT='preds.txt'
SCENE_TXT='/mnt/HD_2/luojiapeng/scene_recognition/data/valset/scenes/labels.txt'

if __name__ == '__main__':
    preds = np.loadtxt(PRED_TXT,dtype=str).tolist()
    correct = 0
    total = 0
    gt = np.loadtxt(SCENE_TXT, dtype=str).tolist()
    gt = dict(gt)

    for p in preds:
        img_name = Path(p.split(':')[0]).name
        pred_scene = p.split(':')[1].split(',')[1]
        if pred_scene == gt[img_name]:
            correct += 1
        total += 1
    print('scene accuracy: %.4f' % (correct / total))


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 11 11:55:38 2021

@author: ndbke
"""

import numpy as np
import xarray as xr
import matplotlib.pyplot as plt

# find depth of bottom based on NaNs in vertical profile
def find_bottom(data,depth):

    # find & count non-NaN observations
    bath_mask = ~np.isnan(data)
    bath_count = np.count_nonzero(bath_mask)

    # if location is on land, depth is NaN
    if bath_count == 0:
        bath_depth = np.nan

    # get bottom depth by index
    else:
        bath_depth = depth[bath_count-1]

    return bath_depth

def bathymetry(lvl):

    print("opening data file")

    # load file that we made holding all data in organized object
    file = 'C:/Users/ndbke/Dropbox/_NDBK/Research/WHOI/whoi_code/NetCDFs/data_all.nc'
    dat = xr.open_dataset(file, decode_times=False, autoclose=True)

    print("building bathymetry")

    # construct bathymetry & plot
    full_bath = xr.apply_ufunc(find_bottom, dat.CT, dat.depth, input_core_dims=[["depth"],["depth"]], vectorize=True)
    print(full_bath)
    full_bath.plot()

    print("masking isobath")

    # introduce isobath at 200m
    iso_bath = full_bath.where(full_bath >= lvl)
    # convert to T/F mask
    bath_mask = ~np.isnan(iso_bath)
    # apply mask to data (all variables!)
    dat = dat.where(bath_mask)

    # add bathymetry & mask layers
    dat["bathymetry"] = full_bath
    dat["bath_mask"] = bath_mask

    print("saving & plotting")

    # save masked data & bathymetry to NetCDF
    dat.to_netcdf('../NetCDFs/bathymetry.nc',mode='w')

    # plot bathymetrically masked datasets (at surface)
    plt.figure()
    dat.CT.isel(depth=0).plot(robust=True)
    plt.figure()
    dat.SA.isel(depth=0).plot(robust=True)
    plt.figure()
    dat.bath_mask.plot()
    plt.title(f"{lvl} m isobath")

    print("done")

bathymetry(200)


import re

from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect

import seaserv

from seahub.notifications.models import Notification
from seahub.notifications.utils import refresh_cache
try:
    from seahub.settings import CLOUD_MODE
except ImportError:
    CLOUD_MODE = False
try:
    from seahub.settings import MULTI_TENANCY
except ImportError:
    MULTI_TENANCY = False
from seahub.settings import SITE_ROOT

class BaseMiddleware(object):
    """
    Middleware that add organization, group info to user.
    """

    def process_request(self, request):
        username = request.user.username
        request.user.org = None

        if CLOUD_MODE:
            request.cloud_mode = True

            if MULTI_TENANCY:
                orgs = seaserv.get_orgs_by_user(username)
                if orgs:
                    request.user.org = orgs[0]
        else:
            request.cloud_mode = False

        if CLOUD_MODE and request.user.org is not None:
            org_id = request.user.org.org_id
            request.user.joined_groups = seaserv.get_org_groups_by_user(
                org_id, username)
        else:
            request.user.joined_groups = seaserv.get_personal_groups_by_user(
                username)

        return None

    def process_response(self, request, response):
        return response

class InfobarMiddleware(object):
    """Query info bar close status, and store into request."""

    def get_from_db(self):
        ret = Notification.objects.all().filter(primary=1)
        refresh_cache()
        return ret

    def process_request(self, request):
        topinfo_close = request.COOKIES.get('info_id', '')

        cur_note = cache.get('CUR_TOPINFO') if cache.get('CUR_TOPINFO') else \
            self.get_from_db()
        if not cur_note:
            request.cur_note = None
        else:
            if str(cur_note[0].id) in topinfo_close.split('_'):
                request.cur_note = None
            else:
                request.cur_note = cur_note[0]

        return None

    def process_response(self, request, response):
        return response

class ForcePasswdChangeMiddleware(object):
    def _request_in_black_list(self, request):
        path = request.path
        black_list = (r'^%s$' % SITE_ROOT, r'home/.+', r'repo/.+',
                      r'[f|d]/[a-f][0-9]{10}', r'group/\d+', r'groups/',
                      r'share/', r'profile/', r'notification/list/')

        for patt in black_list:
            if re.search(patt, path) is not None:
                return True
        return False

    def process_request(self, request):
        if request.session.get('force_passwd_change', False):
            if self._request_in_black_list(request):
                return HttpResponseRedirect(reverse('auth_password_change'))


import csv
file = open("bos2021ModC.csv")
csvreader = csv.reader(file)
header = next(csvreader)
print(header)
rows = []
for row in csvreader:
    rows.append(row)
#print(rows)
print(len(rows))
print(rows[0])
file.close()


from requests import Session
import requests
import os

# NLP_ENGINE_URL = "http://127.0.0.1:5001"
# INTENTS_MANAGMENT_URL = "http://127.0.0.1:5002"
# CONTEXT_MANAGMENT_URL = "http://127.0.0.1:5003"

NLP_ENGINE_URL = os.environ.get('NLP_ENGINE_URL')
INTENTS_MANAGMENT_URL = os.environ.get('INTENTS_MANAGMENT_URL')
CONTEXT_MANAGMENT_URL = os.environ.get('CONTEXT_MANAGMENT_URL')

session = Session()
session.trust_env = False
session.verify = False
session.headers["Accept"] = "application/json"
session.headers["Content-Type"] = "application/json"

def discover_intent(agent, text):
    """
    This method connects to the microservice *NLP Engine* in order to retrieve the intents from a raw text.

    :param agent: The agent name of the chatbot

    :param text: The raw input to discover the intent

    :return: The uri of a intent
    """
    url = NLP_ENGINE_URL + "/intents"
    try:
        r = session.post(url, json={"agent": agent, "sentence": text})
        if r.status_code == 200:
            response = r.json()
            if "intent" in response and len(response["intent"]) > 0:
                return response["intent"][0]["prediction"]
            else:
                return None
    except requests.exceptions.RequestException as e:
        print(e)
        return None

def discover_entities(agent, text):
    """
        This method connects to the microservice *NLP Engine* in order to retrieve the entities from a raw text.

        :param agent: The agent name of the chatbot

        :param text: The raw input to discover the intent

        :return: A list with the URIS of the entities and type of entities.
    """
    url = NLP_ENGINE_URL + "/entities"
    try:
        r = session.post(url, json={"agent": agent, "sentence": text})
        if r.status_code == 200:
            response = r.json()
            entities = []
            if len(response["entities"]) > 0:
                for entity in response["entities"]:
                    entities.append({
                        "type": entity["entity"],
                        "value": entity["prediction"],
                    })
            return entities
    except requests.exceptions.RequestException as e:
        print(e)
        return []

def get_requirements(intent):
    """
    This service connects to the microservice *Intents Managment* in order to retrieve requirements of an intent

    :param intent: The intent from where requirements will be retrieved

    :return: A list of URIS of the different entities needed to complete an intent
    """
    url = INTENTS_MANAGMENT_URL + "/intent/requires"
    try:
        r = session.get(url, json={"intent": intent})
        if r.status_code == 200:
            response = r.json()
            if "requires" in response:
                return response["requires"]
            else:
                return None
    except requests.exceptions.RequestException as e:
        print(e)
        return None

def get_options(entity):
    """
        This service connects to the microservice *Intents Management* in order to retrieve options of an entity type

        :param entity: The entity from where options will be retrieved

        :return: A list of options to complete an entity

        .. todo:: give a list of entities
        """
    url = INTENTS_MANAGMENT_URL + "/entity/options"
    try:
        # print({"entity": entity})
        r = session.get(url, json={"entity": entity})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None

def find_in_context(user, entities):
    """
    This method looks for information in the conversation thread.
     :param user: The id of the user to find information

     :param entities: The entities to be found in context.
    """
    url = CONTEXT_MANAGMENT_URL + "/context/entities"
    try:
        r = session.get(url, json={"user": user, "entities": entities})
        if r.status_code == 200:
            response = r.json()
            if "entities" in response:
                entities = response["entities"]
                return entities
            else:
                return []
    except requests.exceptions.RequestException as e:
        print(e)
        return []

def get_answer(intent, entities):
    """
            This service connects to the microservice *Intents Management* in order to retrieve the answer of an intent.

            :param intent: The intent from where answer will be retrieved

            :param entities: A list of entities used to retrieve answer

            :return: A list of options to complete an entity
            """
    url = INTENTS_MANAGMENT_URL + "/intent/answer"
    try:
        r = session.get(url, json={"intent": intent, "entities": entities})
        if r.status_code == 200:
            response = r.json()
            return response
    except requests.exceptions.RequestException as e:
        print(e)

def get_agent_data(agent):
    """
        This service connects to the microservice *Intents Management* in order to information about and agent

        :param agent: A valid agent name

        :return: A dict containing agent, a description and the different intents.

        """
    url = INTENTS_MANAGMENT_URL + "/agent/info"
    try:
        r = session.get(url, json={"agent": agent})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None

def get_intent_rq(intent, entity):
    """
        This service connects to the microservice *Intents Management* in order to get the resolution question of an intent

        :param intent: A valid intent

        :param entity: A valid entity

        :return: A dict containing intent, a entity and the resolution question.

        """
    url = INTENTS_MANAGMENT_URL + "/intent/rq"
    try:
        r = session.get(url, json={"intent": intent, "entity": entity})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None


#sender
import socket
import hashlib
import os
import struct

serverIP = '127.0.0.1'
serverPort = 3333

clnt_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

print("Sender Socket open..")
print("Receiver IP = " + serverIP)
print("Receiver port = " + str(serverPort))

file_name = input("Input your Message : ")
print("Send File Info(file name, file Size, seqNum) to Server...")
print("Start File send")

seq_num = 0	#sequence_number	0,1을 번갈아 가면서 사용

file_size = os.path.getsize(file_name)

file_info_msg = file_size.to_bytes(4, byteorder = "big") + file_name.encode()
file_info_seq_msg = seq_num.to_bytes(1, byteorder = "big") + file_info_msg
file_info_checksum = hashlib.sha1()
file_info_checksum.update(file_info_seq_msg)
file_info_checksum_hashed = file_info_checksum.digest()
file_info_final = file_info_checksum_hashed + file_info_seq_msg

clnt_sock.sendto(file_info_final,(serverIP, serverPort))

recv_ACK = clnt_sock.recv(4)    #4바이트를 전송받음
recv_ACK_int = struct.unpack("!i", recv_ACK)    #ACK로 받은 메세지 int화

if(((seq_num + 1) % 2) == recv_ACK_int):        #ACK
	seq_num = ((seq_num + 1) % 2)   #값 변경

file = open(file_name, "rb")

count_size = 0	#while문을 돌리기 위한 조건으로 사용

while	count_size < file_size:

	data = file.read(1024)	#파일을 1024바이트씩 읽음
	seq_data = seq_num.to_bytes(1, byteorder = "big") + data	#뒷 부분의 정보

	checksum = hashlib.sha1()
	checksum.update(seq_data)
	checksum_byte = checksum.digest()	#hash함수로 암호화

	msg = checksum_byte + seq_num.to_bytes(1, byteorder = "big") + data	#메세지

	clnt_sock.sendto(msg,(serverIP, serverPort))	#데이터 메세지 전송

	size_msg = '( current size / total size ) = ' + str(count_size) + '/' + str(file_size) + ' , ' + str(round((count_size / file_size), 5) * 100) + '%'
	print(size_msg)

	recv_ACK = clnt_sock.recv(4)	#4바이트를 전송받음
	recv_ACK_int = struct.unpack("!i", recv_ACK)	#ACK로 받은 메세지 int화

	if(((seq_num + 1) % 2) == recv_ACK_int):	#ACK가 맞는 경우
		seq_num = ((seq_num + 1) % 2)	#값 변경

	count_size = count_size + len(data)	#while문을 사용하기 위한 조건으로 사용

size_msg = '( current size / total size ) = ' + str(count_size) + '/' + str(file_size) + ',' + str(round((count_size/ file_size), 5)*100) + '%'
print(size_msg)
print("File Send End.")


__author__ = "XiaoY"

from .data_path import MODEL_DIR
import os

FACE_ALIGN_68_KP = {
    "face": {
        "mouth": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67],
        "right_brow": [17, 18, 19, 20, 21],
        "left_brow": [22, 23, 24, 25, 26],
        "right_eye": [36, 37, 38, 39, 40, 41],
        "left_eye": [42, 43, 44, 45, 46, 47],
        "nose": [27, 28, 29, 30, 31, 32, 33, 34, 35]
    },
    "jaw": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
}

# fan: face alignment net
FACE_ALIGN_68_KP_FAN = {
    "proto": os.path.join(MODEL_DIR, "fan/fan.xml"),
    "weights": os.path.join(MODEL_DIR, "fan/fan.bin"),
    "backend": "DLDT",
    "input_height": 256,
    "input_width": 256,
    "swap_rb": True,
    "crop": False,
    "mean": (0, 0, 0),
    "scale_factor": 1 / 255,
    "num_kpts": 68,
    "heat_map_indices": list(range(68)),
    "threshold": 0.2,
    # "decoding_mode": "hard", # soft or hard
    "decoding_mode": "soft", # soft or hard
}

FACE_ALIGN_70_KP_OPENPOSE = {
    "proto": os.path.join(MODEL_DIR, "openpose/face/pose_deploy.prototxt"),
    "weights": os.path.join(MODEL_DIR, "openpose/face/pose_iter_116000.caffemodel"),
    "backend": "Caffe",
    "input_height": 368,
    "input_width": 368,
    "swap_rb": False,
    "crop": False,
    "mean": (0, 0, 0),
    "scale_factor": 1 / 255,
    "heat_map_indices": list(range(70)),
    "threshold": 0.2,
    "decoding_mode": "hard", # soft or hard
}

# intel facial-landmarks-35-adas-0002
# blob shape: [1, 70]
# [x0, y0, x1, y1, ..., x34, y34]
FACE_ALIGN_35_KP_INTEL_ADAS_0002_FP32 = {
    "proto": os.path.join(MODEL_DIR, "intel/facial-landmarks-35-adas-0002/FP32/facial-landmarks-35-adas-0002.xml"),
    "weights": os.path.join(MODEL_DIR, "intel/facial-landmarks-35-adas-0002/FP32/facial-landmarks-35-adas-0002.bin"),
    "backend": "DLDT",
    "input_height": 60,
    "input_width": 60,
    "swap_rb": False,
    "crop": False,
    "num_kpts": 35,
}

# dlib shape predictor
FACE_ALIGN_68_KP_DLIB = {
    "model": os.path.join(MODEL_DIR, "dlib/shape_predictor_68_face_landmarks.dat"),
    "num_kpts": 68,
}


from collections import deque
class Solution:
    def isValid(self, s: str):
        stack=deque()
        for i in range(len(s)):
            if s[i]=='(' or s[i]=='[' or s[i]=='{':
                stack.append(s[i])
            else:
                if len(stack)==0:
                    return False
                elif (stack[-1]=='(' and s[i]==')') or (stack[-1]=='[' and s[i]==']') or (stack[-1]=='{' and s[i]=='}'):
                    stack.pop()
                else:
                    return False
        return True if len(stack)==0 else False

s=Solution()
d = "([)]"
print(s.isValid(d))


from abc import abstractmethod
from dataclasses import dataclass
from typing import Any, List, Optional, Protocol, Sequence, Tuple

@dataclass
class User:
    id: int
    telegram_id: int
    banned: bool
    available_for_broadcast: bool

@dataclass
class Game:
    id: int
    game_id: str
    group_id: int
    message_id: int
    owner_id: int
    owner_name: str
    word: str
    created_at: int
    finished: bool

class Database(Protocol):
    @abstractmethod
    async def _async__init__(self: "Database") -> "Database":
        """Async initializer"""
        raise NotImplementedError

    def __await__(self):
        return self._async__init__().__await__()

    async def open(self: "Database") -> None:
        """Create connection pool"""
        await self

    @abstractmethod
    async def close(self: "Database") -> None:
        """Close connection pool"""
        raise NotImplementedError

    @abstractmethod
    async def sql(
        self: "Database", sql: str, params: Optional[Sequence[Any]] = None
    ) -> Any:
        """Execute plain sql"""
        raise NotImplementedError

    @abstractmethod
    async def get_user_or_create(self: "Database", user_id: int) -> Tuple[User, bool]:
        """Get or create user"""
        raise NotImplementedError

    async def get_users(self: "Database") -> List[User]:
        """Get all users"""
        raise NotImplementedError

    async def create_game(
        self: "Database",
        game_id: str,
        group_id: int,
        owner_id: int,
        owner_name: str,
        word: str,
    ) -> Game:
        """Create new game"""
        raise NotImplementedError

    async def get_game(self: "Database", game_id: str) -> Optional[Game]:
        """Get game"""
        raise NotImplementedError

    async def get_group_game(self: "Database", group_id: int) -> Optional[Game]:
        """Get current group game"""
        raise NotImplementedError

    async def update_game_message(
        self: "Database", game_id: int, new_message_id: int
    ) -> None:
        """Update game message id"""
        raise NotImplementedError

    async def game_finished(
        self: "Database", game_id: int
    ) -> None:
        """Update finished game: delete"""
        raise NotImplementedError

    async def delete_games(
        self: "Database", group_id: int
    ) -> None:
        """Delete all games in group"""
        raise NotImplementedError


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def mergeKLists(self, lists: List[Optional[ListNode]]) -> Optional[ListNode]:
        if not lists or len(lists) == 0:
            return None
        mergedList = lists
        while len(lists) != 1:
            mergedList = []
            for i in range(0, len(lists), 2):
                l1 = lists[i]
                l2 = lists[i + 1] if (i + 1 < len(lists)) else None
                tmpList = mergeTwoLists(l1, l2)
                mergedList.append(tmpList)
            lists = mergedList

        return lists[0]

def mergeTwoLists(list1: Optional[ListNode], list2: Optional[ListNode]) -> Optional[ListNode]:
        if (not list1 and not list2) or not list2:
            return list1
        if not list1:
            return list2

        res, l1R, l2R = list1, list1, list2
        if list1.val < list2.val:
            res, l1R, l2R = list1, list1, list2
        else:
            res, l1R, l2R = list2, list2, list1

        def insert(head, val):
            tmp = head.next
            head.next = val
            val.next = tmp

        while l1R or l2R:
            if not l2R or not l1R:
                return res

            if l1R.val == l2R.val or (l1R.val < l2R.val and not l1R.next) or (l1R.val < l2R.val and l1R.next and l1R.next.val > l2R.val):
                tmp = l2R
                l2R = l2R.next
                insert(l1R, tmp)
                l1R = l1R.next
            else:
                l1R = l1R.next
        return res


#Práctica número 3 de la Asignatura
#Desarrollo de aplicaciones para internet.
#Autor: Samuel Medina Gutiérrez

from flask import Flask, render_template, request, flash, redirect, url_for, session, jsonify
from bson import ObjectId
from pickleshare import *
from ejercicios.ejercicio2 import burbuja
from ejercicios.ejercicio3 import criba
from ejercicios.ejercicio4 import fibonacci
from model import dataBase
from pymongo import MongoClient

app = Flask(__name__)
app.secret_key = 'claveSuperSecreta'

bd = dataBase()

#Conectamos con el servicio "mongo en su puerto estándar"
client = MongoClient("mongo", 27017)
db = client.SampleCollections

@app.route('/')
@app.route('/index')
def index():
    if not session.get('visited'):
        session['visited'] = []
    if not session.get('logeado'):
        session['logeado'] = False
        if not session.get('usuActual'):
            session['usuActual'] = None

    return render_template('index.html', urls = session['visited'], logeado = session['logeado'])

@app.route('/login', methods =['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        name = request.form['username']
        pssw = request.form['password']
        if  name in bd.getKeys() and bd.getPssw(name) == pssw:
            flash('Te has logeado correctamente')
            session['logeado'] = True
            session['usuActual'] = name
            return redirect (url_for('index'))
        else:
            error = 'Credenciales incorrectas'
    return render_template('login.html', error = error)

@app.route('/signup', methods = ["GET" , "POST"])
def singup():
    error = None
    if request.method == 'POST':
        name = request.form['username']
        pssw = request.form['password']
        if not name or not pssw:
            error = "Has introducido una cadena vacia"
        elif name in bd.getKeys():
            error = "Lo siento, ya existe este usuario"
        else:
            bd.aniade(name,pssw)
            flash('Te has registrado correctamente, por favor intente acceder')
            return redirect (url_for('index'))

    return render_template ('signup.html', error = error)

##Cuandio un usuario se desloguee redirigimos al index

@app.route('/logout')
def logout():
    session['logeado'] = False
    return redirect (url_for('index'))

@app.route('/adddata', methods = ["GET" , "POST"])
def aniade():
    error = None

    usuarioAct = session['usuActual']
    infor = bd.getInformacion(usuarioAct)
    name = infor[0]
    surname = infor[1]
    age = infor[2]
    if request.method == 'POST':
        usuarioAct = session['usuActual']
        name = request.form['name']
        surname = request.form['surname']
        age = request.form['age']

        bd.aniadirInformacion(usuarioAct,name,surname,age)

        return redirect(url_for('index'))
    return render_template('aniade.html', error = error, nombre = name, apellido = surname, edad = age)

@app.route('/userdata')
def mostrar():
    usuarioAct = session['usuActual']
    infor = bd.getInformacion(usuarioAct)
    name = infor[0]
    surname = infor[1]
    age = infor[2]
    return render_template('userdata.html', nombre = name, apellido = surname, edad = age)

@app.route('/ejer1' , methods=['GET','POST'])
def mostrarEjer1():
    if request.method == 'POST':
        numeros = request.form['numbers']
        lista = numeros.split(',')
        ordenado =burbuja(lista)
        return render_template('ejer1.html', ordenado=True, lista=ordenado)
    return render_template('ejer1.html', ordenado=False)

@app.route('/ejer2' , methods=['GET','POST'])
def mostrarEjer2():
    if request.method == 'POST':
        numero = request.form['number']
        lista =criba(numero)
        return render_template('ejer2.html', ordenado=True, lista=lista)
    return render_template('ejer2.html', ordenado=False)

@app.route('/ejer3' , methods=['GET','POST'])
def mostrarEjer3():
    if request.method == 'POST':
        numero = request.form['number']
        numero = fibonacci(numero)
        return render_template('ejer3.html', ordenado=True, lista=numero)
    return render_template('ejer3.html', ordenado=False)

@app.after_request
def visited_url(response):
    if not session.get('visited'):
        session['visited'] = []
    session['visited'].append(request.url)
    if len(session['visited']) > 3:
        session['visited'].pop(0)
    session.modified = True

    return (response)

@app.context_processor
def aniade_url():
    if session.get('visited'):
        lurls = session['visited']
    else:
        lurls=[]
    return dict(urls = lurls)

@app.context_processor
def log():
    logeados = session['logeado']
    return dict(logeado=logeados)

############### PRACTICA 4 ###############################

@app.route('/mongo',  methods=['GET','POST'])
def mongo():
    prueba = db.samples_friends

    if request.method == 'POST':
        titulo = request.form['title']

        episodios = prueba.find({'name': { "$regex": titulo, "$options": 'i'}})

        titulo = []

        for episodio in episodios:
            sumary = episodio['summary'].replace("<p>" , " ").replace("</p>" , " ")

            if episodio['image'] != None :
                tit = [episodio['name'], episodio['airdate'], episodio['airtime'], sumary,
                    episodio['image']['medium']]
            else:
                tit = [episodio['name'], episodio['airdate'], episodio['airtime'], sumary, None]
            titulo.append(tit)

        return render_template('list.html', buscado = True, match=titulo)

    ##return str(lista_episodios)
    return render_template('list.html', buscado = False)

##Práctica 5
@app.route('/api/capitulos', methods = ['GET', 'POST'])
def api_1():
    if request.method == 'GET':
        lista = []
        capitulos = db.samples_friends.find().sort('name')
        for capitulo in capitulos:
            lista.append({
                  'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
                  'title': str(capitulo.get('name')),
                  'airTime':  str(capitulo.get('airtime')),
                  'date':  str(capitulo.get('airdate'))
                })
        return jsonify(lista)

    if request.method == 'POST':
        name = request.form['title']
        result = db.samples_friends.insert_one({'name': name})
        capitulo = db.samples_friends.find_one({'_id':ObjectId(result.inserted_id)})
        return jsonify({
            'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
            'title': capitulo.get('name'),
            'airTime':  capitulo.get('airtime'),
            'date':  capitulo.get('airdate')
        })

@app.route('/api/capitulos/<id>', methods = ['GET', 'PUT', 'DELETE'])
def api_2(id):
    if request.method == 'GET':
        capitulo = db.samples_friends.find_one({'_id':ObjectId(id)})
        return jsonify({
            'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
            'title': capitulo.get('name'),
            'airTime':  capitulo.get('airrime'),
            'date':  capitulo.get('airdate')
        })
    if request.method == 'DELETE':
        db.samples_friend.remove({'_id':ObjectId(id)})
        return jsonify('deleted')

    if requested.method == 'PUT':
        name = request.form['name']


import requests
from typing import Dict
from bs4 import BeautifulSoup
from time import sleep
import json

baseurl = "https://wiki.lesswrong.com/wiki/RAZ_Glossary"

def download_html() -> str:
    """
    Download the html of the relative url
    """
    response = requests.get(baseurl)
    return response.text

def extract_keyword_def_map(html: str) -> Dict[str, str]:
    """
    Create a keyword -> definition dictionary from the html
    """

    soup = BeautifulSoup(html, 'html.parser')
    ret = {}
    for ul in soup.find_all('ul'):
        for li in ul.find_all('li'):
            title = li.find('b')
            if not title:
                continue
            ret[title.string] = li.decode_contents()
    return ret

def save_as_json(dic: Dict[str, str]) -> None:
    with open('keyword_def_map.json', 'w') as fp:
        json.dump(dic, fp)

if __name__ == "__main__":
    html = download_html()
    dic = extract_keyword_def_map(html)
    save_as_json(dic)


#################################################################
# Program:                                                      #
#   This program can flash MTK6580 Serial Number,               #
#   BT and wifi address.                                        #
#   And also can flash image.                                   #
# History:                                                      #
# 2017/11/03 : sample serial writer done                        #
# 2017/11/20 : add flash image function                         #
# 2017/12/25 : fixd flash system.img can't boot, add read cmd.  #
#                                                               #
#                           Neo Lin(starlin2@gmail.com)         #
#################################################################
import sys
import getopt
import os.path
from serial import Serial
from serial_writer import *
from image_writer import *
from common import *
already_in_meta = 0
show_debug_message = 0

bluetooth_addr  = ""
wifi_addr       = ""
serial_number   = ""
read_only       = ""
flash_file      = ""
boot_meta_only  = 0
shutdown_meta   = 0
dump_area       = 0
sn_write        = 0
preloader_reboot= 0
"""
with open("ota_scatter.txt", "r") as f:
    content = f.readlines()
content = [x.strip() for x in content]
index = 0
con = 0
file_data = [[]]
for c in range(len(content)):
    if content[c].find("partition_index")!=-1:
        file_data[index].append((content[c][(content[c].find(":")+2):]))
    if content[c].find("file_name")!=-1:
        file_data[index].append((content[c][(content[c].find(":")+2):]))
    if content[c].find("is_download")!=-1:
        file_data[index].append((content[c][(content[c].find(":")+2):]))
    if content[c].find("physical_start_addr")!=-1:
        file_data[index].append((content[c][(content[c].find(":")+2):]))
    if content[c].find("partition_size")!=-1:
        file_data[index].append((content[c][(content[c].find(":")+2):]))
        index += 1
        file_data.append([])
if os.path.exists("logo.bin"):
    print ("aaa")
for c in range(len(file_data)-1):
    for d in range(len(file_data[1])):
#        print (file_data[c][d],end ="")
        sys.stdout.write(file_data[c][d])
        sys.stdout.write("\t")
        if(d==1):
            sys.stdout.write("\t\t")
    sys.stdout.write("\n")
#    print (file_data)
for c in range(len(file_data)-1):
    if file_data[c][1]!="NONE":
        if os.path.exists(file_data[c][1]):
            print (file_data[c][1],"is exist.")
        for d in range(len(file_data[1])):
#        print (file_data[c][d],end ="")
            sys.stdout.write(file_data[c][d])
            sys.stdout.write("\t")
        sys.stdout.write("\n")
#    print (file_data)
sys.exit()
"""
try:
    opts, args = getopt.getopt(sys.argv[1:], "Rhadrmc:f:b:w:s:")
except getopt.GetoptError:
    print ("Error option:%s" % sys.argv[1:])
    DisplayHelp()
for opt, arg in opts:
    if opt == "-h" or opt =="":
        DisplayHelp()
    elif opt in ("-R"):
        preloader_reboot = 1
    elif opt in ("-r"):
        read_only = 1
    elif opt in ("-a"):
        already_in_meta = 1
        sn_write = 1
    elif opt in ("-d"):
        shutdown_meta = 1
        sn_write = 1
    elif opt in ("-m"):
        boot_meta_only = 1
        sn_write = 1
    elif opt in ("-f"):
        flash_file = arg
    elif opt in ("-b"):
        bluetooth_addr = arg
        sn_write = 1
    elif opt in ("-w"):
        wifi_addr = arg
        sn_write = 1
    elif opt in ("-c"):
        dump_area = 1#arg
    elif opt in ("-s"):
        serial_number = bytes(arg,'utf-8')
        sn_write = 1
if not opts:
    DisplayHelp()
if preloader_reboot:
    s = Connect_Device(MTK_PreLoader_VIDPID,115200)
    reboot(s)
    sys.exit()
if sn_write:
    if not already_in_meta:
        print ("Connect to preloader.")
        s = Connect_Device(MTK_PreLoader_VIDPID,115200)
        Connect_Preloader(s)
        print ("Reboot to meta mode.")
        Reboot_to_MetaMode(s)
        time.sleep(2)
    print ("Connect to meta.")
    s = Connect_Device(MTK_Meta_Mode_VIDPID,921600)
    if shutdown_meta:
        shutdown_meta_mode(s)
    if boot_meta_only:
        sys.exit()
    s.timeout = 0.5
    count = 0
    print ("Watting meta is ready")
    while 1:
        rbyte = s.inWaiting()
        if rbyte:
            print ("read%d" %rbyte)
            s.read(rbyte)#when run at reapberry need read 77 bytes first....><
        read_data,count = SendPrimitive(s, count, FT_IS_ALIVE_REQ, 0, 0, 0, 17)
        if read_data:
            if (len(read_data) == 17) and (read_data[10] == FT_IS_ALIVE_CNF):
                take,local_param,ch=struct.unpack("8s8ss",read_data)
#            show_raw_data(local_param)
                sys.stdout.write("\n")
                break;
        time.sleep(1)
        sys.stdout.write("*"); sys.stdout.flush()
#show_raw_data(read_data)
    print ("meta is ready")

    count = get_version(s,count)
    #count = set_modem(s, count)

#blue tooth write
if bluetooth_addr:
    count,peer_msg = read_nvram(s, count, NVRAM_BLUETOOTH_OFFSET)
    show_raw_data(peer_msg)
    peer_msg_a = bytearray(peer_msg)
#    show_raw_data(peer_msg)
    peer_msg_a[0] = 0x40#64
    peer_msg_a[2] = 1
    bluetooth_addr = bytes.fromhex(bluetooth_addr)
    peer_msg_a[8:14] = bluetooth_addr
#    show_raw_data(peer_msg_a)
    read_data,count = SendPrimitive(s, count, FT_AP_Editor_write_req, NVRAM_BLUETOOTH_OFFSET, 1, peer_msg_a, 19)
    print ("over write bluetooth address done.")

if wifi_addr:
    count,peer_msg = read_nvram(s, count, NVRAM_WIFI_OFFSET)
    peer_msg_a = bytearray(peer_msg)
    peer_msg_a[1] = 0x02#512
    peer_msg_a[2] = 1
    wifi_addr = bytes.fromhex(wifi_addr)
    peer_msg_a[12:18] = wifi_addr
#    show_raw_data(peer_msg_a)
    read_data,count = SendPrimitive(s, count, FT_AP_Editor_write_req, NVRAM_WIFI_OFFSET, 1, peer_msg_a, 19)
#    print ("local_param:")
#    show_raw_data(read_data)
    print ("over write wifi address done.")

if serial_number or bluetooth_addr or wifi_addr:
    count,peer_msg = read_nvram(s, count, NVRAM_ALL_ADDR_OFFSET)
    peer_msg_a = bytearray(peer_msg)
    peer_msg_a[1] = 0x04#1024
    peer_msg_a[2] = 1
    if serial_number:
        peer_msg_a[8:8+len(serial_number)] = serial_number
#        print ("set serial number")
    if bluetooth_addr:
        peer_msg_a[0x70:0x76] = bluetooth_addr
#        print ("set bt addr")
    if wifi_addr:
        peer_msg_a[0x76:0x7c] = wifi_addr
#        print ("set wifi addr")
    show_raw_data(peer_msg_a)
    read_data,count = SendPrimitive(s, count, FT_AP_Editor_write_req, NVRAM_ALL_ADDR_OFFSET, 1, peer_msg_a, 19)
    print ("write all addr done")

if read_only:
    count = get_version(s,count)
    count = set_modem(s, count)
    print ("Current address data is")
    count,peer_msg = read_nvram(s, count, NVRAM_BLUETOOTH_OFFSET)
    count,peer_msg = read_nvram(s, count, NVRAM_WIFI_OFFSET)
    count,peer_msg = read_nvram(s, count, NVRAM_ALL_ADDR_OFFSET)#barcode serial number

if sn_write:
    print ("over write nvram done.")
    shutdown_meta_mode(s)
if dump_area:
    s = connect()
    boot_da2(s)
    print("Reading flash...")
    f_read = open("nvram.bin", "wb")
    start = nvram_address
    size = 0x500000 - 1
#    size = 4096 #1024 * 1024 * 1024
    read_flash(s, start, size, f_read)
    f_read.close()
    print("Done....")

if flash_file:
    s = connect()
    boot_da2(s)
    Flash_Image(s, flash_file)


from __future__ import annotations

from typing import TYPE_CHECKING, Iterable

from public import public

import ibis.expr.operations as ops
from ibis.expr.types.generic import Column, Scalar, Value, literal
from ibis.expr.types.typing import V

if TYPE_CHECKING:
    import ibis.expr.datatypes as dt
    import ibis.expr.types as ir

import ibis.common.exceptions as com

@public
class ArrayValue(Value):
    def length(self) -> ir.IntegerValue:
        """Compute the length of an array.

        Returns
        -------
        IntegerValue
            The integer length of `self`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2, 3])
        >>> a.length()
        ArrayLength((1, 2, 3))
        """
        return ops.ArrayLength(self).to_expr()

    def __getitem__(
        self,
        index: int | ir.IntegerValue | slice,
    ) -> ir.Value:
        """Extract one or more elements of `self`.

        Parameters
        ----------
        index
            Index into `array`

        Returns
        -------
        Value
            - If `index` is an [`int`][int] or
              [`IntegerValue`][ibis.expr.types.IntegerValue] then the return
              type is the element type of `self`.
            - If `index` is a [`slice`][slice] then the return type is the same
              type as the input.

        Examples
        --------
        Extract a single element

        >>> import ibis
        >>> x = ibis.array([1, 2, 3, 4])
        >>> x[2]
        ArrayIndex((1, 2, 3, 4), index=2)

        Extract a range of elements

        >>> import ibis
        >>> x = ibis.array([1, 2, 3, 4])
        >>> x[1:3]
        ArraySlice((1, 2, 3, 4), start=1, stop=3)
        """
        if isinstance(index, slice):
            start = index.start
            stop = index.stop
            step = index.step

            if step is not None and step != 1:
                raise NotImplementedError('step can only be 1')

            op = ops.ArraySlice(self, start if start is not None else 0, stop)
        else:
            op = ops.ArrayIndex(self, index)
        return op.to_expr()

    def __add__(self, other: ArrayValue) -> ArrayValue:
        """Concatenate this array with another.

        Parameters
        ----------
        other
            Array to concat with `self`

        Returns
        -------
        ArrayValue
            `self` concatenated with `other`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2])
        >>> b = ibis.array([3, 4, 5])
        >>> a + b
        ArrayConcat(left=(1, 2), right=(3, 4, 5))
        """
        return ops.ArrayConcat(self, other).to_expr()

    def __radd__(self, other: ArrayValue) -> ArrayValue:
        """Concatenate this array with another.

        Parameters
        ----------
        other
            Array to concat with `self`

        Returns
        -------
        ArrayValue
            `self` concatenated with `other`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2])
        >>> b = ibis.array([3, 4, 5])
        >>> b + a
        ArrayConcat(left=(3, 4, 5), right=(1, 2))
        """
        return ops.ArrayConcat(self, other).to_expr()

    def __mul__(self, n: int | ir.IntegerValue) -> ArrayValue:
        """Repeat this array `n` times.

        Parameters
        ----------
        n
            Number of times to repeat `self`.

        Returns
        -------
        ArrayValue
            `self` repeated `n` times

        Examples
        --------
        >>> import ibis
        >>> x = ibis.array([1, 2])
        >>> x * 2
        ArrayRepeat((1, 2), times=2)
        """
        return ops.ArrayRepeat(self, n).to_expr()

    def __rmul__(self, n: int | ir.IntegerValue) -> ArrayValue:
        """Repeat this array `n` times.

        Parameters
        ----------
        n
            Number of times to repeat `self`.

        Returns
        -------
        ArrayValue
            `self` repeated `n` times

        Examples
        --------
        >>> import ibis
        >>> x = ibis.array([1, 2])
        >>> 2 * x
        ArrayRepeat((1, 2), times=2)
        """
        return ops.ArrayRepeat(self, n).to_expr()

    def unnest(self) -> ir.Value:
        """Unnest an array.

        Returns
        -------
        ir.Value
            Unnested array
        """
        expr = ops.Unnest(self).to_expr()
        try:
            return expr.name(self.get_name())
        except com.ExpressionError:
            return expr

@public
class ArrayScalar(Scalar, ArrayValue):
    pass

@public
class ArrayColumn(Column, ArrayValue):
    pass

@public
def array(
    values: Iterable[V],
    type: str | dt.DataType | None = None,
) -> ArrayValue:
    """Create an array expression.

    If the input expressions are all column expressions, then the output will
    be an `ArrayColumn`. The input columns will be concatenated row-wise to
    produce each array in the output array column. Each array will have length
    _n_, where _n_ is the number of input columns. All input columns should be
    of the same datatype.

    If the input expressions are Python literals, then the output will be a
    single `ArrayScalar` of length _n_, where _n_ is the number of input
    values. This is equivalent to

    ```python
    values = [1, 2, 3]
    ibis.literal(values)
    ```

    Parameters
    ----------
    values
        An iterable of Ibis expressions or a list of Python literals
    type
        An instance of `ibis.expr.datatypes.DataType` or a string indicating
        the ibis type of `value`.

    Returns
    -------
    ArrayValue
        An array column (if the inputs are column expressions), or an array
        scalar (if the inputs are Python literals)

    Examples
    --------
    Create an array column from column expressions
    >>> import ibis
    >>> t = ibis.table([('a', 'int64'), ('b', 'int64')], name='t')
    >>> result = ibis.array([t.a, t.b])

    Create an array scalar from Python literals
    >>> import ibis
    >>> result = ibis.array([1.0, 2.0, 3.0])
    """
    if all([isinstance(value, Column) for value in values]):
        return ops.ArrayColumn(values).to_expr()
    elif any([isinstance(value, Column) for value in values]):
        raise com.IbisTypeError(
            'To create an array column using `array`, all input values must '
            'be column expressions.'
        )
    else:
        try:
            return literal(list(values), type=type)
        except com.IbisTypeError as e:
            raise com.IbisTypeError(
                'Could not create an array scalar from the values provided '
                'to `array`. Ensure that all input values have the same '
                'Python type, or can be casted to a single Python type.'
            ) from e


__author__ = 'Excelle'

IGNORED_DBS = ['information_schema', 'performance_schema', 'mysql']

def isIgnorable(name):
    global IGNORED_DBS
    result = False
    for val in IGNORED_DBS:
        if val == name.lower():
            result = True
    return result


from enum import Enum
import TimeUtils

class PackageEventType(Enum):
    pickup = 0
    delivery = 1
class PackageEvent():

    def __init__(self, package_id, event_type, time, address, driver_id):
        self.package_id = package_id
        self.event_type = event_type
        self.time = time
        self.address = address
        self.driver_id = driver_id

    def __str__(self):
        """
            Returns a string representation of the Location object.
            :return: A string representing the Package object.
        """
        time_str = TimeUtils.get_time_string(self.time)
        event_type_str = "Unknown"
        if self.event_type == PackageEventType.pickup:
            event_type_str = "pickup"
        elif self.event_type == PackageEventType.delivery:
            event_type_str = "delivery"

        #return f'package_id: {self.package_id} event_type: {event_type_str} time: {time_str} driverID: {self.driver_id}'
        return f'{event_type_str} for package {self.package_id} at {time_str} by driver: {self.driver_id}'

    def __repr__(self):
        """
            Returns a string representation of the Location object.
            :return: A string representing the Package object.
        """
        return self.__str__()


from utils.archive.message import MessageArchive
from utils.archive.thread import ThreadArchive
import discord

class ChallengeArchive():
    @classmethod
    async def init(cls, challenge: discord.TextChannel):
        self = ChallengeArchive()

        self.name = challenge.name
        self.link = f"./{self.name}.html"

        self.__challenge: discord.TextChannel = challenge
        self.__messages: list[MessageArchive] = [
            await MessageArchive.init(message)
                async for message in self.__challenge.history(limit=None, oldest_first=True)
        ]
        self.__threads: dict[int,ThreadArchive] = {
            thread.id: await ThreadArchive.init(thread)
                for thread in self.__challenge.threads
        }
        self.__threads.update({
            thread.id: await ThreadArchive.init(thread)
                async for thread in self.__challenge.archived_threads(limit=None)
        })

        return self

    def fetch_data(self, attachment_path):
        challenge_messages = []

        for message in self.__messages:
            challenge_messages.append(message)
            message.download_attachments(attachment_path)

            if message.id in self.__threads:
                challenge_messages.append(self.__threads[message.id].fetch_data(attachment_path))

        return challenge_messages


# <aside>
# ðŸ’¡ **Q3.** Given a sorted array of distinct integers and a target value, return the index if the target is found. If not, return the index where it would be if it were inserted in order.

# You must write an algorithm with O(log n) runtime complexity.

# **Example 1:**
# Input: nums = [1,3,5,6], target = 5

# Output: 2

# </aside>

def searchInsert(nums, target):
    left = 0
    right = len(nums) - 1

    while left <= right:
        mid = (left + right) // 2
        if nums[mid] == target:
            return mid
        elif nums[mid] < target:
            left = mid + 1
        else:
            right = mid - 1

    return left

# Get user input for nums array
nums = input("Enter the numbers separated by spaces: ").split()
nums = [int(num) for num in nums]

# Get user input for target value
target = int(input("Enter the target value: "))

index = searchInsert(nums, target)
print(index)


from imgurpython import ImgurClient
from datetime import datetime
from os import listdir
from os.path import isfile, join
import numpy as np
from time import sleep
from pathlib import Path
import pickle
from pprint import pprint
import credentials

creds_file = Path("./credentials.dict")
if creds_file.is_file():
#pricing import
    pickle_in = open(creds_file,"rb")
    credentials_dict = pickle.load(pickle_in)
    pprint(credentials_dict)
else:
    f=open("credentials.dict","w+") #create file
    f.close()
    credentials_dict = {} #create limits dict and variables
    credentials_dict['client_id'] = credentials.client_id
    credentials_dict['client_secret'] = credentials.client_secret

    pickle_out = open("credentials.dict","wb") #open file
    pickle.dump(credentials_dict, pickle_out) #save limits dict to file
    pickle_out.close()

def get_new_tokens(credentials_dict):
    client_id = credentials_dict['client_id']
    client_secret = credentials_dict['client_secret']
    client = ImgurClient(client_id, client_secret )
    print (client.get_auth_url('pin')) #go to page and copy down pin
    creds = client.authorize(input('Pin: '), 'pin')
    print(creds)
    credentials_dict['access_token'] = creds['access_token']
    credentials_dict['refresh_token'] = creds['refresh_token']

    client.set_user_auth(credentials_dict['access_token'], credentials_dict['refresh_token'])

    print("Access Token")
    print(creds['access_token'])
    print("Refresh Token")
    print(creds['refresh_token'])
    print()

    pickle_out = open("credentials.dict","wb") #open file
    pickle.dump(credentials_dict, pickle_out) #save limits dict to file
    pickle_out.close()

    return credentials_dict

def set_old_tokens(credentials_dict):

    client = ImgurClient(credentials_dict['client_id'],credentials_dict['client_secret'], credentials_dict['refresh_token'])
    client.set_user_auth(credentials_dict['access_token'], credentials_dict['refresh_token'])
    return client

def get_files(mypath):
    onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f)) and f != '.DS_Store']
    image_list = []

    for x in onlyfiles:
        image_list.append(mypath+x)
    pprint(image_list)
    return image_list

def upload_images(image_list):
    image_ids = []
    for picture_path in image_list:
        uploaded_image = client.upload_from_path(picture_path, config=None, anon=False)
        sleep(20)
        print(uploaded_image['link'])
        image_ids.append(uploaded_image['id'])
    return image_ids

def create_album(name):
    #take name, check to see if album name exists
    #if yes, return id
    #if no, create album and return id

    config = {
    'name': name,
    'title': name,
    'description': name
    }

    new_album = client.create_album(config)
    return new_album['id']

def add_to_album(album_id,image_id_list):
    result = client.album_add_images(album_id,image_ids)

def get_user_albums():
    #get_account_albums(self, username, page=0)
    albums = client.get_account_albums(USERNAME,page=0)
    for x in albums:
        print(x.title)
        print(x.id)

mypath = '/Users/Havens/Documents/GitHub/imgur_api/Test_Photos/'
#credentials_dict = get_new_tokens(credentials_dict)
client = set_old_tokens(credentials_dict)
pprint(client.credits)

file_list = get_files(mypath) #get files to upload
image_ids = upload_images(file_list) #upload images and return list of ids
album_id = create_album("Test1") #create album and get album id
add_to_album(album_id,image_ids) #add images to album
print("www.imgur.com/a/"+album_id) #pring album link

#get_user_albums()


# get the input file
f = open("input.txt")
entries = f.read().splitlines()
f.close()

rules = dict()
bags = set()
for x in entries:
    x = x.split("s contain ")
    rules.update({x[0]: x[1]})
    if "shiny gold bag" in x[1]:
        bags.add(x[0])

while True:
    length = len(bags)
    for bag in bags.copy():
        for key, value in rules.items():
            if bag in value:
                bags.add(key)
                txt = "{} - A {}, which contains {}"
                print(txt.format(len(bags), key, value))
    if len(bags) == length:
        break


default_app_config = 'djangae.contrib.googleauth.apps.GoogleauthConfig'

_CLIENT_ID_SETTING = "GOOGLEAUTH_CLIENT_ID"
_CLIENT_SECRET_SETTING = "GOOGLEAUTH_CLIENT_SECRET"
_DEFAULT_SCOPES_SETTING = "GOOGLEAUTH_OAUTH_SCOPES"
_DEFAULT_SCOPES_SETTING = "GOOGLEAUTH_OAUTH_SCOPES"
_OAUTH_REDIRECT_HOST = "GOOGLEAUTH_OAUTH_REDIRECT_HOST"
_IAP_AUDIENCE = "GOOGLEAUTH_IAP_JWT_AUDIENCE"
_JWT_SIGNING_ENABLED_SETTING = "GOOGLEAUTH_IAP_JWT_ENABLED"
_SKEW_SECONDS_SETTING = "GOOGLEAUTH_CLOCK_SKEWS_SECONDS"
_GOOG_AUTHENTICATED_USER_ID_HEADER = "HTTP_X_GOOG_AUTHENTICATED_USER_ID"
_GOOG_AUTHENTICATED_USER_EMAIL_HEADER = "HTTP_X_GOOG_AUTHENTICATED_USER_EMAIL"
_GOOG_JWT_ASSERTION_HEADER = "HTTP_X_GOOG_IAP_JWT_ASSERTION"

_SCOPE_SESSION_KEY = "_googleauth_scopes_requested"

def _stash_scopes(request, scopes, offline):
    """
        Stores requested scopes in the session
    """

    request.session[_SCOPE_SESSION_KEY] = (list(scopes), offline)

def _pop_scopes(request):
    return request.session.pop(_SCOPE_SESSION_KEY, ([], False))