__all__ = [
    "DoccanoTask",
    "DoccanoClientConfig",
    "DoccanoInputConverter",
    "DoccanoOutputConverter",
]

import dataclasses
import enum
import json
import logging
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing_extensions import Self
from zipfile import ZipFile

from medkit.core import Attribute, OperationDescription, ProvTracer
from medkit.core.id import generate_id, generate_deterministic_id
from medkit.core.text import Entity, Relation, Span, TextDocument, span_utils

from medkit.io._common import get_anns_by_type

logger = logging.getLogger(__name__)

class DoccanoTask(enum.Enum):
    """Supported doccano tasks. The task defines
    the type of document to convert.

    Attributes
    ----------
    TEXT_CLASSIFICATION
        Documents with a category
    RELATION_EXTRACTION
        Documents with entities and relations (including IDs)
    SEQUENCE_LABELING
        Documents with entities in tuples
    """

    TEXT_CLASSIFICATION = "text_classification"
    RELATION_EXTRACTION = "relation_extraction"
    SEQUENCE_LABELING = "sequence_labeling"

@dataclasses.dataclass
class DoccanoClientConfig:
    """A class representing the configuration in the doccano client.
    The default values are the default values used by doccano.

    Attributes
    ----------
    column_text:
        Name or key representing the text
    column_label:
        Name or key representing the label
    """

    column_text: str = "text"
    column_label: str = "label"

# FIXME: datamodels to factorize in _doccano_utils
@dataclasses.dataclass()
class _DoccanoEntity:
    id: int
    start_offset: int
    end_offset: int
    label: str

    def to_dict(self) -> Dict[str, Any]:
        entity_dict = dict(
            id=self.id,
            start_offset=self.start_offset,
            end_offset=self.end_offset,
            label=self.label,
        )
        return entity_dict

@dataclasses.dataclass()
class _DoccanoEntityTuple:
    start_offset: int
    end_offset: int
    label: str

    def to_tuple(self) -> Tuple[int, int, str]:
        return (self.start_offset, self.end_offset, self.label)

@dataclasses.dataclass()
class _DoccanoRelation:
    id: int
    from_id: int
    to_id: int
    type: str

    def to_dict(self) -> Dict[str, Any]:
        relation_dict = dict(
            id=self.id,
            from_id=self.from_id,
            to_id=self.to_id,
            type=self.type,
        )
        return relation_dict

@dataclasses.dataclass()
class _DoccanoDocRelationExtraction:
    text: str
    entities: List[_DoccanoEntity]
    relations: List[_DoccanoRelation]
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text: str = doc_line.pop(client_config.column_text)
        entities = [_DoccanoEntity(**ann) for ann in doc_line.pop("entities")]
        relations = [_DoccanoRelation(**ann) for ann in doc_line.pop("relations")]
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, entities=entities, relations=relations, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text)
        doc_dict["entities"] = [ent.to_dict() for ent in self.entities]
        doc_dict["relations"] = [rel.to_dict() for rel in self.relations]
        doc_dict.update(self.metadata)
        return doc_dict

@dataclasses.dataclass()
class _DoccanoDocSeqLabeling:
    text: str
    entities: List[_DoccanoEntityTuple]
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text = doc_line.pop(client_config.column_text)
        entities = [
            _DoccanoEntityTuple(*ann)
            for ann in doc_line.pop(client_config.column_label)
        ]
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, entities=entities, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text)
        doc_dict["label"] = [ent.to_tuple() for ent in self.entities]
        doc_dict.update(self.metadata)
        return doc_dict

@dataclasses.dataclass()
class _DoccanoDocTextClassification:
    text: str
    label: str
    metadata: Dict[str, Any]

    @classmethod
    def from_dict(
        cls, doc_line: Dict[str, Any], client_config: DoccanoClientConfig
    ) -> Self:
        text = doc_line.pop(client_config.column_text)
        label = doc_line.pop(client_config.column_label)[0]

        if not isinstance(label, str):
            raise TypeError(
                "The label must be a string. Please check if the document corresponds"
                " to a text classification task rather than sequence labeling"
            )
        # in doccano, metadata is what remains after removing key fields
        metadata = doc_line
        return cls(text=text, label=label, metadata=metadata)

    def to_dict(self) -> Dict[str, Any]:
        doc_dict = dict(text=self.text, label=[str(self.label)])
        doc_dict.update(self.metadata)
        return doc_dict

class DoccanoInputConverter:
    """Convert doccano files (.JSONL) containing annotations for a given task.

    For each line, a :class:`~.core.text.TextDocument` will be created.
    The doccano files can be loaded from a directory with zip files or from a jsonl file.

    The converter supports custom configuration to define the parameters used by doccano
    when importing the data (c.f. :class:`~.io.doccano.DoccanoClientConfig`)

    .. warning::
        If the option *Count grapheme clusters as one character*  was selected
        when creating the doccano project, the converted documents are
        likely to have alignment problems; the converter does not support this option.
    """

    def __init__(
        self,
        task: DoccanoTask,
        client_config: Optional[DoccanoClientConfig] = None,
        attr_label: str = "doccano_category",
        uid: Optional[str] = None,
    ):
        """
        Parameters
        ----------
        task:
            The doccano task for the input converter
        client_config:
            Optional client configuration to define default values in doccano interface.
            This config can change, for example, the name of the text field or labels.
        attr_label:
            The label to use for the medkit attribute that represents the doccano category.
            This is related to :class:`~.io.DoccanoTask.TEXT_CLASSIFICATION` projects.
        uid:
            Identifier of the converter.
        """
        if uid is None:
            uid = generate_id()

        if client_config is None:
            client_config = DoccanoClientConfig()

        self.uid = uid
        self.client_config = client_config
        self.task = task
        self.attr_label = attr_label
        self._prov_tracer: Optional[ProvTracer] = None

    def set_prov_tracer(self, prov_tracer: ProvTracer):
        """Enable provenance tracing.

        Parameters
        ----------
        prov_tracer:
            The provenance tracer used to trace the provenance.
        """
        self._prov_tracer = prov_tracer

    @property
    def description(self) -> OperationDescription:
        """Contains all the input converter init parameters."""
        return OperationDescription(
            uid=self.uid,
            name=self.__class__.__name__,
            class_name=self.__class__.__name__,
            config=dict(task=self.task.value),
        )

    def load_from_directory_zip(self, dir_path: Union[str, Path]) -> List[TextDocument]:
        """Create a list of TextDocuments from zip files in a directory.
        The zip files should contain a JSONL file coming from doccano.

        Parameters
        ----------
        dir_path:
            The path to the directory containing zip files.

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """
        documents = []
        for path_zip in sorted(Path(dir_path).glob("*.zip")):
            documents.extend(self.load_from_zip(path_zip))

        if len(documents) == 0:
            logger.warning(f"No .zip nor .jsonl found in '{dir_path}'")

        return documents

    def load_from_zip(self, input_file: Union[str, Path]) -> List[TextDocument]:
        """
        Create a list of TextDocuments from a zip file containing a JSONL file
        coming from doccano.

        Parameters
        ----------
        input_file:
            The path to the zip file containing a docanno JSONL file

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """

        with tempfile.TemporaryDirectory() as tmpdir:
            with ZipFile(input_file, mode="r") as zip_file:
                filename = zip_file.namelist()[0]
                unzipped_file = Path(tmpdir) / filename
                zip_file.extract(filename, tmpdir)
            return self.load_from_file(unzipped_file)

    def load_from_file(self, input_file: Union[str, Path]) -> List[TextDocument]:
        """Create a list of TextDocuments from a doccano JSONL file.

        Parameters
        ----------
        input_file:
            The path to the JSONL file containing doccano annotations

        Returns
        -------
        List[TextDocument]
            A list of TextDocuments
        """
        documents = []
        with open(Path(input_file), encoding="utf-8") as fp:
            for line in fp:
                doc_line = json.loads(line)
                doc = self._parse_doc_line(doc_line)
                documents.append(doc)

        self._check_crlf_character(documents)
        return documents

    def _check_crlf_character(self, documents: List[TextDocument]):
        """Check if the list of converted documents contains the CRLF character.
        This character is the only indicator available to warn
        if there are alignment problems in the documents"""
        if (
            self.task == DoccanoTask.RELATION_EXTRACTION
            or self.task == DoccanoTask.SEQUENCE_LABELING
        ):
            nb_docs_with_warning = sum(
                document.text.find("\r\n") != -1 for document in documents
            )

            if nb_docs_with_warning > 0:
                logger.warning(
                    f"{nb_docs_with_warning}/{len(documents)} documents contain"
                    " '\\r\\n' characters. If you have selected 'Count grapheme"
                    " clusters as one character' when creating the doccano project,"
                    " converted documents are likely to have alignment problems.\n"
                    " Please ignore this message if you did not select this option when"
                    " creating the project."
                )

    def _parse_doc_line(self, doc_line: Dict[str, Any]) -> TextDocument:
        """Parse a doc_line into a TextDocument depending on the task

        Parameters
        ----------
        doc_line:
            A dictionary representing an annotation from doccano

        Returns
        -------
        TextDocument
            A document with parsed annotations.
        """
        if self.task == DoccanoTask.RELATION_EXTRACTION:
            return self._parse_doc_line_relation_extraction(doc_line=doc_line)
        if self.task == DoccanoTask.TEXT_CLASSIFICATION:
            return self._parse_doc_line_text_classification(doc_line=doc_line)
        if self.task == DoccanoTask.SEQUENCE_LABELING:
            return self._parse_doc_line_seq_labeling(doc_line=doc_line)

    def _parse_doc_line_relation_extraction(
        self, doc_line: Dict[str, Any]
    ) -> TextDocument:
        """Parse a dictionary and return a TextDocument with entities and relations

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation

        Returns
        -------
        TextDocument
            The document with annotations
        """
        try:
            doccano_doc = _DoccanoDocRelationExtraction.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        ents_by_doccano_id = dict()
        relations = []
        for doccano_entity in doccano_doc.entities:
            text = doccano_doc.text[
                doccano_entity.start_offset : doccano_entity.end_offset
            ]
            entity = Entity(
                text=text,
                label=doccano_entity.label,
                spans=[Span(doccano_entity.start_offset, doccano_entity.end_offset)],
                metadata=dict(doccano_id=doccano_entity.id),
            )
            ents_by_doccano_id[doccano_entity.id] = entity

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    entity, self.description, source_data_items=[]
                )

        for doccano_relation in doccano_doc.relations:
            relation = Relation(
                label=doccano_relation.type,
                source_id=ents_by_doccano_id[doccano_relation.from_id].uid,
                target_id=ents_by_doccano_id[doccano_relation.to_id].uid,
                metadata=dict(doccano_id=doccano_relation.id),
            )
            relations.append(relation)

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    relation, self.description, source_data_items=[]
                )

        anns = list(ents_by_doccano_id.values()) + relations
        doc = TextDocument(
            text=doccano_doc.text,
            anns=anns,
            metadata=doccano_doc.metadata,
        )

        return doc

    def _parse_doc_line_seq_labeling(self, doc_line: Dict[str, Any]) -> TextDocument:
        """Parse a dictionary and return a TextDocument with entities

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        TextDocument
            The document with annotations
        """
        try:
            doccano_doc = _DoccanoDocSeqLabeling.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        entities = []
        for doccano_entity in doccano_doc.entities:
            text = doccano_doc.text[
                doccano_entity.start_offset : doccano_entity.end_offset
            ]
            entity = Entity(
                text=text,
                label=doccano_entity.label,
                spans=[Span(doccano_entity.start_offset, doccano_entity.end_offset)],
            )
            entities.append(entity)

            if self._prov_tracer is not None:
                self._prov_tracer.add_prov(
                    entity, self.description, source_data_items=[]
                )

        doc = TextDocument(
            text=doccano_doc.text,
            anns=entities,
            metadata=doccano_doc.metadata,
        )
        return doc

    def _parse_doc_line_text_classification(
        self, doc_line: Dict[str, Any]
    ) -> TextDocument:
        """Parse a dictionary and return a TextDocument with an attribute.
        The attribute will be in its raw segment.

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        TextDocument
            The document with its category
        """
        try:
            doccano_doc = _DoccanoDocTextClassification.from_dict(
                doc_line, client_config=self.client_config
            )
        except Exception as err:
            raise Exception(
                "Impossible to convert the document. Please check the task"
                " or the client configuration of the converter"
            ) from err

        attr = Attribute(label=self.attr_label, value=doccano_doc.label)

        if self._prov_tracer is not None:
            self._prov_tracer.add_prov(attr, self.description, source_data_items=[])

        doc = TextDocument(text=doccano_doc.text, metadata=doccano_doc.metadata)
        # FIXME: related to issue #39
        # the attribute is added to the 'raw_segment', as doc attributes are not supported
        doc.raw_segment.attrs.add(attr)
        return doc

class DoccanoOutputConverter:
    """Convert medkit files to doccano files (.JSONL) for a given task.

    For each :class:`~medkit.core.text.TextDocument` a jsonline will be created.
    """

    def __init__(
        self,
        task: DoccanoTask,
        anns_labels: Optional[List[str]] = None,
        attr_label: Optional[str] = None,
        ignore_segments: bool = True,
        include_metadata: Optional[bool] = True,
        uid: Optional[str] = None,
    ):
        """
        Parameters
        ----------
        task:
            The doccano task for the input converter
        anns_labels:
            Labels of medkit annotations to convert into doccano annotations.
            If `None` (default) all the entities or relations will be converted.
            Useful for :class:`~.io.DoccanoTask.SEQUENCE_LABELING` or
            :class:`~.io.DoccanoTask.RELATION_EXTRACTION` converters.
        attr_label:
            The label of the medkit attribute that represents the text category.
            Useful for :class:`~.io.DoccanoTask.TEXT_CLASSIFICATION` converters.
        ignore_segments:
            If `True` medkit segments will be ignored. Only entities will be
            converted to Doccano entities.  If `False` the medkit segments will
            be converted to Doccano entities as well.
            Useful for :class:`~.io.DoccanoTask.SEQUENCE_LABELING` or
            :class:`~.io.DoccanoTask.RELATION_EXTRACTION` converters.
        include_metadata:
            Whether include medkit metadata in the converted documents
        uid:
            Identifier of the converter.
        """
        if uid is None:
            uid = generate_id()

        self.uid = uid
        self.task = task
        self.anns_labels = anns_labels
        self.attr_label = attr_label
        self.ignore_segments = ignore_segments
        self.include_metadata = include_metadata

    @property
    def description(self) -> OperationDescription:
        return OperationDescription(
            uid=self.uid,
            name=self.__class__.__name__,
            class_name=self.__class__.__name__,
            config=dict(task=self.task.value),
        )

    def save(self, docs: List[TextDocument], output_file: Union[str, Path]):
        """Convert and save a list of TextDocuments into a doccano file (.JSONL)

        Parameters
        ----------
        docs:
            List of medkit doc objects to convert
        output_file:
            Path or string of the JSONL file where to save the converted documents
        """

        output_file = Path(output_file)

        with open(output_file, mode="w", encoding="utf-8") as fp:
            for medkit_doc in docs:
                doc_line = self._convert_doc_by_task(medkit_doc)
                fp.write(json.dumps(doc_line, ensure_ascii=False) + "\n")

    def _convert_doc_by_task(self, medkit_doc: TextDocument) -> Dict[str, Any]:
        """Convert a TextDocument into a dictionary depending on the task

        Parameters
        ----------
        medkit_doc:
            Document to convert

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation
        """
        if self.task == DoccanoTask.RELATION_EXTRACTION:
            return self._convert_doc_relation_extraction(medkit_doc=medkit_doc)
        if self.task == DoccanoTask.TEXT_CLASSIFICATION:
            return self._convert_doc_text_classification(medkit_doc=medkit_doc)
        if self.task == DoccanoTask.SEQUENCE_LABELING:
            return self._convert_doc_seq_labeling(medkit_doc=medkit_doc)

    def _convert_doc_relation_extraction(
        self, medkit_doc: TextDocument
    ) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible
        with the doccano relation extraction task

        Parameters
        ----------
        medkit_doc:
            Document to convert, it may contain entities and relations

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text, entities and relations
        """
        doccano_ents_by_medkit_uid = dict()
        doccano_relations = []

        anns_by_type = get_anns_by_type(medkit_doc, self.anns_labels)
        medkit_segments = anns_by_type["entities"]
        if not self.ignore_segments:
            medkit_segments += anns_by_type["segments"]

        for medkit_segment in medkit_segments:
            spans = span_utils.normalize_spans(medkit_segment.spans)
            ann_id = generate_deterministic_id(medkit_segment.uid)
            entity = _DoccanoEntity(
                id=ann_id.int,
                start_offset=spans[0].start,
                end_offset=spans[-1].end,
                label=medkit_segment.label,
            )
            doccano_ents_by_medkit_uid[medkit_segment.uid] = entity

        for medkit_relation in anns_by_type["relations"]:
            subj = doccano_ents_by_medkit_uid.get(medkit_relation.source_id)
            obj = doccano_ents_by_medkit_uid.get(medkit_relation.target_id)

            if subj is None or obj is None:
                logger.warning(
                    f"Ignore relation {medkit_relation.uid}. Entity source/target was"
                    " no found"
                )
                continue

            ann_id = generate_deterministic_id(medkit_relation.uid)
            relation = _DoccanoRelation(
                id=ann_id.int,
                from_id=subj.id,
                to_id=obj.id,
                type=medkit_relation.label,
            )
            doccano_relations.append(relation)

        metadata = medkit_doc.metadata if self.include_metadata else {}

        doccano_doc = _DoccanoDocRelationExtraction(
            text=medkit_doc.text,
            entities=list(doccano_ents_by_medkit_uid.values()),
            relations=doccano_relations,
            metadata=metadata,
        )

        return doccano_doc.to_dict()

    def _convert_doc_seq_labeling(self, medkit_doc: TextDocument) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible
        with the doccano sequence labeling task

        Parameters
        ----------
        medkit_doc:
            Document to convert, it may contain entities and relations

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text ans its label (a list of tuples representing entities)
        """
        anns_by_type = get_anns_by_type(medkit_doc, self.anns_labels)
        medkit_segments = anns_by_type["entities"]
        if not self.ignore_segments:
            medkit_segments += anns_by_type["segments"]
        doccano_entities = []
        for medkit_segment in medkit_segments:
            spans = span_utils.normalize_spans(medkit_segment.spans)
            entity = _DoccanoEntityTuple(
                start_offset=spans[0].start,
                end_offset=spans[-1].end,
                label=medkit_segment.label,
            )
            doccano_entities.append(entity)

        metadata = medkit_doc.metadata if self.include_metadata else {}
        doccano_doc = _DoccanoDocSeqLabeling(
            text=medkit_doc.text,
            entities=doccano_entities,
            metadata=metadata,
        )

        return doccano_doc.to_dict()

    def _convert_doc_text_classification(
        self, medkit_doc: TextDocument
    ) -> Dict[str, Any]:
        """Convert a TextDocument to a doc_line compatible with
        the doccano text classification task. The attribute to add as a label
        should be in its raw segment.

        Parameters
        ----------
        doc_line:
            Dictionary with doccano annotation.

        Returns
        -------
        Dict[str,Any]
            Dictionary with doccano annotation. It may contain
            text ans its label (a category(str))
        """
        attributes = medkit_doc.raw_segment.attrs.get(label=self.attr_label)

        if not attributes:
            raise KeyError(
                "The attribute with the corresponding text class was not found. Check"
                f" the 'attr_label' for this converter, {self.attr_label} was provided."
            )

        metadata = medkit_doc.metadata if self.include_metadata else {}
        doccano_doc = _DoccanoDocTextClassification(
            text=medkit_doc.text,
            label=attributes[0].value,
            metadata=metadata,
        )
        return doccano_doc.to_dict()


"""
Implement Online EM algorithm for binary and count data

The idea is that
Y ~ f(phi)
where phi is an imaginary tensor with the same size as Y

Say we can also index the observed entries Y_1, Y_2, Y_3, ... and likewise P_1, P_2, ...

The factorization model is more complicated than MCMC-SSVI since we also learn the lambda vector, call it L
Let the rank be R, and L, ui, vj, tk,... are also vectors of length R

P_i = sum(L * ui * vj * ... )

The likelihood function can be written as
f = exp(P_i)^yi / (1 + exp(P_i))^mi

where yi is the observed entry value, which is 0/1 for binary data and 0, 1, 2,... for count data
mi = yi + eps where eps is the overdispersion parameter

--------------------
1. Probabilistic model
a. Prior
ui, vj, tk has standard Gaussian prior
lambda has normal with 0 mean and tau-variance and tau = prod(delta_l) where delta_l ~Gamma(ac, 1)

---------------------
2. Polya Gamma augmentation
By introducing a polya-gamma variable wi, we can change the likelihood function to become Gaussian

f = exp(Ki * P_i - Wi * Pi**2/2)

where Ki = yi - mi/2 and wi ~ PG(mi, Pi)

----------------
3. Redefinition of terms + new terms
The authors introduced (instead of using C_{ik r}^(k) as in the original paper, go with the easier to understand term)

C_u[i], C_v[j], C_t[k] -> colloquially, first isolate a dimension then the C_{ik r} term is the entry
wise product of >>> Vectorview: C_{ik} =  L * [prod(ui) for all ui vector other than the specified dimension]

D_u[i], D_v[j], D_t[k] >>> Entrywise-view D_{ik r} is the sum(L * prod(all ui)) but except for the specified entry r

These are all vectors such that

phi_i = u_{ik r}^{(k)} * c_{ik r}^{(k)} + d_{ik r}^{(k)} -> This is true regardless of which dimension gets picked

--------------------
4. EM algorithm
Nice thing about the PG distribution is that its expectation has closed form
w ~ PG(b, c) => E[w] = b/2c  tanh(c/2)

For binary data, E[w] = 0.5/P tanh(P/2)
For count data, E[w] = 0.5(y+eps)/P  tanh(P/2)

a) E- step
Estimate all E[w]

b) Maximize expected log likelihood
-> Update the ui vectors as solution to the quadratic equation
-> Update the L vector also as a quadratic equation
-> For count data -> estimate the over-dispersion parameter epsilon of the negative binomial
    ???? -> newton-rhapson procedure (how)

----
May be to keep level playing field -> Not update lambda?

"""
import torch
import torch.nn as nn
import numpy as np
from torch.autograd import Variable
from torch.nn import ModuleList, ParameterList, Parameter
import math

class EM_online(nn.Module):
    def __init__(self, tensor, rank):
        super(EM_online, self).__init__()

        self.dims = tensor.dims
        self.ndim = len(self.dims)
        self.datatype = tensor.datatype
        self.tensor = tensor
        if self.datatype == "count":
            self.overdispersion_param = 0.5
            self.max_count = tensor.max_count
            self.min_count = tensor.min_count

        self.train_entries = tensor.train_entries
        self.train_vals    = tensor.train_vals
        self.test_entries  = tensor.test_entries
        self.test_vals     = tensor.test_vals

        # Modify train and test vals with 0 and 1
        for i, val in enumerate(self.train_vals):
            if val == -1:
                self.train_vals[i] = 0
        for i, val in enumerate(self.test_vals):
            if val == -1:
                self.test_vals[i] = 0

        self.N = len(self.train_vals)
        self.batch_size    = 128
        self.R = rank
        self.U = [] # Hidden vector, following the authors' notations
        for dim, nrow in enumerate(self.dims):
            self.U.append(torch.rand(nrow, rank))

        self.S = [] # Storing the diagonal of matrix S, according to the authors' notations
        # Each S^(r, k) is a diagonal matrix -> storing the diagonal only -> (nrow,) vector
        # But we have to store the similar vector (R times) for each dimension
        # Refer to the paper to assert S.shape
        # Each S^(r,k) is (nrow, nrow) matrix. Note that nrow is the number of columns of the specified dimension
        for dim, nrow in enumerate(self.dims):
            self.S.append(torch.zeros(nrow, rank))

        self.T = [] # Storing the t_vectors, following the authors' notation
        # Similarly, the t_vectors have (R, nrow) for each dimension
        for dim, nrow in enumerate(self.dims):
            self.T.append(torch.zeros(nrow, rank))

        # Latent lambda vector
        self.L = torch.ones(rank)

    def optimize(self, max_iterations=1000, step=lambda x : 0.01):
        """

        -------
        :return:
        """
        start = 0
        for iteration in range(max_iterations):
            if iteration in [1, 5, 10, 50] or iteration % 1000 == 0:
                self.report(iteration)

            end = (start + self.batch_size) % self.N
            if end < start:
                sample_idx = list(range(end)) + list(range(start, self.N))
            else:
                sample_idx = list(range(start, end))
            start = end

            batch_entries = np.take(self.train_entries, sample_idx, axis=0)
            batch_vals    = np.take(self.train_vals, sample_idx)

            # First do the expectation step
            # This will compute the required quantities to compute the sufficient statistics
            wi_expected, c_vectors, d_vectors = self.expectation_step(batch_entries, batch_vals)

            stepsize = step(iteration)
            # Then do the maximization step
            # In the maximization step, the algorithm first compute the sufficient statistics and then
            # update the hidden vectors
            self.maximization_step(wi_expected, batch_entries, batch_vals, c_vectors, d_vectors, stepsize)

    def expectation_step(self, batch_entries, batch_vals):
        """

        ----
        Computes the expectation of the wi term associated with the batch_entries

        E[wi] = 0.5/Pi tanh(Pi/2) for binary data
              = 0.5(yi + eps)/Pi  tanh(Pi/2)  for count data

        So first need to compute the Pi
        Each pi is simply sum(L * [vector])
        """
        phi_batch, c_vectors, d_vectors = self.phi_c_d_batch_compute(batch_entries) # shape = (num_samples)
        wi_expected = None
        if self.datatype == "binary":
            wi_expected = 0.5/ phi_batch * torch.tanh(phi_batch/2)
        elif self.datatype == "count":
            yi = torch.FloatTensor(batch_vals)
            wi_expected = 0.5 * (yi + self.overdispersion_param)/ phi_batch * torch.tanh(phi_batch/2)
        return wi_expected, c_vectors, d_vectors

    def maximization_step(self, wi_expected, batch_entries, batch_vals, c_vectors, d_vectors, stepsize):
        """
        :param wi_expected:
                shape = (num_samples)
        :param batch_entries:
                shape = (num_samples, ndim)
        :param batch_vals:
                shape = (num_samples)
        -----
        :return:

        Updates the hidden vectors by solving the quadratic optimization problem involving
        S and t
        -> In this case, involve the stochastic upate of S and t

        Both requires computing cik and dik
        """
        # Comute the c_ik and d_ik vectors
        # c_vectors, d_vectors = self.c_and_d_vectors_compute(batch_entries)
        ki_batch = self.ki_batch_compute(batch_vals)

        # Do stochastic update on the sufficient statistics S and t
        # self.S_update_stochastic(c_vectors, wi_expected, batch_entries, stepsize)
        # self.t_stochastic_update(c_vectors, d_vectors, wi_expected, ki_batch, batch_entries, stepsize)

        # Update both S and t sufficient statistics
        self.S_t_update_stochastic(c_vectors, d_vectors, wi_expected, ki_batch, batch_entries, stepsize)

        self.ui_vectors_update(batch_entries)

    def S_t_update_stochastic(self, c_vectors, d_vectors, wi_expected, ki_batch, batch_entries, stepsize):
        """

        :param c_vectors:
                shape = (num_samples, ndim, R)

        :param d_vectors:
                shape = (num_samples, ndim, R)

        :param wi_expected:
        :param ki_batch:
        :param batch_entries:
        :param stepsize:
        :return:
        """

        num_samples = len(batch_entries)
        s_sum_update = {}
        t_sum_update = {}

        # Compute all the s_update term
        # Note that the update term for a (dim k, row n) factor is the sum over all the
        # tensor entry values that are associated with that factor
        # That's why we need to compute all the update first and store it in a dictionary

        for num, entry in enumerate(batch_entries):
            for dim, row in enumerate(entry):
                # NOTE that s_update is a R-vector
                s_update = c_vectors[num, dim, :] ** 2 * wi_expected[num]
                # Like wise t_update.shape = (R,)
                t_update = (ki_batch[num] - d_vectors[num, dim, :] * wi_expected[num]) * c_vectors[num, dim, :]

                if (dim, row) not in s_sum_update:
                    s_sum_update[(dim, row)] = torch.zeros(self.R)
                    t_sum_update[(dim, row)] = torch.zeros(self.R)

                t_sum_update[(dim, row)] += t_update
                s_sum_update[(dim, row)] += s_update

        for dim, row in s_sum_update.keys():
            # print(s_update)
            s_update = s_sum_update[(dim, row)]
            s_cur    = self.S[dim][row, :]
            # print(s_cur)
            self.S[dim][row, :] = (1 - stepsize) * s_cur + stepsize * s_update

            t_update = t_sum_update[(dim, row)]
            t_cur    = self.T[dim][row, :]
            self.T[dim][row, :] = (1 - stepsize) * t_cur + stepsize * t_update

    def S_update_stochastic(self, c_vectors, wi_expected, batch_entries, stepsize):
        """
        :param c_vectors
                shape = (num_samples, ndim, R)
        :param wi_expected
                shape = (num_samples)
        :param batch_entries
                shape = (num_samples, ndim)
        :param stepsize
                a scalar

        -----
        :return:

        S is a diagonal matrix and thus we only work with the diagonal part
        S_nn^(r, k) = (1-stepsize) * S_nn ^(r, k) + stepsize * sum(c_ik,r^2 wi)

        # for all entries i that uses row n of dimension k

        For each entry in batch_entries
        -> for a particular row n with dimension k

        >>> S[dim k][rank r][row n] = (1-stepsize) * S[dim k][rank r][row n] \
                                + stepsize * SUM [ c_n[entry_num, k]**2 * wi_expected[entry_num] ]

        """
        num_samples = len(batch_entries)
        s_diagonal_update = {}

        # Compute all the s_update term
        # Note that the update term for a (dim k, row n) factor is the sum over all the
        # tensor entry values that are associated with that factor
        # That's why we need to compute all the update first and store it in a dictionary
        for num, entry in enumerate(batch_entries):
            for dim, row in enumerate(entry):
                s_update = c_vectors[num, dim, :] ** 2 * wi_expected[num]
                if (dim, row) not in s_diagonal_update:
                    s_diagonal_update[(dim, row)] = torch.zeros(self.R)
                s_diagonal_update[(dim, row)] += s_update

        for dim, row in s_diagonal_update.keys():
            s_update = s_diagonal_update[(dim, row)]
            s_cur    = self.S[dim][row, :]
            self.S[dim][row, :] = (1 - stepsize) * s_cur + stepsize * s_update

    def t_stochastic_update(self, c_vectors, d_vectors, wi_expected, ki_batch, batch_entries, stepsize):
        """

        :param c_vectors:
                shape = (num_samples, ndim, R)

        :param d_vectors:
                shape = (num_samples, ndim, R)

        :param wi_expected:
        :param ki_batch:
        :param batch_entries:
        :param stepsize:

        :return:
        # Assert with original paper
        >>> t[dim k][column n] += stepsize * (K[entry_num] - d_vectors[entry_num, dim, :] * wi_expected[entry_num]) \
                                            * c_n[entry_num, k]

        """
        # Refer to S_stochastic_update as above -> we also need to compute all the t_update separately
        # And then apply the updates
        num_samples = len(batch_entries)
        t_diagonal_update = {}

        for num, entry in enumerate(batch_entries):
            for dim, col in enumerate(entry):
                t_update = (ki_batch[num] - d_vectors[num, dim, :] * wi_expected[num]) * c_vectors[num, dim, :]
                if (dim, col) not in t_diagonal_update:
                    t_diagonal_update[(dim, col)] = torch.zeros(rank)

        for dim, col in t_diagonal_update.keys():
            t_update = t_diagonal_update[(dim, col)]
            t_cur    = self.T[dim][col, :]
            self.T[dim][col, :] = (1 - stepsize) * t_cur + stepsize * t_update

    def ki_batch_compute(self, batch_vals):
        """

        :param batch_vals:
        -----
        :return:

        ki = yi - mi/2 where mi = 1 for binary and mi = yi + eps for count data
        """
        yi = torch.FloatTensor(batch_vals)
        if self.datatype == "binary":
            ki_batch = yi - 0.5
        else:
            ki_batch = yi/2 - self.overdispersion_param/2
        return ki_batch

    def phi_batch_compute(self, batch_entries):
        """
        :param batch_entries:
        :return: (phi_batch, c_vectors, d_vectors)

        For each observed entry
        For each dimension k
        For each involved column
        >>> c_vector[col] = self.L * [vectors from other dims] # Size R vector
        # d_vector dfined entry-wise
        >>> d_vector[col][r] = sum(self.L * [all vectors from all dims]) - self.L[r] * (prod[all vectors from all dims])
        c-vector-colletion.shape = (ndim, num_column_batch, R)
        """
        num_samples = len(batch_entries)
        inners = self.L.repeat(num_samples, 1) # Shape = (num_samples, R)
        for dim, _ in enumerate(self.dims):
            all_cols = [x[dim] for x in batch_entries]
            # all_cols = Variable(torch.LongTensor(all_cols))
            vectors  = self.U[dim][all_cols] # -> shape = (num_samples, R)
            # Get the list of vector associated with the column in the
            # specified dimension
            inners *= vectors

        phi_batch = inners.sum(dim=1) # shape = (num_samples,)
        c_vectors, d_vectors = self.c_and_d_vectors_compute(batch_entries)

        return phi_batch, c_vectors, d_vectors

    def c_and_d_vectors_compute(self, batch_entries):
        """

        :param batch_entries
               shape = (num_samples, ndim)

        -----
        :return:

        For each observed entry
        For each dimension k
        For each involved column
        >>> c_vector[dim k][col] = self.L * prod[vectors from OTHER dims] # Size R vector
        >>> d_vector[dim k][col] = sum(self.L * prod[all vectors from ALL dims]) # A scalar
        >>>                       - self.L[r] * (prod[all vectors from all dims]) # A vector

        """
        num_samples = len(batch_entries)
        c_vector = torch.ones(num_samples, self.ndim, self.R)
        d_vector = torch.ones(num_samples, self.ndim, self.R)
        involved_vectors = torch.ones(self.ndim, num_samples, self.R)

        # Get all the involved vectors from the batch_entries
        for dim, _ in enumerate(self.dims):
            all_cols = [x[dim] for x in batch_entries]
            # all_cols = Variable(torch.LongTensor(all_cols))

            vectors  = self.U[dim][all_cols] # -> shape = (num_samples, R)
            involved_vectors[dim, :, :] = vectors
        #
        for num, entry in enumerate(batch_entries):
            for dim, col in enumerate(entry):
                # All other dimensions
                other_dims = list(range(dim)) + list(range(dim+1,self.ndim))
                other_vectors_prod = torch.prod(involved_vectors[other_dims, num, :]) # shape = (R)
                c_vector[num, dim, :] = self.L * other_vectors_prod

                all_products = self.L * torch.prod(involved_vectors[:, num, :])
                d_vector[num, dim, :] = torch.sum(all_products) - all_products

        return c_vector, d_vector

    def phi_c_d_batch_compute(self, batch_entries):
        """
        :param batch_entries:
        :return: (phi_batch, c_vectors, d_vectors)

        For each observed entry
        For each dimension k
        For each involved column col
        >>> c_vectors[dim k][col] = self.L * prod[vectors from OTHER dims] # Size R vector
        >>> d_vectors[dim k][col] = sum(self.L * prod[all vectors from ALL dims]) # A scalar
        >>>                       - self.L[r] * (prod[all vectors from all dims]) # A vector

        """
        num_samples = len(batch_entries)
        c_vectors = torch.zeros(num_samples, self.ndim, self.R)
        d_vectors = torch.zeros(num_samples, self.ndim, self.R)
        involved_vectors = torch.zeros(self.ndim, num_samples, self.R)
        inners = self.L.repeat(num_samples, 1) # Shape = (num_samples, R)

        for dim, _ in enumerate(self.dims):
            all_cols = [x[dim] for x in batch_entries]
            # all_cols = Variable(torch.LongTensor(all_cols))
            vectors  = self.U[dim][all_cols] # -> shape = (num_samples, R)
            # Extract all the vectors associated with the entries in the batch
            involved_vectors[dim, :, :] = vectors
            inners *= vectors

        phi_batch = inners.sum(dim=1) # shape = (num_samples,)

        for num, entry in enumerate(batch_entries):
            for dim, col in enumerate(entry):
                # All other dimensions
                other_dims = list(range(dim)) + list(range(dim+1,self.ndim))
                other_vectors = involved_vectors[other_dims, num, :]
                other_vectors_prod = torch.prod(other_vectors, dim=0) # shape = (R)
                c_vectors[num, dim, :] = self.L * other_vectors_prod

                all_vectors  = involved_vectors[:, num, :]
                all_products = self.L * torch.prod(all_vectors, dim=0)
                d_vectors[num, dim, :] = torch.sum(all_products) - all_products

        return phi_batch, c_vectors, d_vectors

    def ui_vectors_update(self, batch_entries):
        """

        :param batch_entries
        ----
        :return:

        Update ui hidden vectors by solving the quadratic maximization problem
        It has a closed form

        for each entry in batch entries
        For all the involved column (with specific dimension)

        Dimension k, column n
        u_n^(k) = S_n^(k) / t_n^(k)

        """
        num_samples = len(batch_entries)
        for num, entry in enumerate(batch_entries):
            for dim, row in enumerate(entry):
                diag = torch.max(self.S[dim][row, :], torch.FloatTensor([0.00001]))
                # update = self.T[dim][row, :] / self.S[dim][row, :]
                update = self.T[dim][row, :] / diag
                self.U[dim][row, :] =  update

    def report(self, iteration):
        if iteration == 0:
            print(" iteration |  test mae  |  train mae  |")

        train_mae = self.evaluate(self.train_entries, self.train_vals)
        test_mae  = self.evaluate(self.test_entries, self.test_vals)
        print ("{:^10} {:^10} {:^10}".format(iteration, np.around(test_mae,4), np.around(train_mae,4)))

    def evaluate(self, entries, vals):
        """
        ---
        :return:

        Do evaluation
        """
        test_mae = 0.
        num_entries = len(vals)
        # How to do prediction etc
        for i, entry in enumerate(entries):
            val = vals[i]
            if self.datatype == "binary":
                predict = self.binary_predict(entry)
            else:
                predict = self.count_predict(entry)
            # print("actual: ", val, " predict: ", predict)
            test_mae += abs(predict - val)/num_entries
        return test_mae

    def binary_predict(self, entry):
        """

        :param entry:
        :return:

        The likelihood function is

        f = sigmoid(phi_i)
        """
        inners = torch.ones(self.R)
        for dim, col in enumerate(entry):
            inners *= self.U[dim][col, :]
        f = torch.sum(inners)
        if f > 0:
            return 1
        return 0

    def count_predict(self, entry):
        """
        :param entry:
        ---
        :return:

        ll = exp(P)^yi / (1 + exp(P))^(yi + eps)

        Do maximum likelihood estimate? or weighted average
        """
        inners = torch.ones(self.R)
        for dim, col in enumerate(entry):
            inners *= self.U[dim][col, :]
        f = torch.sum(inners)
        return f

        range_vals = self.max_count - self.min_count + 1
        probs = torch.zeros(range_vals)
        for yi in range(range_vals):
            probs[yi] = torch.exp(f)/(1+torch.exp(f))**(yi+self.overdispersion_param)

        reg = torch.sum(probs)
        y   = torch.linspace(self.min_count, self.max_count + 1, steps=self.max_count - self.min_count +1) \
              * probs / reg
        return torch.sum(y)

    # NOTE: to make fair comparisons, not updating Lambda
    def S_lambda_stochastic_compute(self):
        pass

    def t_lambda_stochastic_compute(self):
        pass

class SVI(nn.Module):
    def __init__(self):
        pass

    def factorize(self):
        pass

class SVI_binary(SVI):
    def __init__(self):
        super(SVI_binary).__init__()
        pass

class SVI_count(nn.Module):
    def __init__(self):
        super(SVI_count).__init__()
        pass


import requests

API_URL = "http://localhost:3000/api/v1/prediction/85ca04f1-94c1-4366-8183-a270e07a61dc"

def query(payload):
    response = requests.post(API_URL, json=payload)
    return response.json()

output = query({
    "question": "Hey, how are you?",
})

print(output)


from datetime import datetime
from functools import partial

from bouncebox.util import Object
from  bouncebox.core.series import create_series

CLASS_SERIES_CACHE = {}

class Event(Object):
    repr_attrs = ['timestamp']

    # check __setattr__ immutable == True means immutable
    immutable = False

    def __init__(self, timestamp=None, source_event=None, series=None):
        self.generated = datetime.now()
        self.source_event = source_event
        if timestamp is None and source_event:
            timestamp = source_event.timestamp
        self.timestamp = timestamp

        if series is None:
            series = self.__class__.class_series()
        self.series = series
        self.immutable = True

    @classmethod
    def class_series(self):
        # quick way to get default series
        if not self in CLASS_SERIES_CACHE:
            CLASS_SERIES_CACHE[self] = create_series(self)
        return CLASS_SERIES_CACHE[self]

    def ___setattr__(self, name, value):
        # events are immutable
        if self.immutable:
            assert False, 'boo'
        super(Event, self).__setattr__(name, value)

# Would like this switchable? Eventually remove non cython version?
try:
    from bouncebox.core.event_cython import Event
except:
    print('No cython Event')

class SourceEvent(Event):
    """
        Would like to disintuish events that are generated externally
    """

class EndEvent(Event):
    graphable = False

class StartEvent(Event):
    """
        Used to signal to bouncebox that it is beginning.
    """
    graphable = False

class EventFactory(object):
    """
        Events should come from some sort of factory.
        Not sure if factory is the right word.
        Just a place to keep grouped events metadata together
    """

    def __init__(self, event_cls):
        self.event_cls = event_cls

    def build_event(self, *args, **kwargs):
        event = self.event_cls(*args, **kwargs)
        event.factory = self
        return event

class SeriesEventFactory(EventFactory):
    """
        Takes in a Series object and generates events based of those
    """
    def __init__(self, series):
        self.series = series
        self.event_cls = series.event_cls
        self.event_args = series.event_args
        self.create_event_func()
        super(SeriesEventFactory, self).__init__(series.event_cls)

    def create_event_func(self):
        kw = {}
        kw['series'] = self.series
        for arg in self.event_args:
            kw[arg] = getattr(self.series, arg)
        func = partial(self._build_event, **kw)
        self.build_event = func

    def _build_event(self, *args, **kwargs):
        event = self.event_cls(*args, **kwargs)
        return event


# -*- coding: utf-8 -*-
'''
code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
          --┃      ☃      ┃--
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗II━II┓
                ┃  神兽保佑    ┣┓
                ┃　永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
 @Belong = 'StevenTools'  @MadeBy = 'PyCharm'
 @Author = 'lenovo'   @DateTime = '2018/7/4 14:36'
'''
import keras.backend as K

def visModel(model, to_file):
    '''
    对模型进行可视化
    :param model:model对象
    :param to_file: 可视化图片存放的位置
    :return: 将model可视化后的图像放到to_file位置
    '''
    from keras.utils import plot_model
    plot_model(model, to_file=to_file, show_shapes=True)

def log4GraphCb(logp):
    '''
    为了之后生成训练与测试损失图而进行的对每次epoch的损失记录
    :param logp: 损失记录保存的位置 是txt文件
    :return:
    '''
    from keras.callbacks import CSVLogger
    return CSVLogger(logp, append=True)

def visualLoss(logp, targets=None):
    from matplotlib import pyplot as plt
    import os
    import pandas as pd
    '''可视化loss
    :param logp:log文件地址
    :return:
    '''

    if os.path.exists(logp):
        tableXEpoch = pd.read_csv(logp).set_index('epoch')
        targetVailds = tableXEpoch.keys()[:len(tableXEpoch.keys()) // 2] if targets == None else [i for i in targets if
                                                                                i in tableXEpoch.keys()]
        for _y in targetVailds:
            plt.plot(tableXEpoch[_y].values, label='training ' + _y)
            plt.plot(tableXEpoch['val_' + _y].values, label='validation ' + _y)
            plt.ylabel(_y)
            plt.xlabel('Epoch')
            plt.xlim((0, len(tableXEpoch.index)))
            plt.legend(loc='best')
            plt.savefig(logp[:-4] + '_' + _y + '.jpg')
            plt.close()

import numpy as np

def recall(y_true, y_pred):
    """Recall metric.

    Only computes a batch-wise average of recall.

    Computes the recall, a metric for multi-label classification of
    how many relevant items are selected.
    """
    true_positives = K.sum(K.round(y_true *K.clip(y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall =  (true_positives+ K.epsilon()*(1e-3)) / (possible_positives + K.epsilon())
    return recall
def precision(y_true, y_pred):
    """Precision metric.

    Only computes a batch-wise average of precision.

    Computes the precision, a metric for multi-label classification of
    how many selected items are relevant.
    """
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = (true_positives+ K.epsilon()*(1e-3)) / (predicted_positives + K.epsilon())
    return precision

tpr = recall
sensitivity = tpr

def fpr(y_true, y_pred):
    fp = K.sum(K.round(K.clip((1 - y_true) * y_pred, 0, 1)))
    true_neg = K.sum(K.round(K.clip(1 - y_true, 0, 1)))
    return fp / true_neg

def tversky_loss(y_true, y_pred):
    alpha = 0.5
    beta = 0.5

    ones = K.ones(K.shape(y_true))
    p0 = y_pred  # proba that voxels are class i
    p1 = ones - y_pred  # proba that voxels are not class
    g0 = y_true
    g1 = ones - y_true

    num = K.sum(p0 * g0, (0, 1, 2, 3))
    den = num + alpha * K.sum(p0 * g1, (0, 1, 2, 3)) + beta * K.sum(p1 * g0, (0, 1, 2, 3))

    T = K.sum(num / den)  # when summing over classes, T has dynamic range [0 Ncl]

    Ncl = K.cast(K.shape(y_true)[-1], 'float32')
    return Ncl - T

def tverskyMean(y_true, y_pred):
    alpha = 0.5
    beta = 0.5

    ones = K.ones(K.shape(y_true))
    p0 = y_pred  # proba that voxels are class i
    p1 = ones - y_pred  # proba that voxels are not class
    g0 = y_true
    g1 = ones - y_true

    num = K.sum(p0 * g0, (0, 1, 2, 3))
    den = num + alpha * K.sum(p0 * g1, (0, 1, 2, 3)) + beta * K.sum(p1 * g0, (0, 1, 2, 3))

    T = K.sum(num / den)
    result = T / K.cast(K.shape(y_true)[-1], 'float32')  # when summing over classes, T has dynamic range [0 Ncl]
    return result

def createIndpWeightTversky(t0=0.002, t1=0.749, t2=0.250):
    def independentTverskyLoss(y_true, y_pred, ):
        alpha = 0.5
        beta = 0.5

        ones = K.ones(K.shape(y_true))
        p0 = y_pred  # proba that voxels are class i
        p1 = ones - y_pred  # proba that voxels are not class
        g0 = y_true
        g1 = ones - y_true

        def classTver(classIndex):
            num = K.sum((p0 * g0)[:, :, :, classIndex])
            den = num + alpha * K.sum((p0 * g1)[:, :, :, classIndex]) + beta * K.sum((p1 * g0)[:, :, :, classIndex])
            return K.sum(num / den)

        T = classTver(0) * t0 + classTver(1) * t1 + classTver(2) * t2
        return 1 - T

    return independentTverskyLoss

def tverskyMeanWLoss(y_true, y_pred):
    return 1 - tverskyMeanW(y_true, y_pred)

def tverskyMeanW(y_true, y_pred, t0=0.99992, t1=0.00002, t2=0.00006):
    alpha = 0.5
    beta = 0.5

    ones = K.ones(K.shape(y_true))
    p0 = y_pred  # proba that voxels are class i
    p1 = ones - y_pred  # proba that voxels are not class
    g0 = y_true
    g1 = ones - y_true

    num = K.sum((p0 * g0)[:, :, :, 0] * t0 + (p0 * g0)[:, :, :, 1] * t1 + (p0 * g0)[:, :, :, 2] * t2)
    den = num + alpha * K.sum(
        (p0 * g1)[:, :, :, 0] * t0 + (p0 * g1)[:, :, :, 1] * t1 + (p0 * g1)[:, :, :, 2] * t2) + beta * K.sum(
        (p1 * g0)[:, :, :, 0] * t0 + (p1 * g0)[:, :, :, 1] * t1 + (p1 * g0)[:, :, :, 2] * t2)
    return K.sum(num / den)

def show2DDice(classIndex, smooth=1e-5):
    def classDice(y_true, y_pred):
        intersection = K.sum(K.round((y_true * y_pred)[:, :, :, classIndex]))
        den = K.sum(K.round(y_true[:, :, :, classIndex])) + K.sum(K.round(y_pred[:, :, :, classIndex]))
        return (intersection * 2 + smooth) / (den + smooth)

    return classDice
def weightedLoss(originalLossFunc, weightsList):

    def lossFunc(true, pred):

        axis = -1 #if channels last

        #argmax returns the index of the element with the greatest value
        #done in the class axis, it returns the class index
        classSelectors = K.argmax(true, axis=axis)

        #considering weights are ordered by class, for each class
        #true(1) if the class index is equal to the weight index
        classSelectors = [K.equal(np.int64(i), classSelectors) for i in range(len(weightsList))]

        #casting boolean to float for calculations
        #each tensor in the list contains 1 where ground true class is equal to its index
        #if you sum all these, you will get a tensor full of ones.
        classSelectors = [K.cast(x, K.floatx()) for x in classSelectors]

        #for each of the selections above, multiply their respective weight
        weights = [sel * w for sel,w in zip(classSelectors, weightsList)]

        #sums all the selections
        #result is a tensor with the respective weight for each element in predictions
        weightMultiplier = weights[0]
        for i in range(1, len(weights)):
            weightMultiplier = weightMultiplier + weights[i]

        #make sure your originalLossFunc only collapses the class axis
        #you need the other axes intact to multiply the weights tensor
        loss = originalLossFunc(true,pred)
        loss = loss * weightMultiplier

        return loss
    return lossFunc

def show3DDice(classIndex, smooth=1e-5):
    def classDice(y_true, y_pred):
        intersection = K.sum((y_true * y_pred)[:, :, :, :, classIndex])
        den = K.sum(y_true[:, :, :, :, classIndex]) + K.sum(y_pred[:, :, :, :, classIndex])
        return (intersection * 2 + smooth) / (den + smooth)

    return classDice

def dice_coef(y_true, y_pred, smooth=1):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f * y_true_f) + K.sum(y_pred_f * y_pred_f) + smooth)

def dice_coef_loss(y_true, y_pred, smooth=1.):
    return 1. - dice_coef(y_true, y_pred, smooth)

if __name__ == '__main__':
    visualLoss(r'E:\pyWorkspace\NewCAE\FCDenseNet\myDense\experiment\log\log.txt')


#! /usr/bin/env python3
# coding: utf-8

"""
Ubuntu:
1. pip3 install pdfkit markdown
2. apt-get install wkhtmltopdf
   more platform: https://github.com/JazzCore/python-pdfkit/wiki/Installing-wkhtmltopdf

Windows：
1. scoop install python wkhtmltopdf
2. pip3 install pdfkit markdown
"""

import os
import re
import sys
import string

import pdfkit

def get_title(prefix, current):
    if prefix == "":
        title = "{}".format(current)
    else:
        title = "{}.{}".format(prefix, current)

    return title

def generate(prefix, line_list, current_sharp, start_index):
    size = len(line_list)
    if start_index >= size:
        return size + 1

    current = 1

    head_pattern = re.compile(r"^(#+)\s*(.*)$")
    title_pattern = re.compile(r"^[\s\.\d]+(.*)$")
    code_pattern = '```'
    code_cnt = 0

    i = start_index
    while True:
        if i >= size:
            break

        line = line_list[i]

        line_trip = line.strip()
        if line_trip.startswith(code_pattern):
            code_cnt += 1

        if code_cnt % 2 == 1:
            i += 1
            continue;

        match = head_pattern.match(line)
        if match:
            sharp = match.group(1)
            current_size = len(sharp)
            if current_size > current_sharp:
                i = generate(get_title(prefix, current - 1), line_list, current_size, i)

            elif current_size == current_sharp:
                title = get_title(prefix, current)
                right = match.group(2)
                r = title_pattern.match(right)
                if r:
                    right = r.group(1)

                if right != "":
                    line_list[i] = "{} {} {}".format(sharp, title, right)
                else:
                    line_list[i] = "{} {}".format(sharp, title)

                current += 1
                i += 1

            else:
                break

        else:
            i += 1

    return i

def reformat_md(filename):
    if not os.path.exists(filename):
        print ("%s file is not exist." % (filename))
        return

    with open(filename, mode="r", encoding="utf-8") as f:
        contents = f.read()

    line_list = contents.split("\n")
    generate("", line_list, 1, 0)

    with open(filename, "w", encoding="utf-8") as f:
        size = len(line_list)
        i = 0
        while i < size - 1:
            f.write(line_list[i])
            f.write("\n")
            i += 1

        f.write(line_list[size-1])

# 设置环境变量
def to_pdf(file_name):
    import markdown

    out_file = '.'.join([file_name.rsplit('.', 1)[0], 'pdf'])
    out_html = '.'.join([file_name.rsplit('.', 1)[0], 'html'])

    input_file = open(file_name, mode="r", encoding="utf-8")
    text = input_file.read()

    exts = ['markdown.extensions.extra', 'markdown.extensions.codehilite','markdown.extensions.tables','markdown.extensions.toc', 'markdown.extensions.nl2br']

    html = markdown.markdown(text, extensions=exts)

    output_file = open(out_html, "w",encoding="utf-8",errors="xmlcharrefreplace")

    # 可以根据选择修改css样式， 可以参考github.css和default.css
    css = '''
    <html">
    <head>
    <meta content="text/html; charset=utf-8" http-equiv="content-type" />
        <style>
            font-family: Helvetica, arial, sans-serif;
            font-size: 14px;
            line-height: 1.6;
            padding-top: 10px;
            padding-bottom: 10px;
            background-color: white;
            padding: 30px; }

            h1, h2, h3, h4, h5, h6 {
              margin: 20px 0 10px;
              padding: 0;
              font-weight: bold;
              -webkit-font-smoothing: antialiased;
              cursor: text;
              position: relative; }

            h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, h5:hover a.anchor, h6:hover a.anchor {
              background: url("../../images/modules/styleguide/para.png") no-repeat 10px center;
              text-decoration: none; }

            h1 tt, h1 code {
              font-size: inherit; }

            h2 tt, h2 code {
              font-size: inherit; }

            h3 tt, h3 code {
              font-size: inherit; }

            h4 tt, h4 code {
              font-size: inherit; }

            h5 tt, h5 code {
              font-size: inherit; }

            h6 tt, h6 code {
              font-size: inherit; }

            h1 {
              font-size: 28px;
              color: black; }

            h2 {
              font-size: 24px;
              border-bottom: 1px solid #cccccc;
              color: black; }

            h3 {
              font-size: 18px; }

            h4 {
              font-size: 16px; }

            h5 {
              font-size: 14px; }

            h6 {
              color: #777777;
              font-size: 14px; }

            table {
              padding: 0; }
              table tr {
                border-top: 1px solid #cccccc;
                background-color: white;
                margin: 0;
                padding: 0; }
                table tr:nth-child(2n) {
                  background-color: #f8f8f8; }
                table tr th {
                  font-weight: bold;
                  border: 1px solid #cccccc;
                  text-align: left;
                  margin: 0;
                  padding: 6px 13px; }
                table tr td {
                  border: 1px solid #cccccc;
                  text-align: left;
                  margin: 0;
                  padding: 6px 13px; }
                table tr th :first-child, table tr td :first-child {
                  margin-top: 0; }
                table tr th :last-child, table tr td :last-child {
                  margin-bottom: 0;
        </style>
    </head>
    <body>
    %s
    </body>
    </html>
    '''

    output_file.write(css % html)
    output_file.close()

    pdfkit.from_file(out_html, out_file)

    os.remove(out_html)

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        print ("Help >\n" +
			"> generate sequece number: markdown-tools.py name.md>\n" +
			"> genereate pdf: markdown-tools.py pdf name.md>")
        os._exit(0)

    if sys.argv[1] == 'pdf':
        to_pdf(sys.argv[2])
        os._exit(0)

    reformat_md(sys.argv[1])


from django.contrib import admin
from django.contrib import admin, messages
from django.utils.translation import gettext, gettext_lazy as _, ngettext
from jalali_date.admin import ModelAdminJalaliMixin

from .models import Products, Comments

class CommentsInline(admin.TabularInline):
    model = Comments
    fields = ['user', 'text', 'stars', 'is_active', 'recommend']
    extra = 1

@admin.register(Products)
class ProductsAdmin(ModelAdminJalaliMixin, admin.ModelAdmin):
    list_display = ['title', 'cover_img', 'slug', 'price', 'status', 'active', 'datetime_created']
    search_field = ('title',)
    prepopulated_fields = {'slug': ('title',)}
    actions = ['make_not_available', 'make_available', 'make_active', 'make_de_active']
    inlines = [
        CommentsInline,
    ]

    @admin.action(description=_('Selected products are not available'))
    def make_not_available(self, request, queryset):
        updated = queryset.update(status='not')
        self.message_user(request, ngettext(
            _('%d story was successfully marked as not available.'),
            _('%d stories were successfully marked as not available.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are active'))
    def make_active(self, request, queryset):
        updated = queryset.update(active=True)
        self.message_user(request, ngettext(
            _('%d story was successfully marked as activated.'),
            _('%d stories were successfully marked as activated.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are de_active'))
    def make_de_active(self, request, queryset):
        updated = queryset.update(active=False)
        self.message_user(request, ngettext(
            _('%d story was successfully marked as deactivated.'),
            _('%d stories were successfully marked as deactivated.'),
            updated,
        ) % updated, messages.SUCCESS)

    @admin.action(description=_('Selected products are available'))
    def make_available(self, request, queryset):
        updated = queryset.update(status='ava')
        self.message_user(request, ngettext(
            _('%d story was successfully marked as available.'),
            _('%d stories were successfully marked as available.'),
            updated,
        ) % updated, messages.SUCCESS)

@admin.register(Comments)
class CommentsAdmin(ModelAdminJalaliMixin, admin.ModelAdmin):
    list_display = ['user', 'product', 'is_active', 'recommend', 'stars', 'datetime_created']
    search_field = ('product',)


import torch as tr
import torch.nn as nn
from models.attentions import MultiHeadAttention, PositionwiseFeedForward

class TransformerDecoderBlock(nn.Module):
    def __init__(
            self,
            d_model: int,
            n_heads: int,
            d_ff: int,
            dropout=0.1
    ) -> None:
        super().__init__()
        self.d_model = d_model
        self.attention = MultiHeadAttention(d_model, n_heads)
        self.masked_attention = MultiHeadAttention(d_model, n_heads)
        self.norm_1 = nn.LayerNorm(d_model)
        self.norm_2 = nn.LayerNorm(d_model)
        self.norm_3 = nn.LayerNorm(d_model)
        self.ff = PositionwiseFeedForward(d_model, d_ff)
        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)
        self.dropout_3 = nn.Dropout(dropout)

    def forward(
            self,
            enc_out: tr.Tensor,
            dec_out: tr.Tensor,
            enc_mask: tr.Tensor,
            dec_mask: tr.Tensor,
    ) -> tr.Tensor:
        """

        Args:
            enc_out (tr.Tensor): [batch_size, src_len, d_model]
            dec_out (tr.Tensor): [batch_size, tgt_len, d_model]
                the `dec_out` is the token embedding during the training stage
                (i.e. embedding of `<bos> hello world <eos>`), and the `dec_out`
                is the output of the decoder during the inference stage.
            enc_mask (tr.Tensor, optional): [batch_size, 1, src_len].
            dec_mask (tr.Tensor, optional): [batch_size, tgt_len, tgt_len].
            cache (tr.Tensor, optional): [batch_size, tgt_len-1, d_model]. Defaults to None.

        Returns:
            tr.Tensor: [batch_size, tgt_len, d_model]
        """
        q = dec_out + self.dropout_1(
            self.masked_attention(dec_out, dec_out, dec_out, dec_mask))
        q = self.norm_1(q)
        x = q + self.dropout_2(
            self.attention(q, enc_out, enc_out, enc_mask))
        x = self.norm_2(x)
        x = x + self.dropout_3(self.ff(x))
        x = self.norm_3(x)
        return x

class TransformerDecoder(nn.Module):
    def __init__(
            self,
            d_model: int,
            n_heads: int,
            d_ff: int,
            n_layers: int,
            dropout=0.1
        ) -> None:
        super().__init__()
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_ff = d_ff
        self.n_layers = n_layers
        self.dropout = dropout
        self.residual_blocks = nn.ModuleList([
            TransformerDecoderBlock(d_model, n_heads, d_ff, dropout) for _ in range(n_layers)
        ])

    def forward(
            self,
            enc_out: tr.Tensor,
            dec_out: tr.Tensor,
            src_mask: tr.Tensor,
            tgt_mask: tr.Tensor
        ):
        """
        Args:
            enc_out (tr.Tensor): [batch_size, src_len, d_model]
            dec_out (tr.Tensor): [batch_size, tgt_len, d_model]
                the `dec_out` is the token embedding during the training stage
                (i.e. embedding of `<bos> hello world <eos>`), and the `dec_out`
                is the output of the decoder during the inference stage.
            enc_mask (tr.Tensor, optional): [batch_size, 1, src_len].
            dec_mask (tr.Tensor, optional): [batch_size, tgt_len, tgt_len]
            caches (List, optional): [n_layers, batch_size, tgt_len-1, d_model]

        Returns:
            tr.Tensor: [batch_size, tgt_len, token_size]
        """
        for _, layer in enumerate(self.residual_blocks):
            dec_out = layer(enc_out, dec_out, src_mask, tgt_mask)
        return dec_out


from datasets import sythtextprovider
import tensorflow as tf
import numpy as np
import cv2
import os
import time
from nets import txtbox_384
from processing import ssd_vgg_preprocessing

os.environ['CUDA_VISIBLE_DEVICES'] = '0'
slim = tf.contrib.slim

tf.logging.set_verbosity(tf.logging.INFO)

show_pic_sum = 10
save_dir = 'pic_test_dataset'
tf.app.flags.DEFINE_string(
    'dataset_dir', 'tfrecord_train', 'The directory where the dataset files are stored.')

tf.app.flags.DEFINE_integer(
    'num_readers', 2,
    'The number of parallel readers that read data from the dataset.')

tf.app.flags.DEFINE_integer(
    'batch_size', 2, 'The number of samples in each batch.')

FLAGS = tf.app.flags.FLAGS
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

def draw_polygon(img,x1,y1,x2,y2,x3,y3,x4,y4, color=(255, 0, 0)):
    # print(x1, x2, x3, x4, y1, y2, y3, y4)
    x1 = int(x1)
    x2 = int(x2)
    x3 = int(x3)
    x4 = int(x4)

    y1 = int(y1)
    y2 = int(y2)
    y3 = int(y3)
    y4 = int(y4)
    cv2.line(img,(x1,y1),(x2,y2),color,2)
    cv2.line(img,(x2,y2),(x3,y3),color,2)
    cv2.line(img,(x3,y3),(x4,y4),color,2)
    cv2.line(img,(x4,y4),(x1,y1),color,2)
    # cv2_im = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # cv2_im = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # cv2.imwrite('test.png', img)
    return img

def run():
    if not FLAGS.dataset_dir:
        raise ValueError('You must supply the dataset directory with --dataset_dir')

    print('-----start test-------')
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    with tf.device('/GPU:0'):
        dataset = sythtextprovider.get_datasets(FLAGS.dataset_dir)
        print(dataset)
        provider = slim.dataset_data_provider.DatasetDataProvider(
                        dataset,
                        num_readers=FLAGS.num_readers,
                        common_queue_capacity=20 * FLAGS.batch_size,
                        common_queue_min=10 * FLAGS.batch_size,
                        shuffle=True)
        print('provider:',provider)
        [image, shape, glabels, gbboxes, x1, x2, x3, x4, y1, y2, y3, y4] = provider.get(['image', 'shape',
                                                                 'object/label',
                                                                 'object/bbox',
                                                                 'object/oriented_bbox/x1',
                                                                 'object/oriented_bbox/x2',
                                                                 'object/oriented_bbox/x3',
                                                                 'object/oriented_bbox/x4',
                                                                 'object/oriented_bbox/y1',
                                                                 'object/oriented_bbox/y2',
                                                                 'object/oriented_bbox/y3',
                                                                 'object/oriented_bbox/y4'
                                                                 ])
        print('image:',image)
        print('shape:',shape)
        print('glabel:',glabels)
        print('gboxes:',gbboxes)

        gxs = tf.transpose(tf.stack([x1,x2,x3,x4])) #shape = (N,4)
        gys = tf.transpose(tf.stack([y1, y2, y3, y4]))

        image = tf.identity(image, 'input_image')
        text_shape = (384, 384)
        image, glabels, gbboxes, gxs, gys= ssd_vgg_preprocessing.preprocess_image(image,  glabels,gbboxes,gxs, gys,
                                                            text_shape,is_training=True,
                                                            data_format='NHWC')

        x1, x2 , x3, x4 = tf.unstack(gxs, axis=1)
        y1, y2, y3, y4 = tf.unstack(gys, axis=1)

        text_net = txtbox_384.TextboxNet()
        text_anchors = text_net.anchors(text_shape)
        e_localisations, e_scores, e_labels = text_net.bboxes_encode( glabels, gbboxes, text_anchors, gxs, gys)

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)

    config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options, allow_soft_placement=True)
    with tf.Session(config=config) as sess:
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess, coord)
        j = 0
        all_time = 0
        try:
            while not coord.should_stop() and j < show_pic_sum:
                start_time = time.time()
                image_sess, label_sess, gbbox_sess,  x1_sess, x2_sess, x3_sess, x4_sess, y1_sess, y2_sess, y3_sess, y4_sess,p_localisations, p_scores, p_labels = sess.run([
                    image, glabels, gbboxes, x1, x2, x3, x4, y1, y2, y3, y4,e_localisations , e_scores, e_labels])
                end_time = time.time() - start_time
                all_time += end_time
                image_np = image_sess
                # print(image_np)
                # print('label_sess:',label_sess)

                p_labels_concat = np.concatenate(p_labels)
                p_scores_concat = np.concatenate(p_scores)
                debug = False
                if debug is True:
                    print(p_labels)
                    print('l_labels:',  len(p_labels_concat[p_labels_concat.nonzero()]),p_labels_concat[p_labels_concat.nonzero()] )
                    print('p_socres:', len(p_scores_concat[p_scores_concat.nonzero()]), p_scores_concat[p_scores_concat.nonzero()])
                    # print(img_np.shape)

                    print('label_sess:', np.array(list(label_sess)).shape, list(label_sess))
                img_np = np.array(image_np)
                cv2.imwrite('{}/{}.png'.format(save_dir, j), img_np)
                img_np = cv2.imread('{}/{}.png'.format(save_dir, j))

                h, w, d = img_np.shape

                label_sess = list(label_sess)
                # for i , label in enumerate(label_sess):
                i = 0
                num_correct = 0

                for label in label_sess:
                    # print(int(label) == 1)
                    if int(label) == 1:
                        num_correct += 1
                        img_np = draw_polygon(img_np,x1_sess[i] * w, y1_sess[i]*h, x2_sess[i]*w, y2_sess[i]*h, x3_sess[i]*w, y3_sess[i]*h, x4_sess[i]*w, y4_sess[i]*h)
                    if int(label) == 0:
                        img_np = draw_polygon(img_np,x1_sess[i] * w, y1_sess[i]*h, x2_sess[i]*w, y2_sess[i]*h, x3_sess[i]*w, y3_sess[i]*h, x4_sess[i]*w, y4_sess[i]*h, color=(0, 0, 255))
                    i += 1
                img_np = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)
                cv2.imwrite('{}'.format(os.path.join(save_dir, str(j)+'.png')), img_np)
                j+= 1
                print('correct:', num_correct)
        except tf.errors.OutOfRangeError:
            print('done')
        finally:
            print('done')
            coord.request_stop()
        print('all time:', all_time, 'average:', all_time / show_pic_sum)
        coord.join(threads=threads)

if __name__ == '__main__':
    run()


import json
import logging

from flask import Flask, request
from flask_cors import CORS

from inference_operator import (
    classification_inference,
    create_classification_predictions,
    load_image_for_detection,
    detection_inference,
    draw_result_image,
    load_model_to_memory,
)
from meals_data import MealsData

app = Flask(__name__)  # pylint: disable=C0103
CORS = CORS(app, resources={r"/*": {"origins": "*"}})

@app.before_first_request
def on_startup():
    load_model_to_memory()

logging.basicConfig(
    format="%(asctime)s %(levelname)-8s %(message)s",
    level=logging.INFO,
    datefmt="%Y-%m-%d %H:%M:%S",
)

@app.route("/ping", methods=["GET"], strict_slashes=False)
def app_health_check():
    return app.response_class(
        status=200, response=f"App is running.", content_type="application/json"
    )

@app.route("/catalog", methods=["GET"], strict_slashes=False)
def meals_catalog():
    app.logger.info("Creating meal catalog...")
    catalog = MealsData.get_catalog()
    return app.response_class(
        response=json.dumps(catalog).encode("utf8"), content_type="application/json"
    )

@app.route("/classify", methods=["POST"], strict_slashes=False)
def classify_photo():
    app.logger.info("Running classification on the image...")
    classification_model = classification_inference(image=str(request.json["image"]))
    app.logger.info("Model inference completed")

    app.logger.info("Creating predictions")
    predictions = create_classification_predictions(model=classification_model)
    app.logger.info("Classification finished.")

    return app.response_class(
        response=json.dumps(predictions).encode("utf8"), content_type="application/json"
    )

@app.route("/detection", methods=["POST"], strict_slashes=False)
def detect_meal():  # pylint: disable=R0914
    app.logger.info("Running YOLO classification on the image...")

    app.logger.info("Loading image from base64")
    loaded_image = load_image_for_detection(image=str(request.json["image"]))

    app.logger.info("Sending request to Roboflow")
    inference_result = detection_inference(image=loaded_image["image_for_inference"])
    app.logger.info(f"Response from Roboflow model: {inference_result}")

    detected_meals = []
    for prediction in inference_result["predictions"]:
        if prediction["class"] not in detected_meals:
            detected_meals.append(prediction["class"])

    app.logger.info("Drawing result on the new image")
    predictions = draw_result_image(
        inference_result=inference_result["predictions"],
        image_for_drawing=loaded_image["image_for_drawing"],
        detected_meals=detected_meals,
    )
    app.logger.info("Detection finished.")
    return app.response_class(
        status=200,
        response=json.dumps(predictions).encode("utf8"),
        content_type="application/json",
    )

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5100)  # pragma: no cover


# -*- coding:utf-8 -*-
"""

"""
from functools import partial

import numpy as np
import pandas as pd
from dask import dataframe as dd, array as da

from hypernets.utils import logging
from ..data_cleaner import DataCleaner, _CleanerHelper

logger = logging.get_logger(__name__)

class DaskDataCleaner(DataCleaner):
    @staticmethod
    def get_helper(X, y):
        if isinstance(X, (dd.DataFrame, dd.Series, da.Array)):
            return _DaskCleanerHelper()
        else:
            return DataCleaner.get_helper(X, y)

class _DaskCleanerHelper(_CleanerHelper):
    @staticmethod
    def reduce_mem_usage(df, excludes=None):
        raise NotImplementedError('"reduce_mem_usage" is not supported for Dask DataFrame.')

    @staticmethod
    def _get_duplicated_columns(df):
        duplicates = df.reduction(chunk=lambda c: pd.DataFrame(c.T.duplicated()).T,
                                  aggregate=lambda a: np.all(a, axis=0)).compute()
        return duplicates

    @staticmethod
    def _detect_dtype(dtype, df):
        result = {}
        df = df.copy()
        for col in df.columns.to_list():
            try:
                df[col] = df[col].astype(dtype)
                result[col] = [True]  # as-able
            except:
                result[col] = [False]
        return pd.DataFrame(result)

    def _correct_object_dtype_as(self, X, df_meta):
        for dtype, columns in df_meta.items():
            columns = [c for c in columns if str(X[c].dtype) != dtype]
            if len(columns) == 0:
                continue

            correctable = X[columns].reduction(chunk=partial(self._detect_dtype, dtype),
                                               aggregate=lambda a: np.all(a, axis=0),
                                               meta={c: 'bool' for c in columns}).compute()
            correctable = [i for i, v in correctable.items() if v]
            # for col in correctable:
            #     X[col] = X[col].astype(dtype)
            if correctable:
                X[correctable] = X[correctable].astype(dtype)
            logger.info(f'Correct columns [{",".join(correctable)}] to {dtype}.')

        return X


#!/usr/bin/python3
# -*-coding:utf-8-*-
import csv
import os
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

script_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(script_path + '/..')

from dictionary.utils import get_end_with_jongsung

engine = None
db_session = None
base_dir = os.path.dirname(os.path.realpath(__file__))
target_dir = base_dir + '/../../seed'

def connect_db(db_url):
    global engine
    global db_session

    print('connect to ' + db_url)
    engine = create_engine(db_url)
    Session = sessionmaker(bind=engine)
    db_session = Session()
    return db_session

def generate_dictionary_csv_files():
    names = get_pos_names()
    for name in names:
        rows = select_pos_rows(name)
        create_dictionary_csv_file(rows, name)

    names = get_class_names()
    for name in names:
        rows = select_class_rows(name)
        create_dictionary_csv_file(rows, name)

def get_pos_names():
    output = []
    sql = """
    SELECT distinct(pos) FROM lexicon_2_1 WHERE class IS NULL
    """
    rows = db_session.execute(sql).fetchall()
    for row in rows:
        pos = row['pos']
        output.append(pos)
    return output

def get_class_names():
    output = []
    sql = """
    SELECT distinct(class) FROM lexicon_2_1 WHERE class IS NOT NULL
    """
    rows = db_session.execute(sql).fetchall()
    for row in rows:
        pos = row['class']
        output.append(pos)
    return output

def select_pos_rows(pos):
    sql = """
    SELECT  surface, pos, semantic_class, `read`, `type`, start_pos, end_pos,
        expression
    FROM lexicon_2_1
    WHERE pos=:pos AND class IS NULL AND is_available = 1
    ORDER BY surface, pos, semantic_class ASC
    """
    rows = db_session.execute(sql, {'pos': pos}).fetchall()
    return rows

def select_class_rows(class_name):
    sql = """
    SELECT  surface, pos, semantic_class, `read`, `type`, start_pos, end_pos,
        expression
    FROM lexicon_2_1
    WHERE class=:cls AND is_available = 1
    ORDER BY surface, pos, semantic_class ASC
    """
    rows = db_session.execute(sql, {'cls': class_name}).fetchall()
    return rows

def create_dictionary_csv_file(rows, file_name):
    file_path = target_dir + '/' + file_name + '.csv'
    with open(file_path, 'w') as csv_file:
        writer = csv.writer(csv_file, 'unix', quoting=csv.QUOTE_MINIMAL)
        for row in rows:
            write_dic(writer, row)

def write_dic(writer, row):
    writer.writerow([row['surface'],
                     '0', '0', '0',
                     row['pos'],
                     row['semantic_class'],
                     get_end_with_jongsung(row['read']),
                     row['read'],
                     row['type'],
                     row['start_pos'],
                     row['end_pos'],
                     row['expression']])

def main():
    # db_url = 'sqlite:////home/kayden/eunjeon/eunjeon_lexicon_2_1.sqlite'
    db_url = input('DB URL (ex: sqlite:///...):')
    connect_db(db_url)
    generate_dictionary_csv_files()

if __name__ == '__main__':
    main()


import pygame
import random

def setBuildingOnPoint(point,surface):
    baseSize=2.5
    changeSize=random.uniform(-1.0,1.0)
    size=baseSize+changeSize
    shape=random.randint(0,2)
    R=random.randint(75,150)
    G=random.randint(75,150)
    B=random.randint(75,150)
    if(checkPoint(point,surface)==False):
        return
    if(shape==0):
        pygame.draw.circle(surface,(R,G,B),point,1.5*size)
    elif(shape==1):
        rect=pygame.Rect(point[0],point[1]+2*size,2*size,2*size)
        pygame.draw.rect(surface,(R,G,B),rect)
    else:
        points=[]
        shape=random.randint(3,8)
        for i in range(shape):
            newPoint=[point[0],point[1]]
            x=random.uniform(-1.5*size,1.5*size)
            y=random.uniform(-1.5*size,1.5*size)
            newPoint[0]=newPoint[0]+x
            newPoint[1]=newPoint[1]+y
            if(checkPoint(newPoint,surface)==False):
               return
            points.append(newPoint)
        pygame.draw.polygon(surface,(R,G,B),points)

def checkPoint(point,surface):
    try:
        coordinates = [int(point[0]),int(point[1])]
        color=surface.get_at(coordinates)
        if(color==(0,0,0,255)):
            return False
        return True
    except:
        return True


"""
tag: 数组 双指针
给你两个有序整数数组nums1 和 nums2，请你将 nums2 合并到nums1中，使 nums1 成为一个有序数组。

初始化nums1 和 nums2 的元素数量分别为m 和 n 。你可以假设nums1 的空间大小等于m + n，这样它就有足够的空间保存来自 nums2 的元素。

示例 1：

输入：nums1 = [1,2,3,0,0,0], m = 3, nums2 = [2,5,6], n = 3
输出：[1,2,2,3,5,6]
示例 2：

输入：nums1 = [1], m = 1, nums2 = [], n = 0
输出：[1]

提示：

nums1.length == m + n
nums2.length == n
0 <= m, n <= 200
1 <= m + n <= 200
-109 <= nums1[i], nums2[i] <= 109

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/merge-sorted-array
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
"""
from typing import List

class Solution:
    # 还有一种，把nums2贴到nums1后面然后排序
    def merge(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        """
        Do not return anything, modify nums1 in-place instead.
        """
        tmp = []
        i, j = 0, 0
        while i < m or j < n:
            if i >= m:
                tmp.append(nums2[j])
                j += 1
            elif j >= n:
                tmp.append(nums1[i])
                i += 1
            else:
                n1, n2 = nums1[i], nums2[j]
                if n1 < n2:
                    tmp.append(n1)
                    i += 1
                else:
                    tmp.append(n2)
                    j += 1
        for _ in range(m + n):
            nums1[_] = tmp[_]

    def merge2(self, nums1: List[int], m: int, nums2: List[int], n: int) -> None:
        """
        利用nums1的后部，双指针，从后往前移动，把大的值放在nums1的尾部
        Do not return anything, modify nums1 in-place instead.
        """
        i, j = m - 1, n - 1
        index = m + n - 1
        while i >= 0 or j >= 0:
            if i < 0:
                nums1[index] = nums2[j]
                j -= 1
                index -= 1
            elif j < 0:
                i -= 1
            else:
                n1, n2 = nums1[i], nums2[j]
                if n1 > n2:
                    nums1[index] = n1
                    i -= 1
                    index -= 1
                else:
                    nums1[index] = n2
                    j -= 1
                    index -= 1

if __name__ == '__main__':
    # nums1 = [1, 2, 3, 0, 0, 0]
    nums1 = [1]
    m = 1
    # nums2 = [2, 5, 6]
    nums2 = []
    n = 0
    s = Solution()
    s.merge2(nums1, m, nums2, n)
    print(nums1)


import time
from functools import wraps
from typing import Callable

def time_me(func: Callable) -> Callable:
    """
    A function decorator.

    Given function foo, do
        foo_with_time=timeit(foo)
    and this new function will return a tuple whose 1st element is the normal function result
    and whose second element is the elapsed time (in seconds)

    Usage:
        def double(x):
            return x*2

        result, time= time_me(double)(3)
        print(f"result={result}, time={time}")
        > result=9, time=9.5367431640625e-07
    """

    @wraps(func)
    def _time_it(*args, **kwargs):
        start = time.time()
        try:
            result = func(*args, **kwargs)
        finally:
            end = time.time()
            elapsed = end - start
            return result, elapsed

    return _time_it


"""Add product category constraint

Revision ID: 0bba8953a559
Revises: d94fc87b96fa
Create Date: 2021-10-09 20:59:34.864581

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "0bba8953a559"
down_revision = "d94fc87b96fa"
branch_labels = None
depends_on = None

def upgrade():
    op.create_unique_constraint(None, "category_product", ["product_id", "category_id"])

def downgrade():
    op.drop_constraint(None, "category_product", type_="unique")


from lxml import etree
from cStringIO import StringIO

from w3af.core.data.misc.encoding import smart_str_ignore
from w3af.core.data.constants.encodings import DEFAULT_ENCODING

ROUND = 20.0

def round_N(num):
    return int(round(num / ROUND) * ROUND)

def get_xml_bones(document):
    """
    This function returns the "bones" of an XML document, replacing all the
    text-data and attribute values with a round(length)

    For example:

        <xml></xml>                  --->    xml0/xml
        <xml>hello</xml>             --->    xml0/xml
        <xml attr=123>hello</xml>    --->    xmlattr00/xml

    See test_xml_bones for more examples.

    :param document: An XML document as a string
    :return: The bones of the XML document, which is commonly used as an input
             for a hash function, which is then used as a key for an LRU.
    """
    parser = etree.HTMLParser(target=BoneCollector(),
                              no_network=True,
                              recover=True,
                              encoding=DEFAULT_ENCODING,
                              remove_comments=True,
                              remove_pis=True)

    document = smart_str_ignore(document, encoding=DEFAULT_ENCODING)
    etree.parse(StringIO(document), parser)

    # pylint: disable=E1101
    return ''.join(parser.target.bones)

class BoneCollector(object):

    __slots__ = ('bones',)

    def __init__(self):
        self.bones = []

    def start(self, tag, attrib):
        self.bones.append(tag)

        for attr, value in attrib.iteritems():
            args = (attr, round_N(len(value)))
            self.bones.append('%s%s' % args)

    def end(self, tag):
        self.bones.append(tag)

    def data(self, data):
        self.bones.append('%s' % round_N(len(data)))

    def comment(self, text):
        pass

    def close(self):
        return None

def get_xml_bones_iterparse(document, _round_N=round_N):
    """
    This is the iterparse version of get_xml_bones()

    Reading some documents on lxml performance it felt like a good idea to
    implement it using iterparse... measured... and this is actually slower

    Leaving here as a reference and reminder that get_xml_bones() can not
    be improved this way.

    :param document: An XML document as a string
    :return: The bones of the XML document, which is commonly used as an input
             for a hash function, which is then used as a key for an LRU.
    """
    if not document:
        return ''

    output = []
    append = output.append

    document = smart_str_ignore(document, encoding=DEFAULT_ENCODING)
    document_io = StringIO(document)
    events = {'start', 'end'}

    context = etree.iterparse(document_io,
                              events=events,
                              remove_comments=True,
                              remove_pis=True,
                              html=True,
                              recover=True,
                              encoding=DEFAULT_ENCODING,
                              huge_tree=False,
                              resolve_entities=False)

    for event, elem in context:
        if event == 'start':
            append(elem.tag)

            for attr, value in elem.attrib.iteritems():
                append('%s%s' % (attr, _round_N(len(value))))

            if elem.text is not None:
                append('%s' % _round_N(len(elem.text)))

        else:
            append('/%s' % elem.tag)

    return ''.join(output)


from selenium import webdriver
import datetime
import time
import telegram_send
import os

os.system('printf "1782618485:AAGk6TxOkq0FyW7YWlpDu5lo3aAnszd_uDM\npub\nt.me/amazPT" | telegram-send --configure-channel')

def get_driver():
    """Start web driver"""
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(chrome_options=chrome_options)
    driver.implicitly_wait(10)
    return driver

# chromeWebDriverPath = './chromedriver'
priceXPath = '//*[@id=\"priceblock_ourprice\"]'

# driver = webdriver.Chrome(chromeWebDriverPath)
driver = get_driver()
productFile = open('products.txt', 'r')
Lines = productFile.readlines()

outputFile = open('prices.txt', 'a')
telegram_send.send(messages=["deployed model..."])
while True:
    for line in Lines:
        productName = line.split('@')[0]
        productUrl = line.split('@')[1]
        productThresh = line.split('@')[2]
        driver.get(productUrl)
        element = driver.find_element_by_xpath(priceXPath)
        price = element.text.split(' ')[1]
        if price <= productThresh:
            telegram_send.send(messages=[str(productName) + ': ' + str(price)])
        print(productName, price)
        outputFile.write(str(datetime.datetime.now()) + ': ' + productName+'@'+price+'\n')
    time.sleep(10)

productFile.close()
outputFile.close()
driver.close()