from __future__ import annotations

from copy import deepcopy
from itertools import product
from typing import Optional, Tuple, Union

from diffpy.structure import Lattice
import numpy as np

from orix.vector import Vector3d

class Miller(Vector3d):
    r"""Direct crystal lattice vectors (uvw or UVTW) and reciprocal
    crystal lattice vectors (hkl or hkil), the latter known as Miller
    indices, describing directions with respect to the crystal reference
    frame defined by a phase's crystal lattice and symmetry.

    Exactly one of ``xyz``, ``uvw``, ``UVTW``, ``hkl``, or ``hkil``
    must be passed.

    The vectors are stored internally as cartesian coordinates in
    :attr:`data`.

    Parameters
    ----------
    xyz
        Vector(s) given in cartesian coordinates. Default is ``None``.
    uvw
        Indices of direct lattice vector(s). Default is ``None``.
    UVTW
        Indices of direct lattice vector(s), often preferred over
        ``uvw`` in trigonal and hexagonal lattices. Default is ``None``.
    hkl
        Indices of reciprocal lattice vector(s). Default is ``None``.
    hkil
        Indices of reciprocal lattice vector(s), often preferred
        over ``hkl`` in trigonal and hexagonal lattices. Default is
        ``None``.
    phase
        A phase with a crystal lattice and symmetry. Must be passed
        whenever direct or reciprocal lattice vectors are created.

    Notes
    -----
    The Miller-Bravais indices :math:`UVTW` are defined as

    .. math::
        U &= \frac{2u - v}{3}, \\
        V &= \frac{2v - u}{3}, \\
        T &= -\frac{u + v}{3}, \\
        W &= w.
    """

    def __init__(
        self,
        xyz: Union[np.ndarray, list, tuple, None] = None,
        uvw: Union[np.ndarray, list, tuple, None] = None,
        UVTW: Union[np.ndarray, list, tuple, None] = None,
        hkl: Union[np.ndarray, list, tuple, None] = None,
        hkil: Union[np.ndarray, list, tuple, None] = None,
        phase: Optional["orix.crystal_map.Phase"] = None,
    ):
        """Create a set of direct lattice vectors (uvw or UVTW) or
        reciprocal lattice vectors (hkl or hkil) describing directions
        with respect to a crystal reference frame defined by a phase's
        crystal lattice and symmetry.
        """
        n_passed = np.sum([i is not None for i in [xyz, uvw, UVTW, hkl, hkil]])
        if n_passed == 0 or n_passed > 1:
            raise ValueError(
                "Exactly one of `xyz`, `uvw`, `UVTW`, `hkl`, `hkil` must be passed"
            )
        if xyz is None and phase is None:
            raise ValueError(
                "A phase with a crystal lattice and symmetry must be passed to create "
                "direct or reciprocal lattice vector(s)"
            )

        self.phase = phase
        if xyz is not None:
            xyz = np.asarray(xyz)
            self.coordinate_format = "xyz"
        elif uvw is not None:
            xyz = _transform_space(uvw, "d", "c", phase.structure.lattice)
            self.coordinate_format = "uvw"
        elif UVTW is not None:
            UVTW = np.asarray(UVTW)
            _check_UVTW(UVTW)
            uvw = _UVTW2uvw(UVTW=UVTW)
            xyz = _transform_space(uvw, "d", "c", phase.structure.lattice)
            self.coordinate_format = "UVTW"
        elif hkl is not None:
            xyz = _transform_space(hkl, "r", "c", phase.structure.lattice)
            self.coordinate_format = "hkl"
        elif hkil is not None:
            hkil = np.asarray(hkil)
            _check_hkil(hkil)
            hkl = _hkil2hkl(hkil)
            xyz = _transform_space(hkl, "r", "c", phase.structure.lattice)
            self.coordinate_format = "hkil"
        super().__init__(xyz)

    def __repr__(self) -> str:
        """String representation."""
        name = self.__class__.__name__
        shape = self.shape
        symmetry = None if self.phase is None else self.phase.point_group.name
        coordinate_format = self.coordinate_format
        data = np.array_str(self.coordinates, precision=4, suppress_small=True)
        return (
            f"{name} {shape}, point group {symmetry}, {coordinate_format}\n" f"{data}"
        )

    def __getitem__(self, key) -> Miller:
        """NumPy fancy indexing of vectors."""
        m = self.__class__(xyz=self.data[key], phase=self.phase).deepcopy()
        m.coordinate_format = self.coordinate_format
        return m

    # ---------------------- Unique properties ---------------------- #

    @property
    def coordinate_format(self) -> str:
        """Return or set the vector coordinate format.

        Parameters
        ----------
        value : str
            Vector coordinate format, either ``"xyz"``, ``"uvw"``,
            ``"UVTW"``, ``"hkl"`` or ``"hkil"``.
        """
        return self._coordinate_format

    @coordinate_format.setter
    def coordinate_format(self, value: str):
        """Set the vector coordinate format."""
        formats = ["xyz", "uvw", "UVTW", "hkl", "hkil"]
        if value not in formats:
            raise ValueError(f"Available coordinate formats are {formats}")
        self._coordinate_format = value

    @property
    def coordinates(self) -> np.ndarray:
        """Return the vector coordinates."""
        coordinate_format = self.coordinate_format
        if coordinate_format == "xyz":
            coordinate_format = "data"
        coordinates = self.__getattribute__(coordinate_format)
        return coordinates

    @property
    def hkl(self) -> np.ndarray:
        """Return or set the reciprocal lattice vectors.

        Parameters
        ----------
        value : np.ndarray
            New reciprocal lattice vector array.
        """
        return _transform_space(self.data, "c", "r", self.phase.structure.lattice)

    @hkl.setter
    def hkl(self, value: np.ndarray):
        """Set the reciprocal lattice vectors."""
        self.data = _transform_space(value, "r", "c", self.phase.structure.lattice)

    @property
    def hkil(self):
        r"""Return or set the reciprocal lattice vectors expressed as
        4-index Miller-Bravais indices.

        Parameters
        ----------
        value : np.ndarray
            New reciprocal lattice vector array. The sum of the first
            three indices, :math:`h`, :math:`k`, and :math:`i` must be
            zero.
        """
        return _hkl2hkil(self.hkl)

    @hkil.setter
    def hkil(self, value: np.ndarray):
        """Set the reciprocal lattice vectors expressed as 4-index
        Miller-Bravais indices.
        """
        self.hkl = _hkil2hkl(value)

    @property
    def h(self) -> np.ndarray:
        """Return the first reciprocal lattice vector index."""
        return self.hkl[..., 0]

    @property
    def k(self) -> np.ndarray:
        """Return the second reciprocal lattice vector index."""
        return self.hkl[..., 1]

    @property
    def i(self) -> np.ndarray:
        r"""Return the third reciprocal lattice vector index in 4-index
        Miller-Bravais indices, equal to :math:`-(h + k)`.
        """
        return self.hkil[..., 2]

    @property
    def l(self) -> np.ndarray:
        """Return the third reciprocal lattice vector index, or fourth
        index in 4-index Miller Bravais indices.
        """
        return self.hkl[..., 2]

    @property
    def uvw(self) -> np.ndarray:
        """Return or set the direct lattice vectors.

        Parameters
        ----------
        value : np.ndarray
            New direct lattice vector array.
        """
        return _transform_space(self.data, "c", "d", self.phase.structure.lattice)

    @uvw.setter
    def uvw(self, value: np.ndarray):
        """Set the direct lattice vectors."""
        self.data = _transform_space(value, "d", "c", self.phase.structure.lattice)

    @property
    def UVTW(self):
        r"""Return or set the direct lattice vectors expressed as
        4-index Weber symbols.

        They are defined as

        .. math::
            U &= \frac{2u - v}{3}, \\
            V &= \frac{2v - u}{3}, \\
            T &= -\frac{u + v}{3}, \\
            W &= w.

        Parameters
        ----------
        value : np.ndarray
            New direct lattice vector array. The sum of the first three
            indices, :math:`U`, :math:`V`, and :math:`T` must be zero.
        """
        return _uvw2UVTW(self.uvw)

    @UVTW.setter
    def UVTW(self, value):
        """Set the direct lattice vectors expressed as 4-index Weber
        symbols.
        """
        self.uvw = _UVTW2uvw(value)

    @property
    def u(self) -> np.ndarray:
        """Return the first direct lattice vector index."""
        return self.uvw[..., 0]

    @property
    def v(self) -> np.ndarray:
        """Return the second direct lattice vector index."""
        return self.uvw[..., 1]

    @property
    def w(self) -> np.ndarray:
        """Return the third direct lattice vector index."""
        return self.uvw[..., 2]

    @property
    def U(self) -> np.ndarray:
        r"""Return the first direct lattice vector index in 4-index
        Weber symbols, equal to :math:`(2u - v)/(3)`.
        """
        return self.UVTW[..., 0]

    @property
    def V(self) -> np.ndarray:
        r"""Return the second direct lattice vector index in 4-index
        Weber symbols, equal to :math:`(2v - u)/3`.
        """
        return self.UVTW[..., 1]

    @property
    def T(self) -> np.ndarray:
        r"""Return the third direct lattice vector index in 4-index
        Weber symbols, equal to :math:`-(u + v)/3`.
        """
        return self.UVTW[..., 2]

    @property
    def W(self) -> np.ndarray:
        r"""Return the fourth direct lattice vector index in 4-index
        Weber symbols, equal to :math:`w`.
        """
        return self.UVTW[..., 3]

    @property
    def length(self) -> np.ndarray:
        """Return the length of each vector given in lattice parameter
        units if the :attr:`coordinate_format` attribute equals
        ``"uvw"`` or ``"UVTW"``, and inverse lattice parameter units if
        the attribute equals ``"hkl"`` or ``"hkil"``.

        If the attribute equals ``"xyz"``, the norms of the vectors in
        :attr:`data` are returned.
        """
        if self.coordinate_format in ["hkl", "hkil"]:
            return self.phase.structure.lattice.rnorm(self.hkl)
        elif self.coordinate_format in ["uvw", "UVTW"]:
            return self.phase.structure.lattice.norm(self.uvw)
        else:
            return self.norm

    @property
    def multiplicity(self) -> np.ndarray:
        """Return the number of symmetrically equivalent directions per
        vector.
        """
        _, l = self.symmetrise(unique=True, return_multiplicity=True)
        return l.reshape(self.shape)

    @property
    def space(self) -> str:
        """Return whether the vector is in direct (``"d"``) or
        reciprocal (``"r"``) space.
        """
        if self.coordinate_format in ["xyz", "uvw", "UVTW"]:
            return "d"
        else:
            return "r"

    @property
    def is_hexagonal(self) -> bool:
        """Return whether the crystal reference frame is
        hexagonal/trigonal.
        """
        return self.phase.is_hexagonal

    # ----------- Overwritten Vector3d/Object3d properties ----------- #

    @property
    def unit(self) -> Miller:
        """Return unit vectors."""
        m = self.__class__(xyz=super().unit.data, phase=self.phase)
        m.coordinate_format = self.coordinate_format
        return m

    # ------------------------ Unique methods ------------------------ #

    @classmethod
    def from_highest_indices(
        cls,
        phase: "orix.crystal_map.Phase",
        uvw: Union[np.ndarray, list, tuple, None] = None,
        hkl: Union[np.ndarray, list, tuple, None] = None,
    ) -> Miller:
        """Create a set of unique direct or reciprocal lattice vectors
        from three highest indices and a phase (crystal lattice and
        symmetry).

        Parameters
        ----------
        phase
            A phase with a crystal lattice and symmetry.
        uvw
            Three highest direct lattice vector indices.
        hkl
            Three highest reciprocal lattice vector indices.
        """
        if uvw is not None:
            coordinate_format = "uvw"
            highest_idx = uvw
        elif hkl is not None:
            coordinate_format = "hkl"
            highest_idx = hkl
        else:
            raise ValueError("Either highest `hkl` or `uvw` indices must be passed")
        idx = _get_indices_from_highest(highest_indices=highest_idx)
        init_kw = {coordinate_format: idx, "phase": phase}
        return cls(**init_kw).unique()

    @classmethod
    def from_min_dspacing(
        cls, phase: "orix.crystal_map.Phase", min_dspacing: float = 0.05
    ) -> Miller:
        """Create a set of unique reciprocal lattice vectors with a
        a direct space interplanar spacing greater than a lower
        threshold.

        Parameters
        ----------
        phase
            A phase with a crystal lattice and symmetry.
        min_dspacing
            Smallest interplanar spacing to consider. Default is 0.05,
            in the unit used to define the lattice parameters in
            ``phase``.
        """
        highest_hkl = _get_highest_hkl(
            lattice=phase.structure.lattice, min_dspacing=min_dspacing
        )
        hkl = _get_indices_from_highest(highest_indices=highest_hkl)
        hkl = hkl.astype(float).round(0)
        return cls(hkl=hkl, phase=phase).unique()

    def deepcopy(self) -> Miller:
        """Return a deepcopy of the instance."""
        return deepcopy(self)

    def round(self, max_index: int = 20) -> Miller:
        """Round a set of index triplet (Miller) or quartet
        (Miller-Bravais/Weber) to the *closest* smallest integers.

        Adopted from MTEX' :code:`Miller.round` function.

        Parameters
        ----------
        max_index
            Maximum integer index to round to, by default 20.

        Returns
        -------
        mill
            Rounded set of index triplet(s) or quartet(s).
        """
        if self.coordinate_format == "xyz":
            return self.deepcopy()
        else:
            new_coords = _round_indices(indices=self.coordinates, max_index=max_index)
            init_kw = {self.coordinate_format: new_coords, "phase": self.phase}
            return self.__class__(**init_kw)

    def symmetrise(
        self,
        unique: bool = False,
        return_multiplicity: bool = False,
        return_index: bool = False,
    ) -> Union[
        Miller, Tuple[Miller, np.ndarray], Tuple[Miller, np.ndarray, np.ndarray]
    ]:
        """Return vectors symmetrically equivalent to the vectors.

        Parameters
        ----------
        unique
            Whether to return only unique vectors. Default is ``False``.
        return_multiplicity
            Whether to return the multiplicity of each vector. Default
            is ``False``.
        return_index
            Whether to return the index into the vectors for the
            returned symmetrically equivalent vectors. Default is
            ``False``.

        Returns
        -------
        mill
            Flattened symmetrically equivalent vectors.
        multiplicity
            Multiplicity of each vector. Returned if
            ``return_multiplicity=True``.
        idx
            Index into the vectors for the returned symmetrically
            equivalent vectors. Returned if ``return_index=True``.
        """
        if return_multiplicity and not unique:
            raise ValueError("`unique` must be True when `return_multiplicity` is True")
        elif return_index and not unique:
            raise ValueError("`unique` must be True when `return_index` is True")

        # Symmetrise directions with respect to crystal symmetry on the
        # flattened set of vectors
        operations = self.phase.point_group
        v2 = operations.outer(self.flatten())

        if unique:
            n_v = self.size  # Number of initial vectors in `self`

            # Array for symmetrically equivalent vectors
            v3 = self.zero((n_v, operations.size))

            # Array for multiplicity of initial vectors
            multiplicity = np.zeros(n_v, dtype=int)

            # Array for index into `self` for the returned symmetrically
            # equivalent vectors
            idx = np.ones(v3.size, dtype=int) * -1

            # Loop over initial vectors
            l_accum = 0
            for i in range(n_v):
                # Unique vectors among those symmetrically equivalent
                vi = v2[:, i].unique()
                l = vi.size  # Multiplicity
                v3[i, :l] = vi  # Insert only the unique ones

                # Multiplicity of this initial vector
                multiplicity[i] = l

                # Index into `self` for the unique, symmetrically
                # equivalent vectors
                idx[l_accum : l_accum + l] = i
                l_accum += l

            # Remove entries into `v3` and `idx` not used
            non_zero = np.sum(np.abs(v3.data), axis=-1) != 0
            v2 = v3[non_zero]
            idx = idx[: np.sum(non_zero)]

        v2 = v2.flatten()

        # Carry over crystal structure and coordinate format
        m = self.__class__(xyz=v2.data, phase=self.phase)
        m.coordinate_format = self.coordinate_format

        if return_multiplicity and return_index:
            return m, multiplicity, idx
        elif return_multiplicity and not return_index:
            return m, multiplicity
        elif not return_multiplicity and return_index:
            return m, idx
        else:
            return m

    def _compatible_with(self, other: Miller, raise_error: bool = False) -> bool:
        """Whether ``self`` and ``other`` are the same (the same crystal
        lattice and symmetry) with vectors in the same space.

        Parameters
        ----------
        other
            Another vector instance.
        raise_error
            Whether to raise a ``ValueError`` if the instances are
            incompatible (default is ``False``).

        Returns
        -------
        compatible
            Whether they are compatible.
        """
        same_symmetry = self.phase.point_group == other.phase.point_group
        same_lattice = np.allclose(
            self.phase.structure.lattice.abcABG(),
            other.phase.structure.lattice.abcABG(),
        )
        same_space = self.space == other.space
        compatible = same_symmetry * same_lattice * same_space
        if not compatible and raise_error:
            raise ValueError(
                "The crystal lattices and symmetries must be the same, and the "
                "vector(s) must be in the same space"
            )
        else:
            return compatible

    # ------------- Overwritten Vector3d/Object3d methods ------------ #

    def angle_with(
        self,
        other: Miller,
        use_symmetry: bool = False,
        degrees: bool = False,
    ) -> np.ndarray:
        """Return the angles between these vectors and the other vectors
        possibly using symmetrically equivalent vectors to find the
        smallest angle under symmetry.

        Vectors must have compatible shapes, and be in the same space
        (direct or reciprocal) and crystal reference frames.

        Parameters
        ----------
        other
            Other vectors.
        use_symmetry
            Whether to consider equivalent vectors to find the smallest
            angle under symmetry. Default is ``False``.
        degrees
            If ``True``, the given angles are returned in degrees.
            Default is ``False``.

        Returns
        -------
        angles
            Angles in radians (``degrees=False``)  or degrees
            (``degrees=True``).
        """
        self._compatible_with(other, raise_error=True)

        if use_symmetry:
            other2 = other.symmetrise(unique=True)
            cosines = self.dot_outer(other2) / (
                self.norm[..., np.newaxis] * other2.norm[np.newaxis, ...]
            )
            cosines = np.round(cosines, 12)
            angles = np.min(np.arccos(cosines), axis=-1)
        else:
            angles = super().angle_with(other)

        if degrees:
            angles = np.rad2deg(angles)

        return angles

    def cross(self, other: Miller):
        """Return the cross products of the vectors with the other
        vectors, which is considered the zone axes between the vectors.

        Parameters
        ----------
        other
            Other vectors, which must be in the same space (direct or
            reciprocal) and have the same crystal reference frame.

        Returns
        -------
        mill
            Vectors in reciprocal (direct) space if direct (reciprocal)
            vectors are crossed.
        """
        self._compatible_with(other, raise_error=True)
        new_fmt = dict(hkl="uvw", uvw="hkl", hkil="UVTW", UVTW="hkil")
        mill = self.__class__(xyz=super().cross(other).data, phase=self.phase)
        mill.coordinate_format = new_fmt[self.coordinate_format]
        return mill

    def dot(self, other: Miller) -> np.ndarray:
        """Return the dot products of the vectors and the other vectors.

        Parameters
        ----------
        other
            Other vectors, which must be in the same space (direct or
            reciprocal) and have the same crystal reference frame.

        Returns
        -------
        dot_products
            Dot products.
        """
        self._compatible_with(other, raise_error=True)
        return super().dot(other)

    def dot_outer(self, other: Miller) -> np.ndarray:
        """Return the outer dot products of the vectors and the other
        vectors.

        Parameters
        ----------
        other
            Other vectors, which must be in the same space (direct or
            reciprocal) and have the same crystal reference frame.

        Returns
        -------
        dot_products
            Dot products.
        """
        self._compatible_with(other, raise_error=True)
        return super().dot_outer(other)

    def flatten(self) -> Miller:
        """Return the flattened vectors.

        Returns
        -------
        mill
            Flattened vectors.
        """
        mill = self.__class__(xyz=super().flatten().data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        return mill

    def transpose(self, *axes: Optional[int]) -> Miller:
        """Return a new instance with the data transposed.

        The order may be undefined if :attr:`ndim` is originally 2. In
        this case the first two dimensions are transposed.

        Parameters
        ----------
        axes
            Transposed axes order. Only navigation axes need to be
            defined. May be undefined if self only contains two
            navigation dimensions.

        Returns
        -------
        mill
            New transposed Miller instance of the original instance.
        """
        mill = self.__class__(xyz=super().transpose(*axes).data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        return mill

    def get_nearest(self, *args) -> NotImplemented:
        """NotImplemented."""
        return NotImplemented

    def mean(self, use_symmetry: bool = False) -> Miller:
        """Return the mean vector of the set of vectors.

        Parameters
        ----------
        use_symmetry
            Not implemented yet.

        Returns
        -------
        mill
            Mean vector.
        """
        # TODO: Allow using symmetry by projecting to fundamental sector
        if use_symmetry:
            return NotImplemented
        mill = self.__class__(xyz=super().mean().data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        return mill

    def reshape(self, *shape: Optional[int]) -> Miller:
        """Return a new instance with the vectors reshaped.

        Parameters
        ----------
        *shape
            New shape.

        Returns
        -------
        mill
            New instance.
        """
        mill = self.__class__(xyz=super().reshape(*shape).data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        return mill

    def unique(
        self, use_symmetry: bool = False, return_index: bool = False
    ) -> Union[Miller, Tuple[Miller, np.ndarray]]:
        """Unique vectors in ``self``.

        Parameters
        ----------
        use_symmetry
            Whether to consider equivalent vectors to compute the unique
            vectors. Default is ``False``.
        return_index
            Whether to return the indices of the (flattened) data where
            the unique entries were found. Default is ``False``.

        Returns
        -------
        mill
            Flattened unique vectors.
        idx
            Indices of the unique data in the (flattened) array.
        """
        out = super().unique(return_index=return_index)
        if return_index:
            v, idx = out
        else:
            v = out

        if use_symmetry:
            operations = self.phase.point_group
            n_v = v.size
            v2 = operations.outer(v).flatten().reshape(*(n_v, operations.size))
            data = v2.data.round(10)
            data_sorted = np.zeros_like(data)
            for i in range(n_v):
                a = data[i]
                order = np.lexsort(a.T)  # Sort by column 1, 2, then 3
                data_sorted[i] = a[order]
            _, idx = np.unique(data_sorted, return_index=True, axis=0)
            v = v[idx[::-1]]

        mill = self.__class__(xyz=v.data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        if return_index:
            return mill, idx
        else:
            return mill

    def in_fundamental_sector(
        self, symmetry: Optional["orix.quaternion.Symmetry"] = None
    ) -> Miller:
        """Project Miller indices to a symmetry's fundamental sector
        (inverse pole figure).

        This projection is taken from MTEX'
        :code:`project2FundamentalRegion`.

        Parameters
        ----------
        symmetry
            Symmetry with a fundamental sector, possibly not equal to
            :attr:`~orix.crystal_map.Phase.point_group`. If not given,
            ``point_group`` is used if valid, otherwise an error is
            raised.

        Returns
        -------
        mill
            Vectors within the fundamental sector.

        Examples
        --------
        >>> from orix.crystal_map import Phase
        >>> from orix.quaternion.symmetry import D6h
        >>> from orix.vector import Miller
        >>> mill = Miller(uvw=(-1, 1, 0), phase=Phase(point_group="m-3m"))
        >>> mill.in_fundamental_sector()
        Miller (1,), point group m-3m, uvw
        [[1. 0. 1.]]
        >>> mill.in_fundamental_sector(D6h)
        Miller (1,), point group m-3m, uvw
        [[1.366 0.366 0.   ]]
        """
        if symmetry is None:
            symmetry = self.phase.point_group
            if symmetry is None:
                raise ValueError(
                    "`symmetry` must be passed or `self.phase.point_group` must be a "
                    "`Symmetry` with a `Symmetry.fundamental_sector`"
                )
        v = Vector3d(self.data).in_fundamental_sector(symmetry)
        mill = self.__class__(xyz=v.data, phase=self.phase)
        mill.coordinate_format = self.coordinate_format
        return mill

def _transform_space(
    v_in: np.ndarray, space_in: str, space_out: str, lattice: Lattice
) -> np.ndarray:
    r"""Convert vectors in a unit cell from one space to another.

    Parameters
    ----------
    v_in
        Input vectors.
    space_in, space_out
        ``"d"`` for direct (uvw), ``"r"`` for reciprocal (hkl) or
        ``"c"`` for cartesian (xyz).

    Returns
    -------
    v_out
        Output vectors.

    Notes
    -----
    Conversions between direct lattice vectors :math:`[uvw]`, reciprocal
    lattice vectors :math:`(hkl)` and Cartesian vectors
    :math:`(x, y, z)` using the structure matrix :math:`A` and the
    metric tensor :math:`g_{ij}`, where vectors are row vectors and
    matrices are row matrices:

    .. math::

        (x, y, z) = [uvw] \cdot \mathbf{A}
        (x, y, z) = (hkl) \cdot (\mathbf{A}^{-1})^T
        [uvw] = (x, y, z) \cdot \mathbf{A}^{-1}
        [uvw] = (hkl) \cdot g_{ij}^{-1}
        (hkl) = (x, y, z) \cdot A^T
        (hkl) = [uvw] \cdot g_{ij}
    """
    spaces = ["d", "r", "c"]
    if space_in not in spaces or space_out not in spaces:
        raise ValueError(f"`space_in` and `space_out` must be one of {spaces}")

    if space_in == space_out:
        v_out = np.copy(v_in)
    elif space_in == "d":
        if space_out == "c":
            # xyz = uvw * A
            v_out = np.matmul(v_in, lattice.base)
        else:
            # hkl = uvw * g_ij
            v_out = np.matmul(v_in, lattice.metrics)
    elif space_in == "r":
        if space_out == "c":
            # xyz = hkl * (A^-1)^T
            v_out = np.matmul(v_in, lattice.recbase.T)
        else:
            # uvw = hkl * g_ij^-1
            v_out = np.matmul(v_in, lattice.reciprocal().metrics)
    else:
        if space_out == "d":
            # uvw = xyz * A^-1
            v_out = np.dot(v_in, lattice.recbase)
        else:
            # hkl = xyz * ((A^-1)^T)^-1 = xyz * A^T
            v_out = np.matmul(v_in, lattice.base.T)

    return v_out

def _hkl2hkil(hkl: np.ndarray) -> np.ndarray:
    hkl = np.asarray(hkl)
    hkil = np.zeros(hkl.shape[:-1] + (4,))
    h = hkl[..., 0]
    k = hkl[..., 1]
    hkil[..., 0] = h
    hkil[..., 1] = k
    hkil[..., 2] = -(h + k)
    hkil[..., 3] = hkl[..., 2]
    return hkil

def _hkil2hkl(hkil: np.ndarray) -> np.ndarray:
    hkil = np.asarray(hkil)
    hkl = np.zeros(hkil.shape[:-1] + (3,))
    hkl[..., :2] = hkil[..., :2]
    hkl[..., 2] = hkil[..., 3]
    return hkl

def _check_hkil(hkil: np.ndarray):
    hkil = np.asarray(hkil)
    if not np.allclose(np.sum(hkil[..., :3], axis=-1), 0, atol=1e-4):
        raise ValueError(
            "The Miller-Bravais indices convention h + k + i = 0 is not satisfied"
        )

def _uvw2UVTW(uvw: np.ndarray, convention: Optional[str] = None) -> np.ndarray:
    uvw = np.asarray(uvw)
    UVTW = np.zeros(uvw.shape[:-1] + (4,))
    u = uvw[..., 0]
    v = uvw[..., 1]
    # DeGraef: U = (2u - v) / 3, V = (2v - u) / 3, T = -(u + v) / 3, W = w
    UVTW[..., 0] = (2 * u - v) / 3
    UVTW[..., 1] = (2 * v - u) / 3
    UVTW[..., 2] = -(u + v) / 3
    UVTW[..., 3] = uvw[..., 2]
    if convention is not None and convention.lower() == "mtex":
        # MTEX: U = 2u - v, V = 2v - u, T = -(u + v), W = 3w
        UVTW *= 3
    return UVTW

def _UVTW2uvw(UVTW: np.ndarray, convention: Optional[str] = None) -> np.ndarray:
    UVTW = np.asarray(UVTW)
    uvw = np.zeros(UVTW.shape[:-1] + (3,))
    # DeGraef: u = 2U + V, v = 2V + U, w = W
    U = UVTW[..., 0]
    V = UVTW[..., 1]
    uvw[..., 0] = 2 * U + V
    uvw[..., 1] = U + 2 * V
    uvw[..., 2] = UVTW[..., 3]
    if convention is not None and convention.lower() == "mtex":
        # MTEX: u = 2U + V, v = 2V + U, w = W / 3
        uvw /= 3
    return uvw

def _check_UVTW(UVTW: np.ndarray):
    UVTW = np.asarray(UVTW)
    if not np.allclose(np.sum(UVTW[..., :3], axis=-1), 0, atol=1e-4):
        raise ValueError(
            "The Miller-Bravais indices convention U + V + T = 0 is not satisfied"
        )

def _get_indices_from_highest(
    highest_indices: Union[list, tuple, np.ndarray]
) -> np.ndarray:
    """Return a list of coordinates from a set of highest indices.

    Parameters
    ----------
    highest_indices
        Highest indices to consider.

    Returns
    -------
    indices
        An array of indices sorted from positive to negative in the
        first column.
    """
    highest_indices = np.asarray(highest_indices)
    if not np.all(highest_indices >= 0) or np.all(highest_indices == 0):
        raise ValueError(
            f"All indices {highest_indices} must be positive with at least one non-zero"
        )
    index_ranges = [np.arange(-i, i + 1) for i in highest_indices]
    indices = np.asarray(list(product(*index_ranges)), dtype=int)
    indices = indices[~np.all(indices == 0, axis=1)]  # Remove (000)
    indices = indices[::-1]  # Make e.g. (111) first instead of (-1-1-1)
    return indices

def _get_highest_hkl(lattice: Lattice, min_dspacing: float = 0.05) -> np.ndarray:
    """Return the highest Miller indices hkl of the plane with a direct
    space interplanar spacing closest to a threshold.

    Parameters
    ----------
    lattice
        Crystal lattice.
    min_dspacing
        Smallest interplanar spacing to consider. Default is 0.05.

    Returns
    -------
    highest_hkl
        Highest Miller indices.
    """
    highest_hkl = np.ones(3, dtype=int)
    for i in range(3):
        hkl = np.zeros(3)
        hkl[i] = 1
        d = 1 / lattice.rnorm(hkl)
        while d > min_dspacing:
            hkl[i] += 1
            d = 1 / lattice.rnorm(hkl)
        highest_hkl[i] = hkl[i] - 1
    return highest_hkl

def _round_indices(
    indices: Union[list, tuple, np.ndarray], max_index: int = 12
) -> np.ndarray:
    """Round a set of index triplet (Miller) or quartet (Miller-Bravais)
    to the *closest* smallest integers.

    Adopted from MTEX' :code:`Miller.round` function.

    Parameters
    ----------
    indices
        Set of index triplet(s) or quartet(s) to round.
    max_index
        Maximum integer index to round to, by default 12.

    Return
    ------
    new_indices
        Integer array of rounded set of index triplet(s) or quartet(s).
    """
    # Allow list and tuple input (and don't overwrite `indices`)
    idx = np.asarray(indices)

    # Flatten and remove redundant third index if Miller-Bravais
    n_idx = idx.shape[-1]  # 3 or 4
    idx_flat = np.reshape(idx, (-1, n_idx))
    if n_idx == 4:
        idx_flat = idx_flat[..., [0, 1, 3]]

    # Get number of sets, max. index per set, and all possible integer
    # multipliers between 1 and `max_index`
    n_sets = idx_flat.size // 3
    max_per_set = np.max(np.abs(idx_flat), axis=-1)
    multipliers = np.arange(1, max_index + 1)

    # Divide by highest index, repeat array `max_index` number of times,
    # and multiply with all multipliers
    idx_scaled = (
        np.broadcast_to(idx_flat / max_per_set[..., np.newaxis], (max_index, n_sets, 3))
        * multipliers[..., np.newaxis, np.newaxis]
    )

    # Find the most suitable multiplier per set, which gives the
    # smallest error between the initial set and the scaled and rounded
    # set
    error = 1e-7 * np.round(
        1e7
        * np.sum((idx_scaled - np.round(idx_scaled)) ** 2, axis=-1)
        / np.sum(idx_scaled**2, axis=-1)
    )
    idx_min_error = np.argmin(error, axis=0)
    multiplier = (idx_min_error + 1) / max_per_set

    # Reshape `multiplier` to match indices shape
    multiplier = multiplier.reshape(idx.shape[:-1])[..., np.newaxis]

    # Finally, multiply each set with their most suitable multiplier,
    # and round
    new_indices = np.round(multiplier * idx).astype(int)

    return new_indices


'''

our web server using bokeh

display image:
cf: https://docs.bokeh.org/en/latest/docs/reference/models/glyphs/image_url.html

bokeh.server.contexts
https://docs.bokeh.org/en/latest/docs/reference/server/contexts.html

various widgets
https://docs.bokeh.org/en/latest/docs/user_guide/interaction/widgets.html#div

range slider (default)
https://docs.bokeh.org/en/latest/docs/user_guide/interaction/widgets.html#tabs
custom double ended slider
https://docs.bokeh.org/en/latest/docs/user_guide/extensions_gallery/widget.html

hover tooltip with pic
https://docs.bokeh.org/en/latest/docs/user_guide/tools.html?highlight=custom%20tooltip
https://discourse.bokeh.org/t/hovertool-displaying-image/1198/7

'''
from __future__ import print_function

import copy
import time

import numpy as np

from bokeh.driving import count
from bokeh.layouts import column, gridplot, row
from bokeh.models import Select, Slider, Button, ImageURL, Plot, LinearAxis, Paragraph, Div, Range1d, Text, LabelSet, Title
from bokeh.models import CDSView, IndexFilter, RadioButtonGroup

# palettes https://docs.bokeh.org/en/latest/docs/reference/palettes.html
from bokeh.palettes import RdYlGn10, Spectral6
from bokeh.transform import linear_cmap

from bokeh.plotting import curdoc, figure

from bokeh  .models import ColumnDataSource, DataTable, DateFormatter, TableColumn
from datetime import date
from random import randint

# xzl
import sys
import threading
import logging
from dataclasses import asdict
from functools import partial
from tornado import gen

import grpc
import server_diva_pb2
import server_diva_pb2_grpc

import json
from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2

#import server.tables as tables
#import server.forms as forms
#import server.control as cloud
#
import control
from control import VideoInfo

#from .control import list_videos_cam
#from .control import query_submit

#from .control import list_videos_cam
#from control import query_submit

from variables import *

FORMAT = '%(asctime)-15s %(levelname)8s %(thread)d %(threadName)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=FORMAT)
#logger = logging.getLogger(__name__)
logger = logging.getLogger()

np.random.seed(1)

doc = curdoc()

#######################################

PLOT_IMG_WIDTH = 1000
PLOT_IMG_HEIGHT = 200

# for results
#PER_IMG_WIDTH = 200 # for image spacing
PER_IMG_WIDTH = 300 # for image spacing
PER_IMG_HEIGHT = 100 # only for determining label offset
PER_IMG_SPACING = int(PER_IMG_WIDTH * 1.1)
PLOT_IMG_HEIGHT = PER_IMG_HEIGHT + 20 # some margin for frame info

FULL_IMG_WIDTH = 1280 >> 1
FULL_IMG_HEIGHT = 720 >> 1

PER_PREVIEW_IMG_WIDTH = 128
PER_PREVIEW_IMG_HEIGHT = 72
PER_PREVIEW_IMG_SPACING = int(PER_PREVIEW_IMG_WIDTH * 1.1)
PLOT_PREVIEW_HEIGHT = PER_PREVIEW_IMG_HEIGHT + 20 # some margin for frame info

PER_IMG_720P_HEIGHT = int(720/(1280/PER_IMG_WIDTH)) # 720p image scaled height

N_PREVIEWS = 5
N_RESULTS = 20 # show in table
N_RESULTS_IMG = 5

BUTTON_WIDTH = 100

UPDATE_INTERVAL_MS = 2000   # short interval like 500ms will cause unresponsive UI. XXX opt parsing framemaps
the_interval_ms = UPDATE_INTERVAL_MS

PLACEHOLDER = ['test1', 'test2', 'test3', 'test4', 'test5']

#######################################

MA12, MA26, EMA12, EMA26 = '12-tick Moving Avg', '26-tick Moving Avg', '12-tick EMA', '26-tick EMA'

source = ColumnDataSource(dict(
    time=[], average=[], low=[], high=[], open=[], close=[],
    ma=[], macd=[], macd9=[], macdh=[], color=[]
))

p = figure(plot_height=500, tools="xpan,xwheel_zoom,xbox_zoom,reset", x_axis_type=None, y_axis_location="right")
p.x_range.follow = "end"
p.x_range.follow_interval = 100
p.x_range.range_padding = 0

p.line(x='time', y='average', alpha=0.2, line_width=3, color='navy', source=source)
p.line(x='time', y='ma', alpha=0.8, line_width=2, color='orange', source=source)
p.segment(x0='time', y0='low', x1='time', y1='high', line_width=2, color='black', source=source)
p.segment(x0='time', y0='open', x1='time', y1='close', line_width=8, color='color', source=source)

p2 = figure(plot_height=250, x_range=p.x_range, tools="xpan,xwheel_zoom,xbox_zoom,reset", y_axis_location="right")
p2.line(x='time', y='macd', color='red', source=source)
p2.line(x='time', y='macd9', color='blue', source=source)
p2.segment(x0='time', y0=0, x1='time', y1='macdh', line_width=6, color='black', alpha=0.5, source=source)

mean = Slider(title="mean", value=0, start=-0.01, end=0.01, step=0.001)
stddev = Slider(title="stddev", value=0.04, start=0.01, end=0.1, step=0.01)
mavg = Select(value=MA12, options=[MA12, MA26, EMA12, EMA26])

################
# pause/resume
################

b_pause = Button(label="Pause", button_type="success", width=BUTTON_WIDTH, disabled=True)
b_resume = Button(label="Resume", button_type="success", width=BUTTON_WIDTH, disabled=True)

the_started = False
the_paused = False

the_qid = -1
the_video_name = "chaweng-1_10FPS"
the_video_info = None
the_all_classes = ['car','bus','truck','train','person','bicycle']
the_class = the_all_classes[0]

the_frameskip_choices = [1, 5, 10, 20, 100]
the_frameskip = the_frameskip_choices[0]

the_videos:typing.List[VideoInfo] = None

def cb_resume():
    global the_paused, the_poll_cb
    if the_paused:
        console_write("resume query requested")
        msg = control.query_resume()
        console_append("cam replied:" + msg)
        if msg.startswith("OK"):
            the_paused = False
            the_poll_cb = doc.add_periodic_callback(periodic_callback, 500)
            b_resume.disabled = True
            b_pause.disabled = False
    else:
        logger.error("bug?")

def cb_pause():
    global the_paused, the_poll_cb

    if not the_paused:
        console_write("pause query requested")
        msg = control.query_pause()
        console_append("pause query. cam replied:" + msg)
        if msg.startswith("OK"):
            the_paused = True
            doc.remove_periodic_callback(the_poll_cb)
            b_pause.disabled = True
            b_resume.disabled = False
    else:
        logger.error("bug?")

b_pause.on_click(cb_pause)
b_resume.on_click(cb_resume)

######
# the plot showing progress. eg  n_frames_processed_cam
######

src_cam_init = dict(
    ts=[],
    n_frames_processed_cam=[],
    n_frames_processed_cam_rate =[],
    n_frames_processed_yolo_rate=[],
    n_frames_recv_yolo_rate=[], # means positive results
    n_mbytes_cam_sent_rate=[],
    cam_mem_usage_percent=[]
)

src_cam = ColumnDataSource(src_cam_init)

# tools="xpan,xwheel_zoom,xbox_zoom,reset",
#  y_axis_location="right"
pcam = figure(plot_height=100, plot_width=PLOT_IMG_WIDTH>>3, toolbar_location=None,
              x_axis_type=None, y_axis_type=None, title="PosFr/sec")
pcam.x_range.follow = "end"
pcam.x_range.follow_interval = 100
pcam.x_range.range_padding = 0

# pcam.line(x='ts', y='n_frames_processed_cam', alpha=0.8, line_width=2, color='red', source=src_cam)
pcam.line(x='ts', y='n_frames_recv_yolo_rate', alpha=0.8, line_width=2, color='green', source=src_cam)

xaxis = LinearAxis()
xaxis.minor_tick_line_color = None
pcam.add_layout(xaxis, 'below')

yaxis = LinearAxis()
yaxis.minor_tick_line_color = None
pcam.add_layout(yaxis,'left')

## showing cam processing rate
#  y_axis_location="right"
# x_axis_type=None
pcam1 = figure(plot_height=100, plot_width=PLOT_IMG_WIDTH>>3, toolbar_location=None,
               x_axis_type=None, y_axis_type=None, title="CamProcFr/sec")
pcam1.x_range.follow = "end"
pcam1.x_range.follow_interval = 100
pcam1.x_range.range_padding = 0

xaxis = LinearAxis()
xaxis.minor_tick_line_color = None
pcam1.add_layout(xaxis, 'below')

yaxis = LinearAxis()
yaxis.minor_tick_line_color = None
pcam1.add_layout(yaxis,'left')

pcam1.line(x='ts', y='n_frames_processed_cam_rate', alpha=0.8, line_width=2, color='red', source=src_cam)

#############
## show yolo processing rate
pcam2 = figure(plot_height=100, plot_width=PLOT_IMG_WIDTH>>3, toolbar_location=None,
               x_axis_type=None, y_axis_type=None, title="CloudProcFr/sec")
pcam2.x_range.follow = "end"
pcam2.x_range.follow_interval = 100
pcam2.x_range.range_padding = 0

xaxis = LinearAxis()
xaxis.minor_tick_line_color = None
pcam2.add_layout(xaxis, 'below')

yaxis = LinearAxis()
yaxis.minor_tick_line_color = None
pcam2.add_layout(yaxis,'left')

pcam2.line(x='ts', y='n_frames_processed_yolo_rate', alpha=0.8, line_width=2, color='blue', source=src_cam)

#############
## show cam upload data rate
pcam3 = figure(plot_height=100, plot_width=PLOT_IMG_WIDTH>>3, toolbar_location=None,
               x_axis_type=None, y_axis_type=None, title="CamUpMB/sec")
pcam3.x_range.follow = "end"
pcam3.x_range.follow_interval = 100
pcam3.x_range.range_padding = 0

xaxis = LinearAxis()
xaxis.minor_tick_line_color = None
pcam3.add_layout(xaxis, 'below')

yaxis = LinearAxis()
yaxis.minor_tick_line_color = None
pcam3.add_layout(yaxis,'left')

pcam3.line(x='ts', y='n_mbytes_cam_sent_rate', alpha=0.8, line_width=2, color='red', source=src_cam)

#############
## show cam mem usage
pcam4 = figure(plot_height=100, plot_width=PLOT_IMG_WIDTH>>3, toolbar_location=None,
               x_axis_type=None, y_axis_type=None, title="CamMemPercent")
pcam4.x_range.follow = "end"
pcam4.x_range.follow_interval = 100
pcam4.x_range.range_padding = 0

xaxis = LinearAxis()
xaxis.minor_tick_line_color = None
pcam4.add_layout(xaxis, 'below')

yaxis = LinearAxis()
yaxis.minor_tick_line_color = None
pcam4.add_layout(yaxis,'left')

pcam4.line(x='ts', y='cam_mem_usage_percent', alpha=0.8, line_width=2, color='red', source=src_cam)

# not in use. periodic callback. perhaps simpler
@count()
def update_camsrc(t):
    logger.info(f"update_camsrc checkgin....{t}")
    prog = control.get_latest_query_progress(qid=0)
    if not prog: return

    new_data = dict(
        ts = [prog.ts],
        n_frames_processed_cam = [prog.n_frames_processed_cam],
        #n_frames_recv_yolo = [prog.n_frames_recv_yolo]
        n_frames_recv_yolo=[0]
    )

    print(new_data)
    src_cam.stream(new_data, rollover=300)

# curdoc().add_periodic_callback(update_camsrc, 500)

##############################
# test: using a separate thread to poll progress on demand
# todo: call all progress update from here
##############################

# dirty
the_last_n_frames_processed_cam = -1
the_last_n_bytes_sent_cam = -1
the_last_ts = -1

# update the aggregated stats & framemap
# scheduled by worker thread, executed in the main thread
@gen.coroutine
def cb_update_datasrc(prog, stats, res):  # stats: QueryInfoCloud
    global UPDATE_INTERVAL_MS, the_interval_ms

    logger.info(f">>>>>>>>>>>>> cb_update_datasrc started")

    t0 = time.time()

    global the_last_n_frames_processed_cam, the_last_n_frames_recv_yolo, the_last_n_frames_processed_yolo, the_last_ts, the_start_ts
    global the_last_n_bytes_sent_cam

    if the_last_ts < 0: # once
        n_frames_processed_cam_rate = 0
        n_frames_processed_yolo_rate = 0
        n_frames_recv_yolo_rate = 0
        n_mbytes_cam_sent_rate = 0
        the_start_ts = prog.ts
    else:
        #n_frames_processed_cam_rate = (prog.n_frames_processed_cam - the_last_n_frames_processed_cam) / (prog.ts - the_last_ts)
        n_frames_processed_cam_rate = (stats.n_frames_processed_cam - the_last_n_frames_processed_cam) / (
                    prog.ts - the_last_ts)
        n_frames_processed_yolo_rate = (stats.n_frames_processed_yolo - the_last_n_frames_processed_yolo) / (
                prog.ts - the_last_ts)
        n_frames_recv_yolo_rate = (stats.n_frames_recv_yolo - the_last_n_frames_recv_yolo) / (
            prog.ts - the_last_ts)
        n_mbytes_cam_sent_rate = (stats.n_bytes_sent_cam - the_last_n_bytes_sent_cam) /  (
            prog.ts - the_last_ts) / 1024 / 1024

    the_last_n_frames_processed_cam = stats.n_frames_processed_cam
    the_last_n_frames_processed_yolo = stats.n_frames_processed_yolo
    the_last_n_frames_recv_yolo = stats.n_frames_recv_yolo
    the_last_n_bytes_sent_cam = stats.n_bytes_sent_cam
    the_last_ts = prog.ts

    new_data = dict(
        ts=[prog.ts - the_start_ts],
        n_frames_processed_cam=[stats.n_frames_processed_cam],
        n_frames_processed_cam_rate=[n_frames_processed_cam_rate],
        n_frames_processed_yolo_rate=[n_frames_processed_yolo_rate],
        #n_frames_recv_yolo=[prog.n_frames_recv_yolo]
        n_frames_recv_yolo_rate=[n_frames_recv_yolo_rate],
        n_mbytes_cam_sent_rate=[n_mbytes_cam_sent_rate],
        cam_mem_usage_percent=[stats.cam_mem_usage_percent]
    )
    #print(new_data)
    src_cam.stream(new_data, rollover=300)

    cb_update_span(prog.framestates) # ... expensive

    t1 = time.time()

    cb_update_res(res)

    t2 = time.time()

    logger.info(f"<<<<<< cb_update_datasrc ended {(t1-t0)*1000:.2f} {(t2-t1)*1000:.2f} ms")

    # control the rate of update
    '''
    if t2-t0 > 0.1: # too high
        the_interval_ms *= 1.5
    else:
        the_interval_ms = max(the_interval_ms - 100, UPDATE_INTERVAL_MS)
    '''

ev_progress = threading.Event()

def thread_progress():
    logger.info("started")
    while True:
        ev_progress.wait()
        ev_progress.clear()

        # pull updates from control bkend...
        t0 = time.time()
        prog = control.create_query_progress_snapshot(qid=the_qid)
        stat = control.query_progress(qid=the_qid) # stats
        res = control.query_results(the_qid)
        t1 = time.time()

        if not prog:
            continue

        # update/render on demand
        if prog:
            doc.add_next_tick_callback(partial(cb_update_datasrc, prog, stat, res))
            logger.info(f"pull backend update... {1000*(t1-t0)} ms")
        else:
            logger.info("poll progress... failed")

thread_progress = threading.Thread(target=thread_progress)
thread_progress.start()

the_poll_cb = None

# will be called periodically once query in progress.. from main thread
@count()
def periodic_callback(t):
    global the_poll_cb, the_interval_ms
    ev_progress.set() # async. kick the worker thread, which will pull the query progress from cam
    #cb_update_res(t) # sync. pull & update query res from cloud (not cam). sync
    the_poll_cb = doc.add_timeout_callback(periodic_callback, int(the_interval_ms))
    logger.info(f"schedule next update in {the_interval_ms} ms")

######
# text console
######
#para_log = Paragraph(text="hello world", width=200, height=100)
para_log = Div(text="[log messages]", width=300, height=50, background='lightgrey')

def console_append(msg:str):
    #para_log.text += '
{}
'.format(msg)
    para_log.text += '{}
'.format(msg)

def console_clear():
    para_log.text = ''

def console_write(msg:str):
    console_clear()
    console_append(msg)

######
# start a sample query
######

def cb_newquery():
    global the_poll_cb, the_started, the_qid, the_video_info, the_videos
    global data_span, source_single_res, source_results, src_cam
    '''
    the_videos = control.list_videos()
    for v in the_videos:
        if v.video_name == the_video_name:
            the_video_info = v
            break
    else:
        logger.error(f"cannot find video {the_video_name}")
        raise
    '''

    # abort an ongoing query
    if the_started:
        msg = control.query_reset()
        console_write(f"reset cam requested. cam said {msg}")
        if msg.startswith('OK'):
            the_started = False
            #b_query.label="Query"
            update_b_query_label()
            b_pause.disabled = True
        return

    # new query...
    if not the_video_name or not the_video_info:
        logger.error(f"cannot find video {the_video_name}")
        console_write("cannot run query: no video selected")
        return

    # assumption:
    # video naming: video is named as ${scene}-X_XXfps-...
    #   where ${scene} is like "purdue", "chaweng", ...
    # opname: ops on the camera side are named as ${scene}-0, ${scene}-1, ${scene}-2...
    # op num: 0, 1, 2. .. up to 5

    scene = the_video_name.split('-')[0]
    assert(len(scene) > 0)

    op_names = [f'{scene}-{x}' for x in range(0,6)]  # up to 6 ops

    the_qid = control.query_submit(video_name=the_video_name,
                 op_names=op_names, crop='-1,-1,-1,-1', target_class=the_class, frameskip=the_frameskip)
    # print('qid:', resp)
    console_append(f"new query started. qid {the_qid}")
    #the_poll_cb = doc.add_periodic_callback(periodic_callback, UPDATE_INTERVAL_MS)
    the_poll_cb = doc.add_timeout_callback(periodic_callback, the_interval_ms)

    # clear any res from prev query
    data_span = data_span_init
    source_single_res.data = source_single_res_init
    source_results = source_results_init
    src_cam.data = src_cam_init

    the_started = True
    b_query.label="Abort"

    b_pause.disabled = False

b_query = Button(label="Query", button_type="danger", width=BUTTON_WIDTH*2)
b_query.on_click(cb_newquery)

# will return ??:?? if incomplete info
def frameid_to_vtime(id:int) -> str:
    if the_video_info and the_video_info.fps > 0:
        d = int(id/the_video_info.fps)
        return f'{int(d/3600):02d}:{int((d%3600)/60):02d}:{int(d%60):02d}'
    else:
        return '??:??'

def update_b_query_label():
    v = the_video_name if the_video_name else '??'
    fs = the_frameskip if the_frameskip > 0 else '??'
    c = the_class

    nf = the_video_info.frame_id_max - the_video_info.frame_id_min + 1 - the_video_info.n_missing_frames
    '''
    if the_video_info.fps > 0:
        d = int(nf/the_video_info.fps)
        dstr = f'{int(d/3600)}:{int((d%3600)/60)}:{int(d%60)}'
    else:
        dstr = '??:??'
    '''
    dstr = frameid_to_vtime(nf)
    nf /= fs

    b_query.label=f'Query:{v}/{c}/skip={fs} frames={int(nf)} dur={dstr}'

######
# list videos
######

cds_videos = {}
source_videos = ColumnDataSource(data=cds_videos)

columns = [
        TableColumn(field="video_name", title="Name"),
        TableColumn(field="n_frames", title="Frames"),
        TableColumn(field="n_missing_frames", title="Missing"),
        TableColumn(field="fps", title="fps"),
        TableColumn(field="duration", title="duration"),
        TableColumn(field="frame_id_min", title="min"),
        TableColumn(field="frame_id_max", title="max"),
    ]
table_listvideos = DataTable(source=source_videos, columns=columns,
                             width=400, height=100)

def show_cam_specs():
    console_clear()
    console_append('requested to list videos')
    msg = control.cam_specs()
    console_append(msg)

def cb_listvideos():
    global cds_videos
    global the_videos

    console_clear()
    console_append('requested to list videos')

    the_videos = control.list_videos()
    if len(the_videos) == 0:
        logger.error("nothing!")
        return

    console_append(f'{len(the_videos)} found')

    # convert to dicts
    vds = [asdict(v) for v in the_videos]

    # gen cds
    cds_videos = {}
    for k in vds[0].keys():
        cds_videos[k] = list(x[k] for x in vds)

    print(cds_videos)

    source_videos.data = cds_videos  # commit

b_lv = Button(label="ListVideos", button_type="success", width=BUTTON_WIDTH)
b_lv.on_click(cb_listvideos)

def cb_listvideo_table(attraname, old, new):
    global the_video_name, the_video_info
    # print(f"selected! {attraname} {old} {new}")
    i = source_videos.selected.indices[0]
    the_video_name = the_videos[i].video_name
    the_video_info = the_videos[i]
    console_write(f'selected video {i} {the_video_name}')
    if len(the_videos) >= i + 1:
        load_preview_frames(the_videos[i], N_PREVIEWS)
        update_b_query_label()

source_videos.selected.on_change('indices', cb_listvideo_table)

######
# list queries
######

cds_queries = {}
source_queries = ColumnDataSource(data=cds_queries) # init cds

table_listqueries = DataTable(
    source=source_queries,
    columns = [
        TableColumn(field="qid", title="qid"),
        TableColumn(field="status_cloud", title="Cloud"),
        TableColumn(field="status_cam", title="Cam"),
        TableColumn(field="target_class", title="Class"),
        TableColumn(field="video_name", title="Video"),
        TableColumn(field="n_result_frames", title="#ResFrames"),
    ],
    width=400,
    height=100
)

def cb_listqueries():
    global cds_queries

    console_clear()
    console_append("list queries requested")

    queries = control.list_queries_cloud()
    if len(queries) == 0:
        logger.info("no queries returned")
        console_append("no queries returned ")
        return

    # covert to dicts
    console_append(f"{len(queries)} queries found")

    vds = [asdict(v) for v in queries]

    # gen cds
    cds_queries = {}
    for k in vds[0].keys():
        cds_queries[k] = list(x[k] for x in vds)

    # add one column
    cds_queries['n_result_frames'] = list(len(x['results']) for x in vds)

    # erase columns that cannot be sent to browser
    cds_queries.pop('results', None)
    cds_queries.pop('progress_snapshots', None)

    logger.info(f"got {len(queries)} queries")

    source_queries.data = cds_queries # commit

b_lq = Button(label="ListQueries", button_type="success", width=BUTTON_WIDTH)
b_lq.on_click(cb_listqueries)

######
# preview img
######

# url = "server/static/preview/chaweng-1_10FPS/83999.thumbnail.jpg"
url_logo = "https://static.bokeh.org/logos/logo.png"  # for testing

sourceimg = ColumnDataSource(dict(
    url = [url_logo] * N_PREVIEWS,
    x1  = [i * PER_PREVIEW_IMG_SPACING for i in range(N_PREVIEWS)],
    #y1  = [PER_PREVIEW_IMG_HEIGHT] * N_PREVIEWS,
    y1  = [0] * N_PREVIEWS,
    w1  = [10] * N_PREVIEWS,
    h1  = [10] * N_PREVIEWS,
    #frame_ids = ['preview', '-5', '100000', 'id3', 'id4', 'id5']
    frame_ids = [''] * N_PREVIEWS
))

#xdr = Range1d(start=-100, end=200)
#ydr = Range1d(start=-100, end=200)

# format of the plot
ppreview = Plot(
#    title=None, x_range=xdr, y_range=ydr, plot_width=300, plot_height=300,
    x_range=Range1d(start=0, end=PER_PREVIEW_IMG_SPACING * N_PREVIEWS),
        y_range=Range1d(start=0, end=PLOT_PREVIEW_HEIGHT),
        title=Title(text="Preview"),
        plot_width=PER_PREVIEW_IMG_SPACING * N_PREVIEWS,
        plot_height=int(PLOT_PREVIEW_HEIGHT * 1.5),
        min_border=0, toolbar_location=None
    )

'''
# for layout debugging
yaxis = LinearAxis()
ppreview.add_layout(yaxis, 'left')
xaxis = LinearAxis()
ppreview.add_layout(xaxis, 'above')
'''

#image1 = ImageURL(url="url", x="x1", y="y1", w = 'w1', h='h1', anchor="top_left")
image1 = ImageURL(url="url", x="x1", y="y1", w = 'w1', h='h1', anchor="bottom_left")

# text1 = LabelSet(text="frame_ids", x="x1", y="y1", x_offset=0, y_offset=-50, level='glyph', source=sourceimg)
text1 = Text(text="frame_ids", x="x1", y="y1", x_offset=0, y_offset=-PLOT_PREVIEW_HEIGHT,
             text_color="magenta", text_font='helvetica', text_font_size='10px', text_baseline='bottom')

ppreview.add_glyph(sourceimg, image1) # feed the data
ppreview.add_glyph(sourceimg, text1) # feed the data
#ppreview.add_layout(text1) # feed the data -- nothing shown, why??

def load_preview_frames(v:VideoInfo, n_max:int):
    global the_videolib_preview
    the_videolib_preview.AddVideoStore(v.video_name).CleanStoredFrames()
    ids = control.download_video_preview_frames(v, n_max, res=[PER_PREVIEW_IMG_WIDTH, PER_PREVIEW_IMG_HEIGHT])
    console_write(f'{len(ids)} preview fetched')

    cds_previews = dict(
        url = [the_videolib_preview.GetVideoStore(v.video_name).GetFramePath(id) for id in ids],
        frame_ids = [f'fr:{id} [ {frameid_to_vtime(id)} ]' for id in ids],
        #url = [],
        x1 = [i * PER_PREVIEW_IMG_SPACING for i in range(n_max)],
        #y1 = [PER_PREVIEW_IMG_HEIGHT] * n_max,
        y1=[0] * n_max,
        w1 = [PER_PREVIEW_IMG_WIDTH] * n_max,
        h1 = [PER_PREVIEW_IMG_HEIGHT] * n_max
        #h1 = None,
        #h1 = [100] * n_max
    )

    print(cds_previews)
    sourceimg.data = cds_previews

    '''
    cds_txt = dict(
        text = [f'{id:07d}' for id in ids],
        x1 = [i * 200 for i in range(n_max)],
        y1 = [400] * n_max,
    )
    print(cds_txt)
    sourcetext.data = cds_txt
    '''

######
# show a single image selected
######

source_single_res_init = dict(
    url = [""],
    x1 = [0],
    y1 = [FULL_IMG_HEIGHT],
    w1 = [FULL_IMG_WIDTH],
    h1 = [FULL_IMG_HEIGHT],
)

source_single_res = ColumnDataSource(source_single_res_init)

psingle = Plot(
    x_range=Range1d(start=0, end=FULL_IMG_WIDTH), y_range=Range1d(start=0, end=FULL_IMG_HEIGHT),
    title=None, plot_width=FULL_IMG_WIDTH, plot_height=FULL_IMG_HEIGHT,
    min_border=0, toolbar_location=None)

#image_single = ImageURL(url="url", x="x1", y="y1", h="h1", w="w1", anchor="top_left")
image_single = ImageURL(url="url", x="x1", y="y1", w="w1", h="h1", anchor="top_left")
psingle.add_glyph(source_single_res, image_single)

def single_img_load(url:str):
    new_data = dict (
        url = [url],
        x1 = [0],
        y1 = [FULL_IMG_HEIGHT],
        w1 = [FULL_IMG_WIDTH],
        h1 = [FULL_IMG_HEIGHT],
    )
    source_single_res.data = new_data

######
# result imgs & table
######

source_results_init = ColumnDataSource(dict(
    url = [url_logo] * N_RESULTS_IMG,
    x1 = [i * PER_IMG_SPACING for i in range(N_RESULTS_IMG)],
    y1 = [PER_IMG_HEIGHT] * N_RESULTS_IMG,
    w1  = [10] * N_RESULTS_IMG,
    h1  = [10] * N_RESULTS_IMG,
    #frame_desc = ['desc1'] * N_RESULTS_IMG,
    frame_desc = [''] * N_RESULTS_IMG,
    frame_ids = [0] * N_RESULTS_IMG,
    scores = [0] * N_RESULTS_IMG,
    n_bboxes = [0] * N_RESULTS_IMG
))

source_results = source_results_init

presults = Plot(
    x_range=Range1d(start=0, end=PER_IMG_SPACING * N_RESULTS_IMG), y_range=Range1d(start=0, end=PLOT_IMG_HEIGHT),
    title=None, plot_width=PER_IMG_SPACING * N_RESULTS_IMG,
    plot_height=int(PLOT_IMG_HEIGHT*1.5),
    min_border=0, toolbar_location=None)

#image_results = ImageURL(url="url", x="x1", y="y1", anchor="top_left")
image_results = ImageURL(url="url", x="x1", y="y1", h="h1", w="w1", anchor="top_left")
txt_results = Text(text="frame_desc", x="x1", y="y1")
                   #x_offset=0, y_offset=PER_IMG_720P_HEIGHT+30)

# only show a small subset of result images
view_res = CDSView(source=source_results, filters=[IndexFilter([x for x in range(N_RESULTS_IMG)])])

presults.add_glyph(source_results, image_results, view=view_res)
presults.add_glyph(source_results, txt_results, view=view_res)

table_results = DataTable(
    source=source_results,
    columns = [
        TableColumn(field="frame_ids", title="Frame"),
        TableColumn(field="scores", title="Score"),
        TableColumn(field="n_bboxes", title="BBoxes"),
    ],
    width=400,
    height=200
)

# get updated query res from cloud.
# to be called in ticks of the main thread. rendering as table, spans, etc.
# sync, as it wont talk to the cam
def cb_update_res(res):
    global the_qid, the_videolib_results, the_current_span

    #logger.info(f">>>>>>>>>>>>> cb_update_res started")
    #t0 = time.time()

    #res = control.query_results(the_qid) # won't contact the cam

    #t1 = time.time()

    # filter res based on currently selected timespan
    # filtering using CDSviwe is not working: filters there can only accept simple logic (based on prescribed JS)
    # cannot handle custom data + logic like this
    if the_current_span != (-1,-1): # a span selected
        newres = [f for f in res if the_current_span[0] <= f.frame_id < the_current_span[1]]
        res = newres

    if res and len(res) > 0:
        # top frames
        n = min(len(res), N_RESULTS)
        ids = [f.frame_id for f in res[0:n]]
        scores = [f.high_confidence for f in res[0:n]]
        n_bboxes = [f.n_bboxes for f in res[0:n]]

        new_data = dict(
            url = [the_videolib_results.GetVideoStore(f'{the_qid}').GetFramePath(id) for id in ids],
            frame_desc = [f'{ids[x]} score:{scores[x]:.2f} boxes:{n_bboxes[x]}' for x in range(n)],
            frame_ids = ids,
            scores = scores,
            n_bboxes = n_bboxes,
            x1=[i * PER_IMG_SPACING for i in range(n)],
            y1=[PER_IMG_HEIGHT] * n,
            w1=[PER_IMG_WIDTH] * n,
            h1=[PLOT_IMG_HEIGHT] * n
        )

        #print('---------> update results:', new_data)
        source_results.data = new_data # commit

    '''
    t2 = time.time()
    logger.info(f"<<<<<< cb_update_res ended {t} {(t1-t0)*1000:.2f} {(t2-t1)*1000:.2f} ms")
    if t2-t0 > 0.1: # too high
        the_interval_ms *= 1.5
    else:
        the_interval_ms = max(the_interval_ms - 100, UPDATE_INTERVAL_MS)
    '''

def cb_results_table(attr, old, new):
    i = source_results.selected.indices[0]
    url = source_results.data['url'][i]
    console_write(f"selected frame {url} to display")
    logger.warning(f"selected frame {url} to display")
    single_img_load(url)
    # global view_res
    # view_res.filters = [IndexFilter([x for x in range(1)])] # must update filter like this

source_results.selected.on_change('indices', cb_results_table)

##############################
# class selector
##############################
class_sel = RadioButtonGroup(labels=the_all_classes, active=0, width=int(BUTTON_WIDTH/2))

def cb_class(attr, old, new):
    global the_class, the_all_classes
    the_class = the_all_classes[class_sel.active]
    #print(the_class)
    console_write(f'chose class = {the_class}')
    update_b_query_label()

#class_sel.on_change('active', lambda  attr, old, new: cb_class())
class_sel.on_change('active', cb_class)

##############################
# frameskip selector
# https://www.programcreek.com/python/example/106842/bokeh.models.Select
##############################
frameskip_sel = Select(value=f"{the_frameskip}", options=[f"{k}" for k in the_frameskip_choices], width=40
                       #title = "frameskip"
                       )

def cb_frameskip(attr, old, new):
    global the_frameskip
    #print(frameskip_sel.value)
    the_frameskip = int(frameskip_sel.value)
    console_write(f'set frameskip = {the_frameskip}')
    update_b_query_label()

frameskip_sel.on_change('value', cb_frameskip)

##############################
# vbar progress tracker
# colors: https://docs.bokeh.org/en/latest/docs/reference/colors.html
##############################
NSPANS = 10
timespans = [f'win{d}' for d in range(NSPANS)]
#timespans = [f'{d:4d}min' for d in range(NSPANS)]
#states = ['.', '-', '1', '2', '3', '4', '5', 's', 'r']
states = ['held', 'queued', 'pass1', 'pass2', 'pass3', 'pass4', 'pass5', 'uploaded', 'positive']
colors = ["lightsteelblue", "white", "gainsboro", "lightgray", "silver", "darkgray", "gray", "cyan", "lime"]
data_span_init = {
    'timespans' : timespans,
    'held': [0] * NSPANS,  # backqueue
    'queued' : [0] * NSPANS,    # .
    'pass1' : [0] * NSPANS,    # ditto
    'pass2' : [0] * NSPANS,    # ditto
    'pass3' : [0] * NSPANS,    # ditto
    'pass4' : [0] * NSPANS,    # ditto
    'pass5' : [0] * NSPANS,    # ditto
    'uploaded' : [0] * NSPANS,    # s
    'positive' : [0] * NSPANS,    # r
    'max_confidence' : [0.0] * NSPANS,  # max of confidence in all 'r' frames
    'avg_confidence' : [0.0] * NSPANS,  # ditto
}

data_span = data_span_init

source_span = ColumnDataSource(data=data_span)

pspan = figure(x_range=timespans, plot_height=200, plot_width=int(PLOT_IMG_WIDTH/2.5),
               # title="Fruit Counts by Year", toolbar_location=None,
               title="Progress per win",
               tools="hover,box_select,tap",
               #tooltips="$name @timespans: @$name")
                tooltips="$name frames: @$name")
pspan.vbar_stack(states, x='timespans', width=0.9, color=colors, source=source_span,
             #legend_label=states
                 )

''' # this adds labeles to the top of bars. not what we need
label_span = LabelSet(text="max_confidence", x="timespans", y="r",
                      x_offset=0, y_offset=-50, source=source_span)
pspan.add_layout(label_span)
'''

pspan.y_range.start = 0
pspan.y_range.end = 10
#pspan.x_range.range_padding = 0.1
pspan.xgrid.grid_line_color = None
pspan.axis.minor_tick_line_color = None
pspan.outline_line_color = None
pspan.legend.location = "top_left"
pspan.legend.orientation = "horizontal"

the_spans = None
the_current_span = (-1, -1) # min, max

###########
# callbacks, etc

# https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

def cb_update_span(framestates:typing.Dict[int, str]):
    global the_spans
    fs_list = [(frame_id, state) for frame_id, state in framestates.items()]
    fs_list.sort(key= lambda x: x[0])
    #print(fs_list[0:10])

    '''
    I do not fully understand how npo.array_split() works, but i) the 0th span seems incorrect (e.g. 20k frs, minid 0, maxid 9999)
    and ii) the split arrays are in sophisticated types. i) causes a bug. bad.
    '''
    #the_spans = np.array_split(fs_list, NSPANS)    # slice into N spans
    the_spans = list(chunks(fs_list, int(len(fs_list)/NSPANS)+1))
    assert(len(the_spans) == NSPANS)

    new_data = copy.deepcopy(data_span)

    '''
    ss = [id for id, st in the_spans[0]]
    print(ss[0:10], the_spans[0][0:10], len(the_spans[0]), max(the_spans[0], key=lambda x: x[0]))
    for x in range(1,9999):
        c = ss.count(x)
        if c>0:
            print(x, c)
    '''

    #for x in the_spans[0][0:10]:
    #    print(x[0], sep=' ')

    # new_data['timespans'] = [f'winXXX{d}' for d in range(NSPANS)]

    # sp = [ (frame_id, state) ... ]
    # f is a (frame_id, state) tuple

    for idx, sp in enumerate(the_spans):
        new_data['held'][idx] = sum(1 for f in sp if f[1] == '-')
        new_data['queued'][idx] = sum(1 for f in sp if f[1] == '.')
        new_data['pass1'][idx] = sum(1 for f in sp if f[1] == '1')
        new_data['pass2'][idx] = sum(1 for f in sp if f[1] == '2')
        new_data['pass3'][idx] = sum(1 for f in sp if f[1] == '3')
        new_data['pass4'][idx] = sum(1 for f in sp if f[1] == '4')
        new_data['pass5'][idx] = sum(1 for f in sp if f[1] == '5')
        new_data['uploaded'][idx] = sum(1 for f in sp if f[1] == 's')
        #new_data['r'][idx] = sum(1 for f in sp if f[1] == 'r')
        # a bit hack. for 'r' frames, we store the confidence as its framestate
        new_data['positive'][idx] = sum(1 for f in sp if f[1][0] == '0')

        max_conf_frame, new_data['max_confidence'][idx] = max(sp, key=lambda x: float(x[1]) if x[1][0] == '0' else 0)
        # todo: later...
        #new_data['avg_confidence'][idx] = mean(sp, key=lambda x: float(x[1]) if x[1][0] == '0' or x[1][0] == '1' else 0)

    #print(new_data)
    #pspan.x_range.factors = [f'winXXX{d}' for d in range(NSPANS)] # works

    #factors = [f'winXXX{d}' for d in range(NSPANS)]  # working

    # factors = [f'winXXX{len(sp)}' for sp in the_spans] # not working?
    #factors = [f'winXXX{idx}' for idx, sp in enumerate(the_spans)]  # working
    # factors = [f'winXXX{len(sp)/10000}' for idx, sp in enumerate(the_spans)]  # not working

    #factors = [f'{d+1}min' for d in range(NSPANS)]  #  working
    #factors = [f'3min' for d in range(NSPANS)]  # working
    #factors = [f"{int(len(sp) * the_frameskip/10/60)}min" for sp in the_spans] # not working?

    # update pspan -- XXX inefficeint. should do it only once when query starts
    '''
    factors = []
    offset = 0
    if the_video_info and the_video_info.fps > 0:
        fps = the_video_info.fps
        for sp in the_spans:
            #factors.append(f"{int(len(sp) * the_frameskip/fps/60)}min")
            factors.append(f"{offset}-{offset+int(len(sp) * the_frameskip / fps / 60)}min")
            offset+=int(len(sp) * the_frameskip / fps / 60)
    else: # we have no fps info
        for sp in the_spans:
            factors.append(f"{int(len(sp)/1000)}k")

    pspan.x_range.factors = factors
    '''

    title = f"Each win: {int(len(sp))} frames in "
    if the_video_info and the_video_info.fps > 0:
        fps = the_video_info.fps
        title += f"{int(len(sp) * the_frameskip / fps / 60)} mins"

    pspan.title.text= title

    #print(len(sp), the_frameskip, fps)
    #print(factors)

    source_span.data = new_data

def cb_span(attr, old, new):
    global the_spans, the_current_span
    nsel = len(source_span.selected.indices)
    if nsel == 0 or not the_spans:
        the_current_span = (-1, -1)
        return
    i = source_span.selected.indices[0]
    the_current_span = (int(min(the_spans[i], key=lambda x: x[0])[0]),
                        int(max(the_spans[i], key=lambda x: x[0])[0]))

    logger.warning(f"selected span {i}. fr {the_current_span} frs {len(the_spans[i])}")

source_span.selected.on_change('indices', cb_span)

####
# "promote" "demote" buttons
####

def cb_promo(is_promote:bool):
    global the_spans
    nsel = len(source_span.selected.indices)
    #print(f'{nsel} total selected')

    if is_promote:
        action = "promote"
    else:
        action = "demote"

    i = source_span.selected.indices[0]
    minid = min(the_spans[i], key=lambda x: x[0])[0]
    maxid = max(the_spans[i], key=lambda x: x[0])[0]

    console_write(f"{action} frames {minid} -- {maxid}")
    logger.warning(f"{action} span {i} frs {len(the_spans[i])} {minid} -- {maxid}")
    msg = control.promote_frames(-1, int(minid), int(maxid), is_promote=is_promote)
    console_append("cam replied:" + msg)

b_promo = Button(label="Promo", button_type="success", width=BUTTON_WIDTH)
b_promo.on_click(partial(cb_promo, is_promote=True))

b_demo = Button(label="Demote", button_type="danger", width=BUTTON_WIDTH)
b_demo.on_click(partial(cb_promo, is_promote=False))

###########
# bar graph with only "results"

pspan0 = figure(x_range=timespans, y_range=(0,10), plot_height=200, plot_width=int(PLOT_IMG_WIDTH/2.5),
               # title="Fruit Counts by Year", toolbar_location=None,
                title="Positives per win",
               tools="hover,box_select,tap", tooltips="pos: @positive; max_confidence: @max_confidence")

mapper = linear_cmap(field_name='max_confidence', palette=RdYlGn10, low=1, high=0) # reverse a palaette
pspan0.vbar(x='timespans', top='positive', width=0.9, color=mapper, source=source_span)

##############################

def _create_prices(t):
    last_average = 100 if t==0 else source.data['average'][-1]
    returns = np.asarray(np.random.lognormal(mean.value, stddev.value, 1))
    average =  last_average * np.cumprod(returns)
    high = average * np.exp(abs(np.random.gamma(1, 0.03, size=1)))
    low = average / np.exp(abs(np.random.gamma(1, 0.03, size=1)))
    delta = high - low
    open = low + delta * np.random.uniform(0.05, 0.95, size=1)
    close = low + delta * np.random.uniform(0.05, 0.95, size=1)
    return open[0], high[0], low[0], close[0], average[0]

def _moving_avg(prices, days=10):
    if len(prices) < days: return [100]
    return np.convolve(prices[-days:], np.ones(days, dtype=float), mode="valid") / days

def _ema(prices, days=10):
    if len(prices) < days or days < 2: return [prices[-1]]
    a = 2.0 / (days+1)
    kernel = np.ones(days, dtype=float)
    kernel[1:] = 1 - a
    kernel = a * np.cumprod(kernel)
    # The 0.8647 normalizes out that we stop the EMA after a finite number of terms
    return np.convolve(prices[-days:], kernel, mode="valid") / (0.8647)

# xzl
data1 = dict(
        dates=[date(2014, 3, i+1) for i in range(10)],
        downloads=[randint(0, 100) for i in range(10)],
        test=[randint(0, 100) for i in range(10)]
    )
source1 = ColumnDataSource(data1)

columns = [
        TableColumn(field="dates", title="Date", formatter=DateFormatter()),
        TableColumn(field="downloads", title="Downloads"),
        TableColumn(field="test", title="xxx"),
    ]
data_table = DataTable(source=source1, columns=columns, width=400, height=280)

@count()
def update(t):
    #print(t)
    if paused:
        return

    open, high, low, close, average = _create_prices(t)
    color = "green" if open < close else "red"

    new_data = dict(
        time=[t],
        open=[open],
        high=[high],
        low=[low],
        close=[close],
        average=[average],
        color=[color],
    )

    close = source.data['close'] + [close]
    ma12 = _moving_avg(close[-12:], 12)[0]
    ma26 = _moving_avg(close[-26:], 26)[0]
    ema12 = _ema(close[-12:], 12)[0]
    ema26 = _ema(close[-26:], 26)[0]

    if   mavg.value == MA12:  new_data['ma'] = [ma12]
    elif mavg.value == MA26:  new_data['ma'] = [ma26]
    elif mavg.value == EMA12: new_data['ma'] = [ema12]
    elif mavg.value == EMA26: new_data['ma'] = [ema26]

    macd = ema12 - ema26
    new_data['macd'] = [macd]

    macd_series = source.data['macd'] + [macd]
    macd9 = _ema(macd_series[-26:], 9)[0]
    new_data['macd9'] = [macd9]
    new_data['macdh'] = [macd - macd9]

    source.stream(new_data, 300)

######################## UI layout #####################
#curdoc().add_root(column(row(mean, stddev, mavg), gridplot([[p], [p2]], toolbar_location="left", plot_width=1000)))
#curdoc().add_root(column(row(mean, stddev, mavg, b),
    #gridplot([[p]], toolbar_location="left", plot_width=1000))
    #)

'''
curdoc().add_root(column(row(mean, stddev, mavg, b, b_lv),
    gridplot([[p]], toolbar_location="left", plot_width=1000), data_table)
    )
'''

'''
doc.add_root(column(row(b, b_lv, b_query),
    gridplot([[pcam, ppreview]], toolbar_location="left", plot_width=1000), table_listvideos)
    )
'''

'''
doc.add_root(column(
    row(para_log, frameskip_sel, class_sel),
    row(table_listvideos, pcam, pcam1, pcam2, table_listqueries),
    row(b_pause, b_resume, b_lv, b_lq, b_query),
    #row(pcam, table_results),
    row(column(pspan, pspan0, row(b_promo, b_demo)), table_results),
    #row(pspan, b_promo, b_demo),
    #pspan0,
    presults,
    psingle,
    ppreview
))
'''

doc.add_root(column(
    row(table_listvideos, ppreview),
    row(frameskip_sel, class_sel),
    row(b_pause, b_resume, b_lv, b_lq, b_query),
    #row(pcam, table_results),
    row(column(pspan, pspan0, row(b_promo, b_demo)), column(table_results, row(pcam, pcam2), row(pcam1, pcam3, pcam4), para_log), psingle, table_listqueries),
    #row(pspan, b_promo, b_demo),
    #pspan0,
    presults
))

# curdoc().add_periodic_callback(update, 500)
doc.title = "ðŸ“·âš™ï¸�"

logger.info("main execed!")

# cb_listvideos()   # can do it here.
show_cam_specs()


from didww_encrypt.fingerprint import calculate_fingerprint
from didww_encrypt.encryption import encrypt
from didww_encrypt.fetching import fetch_public_keys
from . import MODE_SANDBOX, MODE_PRODUCTION, URI_SANDBOX, URI_PRODUCTION

class DIDWW_Encrypt:
    def __init__(self, pubkey_a: str, pubkey_b: str):
        self.pubkey_a = pubkey_a
        self.pubkey_b = pubkey_b
        self.fingerprint = calculate_fingerprint(self.pubkey_a, self.pubkey_b)

    def encrypt(self, data_bytes: bytes) -> bytes:
        return encrypt(data_bytes, self.pubkey_a, self.pubkey_b)

def new(mode: str = None, uri: str = None) -> DIDWW_Encrypt:
    if uri is not None:
        pubkey_a, pubkey_b = fetch_public_keys(uri)
    elif mode == MODE_SANDBOX:
        pubkey_a, pubkey_b = fetch_public_keys(URI_SANDBOX)
    elif mode == MODE_PRODUCTION:
        pubkey_a, pubkey_b = fetch_public_keys(URI_PRODUCTION)
    else:
        raise ValueError("Valid mode, or uri must be provided")

    return DIDWW_Encrypt(pubkey_a, pubkey_b)


import random
from Queue import Queue
import json
from threading import Thread, Lock, Semaphore
import time
import uuid

import datetime
from boto.dynamodb2.exceptions import ItemNotFound, ConditionalCheckFailedException, \
    ProvisionedThroughputExceededException, LimitExceededException
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import STRING
from boto.exception import JSONResponseError
from motorway.messages import Message
from motorway.ramp import Ramp
import boto.kinesis
import boto.ec2.cloudwatch
import boto.kinesis.exceptions
import logging

shard_election_logger = logging.getLogger("motorway.contrib.amazon_kinesis.shard_election")

class KinesisRamp(Ramp):
    stream_name = None
    heartbeat_timeout = 30  # Wait 10 seconds for a heartbeat update, or kill it

    def __init__(self, shard_threads_enabled=True, **kwargs):
        super(KinesisRamp, self).__init__(**kwargs)
        self.conn = boto.kinesis.connect_to_region(**self.connection_parameters())
        assert self.stream_name, "Please define attribute stream_name on your KinesisRamp"

        control_table_name = 'pipeline-control-%s' % self.stream_name

        self.worker_id = str(uuid.uuid4())
        self.semaphore = Semaphore()
        self.uncompleted_ids = {}

        if shard_threads_enabled:
            dynamodb_connection = boto.dynamodb2.connect_to_region(**self.connection_parameters())

            table_config = dict(
                connection=dynamodb_connection,
                schema=[
                    HashKey('shard_id', data_type=STRING),
                ],
                throughput={
                    'read': 10,
                    'write': 10,
                }
            )

            try:
                dynamodb_connection.describe_table(control_table_name)
            except JSONResponseError:
                Table.create(
                    control_table_name,
                    **table_config
                )

            self.control_table = Table(
                control_table_name,
                **table_config
            )

            shards = self.conn.describe_stream(self.stream_name)['StreamDescription']['Shards']
            threads = []
            self.insertion_queue = Queue()
            for i, shard in enumerate(shards):
                self.uncompleted_ids[shard['ShardId']] = set()
                t = Thread(target=self.process_shard, name="%s-%s" % (self.__class__.__name__, i), args=(shard['ShardId'], ))
                threads.append(t)
                t.start()

    def claim_shard(self, shard_id):
        shard_election_logger.info("Claiming shard %s" % shard_id)
        control_record = self.control_table.get_item(shard_id=shard_id)
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            control_record.save()
        except ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True

    def can_claim_shard(self, shard_id):
        control_record = self.control_table.get_item(shard_id=shard_id)
        heartbeat = control_record['heartbeat']
        time.sleep(self.heartbeat_timeout)
        if heartbeat == self.control_table.get_item(shard_id=shard_id)['heartbeat']:  # TODO: check worker_id as well
            shard_election_logger.debug("Heartbeat remained unchanged for defined time, taking over")
            return True
        else:
            shard_election_logger.debug("Heartbeat changed, continue sleeping")
        # Balance, if possible
        active_workers = {
            self.worker_id: True
        }
        heartbeats = {
            self.worker_id: 0
        }

        shards = list(self.control_table.scan())
        for shard in shards:  # Update active worker cache
            if shard['worker_id'] not in active_workers:
                heartbeats[shard['worker_id']] = shard['heartbeat']
            if heartbeats[shard['worker_id']] == shard['heartbeat']:
                active_workers[shard['worker_id']] = False
            else:
                active_workers[shard['worker_id']] = True
        number_of_active_workers = sum([1 for is_active in active_workers.values() if is_active])
        number_of_shards = len(shards)
        optimal_number_of_shards_per_worker = number_of_shards / number_of_active_workers
        workers = set([shard['worker_id'] for shard in shards])
        shards_per_worker = {worker: sum([1 for shard in shards if shard['worker_id'] == worker]) for worker in workers}
        for shard in shards:
            if shards_per_worker.get(shard['worker_id'], 0) > optimal_number_of_shards_per_worker:
                if shards_per_worker.get(self.worker_id, 0) < optimal_number_of_shards_per_worker:
                    if shard['shard_id'] == shard_id:
                        shard_election_logger.debug("Taking over %s from %s" % (shard_id, shard['worker_id']))
                        return True
        return False

    def process_shard(self, shard_id):
        while True:
            try:
                try:
                    while True:
                        with self.semaphore:
                            if self.can_claim_shard(shard_id):
                                if self.claim_shard(shard_id):
                                    break
                        time.sleep(random.randrange(2, 15))
                except ItemNotFound:
                    control_record = Item(self.control_table, data={
                        'shard_id': shard_id,
                        'checkpoint': 0,
                        'worker_id': self.worker_id,
                        'heartbeat': 0,
                    })
                    control_record.save()

                control_record = self.control_table.get_item(shard_id=shard_id)
                if control_record['checkpoint'] > 0:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "AT_SEQUENCE_NUMBER",
                        starting_sequence_number=str(control_record['checkpoint'])
                    )['ShardIterator']
                else:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "LATEST",
                    )['ShardIterator']

                cloudwatch = boto.ec2.cloudwatch.connect_to_region(**self.connection_parameters())
                current_minute = lambda: datetime.datetime.now().minute
                minute = None
                latest_item = None
                while True:
                    control_record = self.control_table.get_item(shard_id=shard_id)
                    if not control_record['worker_id'] == self.worker_id:
                        shard_election_logger.info("Lost shard %s, going back to standby" % shard_id)
                        break
                    control_record['heartbeat'] += 1
                    if len(self.uncompleted_ids[shard_id]):  # Get the "youngest" uncompleted sequence number
                        control_record['checkpoint'] = min(self.uncompleted_ids[shard_id])
                    elif latest_item:
                        control_record['checkpoint'] = latest_item

                    control_record.save()  # Will fail if someone else modified it - ConditionalCheckFailedException

                    result = self.conn.get_records(iterator)
                    for record in result['Records']:
                        self.uncompleted_ids[shard_id].add(record['SequenceNumber'])
                        latest_item = record['SequenceNumber']
                        self.insertion_queue.put(record)
                    # Push metrics to CloudWatch
                    iterator = result['NextShardIterator']
                    delay = result['MillisBehindLatest']
                    if minute != current_minute():  # push once per minute to CloudWatch.
                        minute = current_minute()
                        cloudwatch.put_metric_data(
                            'Motorway/Kinesis',
                            'MillisecondsBehind',
                            value=delay,
                            unit='Milliseconds',
                            dimensions={
                                'Stream': self.stream_name,
                                'Shard': shard_id
                            }
                        )
                    time.sleep(1)  # recommended pause between fetches from AWS
            except ConditionalCheckFailedException:
                pass  # we're no longer worker for this shard
            except (ProvisionedThroughputExceededException, LimitExceededException, boto.kinesis.exceptions.ProvisionedThroughputExceededException, boto.kinesis.exceptions.LimitExceededException) as e:
                time.sleep(random.randrange(5, self.heartbeat_timeout/2))  # back off for a while

    def connection_parameters(self):
        return {
            'region_name': 'eu-west-1',
            # Add this or use ENV VARS
            # 'aws_access_key_id': '',
            # 'aws_secret_access_key': ''
        }

    def next(self):
        msg = self.insertion_queue.get()
        yield Message(msg['SequenceNumber'], json.loads(msg['Data']), grouping_value=msg['PartitionKey'])

    def success(self, _id):
        for uncompleted_ids in self.uncompleted_ids.values():
            if _id in uncompleted_ids:
                uncompleted_ids.remove(_id)


print('<>' * 35)
print('{:^70}'.format('\033[31mEXERCICIO 66\033[m'))
print('<>' * 35)
soma = contador = 0
while True:
    valor = int(input('Digite uma valor [999 para parar]: '))
    if valor == 999:
        break
    soma += valor
    contador += 1
print(f'A soma dos {contador} valores é {soma}')


import os
import gensim
import keras
import sklearn
import numpy as np
from keras import Sequential
from keras.layers import Masking, LSTM, Dense, Bidirectional, Embedding, Dropout, Flatten, BatchNormalization, GRU
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import ReduceLROnPlateau, EarlyStopping
from keras import regularizers
from keras import backend as K
from keras.layers import Layer
from keras import initializers, regularizers, constraints

files = {"Spyware", "Downloader", "Trojan", "Worms", "Adware", "Dropper", "Virus", "Backdoor"}
classes = {"Spyware": 0, "Downloader": 1, "Trojan": 2, "Worms": 3, "Adware": 4, "Dropper": 5, "Virus": 6, "Backdoor": 7}

batch_size = 128
num_step = 150
class_num = 8
sequence_length = 5000
embedding = 50
dev_sample_percentage = 0.2

def dot_product(x, kernel):
    """
    Wrapper for dot product operation, in order to be compatible with both
    Theano and Tensorflow
    Args:
        x (): input
        kernel (): weights
    Returns:
    """
    if K.backend() == 'tensorflow':
        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
    else:
        return K.dot(x, kernel)

class AttentionWithContext(Layer):
    """
    Attention operation, with a context/query vector, for temporal data.
    Supports Masking.
    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]
    "Hierarchical Attention Networks for Document Classification"
    by using a context vector to assist the attention
    # Input shape
        3D tensor with shape: `(samples, steps, features)`.
    # Output shape
        2D tensor with shape: `(samples, features)`.
    How to use:
    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.
    The dimensions are inferred based on the output shape of the RNN.
    Note: The layer has been tested with Keras 2.0.6
    Example:
        model.add(LSTM(64, return_sequences=True))
        model.add(AttentionWithContext())
        # next add a Dense layer (for classification/regression) or whatever...
    """

    def __init__(self,
                 W_regularizer=None, u_regularizer=None, b_regularizer=None,
                 W_constraint=None, u_constraint=None, b_constraint=None,
                 bias=True, **kwargs):

        self.supports_masking = True
        self.init = initializers.get('glorot_uniform')

        self.W_regularizer = regularizers.get(W_regularizer)
        self.u_regularizer = regularizers.get(u_regularizer)
        self.b_regularizer = regularizers.get(b_regularizer)

        self.W_constraint = constraints.get(W_constraint)
        self.u_constraint = constraints.get(u_constraint)
        self.b_constraint = constraints.get(b_constraint)

        self.bias = bias
        super(AttentionWithContext, self).__init__(**kwargs)

    def build(self, input_shape):
        assert len(input_shape) == 3

        self.W = self.add_weight((input_shape[-1], input_shape[-1],),
                                 initializer=self.init,
                                 regularizer=self.W_regularizer,
                                 constraint=self.W_constraint)
        if self.bias:
            self.b = self.add_weight((input_shape[-1],),
                                     initializer='zero',
                                     regularizer=self.b_regularizer,
                                     constraint=self.b_constraint)

        self.u = self.add_weight((input_shape[-1],),
                                 initializer=self.init,
                                 regularizer=self.u_regularizer,
                                 constraint=self.u_constraint)

        super(AttentionWithContext, self).build(input_shape)

    def compute_mask(self, input, input_mask=None):
        # do not pass the mask to the next layers
        return None

    def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)
        ait = dot_product(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1)

    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[-1]

class MySentences(object):
    def __init__(self, dirname):
        self.dirname = dirname

    def __iter__(self):
        for fname in os.listdir(self.dirname):
            print(str(fname) + " is doing")
            for line in open(os.path.join(self.dirname, fname)):
                yield line.split()

def shuffleData(datas, labels, tokenflag=False):
    all_text_seq = []
    seq = []
    num = 1
    if (tokenflag):
        sentences = MySentences('data/')  # a memory-friendly iterator
        model = gensim.models.Word2Vec(sentences, min_count=0, size=50)
        model.save('myWord2VecModel.model')

    gensim_model = gensim.models.Word2Vec.load('myWord2VecModel.model')
    word2idx = {"_PAD": 0}

    vocab_list = [(k, gensim_model.wv[k]) for k, v in gensim_model.wv.vocab.items()]

    embeddings_matrix = np.zeros((len(gensim_model.wv.vocab.items()) + 1, gensim_model.vector_size))
    for i in range(len(vocab_list)):
        word = vocab_list[i][0]
        word2idx[word] = i + 1
        embeddings_matrix[i + 1] = vocab_list[i][1]
    vocab_list = list(gensim_model.wv.vocab.keys())
    word_index = {word: index for index, word in enumerate(vocab_list)}
    for line in datas:
        lineWords = line.split(" ")
        for index in range(len(lineWords)-1):
            if num < sequence_length:
                if lineWords[index] in word_index and lineWords[index]!=lineWords[index+1] :
                    seq.append(word_index[lineWords[index]])
                    num += 1
                else:
                    pass
        all_text_seq.append(seq)
        seq = []
        num =1
    print(all_text_seq)

    all_text_test = pad_sequences(all_text_seq, maxlen=sequence_length, padding='post')
    # print(all_text_test)
    np.random.seed(1226)
    del all_text_seq
    shuffle_indices = np.random.permutation(np.arange(len(labels)))
    x_shuffled = np.array(all_text_test)[shuffle_indices.astype(int)]
    y_shuffled = np.array(labels)[shuffle_indices.astype(int)]

    dev_sample_index = -1 * int(dev_sample_percentage * len(labels))
    train, val = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
    train_labels, val_labels = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]
    del datas, labels, x_shuffled, y_shuffled, all_text_test
    with open("test_data.txt","w") as f:
        for line in val:
            f.write(line)
            f.write("\n")
    with open("test_label.txt","w") as f:
        for line in val_labels:
            f.write(line)
            f.write("\n")
    return train, val, train_labels, val_labels,embeddings_matrix

def lstm(embeddings_matrix):
    # embedding_matrix = gensim_model.wv.vectors
    model = Sequential()
    model.add(Masking(input_shape=(sequence_length,)))

    # model.add(Flatten())
    model.add(Embedding(
        input_dim=embeddings_matrix.shape[0],
        output_dim=embeddings_matrix.shape[1],
        input_length=sequence_length,
        weights=[embeddings_matrix],
        trainable=False))

    model.add(Bidirectional(GRU(128, kernel_regularizer=regularizers.l2(0.01), recurrent_regularizer=regularizers.l2(0.01),
                                 recurrent_dropout=0.3,return_sequences=False)))
    model.add(Dropout(0.25))
    # model.add(AttentionWithContext())
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.4))
    model.add(Dense(8))
    model.add(BatchNormalization())
    model.add(keras.layers.core.Activation('softmax'))
    model.compile(loss="sparse_categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
    return model

def main():
    datas = []
    labels = []
    for file in files:
        with open("data/"+file + ".txt", 'r') as f:
            for line in f.readlines():
                datas.append(line.strip("\n"))
                labels.append(classes[file])

    # print(len(datas[100]))
    shuffleData(datas,labels,tokenflag=False)
    train, val, train_labels, val_labels,embeddings_matrix = shuffleData(datas, labels, tokenflag=False)

    # train = train.reshape(len(train), sequence_length, 1)
    # val = val.reshape(len(val), sequence_length, 1)
    model = lstm(embeddings_matrix)
    early_stopping=EarlyStopping(monitor='val_loss',
                              patience=10, verbose=0, mode='auto')
    # reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=10, mode='auto')
    history = model.fit(train, train_labels, batch_size=1, epochs=num_step, shuffle=True,
                        validation_data=(val, val_labels),callbacks=[early_stopping])

    loss, acc = model.evaluate(val, val_labels, batch_size=batch_size)

    model.save("my2VecModelmodiall.h5")
    print("\nTest score: %.3f, accuracy: %.3f" % (loss, acc))
    with open("./log/alltest.log", "a+") as f:
        f.write("-----------------------------new start------------------------------")
        f.write("\nval_loss-----------------------\n")
        f.write(str(history.history['val_loss']))
        f.write("\nval_acc-----------------------\n")
        f.write(str(history.history['val_accuracy']))
        f.write("\nloss--------------------------\n")
        f.write(str(history.history['loss']))
        f.write("\nacc---------------------------\n")
        f.write(str(history.history['accuracy']))

        f.write("\nTest loss:" + str(round(loss, 3)) + ", accuracy:" + str(round(acc, 3)))
    model.summary()
    from keras.utils import plot_model
    plot_model(model, to_file='graphics_model.jpg')

if __name__ == '__main__':
    main()