"""
Tafra: the innards of a dataframe

Author
------
Derrick W. Turk
David S. Fulford

Notes
-----
Created on April 25, 2020
"""
__all__ = ['Tafra']

import sys
import re
import warnings
import pprint as pprint
from datetime import date, datetime
from itertools import chain
import dataclasses as dc

import numpy as np
from .protocol import Series, DataFrame  # just for mypy...

from typing import (Any, Callable, Dict, Mapping, List, Tuple, Optional, Union as _Union, Sequence,
                    NamedTuple, Sized, Iterable, Iterator, Type, KeysView, ValuesView, ItemsView)
from typing import cast
from typing_extensions import Protocol

from .formatter import ObjectFormatter

object_formatter = ObjectFormatter()
object_formatter['Decimal'] = lambda x: x.astype(float)

TAFRA_TYPE: Dict[str, Callable[[np.ndarray], np.ndarray]] = {
    'int': lambda x: x.astype(int),
    'float': lambda x: x.astype(float),
    'decimal': lambda x: x.astype(float),
    'bool': lambda x: x.astype(bool),
    'str': lambda x: x.astype(str),
    'date': lambda x: x.astype('datetime64'),
    'object': lambda x: x.astype(object),
}

NAMEDTUPLE_TYPE: Dict[str, Type[Any]] = {
    'int': int,
    'float': float,
    'decimal': float,
    'bool': bool,
    'str': str,
    'date': datetime,
    'object': str,
}

RECORD_TYPE: Dict[str, Callable[[Any], Any]] = {
    'int': lambda x: int(x),
    'float': lambda x: float(x),
    'bool': lambda x: bool(x),
    'str': lambda x: str(x),
    'date': lambda x: x.strftime(r'%Y-%m-%d'),
    'object': lambda x: str(x),
}

@dc.dataclass
class Tafra:
    """
    The innards of a dataframe.

    Parameters
    ----------
        _data: Dict[str, np.ndarray]
            The data used to build the :class:`Tafra`.

        _dtypes: Dict[str, str] = {}
            The dtypes of the ``_data``. If not given, will be inferred from
            the ``_data``.

    """

    Scalar = _Union[str, int, float, bool]
    _Mapping = _Union[
        Mapping[str, Any],
        Mapping[int, Any],
        Mapping[float, Any],
        Mapping[bool, Any],
    ]
    InitVar = _Union[
        Tuple[Scalar, Any],
        _Mapping,
        Iterator[Tuple[Scalar, Any]],
        Iterator[_Mapping]
    ]

    data: dc.InitVar[InitVar]
    dtypes: dc.InitVar[InitVar] = None

    _data: Dict[str, np.ndarray] = dc.field(init=False)
    _dtypes: Dict[str, str] = dc.field(init=False)

    def __post_init__(self, data: InitVar, dtypes: Optional[InitVar]) -> None:
        # TODO: enable this?
        # if isinstance(self._data, DataFrame):
        #     tf = self.from_dataframe(df=self._data)
        #     self._data = tf._data
        #     self._dtypes = tf._dtypes
        #     self._rows = tf._rows
        #     return

        rows: Optional[int] = None
        self._data = cast(Dict[str, np.ndarray], self._check_initvar(data))
        if dtypes is None or isinstance(dtypes, property):
            self._dtypes = {}
        else:
            self._dtypes = cast(Dict[str, str], self._check_initvar(dtypes))

        for column, value in self._data.items():
            value, modified = self._validate_value(value, check_rows=False)
            if modified:
                self._data[column] = value

            if rows is None:
                rows = len(value)
            elif rows != len(value):
                raise ValueError('`Tafra` must have consistent row counts.')

        if rows is None:
            raise ValueError('No data provided in constructor statement.')

        self._update_rows()
        self._coalesce_dtypes()
        self.update_dtypes_inplace(self._dtypes)

    def _check_initvar(self, values: InitVar) -> Dict[str, _Union[str, np.ndarray]]:
        if isinstance(values, Iterator):
            head = next(values)
            if isinstance(head, Dict):
                for _dict in values:
                    head.update(cast(Mapping[Any, Any], _dict))
                _values = head
            elif isinstance(head, Sequence) and len(head) == 2:
                # maybe an iterator of mapping? Cast and try it.
                _values = dict(chain(
                    cast(Iterable[Tuple[str, Any]], [head]),
                    cast(Iterator[Tuple[str, Any]], values)))
        else:
            _values = cast(Dict[str, _Union[str, np.ndarray]], values)

        if not isinstance(_values, Dict):
            raise TypeError(f'Must supply a `Dict` or `Mapping`, got `{type(_values)}`')

        # must copy first as mutating the dict changes next(iterator)
        columns = [c for c in _values.keys() if not isinstance(c, str)]
        for column in columns:
            _values[str(column)] = _values.pop(column)

        return _values

    def __getitem__(
            self,
            item: _Union[str, int, slice, Sequence[_Union[str, int, bool]], np.ndarray]) -> Any:
        # return type is actually _Union[np.ndarray, 'Tafra'] but mypy goes insane
        if isinstance(item, str):
            return self._data[item]

        elif isinstance(item, int):
            return self._slice(item)

        elif isinstance(item, slice):
            return self._slice(item)

        elif isinstance(item, np.ndarray):
            return self._ndindex(item)

        elif isinstance(item, Sequence):
            if isinstance(item[0], str):
                return self.select(cast(Sequence[str], item))
            else:
                return self._index(item)

        else:
            raise TypeError(f'Type {type(item)} not supported.')

    def __setitem__(self, item: str, value: _Union[np.ndarray, Sequence[Any], Any]) -> None:
        _value, _ = self._validate_value(value)
        self._data[item] = _value
        self._dtypes[item] = self._format_type(_value.dtype)

    def __len__(self) -> int:
        assert self._data is not None, 'Cannot construct a Tafra with no data.'
        return self._rows

    def __iter__(self) -> Iterator['Tafra']:
        for i in range(self._rows):
            yield self[i]

    def iterrows(self) -> Iterator['Tafra']:
        """
        Yield rows as :class:`Tafra`.

        Returns
        -------
            tafras: Iterator[Tafra]
                An iterator of :class:`Tafra`.
        """
        yield from self.__iter__()

    def itertuples(self, name: str = 'Tafra') -> Iterator[Tuple[Any, ...]]:
        """
        Yield rows as :class:`NamedTuple`.

        Parameters
        ----------
            name: str = 'Tafra'
                The name for the :class:`NamedTuple`.

        Returns
        -------
            tuples: Iterator[NamedTuple[Any, ...]]
                An iterator of :class:`NamedTuple`.
        """
        TafraNT = NamedTuple(name, **{  # type: ignore
            column: NAMEDTUPLE_TYPE[dtype] for column, dtype in self._dtypes.items()})

        for tf in self.__iter__():
            yield TafraNT(*(value.item() for value in tf._data.values()))

    def itercols(self) -> Iterator[Tuple[str, np.ndarray]]:
        """
        Yield columns as :class:`Tuple[str, np.ndarray]` .

        Returns
        -------
            tuples: Iterator[Tuple[str, np.ndarray]]
                An iterator of :class:`Tafra`.
        """
        for column, value in self._data.items():
            yield column, value

    def __str__(self) -> str:
        return self.__repr__()

    def __repr__(self) -> str:
        return f'Tafra(data={self._data}, dtypes={self._dtypes}, rows={self._rows})'

    def _update_rows(self) -> None:
        iter_values = iter(self._data.values())
        self._rows = len(next(iter_values))
        if not all(len(v) == self._rows for v in iter_values):
            raise TypeError('Uneven length of data.')

    def _slice(self, _slice: _Union[int, slice]) -> 'Tafra':
        """
        Use slice object to slice np.ndarray.

        Parameters
        ----------
            _slice: Union[int, slice]
                The ``slice`` object.

        Returns
        -------
            tafra: Tafra
                The sliced :class:`Tafra`.
        """
        return Tafra(
            {column: value[_slice]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _index(self, index: Sequence[_Union[str, int, bool]]) -> 'Tafra':
        """
        Use numpy indexing to slice the data :class:`np.ndarray`.

        Parameters
        ----------
            index: Sequence[Union[str, int, bool]]

        """
        # TODO: just let numpy handle errors?
        # head = index[0]

        # if isinstance(index[0], str):
        #     if all(isinstance(item, str) for item in index):
                # return self.select(cast(Sequence[str], index))
        #     else:
        #         raise IndexError('Index `Sequence` does not contain all `str`.')

        # elif (isinstance(head, bool)
        #         and not(all(isinstance(item, bool) for item in index))):
        #     raise IndexError('Index `Sequence` does not contain all `bool`.')

        # elif (isinstance(head, int)   # type: ignore
        #         and not(all(isinstance(item, int) for item in index))):
        #     raise IndexError('Index `Sequence` does not contain all `int`.')

        return Tafra(
            {column: value[index]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _ndindex(self, index: np.ndarray) -> 'Tafra':
        """
        Use :class:`numpy.ndarray` indexing to slice the data :class:`np.ndarray`.

        Parameters
        ----------
            index: np.ndarray

        """
        # TODO: just let numpy handle errors?
        # if not (index.dtype == np.int or index.dtype == np.bool):
        #     raise IndexError(
        #         f'Index array is of dtype={index.dtype}, '
        #         'must subtype of `np.int` or `np.bool`.')
        if index.ndim != 1:
            raise IndexError(f'Indexing np.ndarray must ndim == 1, got ndim == {index.ndim}')

        return Tafra(
            {column: value[index]
                for column, value in self._data.items()},
            self._dtypes
        )

    def _repr_pretty_(self, p: 'IPython.lib.pretty.RepresentationPrinter',  # type: ignore # noqa
                      cycle: bool) -> None:
        """
        A dunder method for IPython to pretty print.

        Parameters
        ----------
            p: IPython.lib.pretty.RepresentationPrinter
                IPython provides this class to handle the object representation.

            cycle: bool
                IPython has detected an infinite loop. Print an alternative
                represenation and return.

        Returns
        -------
            None
                Calls p.text and returns.
        """
        if cycle:
            p.text('Tafra(...)')
        else:
            p.text(self._pretty_format(lambda s: ' ' + pprint.pformat(s, indent=1)[1:].strip()))

    def _repr_html_(self) -> str:
        """
        a dunder moethod for Jupyter Notebook to print HTML.
        """
        return self.to_html()

    def _pretty_format(self, formatter: Callable[[object], str]) -> str:
        """
        Format _data and _dtypes for pretty printing.

        Parameters
        ----------
            formatter: Callabke[[object], str]
                A formatter that operates on the _data and _dtypes :class:`dict`.

        Returns
        -------
            string: str
                The formatted string for printing.
        """
        PATTERN = r'(, dtype=[a-z]+)(?=\))'

        return '\n'.join([
            'Tafra(data = {',
            f'{re.sub(PATTERN, "", formatter(self._data))},',
            'dtypes = {',
            f'{re.sub(PATTERN, "", formatter(self._dtypes))},',
            f'rows = {self._rows})'
        ])

    def pformat(self, indent: int = 1, width: int = 80, depth: Optional[int] = None,
                compact: bool = False) -> str:
        """
        Format for pretty printing. Parameters are passed to :class:`pprint.PrettyPrinter`.

        Parameters
        ----------
            indent: int
                Number of spaces to indent for each level of nesting.

            width: int
                Attempted maximum number of columns in the output.

            depth: Optional[int]
                The maximum depth to print out nested structures.

            compact: bool
                If true, several items will be combined in one line.

        Returns
        -------
            formatted string: str
                A formatted string for pretty printing.
        """
        return self._pretty_format(
            lambda s: indent * ' ' + pprint.pformat(
                s, indent, width, depth, compact=compact)[1:].strip())

    def pprint(self, indent: int = 1, width: int = 80, depth: Optional[int] = None,
               compact: bool = False) -> None:
        """
        Pretty print. Parameters are passed to :class:`pprint.PrettyPrinter`.

        Parameters
        ----------
            indent: int
                Number of spaces to indent for each level of nesting.

            width: int
                Attempted maximum number of columns in the output.

            depth: Optional[int]
                The maximum depth to print out nested structures.

            compact: bool
                If true, several items will be combined in one line.

        Returns
        -------
            None: None
        """
        print(self.pformat(indent, width, depth, compact=compact))

    @staticmethod
    def _html_thead(columns: Iterable[Any]) -> str:
        """
        Construct the table head of the HTML representation.

        Parameters
        ----------
            columns: Iterable[Any]
                An iterable of items with defined func:`__repr__` methods.

        Returns
        -------
            HTML: str
                The HTML table head.
        """
        return '
\n
\n{th}\n
\n
' \
            .format(th='\n'.join(f'{c}	' for c in columns))

    @staticmethod
    def _html_tr(row: Iterable[Any]) -> str:
        """
        Construct each table row of the HTML representation.

        Parameters
        ----------
            row: Iterable[Any]
                An iterable of items with defined func:`__repr__` methods.

        Returns
        -------
            HTML: str
                The HTML table row.
        """
        return '
\n{td}\n
' \
            .format(td='\n'.join(f'{td}	' for td in row))

    @staticmethod
    def _html_tbody(tr: Iterable[str]) -> str:
        """
        Construct the table body of the HTML representation.

        Parameters
        ----------
            tr: Iterable[str]
                An iterable of HTML table rows.

        Returns
        -------
            HTML: str
                The HTML table body.
        """
        return '
\n{tr}\n
' \
            .format(tr='\n'.join(tr))

    @staticmethod
    def _html_table(thead: str, tbody: str) -> str:
        """
        Construct the final table of the HTML representation.

        Parameters
        ----------
            thead: str
                An HTML representation of the table head.

            tbody: str
                An HTML representation of the table body.

        Returns
        -------
            HTML: str
                The HTML table.
        """
        return f'
\n{thead}\n{tbody}\n
'

    def to_html(self, n: int = 20) -> str:
        """
        Construct an HTML table representation of the :class:`Tafra` data.

        Parameters
        ----------
            n: int = 20
                Number of items to print.

        Returns
        -------
            HTML: str
                The HTML table representation.
        """
        thead = self._html_thead(chain([''], self._data.keys()))
        tr = chain(
            [self._html_tr(chain(
                ['dtype'],
                self._dtypes.values()
            ))],
            (self._html_tr(chain(
                [i],
                (v[i] for v in self._data.values())
            ))
                for i in range(min(n, self._rows)))
        )
        tbody = self._html_tbody(tr)
        return self._html_table(thead, tbody)

    def _validate_value(self, value: _Union[np.ndarray, Sequence[Any], Any],
                        check_rows: bool = True) -> Tuple[np.ndarray, bool]:
        """
        Validate values as an :class:`np.ndarray` of equal length to
        :attr:`rows` before assignment. Will attempt to create a
        :class:`np.ndarray` if ``value`` is not one already, and will check
        that :attr`np.ndarray.ndim` is ``1``.
        If :attr:`np.ndarray.ndim` ``> 1`` it will attempt :meth:`np.squeeze`
        on ``value``.

        Parameters
        ----------
            value: Union[np.ndarray, Sequence[Any], Any]
                The value to be assigned.

        Returns
        -------
            value: np.ndarray
                The validated value.
        """
        modified = False
        rows = self._rows if check_rows else 1

        if isinstance(value, np.ndarray) and len(value.shape) == 0:
            value = np.full(rows, value.item())
            modified = True

        elif isinstance(value, str) or not isinstance(value, Sized):
            value = np.full(rows, value)
            modified = True

        elif isinstance(value, Iterable):
            value = np.array(value)
            modified = True

        assert isinstance(value, np.ndarray), '`Tafra` only supports assigning `ndarray`.'

        if value.ndim > 1:
            sq_value = value.squeeze()
            if sq_value.ndim > 1:
                raise ValueError('`ndarray` or `np.squeeze(ndarray)` must have ndim == 1.')
            elif sq_value.ndim == 1:
                # if value was a single item, squeeze returns zero length item
                warnings.warn('`np.squeeze(ndarray)` applied to set ndim == 1.')
                warnings.resetwarnings()
                value = sq_value

            modified = True

        # special parsing of various object types
        if value.dtype == np.dtype(object):
            type_name = type(value[0]).__name__
            if type_name in object_formatter:
                value = object_formatter[type_name](value)
                modified = True

        assert value.ndim >= 1, '`Tafra` only supports assigning ndim >= 1.'

        if check_rows and len(value) != rows:
            raise ValueError(
                '`Tafra` must have consistent row counts.\n'
                f'This `Tafra` has {rows} rows. Assigned np.ndarray has {len(value)} rows.')

        return value, modified

    def _validate_columns(self, columns: Iterable[str]) -> None:
        """
        Validate that the column name(s) exists in :attr:`_data`.

        Parameters
        ----------
            columns: Iterable[str]
                The column names to validate.

        Returns
        -------
            None: None
        """
        for column in columns:
            if column not in self._data.keys():
                raise ValueError(f'Column {column} does not exist in `tafra`.')

    def _validate_dtypes(self, dtypes: Dict[str, Any]) -> Dict[str, str]:
        """
        Validate that the dtypes are defined as valid dtypes in
        :const:`TAFRA_TYPE` and the columns exists in :attr:`_data`.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to validate.

        Returns
        -------
            dtypes: Dict[str, str]
                The validated types.
        """
        msg = ''
        _dtypes = {}

        self._validate_columns(dtypes.keys())

        for column, _dtype in dtypes.items():
            _dtypes[column] = self._format_type(_dtype)
            if _dtypes[column] not in TAFRA_TYPE:
                msg += f'`{_dtypes[column]}` is not a valid dtype for `{column}.`\n'

        if len(msg) > 0:
            # should be KeyError value Python 3.7.x has a bug with '\n'
            raise ValueError(msg)

        return _dtypes

    @staticmethod
    def _format_type(dtype: Any) -> str:
        """
        Parse a dtype into the internally used string representation.

        Parameters
        ----------
            dtype: Any
                The dtype to parse.

        Returns
        -------
            dtype: str
                The parsed dtype.
        """
        _dtype = str(dtype)

        if 'int' in _dtype:
            return 'int'
        if 'float' in _dtype:
            return 'float'
        if 'bool' in _dtype:
            return 'bool'
        if 'str' in _dtype:
            return 'str'
        if '<U' in _dtype:
            return 'str'
        if 'date' in _dtype:
            return 'date'
        if '<M' in _dtype:
            return 'date'
        if 'object' in _dtype:
            return 'object'
        if 'O' in _dtype:
            return 'object'

        return _dtype

    @staticmethod
    def _apply_type(dtype: str, array: np.ndarray) -> np.ndarray:
        """
        Apply the dtype to the :class:`np.ndarray`.

        Parameters
        ----------
            dtype: str
                The parsed dtype.

            array: np.ndaray
                The array to which the dtype is applied.

        Returns
        -------
            array:
                The array with updated dtype.
        """
        return TAFRA_TYPE[dtype](array)

    @classmethod
    def from_series(cls, s: Series, dtype: Optional[str] = None) -> 'Tafra':
        """
        Construct a :class:`Tafra` from a :class:`pd.DataFrame`.

        Parameters
        ----------
            df: pd.DataFrame
                The dataframe used to build the :class:`Tafra`.

            dtypes: Optional[Dict[str, str]] = None
                The dtypes of the columns.

        Returns
        -------
            tafra: Tafra
                The constructed :class:`Tafra`.
        """
        if dtype is None:
            dtype = s.dtype
        dtypes = {s.name: cls._format_type(dtype)}

        return cls(
            {s.name: cls._apply_type(dtypes[s.name], s.values)},
            dtypes
        )

    @classmethod
    def from_dataframe(cls, df: DataFrame, dtypes: Optional[Dict[str, str]] = None) -> 'Tafra':
        """
        Construct a :class:`Tafra` from a :class:`pd.DataFrame`.

        Parameters
        ----------
            df: pd.DataFrame
                The dataframe used to build the :class:`Tafra`.

            dtypes: Optional[Dict[str, str]] = None
                The dtypes of the columns.

        Returns
        -------
            tafra: Tafra
                The constructed :class:`Tafra`.
        """
        if dtypes is None:
            dtypes = {c: t for c, t in zip(df.columns, df.dtypes)}
        dtypes = {c: cls._format_type(t) for c, t in dtypes.items()}

        return cls(
            {c: cls._apply_type(dtypes[c], df[c].values) for c in df.columns},
            {c: dtypes[c] for c in df.columns}
        )

    @classmethod
    def as_tafra(cls, maybe_tafra: _Union['Tafra', DataFrame, Series, Dict[str, Any], Any]
                 ) -> Optional['Tafra']:
        """
        Returns the unmodified `tafra`` if already a `Tafra`, else construct
        a `Tafra` from known types or subtypes of :class:`DataFrame` or `dict`.
        Structural subtypes of :class:`DataFrame` or :class:`Series` are also
        valid, as are classes that have ``cls.__name__ == 'DataFrame'`` or
        ``cls.__name__ == 'Series'``.

        Parameters
        ----------
            maybe_tafra: Union['tafra', DataFrame]
                The object to ensure is a :class:`Tafra`.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra`, or None is ``maybe_tafra`` is an unknown
                type.
        """
        if isinstance(maybe_tafra, Tafra):
            return maybe_tafra

        elif isinstance(maybe_tafra, Series):
            return cls.from_series(maybe_tafra)  # pragma: no cover

        elif type(maybe_tafra).__name__ == 'Series':
            return cls.from_series(cast(Series, maybe_tafra))  # pragma: no cover

        elif isinstance(maybe_tafra, DataFrame):
            return cls.from_dataframe(maybe_tafra)  # pragma: no cover

        elif type(maybe_tafra).__name__ == 'DataFrame':
            return cls.from_dataframe(cast(DataFrame, maybe_tafra))  # pragma: no cover

        elif isinstance(maybe_tafra, dict):
            return cls(maybe_tafra)

        raise TypeError(f'Unknown type `{type(maybe_tafra)}` for conversion to `Tafra`')

    @property
    def columns(self) -> Tuple[str, ...]:
        """
        The names of the columns. Equivalent to `Tafra`.keys().

        Returns
        -------
            columns: Tuple[str, ...]
                The column names.
        """
        return tuple(self._data.keys())

    @columns.setter
    def columns(self, value: Any) -> None:
        raise ValueError('Assignment to `columns` is forbidden.')

    @property
    def rows(self) -> int:
        """
        The number of rows of the first item in :attr:`data`. The :func:`len()`
        of all items have been previously validated.

        Returns
        -------
            rows: int
                The number of rows of the :class:`Tafra`.
        """
        return self.__len__()

    @rows.setter
    def rows(self, value: Any) -> None:
        raise ValueError('Assignment to `rows` is forbidden.')

    @property  # type: ignore
    def data(self) -> Dict[str, np.ndarray]:
        """
        The :class:`Tafra` data.

        Returns
        -------
            data: Dict[str, np.ndarray]
                The data.
        """
        return self._data

    @data.setter
    def data(self, value: Any) -> None:
        raise ValueError('Assignment to `data` is forbidden.')

    @property  # type: ignore
    def dtypes(self) -> Dict[str, str]:
        """
        The :class:`Tafra` dtypes.

        Returns
        -------
            dtypes: Dict[str, str]
                The dtypes.
        """
        return self._dtypes

    @dtypes.setter
    def dtypes(self, value: Any) -> None:
        raise ValueError('Assignment to `dtypes` is forbidden.')

    @property
    def size(self) -> int:
        """
        The :class:`Tafra` size.

        Returns
        -------
            size: int
                The size.
        """
        return self.rows * len(self.columns)

    @size.setter
    def size(self, value: Any) -> None:
        raise ValueError('Assignment to `size` is forbidden.')

    @property
    def ndim(self) -> int:
        """
        The :class:`Tafra` number of dimensions.

        Returns
        -------
            ndim: int
                The number of dimensions.
        """
        return max(2, len(self.columns))

    @ndim.setter
    def ndim(self, value: Any) -> None:
        raise ValueError('Assignment to `ndim` is forbidden.')

    @property
    def shape(self) -> Tuple[int, int]:
        """
        The :class:`Tafra` shape.

        Returns
        -------
            shape: int
                The shape.
        """
        return self.rows, len(self.columns)

    @shape.setter
    def shape(self, value: Any) -> None:
        raise ValueError('Assignment to `shape` is forbidden.')

    def row_map(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Iterator[Any]:
        """
        Map a function over rows. To apply to specific columns, use
        :meth:`select` first. The function must operate on :class:`Tafra`.

        Parameters
        ----------
            fn: Callable[..., Any]
                The function to map.

            *args: Any
                Additional positional arguments to ``fn``.

            **kwargs: Any
                Additional keyword arguments to ``fn``.

        Returns
        -------
            iter_tf: Iterator[Any]
                An iterator to map the function.
        """
        for tf in self.__iter__():
            yield fn(tf, *args, **kwargs)

    def col_map(self, fn: Callable[..., Any], name: bool = True,
                *args: Any, **kwargs: Any) -> Iterator[_Union[Any, Tuple[str, Any]]]:
        """
        Map a function over columns. To apply to specific columns, use
        :meth:`select` first. The function must operate on :class:`Tuple[str, np.ndarray]`.

        Parameters
        ----------
            fn: Callable[..., Any]
                The function to map.

            name: bool
                Return the column name.

            *args: Any
                Additional positional arguments to ``fn``.

            **kwargs: Any
                Additional keyword arguments to ``fn``.

        Returns
        -------
            iter_tf: Iterator[Any]
                An iterator to map the function.
        """
        if name:
            for column, value in self.itercols():
                yield column, fn(value, *args, **kwargs)
            return

        for column, value in self.itercols():
            yield fn(value, *args, **kwargs)

    def head(self, n: int = 5) -> None:  # pragma: no cover
        """
        Display the head of the :class:`Tafra`.

        Parameters
        ----------
            n: int = 5
                The number of rows to display.

        Returns
        -------
            None: None
        """
        if _in_notebook():
            try:
                from IPython.display import display  # type: ignore # noqa
                display(self[:min(self._rows, n)])
                return
            except Exception as e:
                pass

        print(self[:min(self._rows, n)].pformat())

    def select(self, columns: Sequence[str]) -> 'Tafra':
        """
        Use column names to slice the :class:`Tafra` columns analogous to
        SQL SELECT.
        This does not copy the data. Call :meth:`copy` to obtain a copy of
        the sliced data.

        Parameters
        ----------
            columns: Sequence[str]
                The column names to slice from the :class:`Tafra`.

        Returns
        -------
            tafra: Tafra
                the :class:`Tafra` with the sliced columns.
        """
        self._validate_columns(columns)

        return Tafra(
            {column: value for column, value in self._data.items()
             if column in columns},
            {column: value for column, value in self._dtypes.items()
             if column in columns},
        )

    def keys(self) -> KeysView[str]:
        """
        Return the keys of :attr:`data`, i.e. like :meth:`dict.keys()`.

        Returns
        -------
            data keys: KeysView[str]
                The keys of the data property.
        """
        return self._data.keys()

    def values(self) -> ValuesView[np.ndarray]:
        """
        Return the values of :attr:`data`, i.e. like :meth:`dict.values()`.

        Returns
        -------
            data values: ValuesView[np.ndarray]
                The values of the data property.
        """
        return self._data.values()

    def items(self) -> ItemsView[str, np.ndarray]:
        """
        Return the items of :attr:`data`, i.e. like :meth:`dict.items()`.

        Returns
        -------
            items: ItemsView[str, np.ndarray]
                The data items.
        """
        return self._data.items()

    def get(self, key: str, default: Any = None) -> Any:
        """
        Return from the :meth:`get` function of :attr:`data`, i.e. like
        :meth:`dict.get()`.

        Parameters
        ----------
            key: str
                The key value in the data property.

            default: Any
                The default to return if the key does not exist.

        Returns
        -------
            value: Any
                The value for the key, or the default if the key does not
                exist.
        """
        return self._data.get(key, default)

    def update(self, other: 'Tafra') -> 'Tafra':
        """
        Update the data and dtypes of this :class:`Tafra` with another
        :class:`Tafra`. Length of rows must match, while data of different
        ``dtype`` will overwrite.

        Parameters
        ----------
            other: Tafra
                The other :class:`Tafra` from which to update.

        Returns
        -------
            None: None
        """
        tafra = self.copy()
        rows = tafra._rows

        for column, value in other._data.items():
            if len(value) != rows:
                raise ValueError(
                    'Other `Tafra` must have consistent row count. '
                    f'This `Tafra` has {rows} rows, other `Tafra` has {len(value)} rows.')
            tafra._data[column] = value

        tafra.update_dtypes_inplace(other._dtypes)
        return tafra

    def update_inplace(self, other: 'Tafra') -> None:
        """
        Update the data and dtypes of this :class:`Tafra` with another
        :class:`Tafra`. Length of rows must match, while data of different
        ``dtype`` will overwrite.

        Parameters
        ----------
            other: Tafra
                The other :class:`Tafra` from which to update.

        Returns
        -------
            None: None
        """
        rows = self._rows

        for column, value in other._data.items():
            if len(value) != rows:
                raise ValueError(
                    'Other `Tafra` must have consistent row count. '
                    f'This `Tafra` has {rows} rows, other `Tafra` has {len(value)} rows.')
            self._data[column] = value

        self.update_dtypes_inplace(other._dtypes)

    def _coalesce_dtypes(self) -> None:
        """
        Update :attr:`_dtypes` with missing keys that exist in :attr:`_data`.
        **Must be called if :attr:`data` or :attr`:_data` is directly
        modified!**

        Returns
        -------
            None: None
        """
        for column in self._data.keys():
            if column not in self._dtypes:
                self._dtypes[column] = self._format_type(self._data[column].dtype)

    def update_dtypes(self, dtypes: Dict[str, Any]) -> 'Tafra':
        """
        Apply new dtypes.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to update. If ``None``, create from entries in
                :attr:`data`.

        Returns
        -------
            tafra: Optional[Tafra]
                The updated :class:`Tafra`.
        """
        tafra = self.copy()

        tafra._validate_columns(dtypes.keys())
        dtypes = tafra._validate_dtypes(dtypes)
        tafra._dtypes.update(dtypes)

        for column in tafra._dtypes.keys():
            if tafra._format_type(tafra._data[column].dtype) != tafra._dtypes[column]:
                tafra._data[column] = tafra._apply_type(tafra._dtypes[column], tafra._data[column])

        return tafra

    def update_dtypes_inplace(self, dtypes: Dict[str, Any]) -> None:
        """
        Apply new dtypes.

        Parameters
        ----------
            dtypes: Dict[str, Any]
                The dtypes to update. If ``None``, create from entries in
                :attr:`data`.

        Returns
        -------
            tafra: Optional[Tafra]
                The updated :class:`Tafra`.
        """
        self._validate_columns(dtypes.keys())
        dtypes = self._validate_dtypes(dtypes)
        self._dtypes.update(dtypes)

        for column in self._dtypes.keys():
            if self._format_type(self._data[column].dtype) != self._dtypes[column]:
                self._data[column] = self._apply_type(self._dtypes[column], self._data[column])

    def rename(self, renames: Dict[str, str]) -> 'Tafra':
        """
        Rename columns in the :class:`Tafra` from a :class:`dict`.

        Parameters
        ----------
            renames: Dict[str, str]
                The map from current names to new names.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with update names.
        """

        tafra = self.copy()
        for cur, new in renames.items():
            tafra._data[new] = tafra._data.pop(cur)
            tafra._dtypes[new] = tafra._dtypes.pop(cur)
        return tafra

    def rename_inplace(self, renames: Dict[str, str]) -> None:
        """
        In-place version.

        Rename columns in the :class:`Tafra` from a :class:`dict`.

        Parameters
        ----------
            renames: Dict[str, str]
                The map from current names to new names.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with update names.
        """
        self._validate_columns(renames.keys())

        for cur, new in renames.items():
            self._data[new] = self._data.pop(cur)
            self._dtypes[new] = self._dtypes.pop(cur)
        return None

    def delete(self, columns: Iterable[str]) -> 'Tafra':
        """
        Remove a column from :attr:`data` and :attr:`dtypes`.

        Parameters
        ----------
            column: str
                The column to remove.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with the deleted column.
        """
        if isinstance(columns, str):
            columns = [columns]

        self._validate_columns(columns)

        return Tafra(
            {column: value.copy() for column, value in self._data.items()
             if column not in columns},
            {column: value for column, value in self._dtypes.items()
             if column not in columns}
        )

    def delete_inplace(self, columns: Iterable[str]) -> None:
        """
        In-place version.

        Remove a column from :attr:`data` and :attr:`dtypes`.

        Parameters
        ----------
            column: str
                The column to remove.

        Returns
        -------
            tafra: Optional[Tafra]
                The :class:`Tafra` with the deleted column.
        """
        if isinstance(columns, str):
            columns = [columns]

        self._validate_columns(columns)

        for column in columns:
            _ = self._data.pop(column, None)
            _ = self._dtypes.pop(column, None)

    def copy(self, order: str = 'C') -> 'Tafra':
        """
        Create a copy of a :class:`Tafra`.

        Parameters
        ----------
            order: str = 'C' {‘C’, ‘F’, ‘A’, ‘K’}
                Controls the memory layout of the copy. ‘C’ means C-order,
                ‘F’ means F-order, ‘A’ means ‘F’ if a is Fortran contiguous,
                ‘C’ otherwise. ‘K’ means match the layout of a as closely as
                possible.

        Returns
        -------
            tafra: Tafra
                A copied :class:`Tafra`.
        """
        return Tafra(
            {column: value.copy(order=order)
                for column, value in self._data.items()},
            self._dtypes.copy()
        )

    def coalesce(self, column: str,
                 fills: Iterable[_Union[None, str, int, float, bool, np.ndarray]]) -> np.ndarray:
        """
        Fill ``None`` values from ``fills``. Analogous to ``SQL COALESCE`` or
        :meth:`pd.fillna`.

        Parameters
        ----------
            column: str
                The column to coalesce.

            fills: Iterable[Union[str, int, float, bool, np.ndarray]:

        Returns
        -------
            data: np.ndarray
                The coalesced data.
        """
        #TODO: handle dtype?
        iter_fills = iter(fills)
        head = next(iter_fills)

        if column in self._data.keys():
            value = self._data[column].copy()
        else:
            value = np.empty(self._rows, np.asarray(head).dtype)

        for fill in chain([head], iter_fills):
            f = np.atleast_1d(fill)
            where_na = np.full(self._rows, False)
            where_na |= value == np.array([None])
            try:
                where_na |= np.isnan(value)
            except:
                pass
                # pass

            for w in where_na:
                if len(f) == 1:
                    value[where_na] = f
                else:
                    value[where_na] = f[where_na]

        return value

    def coalesce_inplace(self, column: str,
                         fills: Iterable[_Union[None, str, int, float, bool, np.ndarray]]) -> None:
        """
        In-place version.

        Fill ``None`` values from ``fills``. Analogous to ``SQL COALESCE`` or
        :meth:`pd.fillna`.

        Parameters
        ----------
            column: str
                The column to coalesce.

            fills: Iterable[Union[str, int, float, bool, np.ndarray]:

        Returns
        -------
            data: np.ndarray
                The coalesced data.
        """
        self._data[column] = self.coalesce(column, fills)
        self.update_dtypes_inplace({column: self._data[column].dtype})

    def union(self, other: 'Tafra') -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        return Union().apply(self, other)

    def union_inplace(self, other: 'Tafra') -> None:
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        Union().apply_inplace(self, other)

    @staticmethod
    def _cast_record(dtype: str, data: np.ndarray, cast_null: bool) -> Optional[float]:
        """
        Casts needed to generate records for database insert.
        Will cast ``np.nan`` to ``None``. Requires changing ``dtype`` to
        ``object``.

        Parameters
        ----------
            dtype: str
                The dtype of the data value.

            data: np.ndarray
                The data to have its values cast.

            cast_null: bool
                Perform the cast for ``np.nan``

        Returns
        -------
            value: Any
                The cast value.
        """
        value: Any = RECORD_TYPE[dtype](data.item())
        if cast_null and dtype == 'float' and np.isnan(data.item()):
            return None
        return value

    def to_records(self, columns: Optional[Sequence[str]] = None,
                   cast_null: bool = True) -> Iterator[Tuple[Any, ...]]:
        """
        Return a :class:`Iterator` of :class:`Tuple`, each being a record
        (i.e. row) and allowing heterogeneous typing. Useful for e.g. sending
        records back to a database.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

            cast_null: bool
                Cast ``np.nan`` to None. Necessary for :mod:``pyodbc``

        Returns
        -------
            records: Iterator[Tuple[Any, ...]]
        """

        if columns is None:
            columns = self.columns
        else:
            if isinstance(columns, str):
                columns = [columns]
            self._validate_columns(columns)

        for row in range(self._rows):
            yield tuple(self._cast_record(
                self._dtypes[c], self._data[c][[row]],
                cast_null
            ) for c in columns)
        return

    def to_list(self, columns: Optional[Sequence[str]] = None,
                inner: bool = False) -> _Union[List[np.ndarray], List[List[Any]]]:
        """
        Return a list of homogeneously typed columns (as np.ndarrays). If a
        generator is needed, use `Tafra.to_records()`. If `inner == True`
        each column will be cast from :class:`np.ndarray` to a :class:`List`.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

            inner: bool = False
                Cast all :class:`np.ndarray` to :class`List`.

        Returns
        -------
            list: Union[List[np.ndarray], List[List[Any]]]
        """
        if columns is None:
            if inner:
                return [list(v) for v in self._data.values()]
            return list(self._data.values())

        else:
            if isinstance(columns, str):
                columns = [columns]
            if inner:
                return [list(self._data[c]) for c in columns]
            return list(self._data[c] for c in columns)

    def to_array(self, columns: Optional[Sequence[str]] = None) -> np.ndarray:
        """
        Return an object array.

        Parameters
        ----------
            columns: Optional[Sequence[str]] = None
                The columns to extract. If ``None``, extract all columns.

        Returns
        -------
            array: np.ndarray
        """
        if columns is None:
            return np.array(list(self._data.values())).T

        else:
            if isinstance(columns, str):
                columns = [columns]
            return np.array(list(self._data[c] for c in columns)).T

    def group_by(self, group_by: Sequence[str], aggregation: 'InitAggregation' = {},
                 iter_fn: Mapping[str, Callable[[np.ndarray], Any]] = dict()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.GroupBy`.
        """
        return GroupBy(group_by, aggregation, iter_fn).apply(self)

    def transform(self, group_by: Sequence[str], aggregation: 'InitAggregation' = {},
                  iter_fn: Dict[str, Callable[[np.ndarray], Any]] = dict()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.Transform`.
        """
        return Transform(group_by, aggregation, iter_fn).apply(self)

    def iterate_by(self, group_by: Sequence[str]) -> Iterator['GroupDescription']:
        """
        Helper function to implement :class:`tafra.groups.IterateBy`.
        """
        yield from IterateBy(group_by).apply(self)

    def inner_join(self, right: 'Tafra', on: Sequence[Tuple[str, str, str]],
                   select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.InnerJoin`.
        """
        return InnerJoin(on, select).apply(self, right)

    def left_join(self, right: 'Tafra', on: Sequence[Tuple[str, str, str]],
                  select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.LeftJoin`.
        """
        return LeftJoin(on, select).apply(self, right)

    def cross_join(self, right: 'Tafra',
                   select: Sequence[str] = list()) -> 'Tafra':
        """
        Helper function to implement :class:`tafra.groups.CrossJoin`.
        """
        return CrossJoin([], select).apply(self, right)

def _in_notebook() -> bool:  # pragma: no cover
    """
    Checks if running in a Jupyter Notebook.

    Returns
    -------
        in_notebook: bool
    """
    try:
        from IPython import get_ipython  # type: ignore
        if 'IPKernelApp' in get_ipython().config:  # pragma: no cover
            return True
    except Exception as e:
        pass
    return False

# Import here to resolve circular dependency
from .group import (GroupBy, Transform, IterateBy, InnerJoin, LeftJoin, CrossJoin, Union,
                    InitAggregation, GroupDescription)

Tafra.group_by.__doc__ += GroupBy.__doc__  # type: ignore
Tafra.transform.__doc__ += Transform.__doc__  # type: ignore
Tafra.iterate_by.__doc__ += IterateBy.__doc__  # type: ignore
Tafra.inner_join.__doc__ += InnerJoin.__doc__  # type: ignore
Tafra.left_join.__doc__ += LeftJoin.__doc__  # type: ignore
Tafra.cross_join.__doc__ += CrossJoin.__doc__  # type: ignore


import urllib

from bs4 import BeautifulSoup
import CifFile
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
from django.contrib.auth.models import Group

from tardis.tardis_portal.models import ExperimentAuthor

from . import default_settings

class CifHelper(object):

    def __init__(self, cif_url):
        self.cf = CifFile.ReadCif(cif_url)

    def get_cif_file_object(self):
        return self.cf

    def _is_list(self, obj):
        return isinstance(obj, list)

    def as_list(self, obj):
        return obj if self._is_list(obj) else [obj]

class PDBCifHelper(CifHelper):

    def __init__(self, pdb_id):
        super(PDBCifHelper, self).__init__(
            'http://www.pdb.org/pdb/files/' + urllib.quote(pdb_id) + '.cif')
        self.pdb_id = pdb_id

    def __getitem__(self, key):
        return super(PDBCifHelper, self).get_cif_file_object()[
            self.pdb_id][key]

    def get_pdb_id(self):
        return self['_entry.id']

    def get_pdb_title(self):
        return self['_struct.title']

    def get_pdb_url(self):
        return 'http://www.pdb.org/pdb/search/structidSearch.do?structureId=' \
               + urllib.quote(self.get_pdb_id())

    def get_obs_r_value(self):
        return float(self['_refine.ls_R_factor_obs'])

    def get_free_r_value(self):
        return float(self['_refine.ls_R_factor_R_free'])

    def get_resolution(self):
        return float(self['_reflns.d_resolution_high'])

    def get_spacegroup(self):
        return self['_symmetry.space_group_name_H-M']

    def get_unit_cell(self):
        length_a = self['_cell.length_a']
        length_b = self['_cell.length_b']
        length_c = self['_cell.length_c']
        angle_a = self['_cell.angle_alpha']
        angle_b = self['_cell.angle_beta']
        angle_c = self['_cell.angle_gamma']
        return '(a = %s, b = %s, c = %s),(alpha = %s, beta = %s, gamma = %s)' \
               % (length_a, length_b, length_c, angle_a, angle_b, angle_c)

    def get_citations(self):
        ids = self.as_list(self['_citation.id'])
        titles = self.as_list(self['_citation.title'])
        journals = self.as_list(self['_citation.journal_abbrev'])
        volumes = self.as_list(self['_citation.journal_volume'])
        pages_first = self.as_list(self['_citation.page_first'])
        pages_last = self.as_list(self['_citation.page_last'])
        years = self.as_list(self['_citation.year'])
        dois = self.as_list(self['_citation.pdbx_database_id_DOI'])
        author_citation_ids = self.as_list(
            self['_citation_author.citation_id'])
        author_citation_names = self.as_list(self['_citation_author.name'])

        citations = []
        for pub_id, title, journal, volume, page_first, page_last, year, doi \
                in zip(ids, titles, journals, volumes, pages_first, pages_last,
                       years, dois):
            citation = {'_id': pub_id,
                        'title': title,
                        'journal': journal,
                        'volume': volume,
                        'page_first': page_first,
                        'page_last': page_last,
                        'year': year,
                        'doi': doi}

            authors = []
            for auth_pub_id, auth_name in zip(
                    author_citation_ids, author_citation_names):
                if auth_pub_id == pub_id:
                    authors.append(auth_name)
            citation['authors'] = authors
            citations.append(citation)

        return citations

    def get_sequence_info(self):
        try:
            seqs_gen_id = self.as_list(self['_entity_src_gen.entity_id'])
            seqs_gen_org = self.as_list(
                self['_entity_src_gen.pdbx_gene_src_scientific_name'])
            seqs_gen_exp_sys = self.as_list(
                self['_entity_src_gen.pdbx_host_org_scientific_name'])
            seqs_gen = dict(  # R0204:116,12:PDBCifHelper.get_sequence_info: Redefinition of seqs_gen type from list to dict
                zip(seqs_gen_id, zip(seqs_gen_org, seqs_gen_exp_sys)))
        except KeyError:
            seqs_gen = dict()

        try:
            seqs_nat_id = self.as_list(self['_entity_src_nat.entity_id'])
            seqs_nat_org = self.as_list(
                self['_entity_src_nat.pdbx_organism_scientific'])
            seqs_nat = dict(zip(seqs_nat_id, seqs_nat_org))
        except KeyError:
            seqs_nat = dict()

        try:
            seqs_code_id = self.as_list(self['_entity_poly.entity_id'])
            seqs_code = self.as_list(
                self['_entity_poly.pdbx_seq_one_letter_code_can'])
            seqs_code = dict(zip(seqs_code_id, seqs_code))
        except KeyError:
            seqs_code = dict()

        try:
            seqs_name_id = self.as_list(self['_entity_name_com.entity_id'])
            seqs_names = self.as_list(self['_entity_name_com.name'])
            seqs_name = dict(zip(seqs_name_id, seqs_names))
        except KeyError:
            seqs_name = dict()

        sequences = []
        # Only look at the poly entities
        for entity_id in seqs_code.keys():
            seq = {
                'name': '',
                'sequence': seqs_code[entity_id],
                'organism': '',
                'expression_system': ''
            }

            # Protein isolated directly from organism?
            if entity_id in seqs_nat:
                seq['organism'] = seqs_nat[entity_id]
            # Protein isolated using an expression system?
            elif entity_id in seqs_gen:
                seq['organism'], seq['expression_system'] = seqs_gen[entity_id]

            # Has a name?
            if entity_id in seqs_name:
                seq['name'] = seqs_name[entity_id]

            sequences.append(seq)

        return sequences

def check_pdb_status(pdb_id):
    status_page = urllib.urlopen(
        'http://www.rcsb.org/pdb/rest/idStatus?structureId=' +
        urllib.quote(pdb_id.upper())).read()
    soup = BeautifulSoup(status_page)
    if soup.record:
        pdb_status = soup.record['status']
    else:
        pdb_status = 'INVALID'

    return pdb_status

def get_unreleased_pdb_info(pdb_id):
    info_page = urllib.urlopen(
        'http://www.rcsb.org/pdb/rest/getUnreleased?structureId=' +
        urllib.quote(pdb_id.upper())).read()
    soup = BeautifulSoup(info_page)

    info = {}

    info['title'] = soup.record.title.string
    info['authors'] = soup.record.authors.string

    return info

def get_pub_admin_email_addresses():
    return [
        user.email
        for user in Group.objects.get(
            name=getattr(settings, 'PUBLICATION_OWNER_GROUP',
                         default_settings.PUBLICATION_OWNER_GROUP))
            .user_set.all()
        if user.email]

def send_mail_to_authors(publication, subject, message):
    recipients = [author.email for author in
                  ExperimentAuthor.objects.filter(experiment=publication)]
    from_email = getattr(settings, 'PUBLICATION_NOTIFICATION_SENDER_EMAIL',
                         default_settings.PUBLICATION_NOTIFICATION_SENDER_EMAIL)
    msg = EmailMultiAlternatives(
        subject,
        message,
     from_email,
     recipients,
     cc=get_pub_admin_email_addresses())
    msg.send()


__author__ = 'roopalgarg'

import fileinput
from numpy import *
from ParseLabels import ParseLabels
from WriteResults import WriteResults

class ParseTest_BofW:
	testResults=[]

	@staticmethod
	def test(dictWords):
		skippedInitialLine=False
		for line in fileinput.input('../InputFiles/test.tsv'):
			if not skippedInitialLine:
				skippedInitialLine=True
				continue
			line=line.split('\t')
			phraseId=line[0]
			sentenceId=line[1]
			testStr=line[2].strip()
			words=[(word.strip()).lower() for word in testStr.split()]

			sentenceSentimentVec=zeros(ParseLabels.NumLabels)
			for sentiment in range(ParseLabels.NumLabels):
				sentimentProb=0.0
				for word in words:
					if dictWords.has_key(word):
						wordItem=dictWords[word]
						sentimentProb+=wordItem.Sentiments[sentiment]
				sentenceSentimentVec[sentiment]=sentimentProb

			maxSentiment=ParseTest_BofW.GetMaxSentiment(sentenceSentimentVec)
			testResult=[phraseId, maxSentiment]
			ParseTest_BofW.testResults.append(testResult)

		WriteResults.Write(ParseTest_BofW.testResults, "result.csv")

	@staticmethod
	def GetMaxSentiment(sentenceSentimentVec):
		return sentenceSentimentVec.argmax()

Sentiment Analysis on Movie Reviews/Codes/ParseTest_BofW.py


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random

def resample_data(df,frequency):
    df['Date'] = pd.to_datetime(df['Date'])

    df = (df.set_index('Date')
            .resample(frequency).first()
            .reset_index()
           .reindex(columns=df.columns)).dropna()
    return df

def log_return(Data):
    Returns = np.log(Data/Data.shift(1))
    Returns.dropna(inplace=True)
    return Returns

def inverse_log_return(prices, returns):
    returns = np.exp(returns)
    predicted_prices = np.multiply(np.reshape(prices, np.size(prices)), np.reshape(returns, np.size(returns)))
    return predicted_prices

def creating_study_periods(frequencies, frequencies_number_of_samples, frequency_index):
    # Import data
    Data = pd.read_csv('Data_1min.csv', dtype=str)
    Data = resample_data(Data, frequencies[frequency_index])

    # Create datasets
    dates = Data['Date']
    Data.drop('Date', inplace=True, axis=1)

    Data = Data.astype('float64')

    # Log return
    Returns = log_return(Data).to_numpy()

    # Create study peridos
    number_of_study_periods = np.floor(Returns.shape[0]/(frequencies_number_of_samples[frequency_index]/2)).astype(int)-3
    study_periods = np.zeros((2,number_of_study_periods, frequencies_number_of_samples[frequency_index]*2))
    for i in range(number_of_study_periods):
        study_periods[0,i] = Returns[(i*frequencies_number_of_samples[frequency_index]/2).astype(int):\
                                        ((i+4)*frequencies_number_of_samples[frequency_index]/2).astype(int)].flatten()
        study_periods[1,i] = dates.iloc[(i*frequencies_number_of_samples[frequency_index]/2).astype(int):\
                                        ((i+4)*frequencies_number_of_samples[frequency_index]/2).astype(int)].to_numpy().flatten()

    dates = dates[:((number_of_study_periods+3)*frequencies_number_of_samples[frequency_index]/2).astype(int)]
    return number_of_study_periods, study_periods, Data, dates

def visualize_results(mse):
    fig = plt.figure(figsize=(14,8))
    plt.plot(mse)
    plt.title('MSE of models')
    plt.legend(['ARMA', 'LSTM', 'GRU', 'RF', 'Zero return'])
    plt.show()

def visualize_data(Data, Returns):
    Data.plot(subplots=True, legend=False)
    plt.show()
    Returns.plot(subplots=True, legend=False)
    plt.show()
    Returns.hist(bins=100)
    plt.show()
    Returns.boxplot()
    plt.show()
    plt.matshow(Returns.T, interpolation=None, aspect='auto', cmap='Greys')
    plt.show()

def data_split(study_periods):
    train_ratio = 0.5
    valid_ratio = 0.25

    train_size = np.round(np.size(study_periods[0], 1) * train_ratio).astype(int)
    valid_size = np.round(np.size(study_periods[0], 1) * valid_ratio).astype(int)
    test_size = (np.size(study_periods[0], 1) - train_size - valid_size).astype(int)

    return train_size, valid_size, test_size

def append_periods(model_names, frequencies, frequencies_number_of_samples):
    study_periods_predictions = list()
    study_periods_returns = list()
    study_periods_dates = list()
    study_periods_number_of_study_periods = list()
    study_periods_study_periods = list()

    for frequency_index in range(5):
        print(f'Frequency: {frequencies[frequency_index]}')
        number_of_study_periods, study_periods, Data, dates = creating_study_periods(frequencies,\
                                                                                frequencies_number_of_samples,\
                                                                                frequency_index)
        train_size, valid_size, test_size = data_split(study_periods)

        predictions = np.zeros((len(model_names)+1, number_of_study_periods*test_size))

        for model_index in range(len(model_names)):
    #         print(model_names[model_index])
            predictions[model_index] = pd.read_csv('results/'+str(model_names[model_index])+\
                                                    '_predictions_frequency_'+str(frequencies[frequency_index])+\
                                                    '.csv', header=None).dropna(axis='columns').values.flatten()

        predictions[-1] = np.mean(predictions[:-1], axis=0)
        study_periods_predictions.append(predictions)
        study_periods_returns.append(study_periods[0,:,-test_size:].flatten())
        study_periods_dates.append(dates[-predictions.shape[1]:])
        study_periods_number_of_study_periods.append(number_of_study_periods)
        study_periods_study_periods.append(study_periods)
    return  study_periods_predictions, study_periods_returns, study_periods_dates,\
            study_periods_number_of_study_periods, study_periods_study_periods

def np_to_latex_table(data, name, calculate_mean=False, accuracy=2):
    if calculate_mean:
        table = np.zeros((data.shape[0]+1, data.shape[1]+1))
        table[:-1, :-1] = data
        table[-1, :-1] = np.mean(data, axis=0)
        table[:-1, -1] = np.mean(data, axis=1)
        table[-1, -1] = np.mean(np.mean(data))
    else:
        table = data

    if 0.1<np.mean(np.mean(table)):
        fmt = '%1.'+str(accuracy)+'f'
    else:
        fmt = '%2.'+str(accuracy)+'e'

#     print(table)
    np.savetxt(name, table, delimiter=' & ', fmt=fmt, newline=' \\\\\n')


# project/views/api.py

from flask import Blueprint, jsonify
import project.models

api = Blueprint('api', __name__)

@api.route('/api')
def api_test():

    # Get all best friends
    table_data = project.models.best_friends.query.all()

    # Create an object
    obj = {}

    # Create a list in the object
    obj['my_best_friends'] = []

    # For every best friends we found
    for i in table_data:
        # Create an object with the friend information
        friend = {'firstname': i.firstname, 'lastname': i.lastname}
        # Add the friend to our list
        obj['my_best_friends'].append(friend)

    # Return our object in json format for our api users
    return jsonify(obj)


import numpy as np
import vtk

major_radius = 1
band_width = 0.8
turns = 1
rad_samples = 90
band_samples = 4

xs = rad_samples+1
ys = band_samples
num_pts_cover = xs * ys

xy_to_id = dict()
for foo in range(num_pts_cover):
    y = foo % ys
    x = foo // ys
    xy_to_id[(x,y)] = foo

equivalence_classes = [ ( (0,y), (xs-1, (-y-1)%ys) ) for y in range(ys)]

for equiv_class in equivalence_classes:
    mm = min(equiv_class)
    for mem in equiv_class:
        xy_to_id[mem] = xy_to_id[mm]

cell_list = []
for foo in range(rad_samples):
    for bar in range(band_samples - 1):
        cell = [xy_to_id[(foo, bar)],
                xy_to_id[(foo, bar+1)],
                xy_to_id[(foo+1, bar+1)]]
        cell_list.append(cell)
        cell2= [xy_to_id[(foo, bar)],
                xy_to_id[(foo+1, bar+1)],
                xy_to_id[(foo+1, bar)]]
        cell_list.append(cell2)
print(cell_list)

num_pts = max(xy_to_id.values()) + 1

geo_pts = dict()
for foo in range(rad_samples):
    for bar in range(band_samples):
        theta = 2*np.pi*foo / rad_samples
        bb = band_width * ((bar/(band_samples-1)) - 0.5)
        x = np.cos(theta) * ( major_radius + bb * np.sin(theta/2 * turns))
        y = np.sin(theta) * ( major_radius + bb * np.sin(theta/2 * turns))
        z = bb * np.cos(theta/2 * turns)
        geo_pts[xy_to_id[(foo,bar)]] = (x,y,z)

if num_pts != len(set(geo_pts)):
    print(num_pts)
    print(geo_pts)
    print(len(set(geo_pts)))
    raise Exception("POINT NUMBER MISMATCH!")

points = vtk.vtkPoints()
triangles = vtk.vtkCellArray()
# verts = vtk.vtkCellArray()

for foo in range(num_pts):
    pt = geo_pts[foo]
    points.InsertNextPoint(pt[0],pt[1],pt[2])
    print(pt[0], pt[1], pt[2])
    # verts.InsertNextCell(1, [foo])

for cell in cell_list:
    # vtkcell = vtk.vtkTriangle()
    # for bar in cell:
    #     vtkcell.InsertNextPoint(bar)
    triangles.InsertNextCell(3, cell)
    # for p in cell:
    #     triangles.InsertNextCellPoint(p)

simplicial_complex = vtk.vtkPolyData()
simplicial_complex.SetPoints(points)
# simplicial_complex.SetVerts(verts)
simplicial_complex.SetPolys(triangles)
simplicial_complex.BuildCells()
simplicial_complex.BuildLinks()

print("cells", simplicial_complex.GetNumberOfCells())
print("points", simplicial_complex.GetNumberOfPoints())

ww = vtk.vtkSTLWriter()
ww.SetFileName("C:\\Users\\sscott\\Pictures\\fat_mobius.stl")
ww.SetInputData(simplicial_complex)
ww.Update()
ww.Write()

# xtris = rad_samples
# ytris = band_samples
# num_pts = xtris*(ytris+1)
# num_tris = 2 * rad_samples * band_samples;
# # vector < array < int, 3 > > cell_list;
#
# parametric_pts = []
# for foo in range(num_pts):
#     pt_col = foo % xtris
#     pt_row = foo // xtris
#     parametric_pts.append([ pt_row/ytris, pt_col/xtris])
#
# for (t,b) in parametric_pts:
#     theta = 2*np.pi * t
#     ll = (b-.5)
#
# cell_list = []
# for foo in range(num_tris):
#     sq = foo // 2;
#     is_down = foo % 2;
#     pt_col = sq % xtris;
#     pt_row = sq // xtris;
#     if ( not is_down):
#         cell = []
#         cell.append( pt_col  + xtris * pt_row );
#         cell.append( (1 + pt_col) % xtris  + xtris * (pt_row) )
#         cell.append( 0 + pt_col + xtris * (pt_row+1) )
#         cell_list.append(cell)
#     else:
#         cell = []
#         cell.append( (1 + pt_col) % xtris + xtris * pt_row );
#         cell.append( (1 + pt_col) % xtris + xtris * (pt_row+1))
#         cell.append( 0 + pt_col  + xtris * (pt_row+1) )
#         cell_list.append(cell)


from django.urls import include, path
from rest_framework.routers import SimpleRouter
from survey.views import OperatingSystemViewSet, SurveyResultViewSet

app_name = 'survey'

router = SimpleRouter()
router.register('survey', SurveyResultViewSet, basename='survey')
router.register('os', OperatingSystemViewSet, basename='os')

urlpatterns = [
    path('', include((router.urls))),
]


# coding=utf-8

import tool as t
from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, String, Boolean, DateTime, Unicode
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.expression import update

engine = create_engine('mysql+mysqldb://root:laso2401@localhost:3306/fanboflask')

Base = declarative_base()

class zhubo(Base):
    __tablename__ = 'zhubo'
    idzhubo = Column(Integer, primary_key=True)
    room = Column(String(128), unique=True)
    nick_name = Column(Unicode(128))
    avatar = Column(String(1024))
    room_url = Column(String(1024))
    cover_picture = Column(String(1024))
    is_online = Column(Boolean)
    update_time = Column(DateTime)
    last_kaibo_time = Column(DateTime)

Base.metadata.create_all(engine)

smaker = sessionmaker(bind=engine)
session = smaker()
conn = engine.connect()

def commit_change(model_obj):
    session.add(model_obj)
    session.commit()

def find_zhubo(room):
    try:
        zb = session.query(zhubo).filter(zhubo.room==room).first()
        if zb is None:
            zb = zhubo(room=room)
            commit_change(zb)
        return zb
    except Exception as ex:
        t.err()
        t.log('can not find or create zhubo, room number: %s' % room)
        return None

def set_zhubo_info(room, **kwargs):
    zb = find_zhubo(room)
    if zb is None:
        return
    for key, value in kwargs.items():
        if hasattr(zb, key):
            setattr(zb, key, value)
    zb.update_time = t.now()
    commit_change(zb)

def update_other_invalid(time_before):
    tupdate = update(zhubo).where(zhubo.update_time<time_before).values(is_online=False)
    conn.execute(tupdate)


import random
a="abcdefghijklmnopqrstuvxz"
cri=a+"@*&#%$"
v=str()
while (v !="sair"):
 c=str(input(" entre ?"))
 v=c
 if c=="sair":
 	break
 for i in a:
    for f in c:
    	if f==i:
     	  t=random.choice(cri)
     	  c=c.replace("a",t)

 print("letra encontrada", c)
 print("pavra escrita",v)


MENU = """C - Convert Celsius to Fahrenheit
          F - Convert Fahrenheit to Celsius
          Q - Quit"""
print(MENU)

def main():
 choice = input(">>> ").upper()

 while choice != "Q":
   if choice == "C":
       celsius = float(input("Celsius: "))
       Fahrenheit = celsius_to_fahrenheit(celsius)
       print ( "Fahrenheit is {:.2f} F".format (Fahrenheit))       # fahrenheit = celsius * 9.0 / 5 + 32
   elif choice == "F":
      fahrenheit = float(input("Fahrenheit:"))
      celsius = fahrenheit_to_celsius(fahrenheit)
      print("Celsius is {:.2f} C".format(celsius))
   else:
     print("Invalid option")
   print(MENU)
   choice = input(">>> ").upper()
 print("Thank you.")

def celsius_to_fahrenheit(c):
 return c * 9.0 / 5 + 32

def fahrenheit_to_celsius(f):
 return (5 / 9 * (f - 32))
main()


class NodoTris:
    def __init__(self, griglia):
        self.name = griglia
        self.issue, self.next_one, self.free_one = self.obtain_infos()
        self.sons = self.find_sons()

    def tipo(self):
        return self.issue

    def vittorie_livello(self, giocatore, h):
        numb = 0
        ls_numb = [0]
        finale = self.lvl_winning(giocatore, numb, h, ls_numb)
        return finale

    def find_issues(self, ls_num):
        if self.issue is '-':
            ls_num[0] += 1
        if self.issue is 'o':
            ls_num[1] += 1
        if self.issue is 'x':
            ls_num[2] += 1
        for schema in self.sons:
            schema.find_issues(ls_num)
        return ls_num

    def find_right_strategy(self, plr):
        if self.issue != '?':
            s = self.issue != plr
            return s
        if self.next_one != plr:
            return self.wrong_strategy(plr)
        else:
            return self.right_strategy(plr)

    def right_strategy(self,ply):
        for crux in self.sons:
                if crux.find_right_strategy(ply) == True:
                    return True
        return False

    def wrong_strategy(self,ply):
        for crux in self.sons:
                if crux.find_right_strategy(ply) == False:
                    return False
        return True

    def esiti(self):
        tupla_list_num = [0, 0, 0]
        self.find_issues(tupla_list_num)
        return tuple(tupla_list_num)

    def obtain_infos(self):
        x_symbol = 0
        o_symbol = 0
        prox_str=""
        advice=""
        col_tic_toe = ['', '', '']
        rig_tic_toe = ['', '', '']
        free_one = []
        rig_tic_toe,col_tic_toe,free_one,x_symbol,o_symbol = self.get_info(col_tic_toe,rig_tic_toe,x_symbol,o_symbol,free_one)
        rig_tic_toe,col_tic_toe=self.get_info_sup(col_tic_toe,rig_tic_toe)
        advice,prox_str=self.get_inf_sup2(x_symbol,o_symbol,rig_tic_toe,col_tic_toe,prox_str,advice,free_one)
        return advice, prox_str, free_one

    def strategia_vincente(self, giocatore):
        return self.find_right_strategy(giocatore)

    def lvl_winning(self, player, lvl, finished, s):
        if lvl == finished:
            if self.issue is player:
                s[0] += 1
        else:
            for crux in self.sons:
                lvl = lvl + 1
                crux.lvl_winning(player, lvl, finished, s)

        return s[0]

    def find_sons_sup(self,ls_ret):
        for box in self.free_one:
                winter_scheme = []
                winter_scheme.append(self.name[0][:])
                winter_scheme.append(self.name[1][:])
                winter_scheme.append(self.name[2][:])
                winter_scheme[box[0]][box[1]] = self.next_one
                ls_ret.append(NodoTris(winter_scheme))
        return ls_ret

    def find_sons(self):
        return_list = []
        if self.issue == "?":
            return_list=self.find_sons_sup(return_list)
        return return_list

    def get_inf_sup2(self,X,O,rig,col,prox,adc,free):
        if X == O:
            prox = "o"
        else:
            prox = "x"
        if "ooo" in col or "ooo" in rig:
            adc = "o"
        elif "xxx" in col or "xxx" in rig:
            adc = "x"
        elif len(free) != 0:
            adc = "?"
        else:
            adc = "-"
        return adc,prox

    def get_info1(self,rig,col,V,val,x,y):
        rig[y] += val
        col[x] += val
        V += 1
        return rig,col,V

    def get_info(self,col,rig,X,O,free):
        xx='x'
        oo='o'
        for y in range(3):
            for x in range(3):
                if self.name[y][x] is xx:
                    rig,col,X=self.get_info1(rig,col,X,xx,x,y)
                if self.name[y][x] is oo:
                    rig,col,O=self.get_info1(rig,col,O,oo,x,y)
                if self.name[y][x] is '':
                    free.append((y, x))
        return rig,col,free,X,O
    def get_info_sup(self,col,rig):
        rig.append("")
        col.append("")
        col[3] += self.name[0][0]
        col[3] += self.name[1][1]
        col[3] += self.name[2][2]
        rig[3] += self.name[0][2]
        rig[3] += self.name[1][1]
        rig[3] += self.name[2][0]
        return rig,col

def gen_tree(griglia):
    return NodoTris(griglia)


# SPDX-FileCopyrightText: Copyright (c) 2021 Artёm IG <github.com/rtmigo>
# SPDX-License-Identifier: BSD-3-Clause

import re

from commie.x01_common import Comment, Span

def matchGroupToComment(match: re.Match, groupName: str, multiline: bool) -> Comment:
	fullSpan = match.span()

	fullText = match.group(0)
	innerText = match.group(groupName)
	textStart = fullText.index(innerText)
	assert textStart >= 0

	return Comment(
		match.string,
		text_span=Span(fullSpan[0] + textStart, fullSpan[0] + textStart + len(innerText)),
		code_span=Span(fullSpan[0], fullSpan[1]),
		multiline=multiline)


#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#

class MlosTelemetryAggregator:
    """ An abstract class to be implemented by all classes that need to enable and consume mlos telemetry.

    """

    @property
    def requested_message_types(self):
        return self.callbacks.keys()

    @staticmethod
    def register_callback(component_type, message_type):
        """ Associates the message_type for which the decorated function should be called.

        :param message_type:
        :return:
        """
        def decorator(wrapped_function):
            wrapped_function.component_type = component_type
            wrapped_function.message_type = message_type
            return wrapped_function
        return decorator

    def __init__(self):
        """ Iterates over all methods defined in the subclass and finds ones that have been decorated.

        """

        # Callbacks is a dictionary where:
        # * key is the message type
        # * value is a List[Tuple[MlosSmartComponentSelector, callback]]
        self.callbacks = dict()

        for component_type, message_type, method in self._find_decorated_functions():
            existing_callbacks = self.callbacks.get((component_type, message_type), [])
            existing_callbacks.append(method)
            self.callbacks[(component_type, message_type)] = existing_callbacks

    def _find_decorated_functions(self):
        for attribute_name in dir(self):
            attribute = getattr(self, attribute_name)
            if callable(attribute):
                method = attribute
                if hasattr(method, 'message_type'):
                    message_type = method.message_type
                    component_type = method.component_type
                    yield component_type, message_type, method


from multiprocessing import Process, Queue
import time
import psutil
from os import getpid
import numpy as np

def collect_stats(ppid,childq,tts):
    p = psutil.Process(ppid)
    collector = XPerfStats()
    while(1):
        stats = collector.read_raw_stats(p)
        childq.put(stats)
        time.sleep(tts)

#Open a new process and start stat collection periodically
class XPerfStatsTask:
    def __init__(self,sampling_rate_sec):
        self._sleep = sampling_rate_sec
        self._pid = getpid()
        self._q = Queue(10)
        self._p = Process(target=collect_stats, args=(self._pid,self._q,self._sleep))

    def start(self):
        self._p.start()

    def stop(self):
        self._p.terminate()
        tmp = []
        while not self._q.empty():
            tmp.append(self._q.get())

        #Pass this list to XPerfStats object asking it to give back formatted dictionary object
        stats = XPerfStats().format_raw_stats(tmp)
        return stats

class XPerfStats():
    def __init__(self,extstats=None):
        self.valid = False
        self._ncores = psutil.cpu_count()

    #Utility function to collect stats
    def read_raw_stats(self,phdl):
        if self._ncores > 4:
            sys_cpu_per = [psutil.cpu_percent(interval=1,percpu=False)]
        else:
            sys_cpu_per = psutil.cpu_percent(interval=1,percpu=True)

        cput = psutil.cpu_times()
        total_time = sum(cput)
        sys_cpu_times = [round((cput.user/total_time)*100,2),
                         round((cput.system/total_time)*100,2),
                         round((cput.idle/total_time)*100,2)]

        vm = psutil.virtual_memory()
        sys_vm_data = [vm.percent, 100.0-vm.percent]

        #Process specific collection
        pp_cpu_per = phdl.cpu_percent(interval=1)

        pcput = phdl.cpu_times()
        pp_cpu_times = [pcput.user,pcput.system]

        pp_vm_data = round(phdl.memory_percent(),2)

        csw = phdl.num_ctx_switches()
        pp_ctx_switches = [csw.voluntary,csw.involuntary]

        return [sys_cpu_per,sys_cpu_times,sys_vm_data,pp_cpu_per,pp_cpu_times,pp_vm_data,pp_ctx_switches]

    #utility function to make a dictionary from list of stats
    def format_raw_stats(self,extstats):
        dict_data = {}
        dict_data["Sys_CPU%"] = []
        dict_data["Sys_user_time%"] = []
        dict_data["Sys_system_time%"] = []
        dict_data["Sys_idle_time%"] = []
        dict_data["Sys_used_VM%"] = []
        dict_data["Sys_available_VM%"] = []
        dict_data["Proc_CPU%"] = []
        dict_data["Proc_user_time_sec"] = []
        dict_data["Proc_system_time_sec"] = []
        dict_data["Proc_used_VM%"] = []
        dict_data["Proc_vol_ctx_switch"] = []
        dict_data["Proc_invol_ctx_switch"] = []

        for data in extstats:
            dict_data["Sys_CPU%"].append(data[0])
            dict_data["Sys_user_time%"].append(data[1][0])
            dict_data["Sys_system_time%"].append(data[1][1])
            dict_data["Sys_idle_time%"].append(data[1][2])
            dict_data["Sys_used_VM%"].append(data[2][0])
            dict_data["Sys_available_VM%"].append(data[2][1])
            dict_data["Proc_CPU%"].append(data[3])
            dict_data["Proc_user_time_sec"].append(data[4][0])
            dict_data["Proc_system_time_sec"].append(data[4][1])
            dict_data["Proc_used_VM%"].append(data[5])
            dict_data["Proc_vol_ctx_switch"].append(data[6][0])
            dict_data["Proc_invol_ctx_switch"].append(data[6][1])

        return dict_data

    #Read a dictionary and save lists of different stats
    def parse_formatted_stats(self,extstats):
        #Input is unfortunately compilcated..simlify it a bit
        #Make a new list of dictionaries
        nvalues = len(extstats)
        if nvalues == 0:
            self.valid = False
            return
        else:
            self.valid = True

        raw_stats = []
        for data in extstats:
            raw_stats.append(data[0][0])

        ###############

        first = 12*[True]
        for data in raw_stats:
             for cpu_data in data["Sys_CPU%"]:
                 if first[0]:
                     self.cpu_load_np_arr = np.array([cpu_data])
                     first[0] = False
                 else:
                     self.cpu_load_np_arr = np.append(self.cpu_load_np_arr,[cpu_data] ,axis=0)

             for cpu_data in data["Proc_CPU%"]:
                 if first[1]:
                     self.cpu_proc_load_np_arr = np.array([cpu_data])
                     first[1] = False
                 else:
                     self.cpu_proc_load_np_arr = np.append(self.cpu_proc_load_np_arr,[cpu_data])

             for cpu_data in data["Sys_user_time%"]:
                 if first[2]:
                    self.cpu_user_util_np_arr = np.array([cpu_data])
                    first[2] = False
                 else:
                    self.cpu_user_util_np_arr = np.append(self.cpu_user_util_np_arr,[cpu_data])

             for cpu_data in data["Sys_system_time%"]:
                 if first[3]:
                    self.cpu_sys_util_np_arr = np.array([cpu_data])
                    first[3] = False
                 else:
                    self.cpu_sys_util_np_arr = np.append(self.cpu_sys_util_np_arr,[cpu_data])

             for cpu_data in data["Sys_idle_time%"]:
                 if first[4]:
                    self.cpu_idle_util_np_arr = np.array([cpu_data])
                    first[4] = False
                 else:
                    self.cpu_idle_util_np_arr = np.append(self.cpu_idle_util_np_arr,[cpu_data])

             for cpu_data in data["Proc_user_time_sec"]:
                 if first[5]:
                    self.cpu_proc_user_util_np_arr = np.array([cpu_data])
                    first[5] = False
                 else:
                    self.cpu_proc_user_util_np_arr = np.append(self.cpu_proc_user_util_np_arr,[cpu_data])

             for cpu_data in data["Proc_system_time_sec"]:
                 if first[6]:
                    self.cpu_proc_sys_util_np_arr = np.array([cpu_data])
                    first[6] = False
                 else:
                    self.cpu_proc_sys_util_np_arr = np.append(self.cpu_proc_sys_util_np_arr,[cpu_data])

             for vm_data in data["Sys_used_VM%"]:
                 if first[7]:
                    self.sys_used_vm_np_arr = np.array([vm_data])
                    first[7] = False
                 else:
                    self.sys_used_vm_np_arr = np.append(self.sys_used_vm_np_arr,[vm_data])

             for vm_data in data["Sys_available_VM%"]:
                 if first[8]:
                    self.sys_free_vm_np_arr = np.array([vm_data])
                    first[8] = False
                 else:
                    self.sys_free_vm_np_arr = np.append(self.sys_free_vm_np_arr,[vm_data])

             for vm_data in data["Proc_used_VM%"]:
                 if first[9]:
                    self.proc_used_vm_np_arr = np.array([vm_data])
                    first[9] = False
                 else:
                    self.proc_used_vm_np_arr = np.append(self.proc_used_vm_np_arr,[vm_data])

             for ctx_data in data["Proc_vol_ctx_switch"]:
                 if first[10]:
                    self.proc_vol_ctx_np_arr = np.array([ctx_data])
                    first[10] = False
                 else:
                    self.proc_vol_ctx_np_arr = np.append(self.proc_vol_ctx_np_arr,[ctx_data])

             for ctx_data in data["Proc_invol_ctx_switch"]:
                 if first[11]:
                    self.proc_invol_ctx_np_arr = np.array([ctx_data])
                    first[11] = False
                 else:
                    self.proc_invol_ctx_np_arr = np.append(self.proc_invol_ctx_np_arr,[ctx_data])

    def xperf_stat(self):
        lines = []
        if not self.valid:
            lines.append("-------------------")
            lines.append("No external data available to display!!")
            lines.append("-------------------")
            return lines

        lines.append("\nSystem wide external stats:-")
        lines.append("CPU core binding - TBD ")

        ############
        lines.append("CPU loading in %:")
        tmp = "             C0"
        for i in range(self._ncores-1):
            tmp = tmp + "   C" + str(i+1) + " "
        lines.append(tmp)

        tmp = "   max      " + str(np.max(self.cpu_load_np_arr,axis=0))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.cpu_load_np_arr,axis=0))
        lines.append(tmp)
        ############

        ############
        lines.append("\n CPU utilization in %:")
        lines.append("            User    Sys    Idle")
        tmp = "   max      " + str(np.max(self.cpu_user_util_np_arr)) + "    " + str(np.max(self.cpu_sys_util_np_arr)) + "    " + str(np.max(self.cpu_idle_util_np_arr))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.cpu_user_util_np_arr)) + "    " + str(np.min(self.cpu_sys_util_np_arr)) + "    " + str(np.min(self.cpu_idle_util_np_arr))
        lines.append(tmp)
        ############

        ############
        lines.append("\n Virtual memory utilization in %:")
        lines.append("            Used    Available")
        tmp = "   max      " + str(np.max(self.sys_used_vm_np_arr)) + "    " + str(np.max(self.sys_free_vm_np_arr))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.sys_used_vm_np_arr)) + "    " + str(np.min(self.sys_free_vm_np_arr))
        lines.append(tmp)
        ############

        lines.append("\nProcess specific external stats :-")
        ############
        lines.append("CPU loading in %:")
        tmp = "   max      " + str(np.max(self.cpu_proc_load_np_arr,axis=0))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.cpu_proc_load_np_arr,axis=0))
        lines.append(tmp)
        ############

        ############
        lines.append("\n CPU utilization in sec:")
        lines.append("            User    Sys")
        tmp = "   max      " + str(np.max(self.cpu_proc_user_util_np_arr)) + "    " + str(np.max(self.cpu_proc_sys_util_np_arr))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.cpu_proc_user_util_np_arr)) + "    " + str(np.min(self.cpu_proc_sys_util_np_arr))
        lines.append(tmp)
        ############

        ############
        lines.append("\n Context switch count:")
        lines.append("            Vol    Invol")
        tmp = "   max      " + str(np.max(self.proc_vol_ctx_np_arr)) + "    " + str(np.max(self.proc_invol_ctx_np_arr))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.proc_vol_ctx_np_arr)) + "    " + str(np.min(self.proc_invol_ctx_np_arr))
        lines.append(tmp)
        ############

        ############
        lines.append("\n Virtual memory utilization in %:")
        lines.append("            Used")
        tmp = "   max      " + str(np.max(self.proc_used_vm_np_arr))
        lines.append(tmp)
        tmp = "   min      " + str(np.min(self.proc_used_vm_np_arr))
        lines.append(tmp)
        ############
        return lines

    def xperf_dump(self):
        lines = []
        if not self.valid:
            lines.append("-------------------")
            lines.append("No external data available to display!!")
            lines.append("-------------------")
            return lines

        lines.append("- System wide external stats:")

        ############
        lines.append("    - CPU loading in % :")
        tmp = "     C0"
        for i in range(self._ncores-1):
            tmp = tmp + "   C" + str(i+1) + " "
        lines.append(tmp)
        for i in self.cpu_load_np_arr:
            lines.append("    " + str(i))

        ############
        lines.append("    - CPU utilization in %")
        lines.append("    User    Sys    Idle")
        cnt = len(self.cpu_user_util_np_arr)
        for i in range(cnt):
            lines.append("    " + str(self.cpu_user_util_np_arr[i]) +
                         "    " + str(self.cpu_sys_util_np_arr[i]) +
                         "    " + str(self.cpu_idle_util_np_arr[i]))
        ############

        ############
        lines.append("    - Virtual memory utilization in %")
        lines.append("     Used    Available")
        cnt = len(self.sys_used_vm_np_arr)
        for i in range(cnt):
            lines.append("    " + str(self.sys_used_vm_np_arr[i]) +
                         "    " + str(self.sys_free_vm_np_arr[i]))
        ############
        lines.append("- Process wide external stats:")

        ############
        lines.append("    - CPU loading in % :")
        for i in self.cpu_proc_load_np_arr:
            lines.append("    " + str(i))

        lines.append("    - CPU utilization in sec")
        lines.append("    User    Sys")
        cnt = len(self.cpu_proc_user_util_np_arr)
        for i in range(cnt):
            lines.append("    " + str(self.cpu_proc_user_util_np_arr[i]) +
                         "    " + str(self.cpu_proc_sys_util_np_arr[i]))

        lines.append("    - Context switch count")
        lines.append("    Vol    Invol")
        cnt = len(self.proc_vol_ctx_np_arr)
        for i in range(cnt):
            lines.append("    " + str(self.proc_vol_ctx_np_arr[i]) +
                         "    " + str(self.proc_invol_ctx_np_arr[i]))

        lines.append("    - Virtual memory utilization in %")
        lines.append("    Used")
        cnt = len(self.proc_used_vm_np_arr)
        for i in range(cnt):
            lines.append("    " + str(self.proc_used_vm_np_arr[i]))
        ############
        return lines

    def xperf_get_values(self,sys=False,proc=False):
        values = {}

        if (not self.valid) or (sys is False and proc is False):
            print("No external data available to display!!")
            return values

        if sys is True:
            #TODO: There should be better error handling in places like here
            #We hope that we are not so unlucky to have no CPU load data
            data_cnt = len(self.cpu_load_np_arr[0])
            values["syscpuload"] = []
            for i in range(data_cnt):
                cpu_arr = self.cpu_load_np_arr[:,i]
                values["syscpuload"].append(cpu_arr.tolist());

            values["syscpuutil"] = [] #sys,user,idle
            cpu_arr = self.cpu_sys_util_np_arr
            values["syscpuutil"].append(cpu_arr.tolist())
            cpu_arr = self.cpu_user_util_np_arr
            values["syscpuutil"].append(cpu_arr.tolist())
            cpu_arr = self.cpu_idle_util_np_arr
            values["syscpuutil"].append(cpu_arr.tolist())

            values["sysvm"] = [] #used, available
            vm_arr = self.sys_used_vm_np_arr
            values["sysvm"].append(vm_arr.tolist())
            vm_arr = self.sys_free_vm_np_arr
            values["sysvm"].append(vm_arr.tolist())

        if proc is True:
            values["procpuload"] = []
            cpu_arr = self.cpu_proc_load_np_arr
            values["procpuload"].append(cpu_arr.tolist())

            values["proccpuutil"] = []
            cpu_arr = self.cpu_proc_sys_util_np_arr
            values["proccpuutil"].append(cpu_arr.tolist())
            cpu_arr = self.cpu_proc_user_util_np_arr
            values["proccpuutil"].append(cpu_arr.tolist())

            vm_arr = self.proc_used_vm_np_arr
            values["procvm"] = vm_arr.tolist()

            values["procctx"] = [] #Voluntary, involuntary
            cpu_arr = self.proc_vol_ctx_np_arr
            values["procctx"].append(cpu_arr.tolist())
            cpu_arr = self.proc_invol_ctx_np_arr
            values["procctx"].append(cpu_arr.tolist())

        return values


# -*- coding: utf-8 -*-
#################
import numpy as np
#################

"""
mysmps.util.mathfuncs
================

Basic mathematical equations
    roundup

Created on Thu Aug 6 18:00 2020

@author: flovan / fvanden

Revision history:   06.08.2020 - Created

"""
## -------------------------------------------------------------------------- ##

def roundup(x):
    """
    Rounds value up to nearest 10, 100, 1000, ...

    Parameters
    ----------
    x : float
        input number

    Returns
    -------
    y : float
        x rounded up to nearest 10, 100, 1000 etc
    """
    d = np.round(100*np.log(x)/np.log(10))/100
    d = np.floor(d)
    y = int(np.ceil(x / 10**d)) * 10**d
    return(y)


#!/usr/bin/env python

import roslib
import rospy
import time
import math
import cv2
from sensor_msgs.msg import Image
from cv_bridge import CvBridge
import numpy as np

PATH = rospy.get_param("path")
PAUSE = rospy.get_param("pause")

class NodeRawImage():
	def __init__(self):
		self.seq = 1
		rospy.init_node('image_raw_publisher')
		self.publisher = rospy.Publisher('/image_raw', Image, queue_size = 1)
		self.bridge = CvBridge()

	def waitInput(self):
		if PAUSE:
			t = raw_input()
			if t == 'q':
				self.seq -= 1
			elif t == 'qq':
				self.seq -= 2
		self.publish()

	def publish(self):
		img = cv2.imread(PATH + "images/" + str(self.seq) + ".jpg", cv2.CV_LOAD_IMAGE_COLOR)
		# msg = self.bridge.cv2_to_imgmsg(img, encoding="passthrough")
		msg = self.bridge.cv2_to_imgmsg(img, encoding="bgr8")
		msg.header.stamp = rospy.Time.now()
		msg.header.frame_id = 'image_raw_publisher'
		msg.header.seq = self.seq
		self.seq += 1
		self.publisher.publish(msg)

if __name__ == '__main__':
	try:
		n = NodeRawImage()
		#rospy.spin()
		while not rospy.is_shutdown():
			n.waitInput()
	except rospy.ROSInterruptException:
		pass


import gym

from ray.rllib.agents.dqn import DQNTrainer

from smarts.core.controllers import ActionSpaceType
from smarts.core.agent_interface import RGB, Waypoints, NeighborhoodVehicles

from utils import common

NAME = "DQN"
TRAINER = DQNTrainer

def make_config(**kwargs):
    use_stacked_observation = kwargs.get("use_stacked_observation", False)
    use_rgb = kwargs.get("use_rgb", False)
    closest_neighbor_num = kwargs.get("max_observed_neighbors", 8)

    img_resolution = 24
    observe_lane_num = 3
    look_ahead = 10
    stack_size = 3 if use_stacked_observation else 1
    subscribed_features = dict(
        goal_relative_pos=(stack_size, 2),
        distance_to_center=(stack_size, 1),
        speed=(stack_size, 1),
        steering=(stack_size, 1),
        heading_errors=(stack_size, look_ahead),
        neighbor=(stack_size, closest_neighbor_num * 5),  # dist, speed, ttc
        img_gray=(stack_size, img_resolution, img_resolution) if use_rgb else False,
    )

    action_space = common.ActionSpace.from_type(1)
    observation_space = gym.spaces.Dict(
        common.subscribe_features(**subscribed_features)
    )

    policy_config = (
        None,
        observation_space,
        action_space,
        dict(model=dict(custom_model_config=dict(obs_space_dict=observation_space),)),
    )

    interface_config = dict(
        obs_stack_size=stack_size,
        neighborhood_vehicles=NeighborhoodVehicles(radius=100),
        waypoints=Waypoints(lookahead=10),
        rgb=RGB(width=256, height=256, resolution=img_resolution / 256)
        if use_rgb
        else None,
        action=ActionSpaceType.Lane,
        road_waypoints=None,
        drivable_area_grid_map=None,
        ogm=None,
        lidar=None,
        debug=False,
    )

    observation_adapter = common.get_observation_adapter(
        observation_space,
        look_ahead=look_ahead,
        observe_lane_num=observe_lane_num,
        resize=(img_resolution, img_resolution),
        closest_neighbor_num=closest_neighbor_num,
    )

    agent_config = dict(
        observation_adapter=observation_adapter,
        reward_adapter=get_reward_adapter(observation_adapter),
        action_adapter=common.ActionAdapter.from_type(1),
        info_adapter=common.default_info_adapter,
    )

    learning_config = dict()

    other_config = dict(
        stop={"time_total_s": 2 * 60 * 60},
        checkpoint_freq=40,
        checkpoint_at_end=True,
        max_failures=1000,
    )

    return common.Config(
        name=NAME,
        agent=agent_config,
        interface=interface_config,
        policy=policy_config,
        learning=learning_config,
        other=other_config,
        trainer=DQNTrainer,
        spec={"obs": observation_space, "act": action_space},
    )

def get_reward_adapter(observation_adapter, **kwargs):
    def reward_adapter(env_obs, env_reward):
        return 0.05 * env_reward

    return reward_adapter


from django.utils.translation import ugettext_lazy as _
from django.contrib import admin
from django.utils.safestring import mark_safe
from django import forms
from django.contrib.admin.sites import site

from modeltranslation.admin import TranslationAdmin

from .models import Event, Template, TemplateVars
from libs.linked_in_admin.widgets import LinkedManyToManyRawIdWidget

class EventAdmin(admin.ModelAdmin):
    list_display = ('event_name', 'status', 'name', 'template_count', )
    search_fields = ('event_name', 'name',)

    @staticmethod
    def template_count(obj):
        return obj.templates.count()
    template_count.short_description = _('Количество шаблонов')

class TemplateAdminForm(forms.ModelForm):

    @staticmethod
    def set_message_help_text():
        output = ['
',
                  '%s' % _('Доступные переменные drag&drop:'), '
']
        for item in TemplateVars.objects.filter(status=True):
            output.append(
                '
{{ %s|safe }} :: %s
' % (
                item.var_name, item.id, item.description))
        output.append('

')
        output.append('
')
        return mark_safe(u'\n'.join(output))

    def __init__(self, data=None, *args, **kwargs):
        super(TemplateAdminForm, self).__init__(data, *args, **kwargs)
        self.fields['content_ru'].help_text = self.set_message_help_text()

    class Meta:
        widgets = {
            'recipient_list': LinkedManyToManyRawIdWidget(Template._meta.get_field('recipient_list').rel, site),
        }

class TemplateAdmin(TranslationAdmin):
    form = TemplateAdminForm
    list_display = (
        'date_of_change',
        # 'email_to',
        # 'email_from',
        'subject',
        'type_message',
        # 'delay_day',
        # 'delay_hour',
        # 'delay_minute',
        'status',
        'email_event'
    )
    list_filter = ('email_event',)
    # ordering = ('-delay_day', '-delay_hour', '-delay_minute', )
    # fieldsets = (
    #     (_('Отложить оправку - (если все поля содержат 0 то отправка произойдет сразу)'), {
    #         'classes': ('grp-collapse grp-closed',),
    #         'fields': ('delay_day', 'delay_hour', 'delay_minute', )
    #     }
    #     ),
    #     (_('Заголовки шаблона'), {
    #         'classes': ('grp-collapse grp-closed',),
    #         'fields': ('email_to', 'email_from', 'subject', )
    #     }
    #     ),
    #     ('', {
    #         'fields': ('status', 'email_event', 'type_message', 'content', )
    #     }
    #     ),
    # )

    # class Media:
    #     js = ('email_notices/js/admin.js', )

    class Media:
        js = (
            '/static/modeltranslation/js/force_jquery.js',
            'http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.2/jquery-ui.min.js',
            '/static/modeltranslation/js/tabbed_translation_fields.js',
        )
        css = {
            'screen': ('/static/modeltranslation/css/tabbed_translation_fields.css',),
        }

class TemplateVarsAdmin(admin.ModelAdmin):
    list_display = ('date_of_change', 'status', 'var_name', 'description', )

admin.site.register(Event, EventAdmin)
admin.site.register(Template, TemplateAdmin)
admin.site.register(TemplateVars, TemplateVarsAdmin)