import sys

import vtk
from vtk.util import vtkConstants
try:
    from vtk.util import numpy_support
except ImportError:
    numpy_support = None

import numpy

# Enthought library imports.
try:
    from tvtk.array_ext import set_id_type_array
    HAS_ARRAY_EXT = True
except ImportError:
    HAS_ARRAY_EXT = False

# Useful constants for VTK arrays.
VTK_ID_TYPE_SIZE = vtk.vtkIdTypeArray().GetDataTypeSize()
if VTK_ID_TYPE_SIZE == 4:
    ID_TYPE_CODE = numpy.int32
elif VTK_ID_TYPE_SIZE == 8:
    ID_TYPE_CODE = numpy.int64

VTK_LONG_TYPE_SIZE = vtk.vtkLongArray().GetDataTypeSize()
if VTK_LONG_TYPE_SIZE == 4:
    LONG_TYPE_CODE = numpy.int32
    ULONG_TYPE_CODE = numpy.uint32
elif VTK_LONG_TYPE_SIZE == 8:
    LONG_TYPE_CODE = numpy.int64
    ULONG_TYPE_CODE = numpy.uint64

BASE_REFERENCE_COUNT = vtk.vtkObject().GetReferenceCount()

def getbuffer(array):
    return getattr(numpy, 'getbuffer', memoryview)(array)

def set_id_type_array_py(id_array, out_array):
    """Given a 2D Int array (`id_array`), and a contiguous 1D numarray array
    (`out_array`) having the correct size, this function sets the data from
    `id_array` into `out_array` so that it can be used in place of a
    `vtkIdTypeArray` in order to set the cells of a `vtkCellArray`.

    Note that if `shape = id_array.shape` then `size(out_array) ==
    shape[0]*(shape[1] + 1)` should be true. If not you'll get an
    `AssertionError`.

    `id_array` need not be contiguous but `out_array` must be.

    """
    assert numpy.issubdtype(id_array.dtype, numpy.signedinteger)
    assert out_array.flags.contiguous == 1, \
        "out_array must be contiguous."
    shp = id_array.shape
    assert len(shp) == 2, "id_array must be a two dimensional array."
    sz = out_array.size
    e_sz = shp[0]*(shp[1]+1)
    assert sz == e_sz, \
        "out_array size is incorrect, expected: %s, given: %s" % (e_sz, sz)

    # we are guaranteed contiguous, so these just change the view (no copy)
    out_shp = out_array.shape
    out_array.shape = (shp[0], shp[1] + 1)
    out_array[:, 0] = shp[1]
    out_array[:, 1:] = id_array
    out_array.shape = out_shp

if not HAS_ARRAY_EXT:
    set_id_type_array = set_id_type_array_py

######################################################################
# The array cache.
######################################################################
class ArrayCache(object):

    """Caches references to numpy arrays that are not copied but views
    of which are converted to VTK arrays.  The caching prevents the user
    from deleting or resizing the numpy array after it has been sent
    down to VTK.  The cached arrays are automatically removed when the
    VTK array destructs."""

    ######################################################################
    # `object` interface.
    ######################################################################
    def __init__(self):
        # The cache.
        self._cache = {}

    def __len__(self):
        return len(self._cache)

    def __contains__(self, vtk_arr):
        key = vtk_arr.__this__
        return key in self._cache

    ######################################################################
    # `ArrayCache` interface.
    ######################################################################
    def add(self, vtk_arr, np_arr):
        """Add numpy array corresponding to the vtk array to the
        cache."""
        key = vtk_arr.__this__
        cache = self._cache

        # Setup a callback so this cached array reference is removed
        # when the VTK array is destroyed.  Passing the key to the
        # `lambda` function is necessary because the callback will not
        # receive the object (it will receive `None`) and thus there
        # is no way to know which array reference one has to remove.
        vtk_arr.AddObserver(
            'DeleteEvent', lambda o, e, key=key: self._remove_array(key)
        )

        # Cache the array
        cache[key] = np_arr

    def get(self, vtk_arr):
        """Return the cached numpy array given a VTK array."""
        key = vtk_arr.__this__
        return self._cache[key]

    ######################################################################
    # Non-public interface.
    ######################################################################
    def _remove_array(self, key):
        """Private function that removes the cached array.  Do not
        call this unless you know what you are doing."""
        try:
            del self._cache[key]
        except KeyError:
            pass

######################################################################
# Setup a global `_array_cache`.  The array object cache caches all the
# converted numpy arrays that are not copied.  This prevents the user
# from deleting or resizing the numpy array after it has been sent down
# to VTK.
######################################################################

_dummy = None
# This makes the cache work even when the module is reloaded.
for name in ['array_handler', 'tvtk.array_handler']:
    if name in sys.modules:
        mod = sys.modules[name]
        if hasattr(mod, '_array_cache'):
            _dummy = mod._array_cache
        del mod
        break

if _dummy:
    _array_cache = _dummy
else:
    _array_cache = ArrayCache()
del _dummy

def get_vtk_array_type(numeric_array_type):
    """Returns a VTK typecode given a numpy array."""
    # This is a Mapping from numpy array types to VTK array types.
    _arr_vtk = {
        numpy.dtype('S'): vtkConstants.VTK_UNSIGNED_CHAR,  # numpy.character
        numpy.dtype(numpy.uint8): vtkConstants.VTK_UNSIGNED_CHAR,
        numpy.dtype(numpy.uint16): vtkConstants.VTK_UNSIGNED_SHORT,
        numpy.dtype(numpy.int8): vtkConstants.VTK_CHAR,
        numpy.dtype(numpy.int16): vtkConstants.VTK_SHORT,
        numpy.dtype(numpy.int32): vtkConstants.VTK_INT,
        numpy.dtype(numpy.uint32): vtkConstants.VTK_UNSIGNED_INT,
        numpy.dtype(numpy.uint64): vtkConstants.VTK_UNSIGNED_LONG,
        numpy.dtype(numpy.float32): vtkConstants.VTK_FLOAT,
        numpy.dtype(numpy.float64): vtkConstants.VTK_DOUBLE,
        numpy.dtype(numpy.complex64): vtkConstants.VTK_FLOAT,
        numpy.dtype(numpy.complex128): vtkConstants.VTK_DOUBLE,
    }
    _extra = {
        numpy.dtype(ID_TYPE_CODE): vtkConstants.VTK_ID_TYPE,
        numpy.dtype(ULONG_TYPE_CODE): vtkConstants.VTK_UNSIGNED_LONG,
        numpy.dtype(LONG_TYPE_CODE): vtkConstants.VTK_LONG,
    }
    for t in _extra:
        if t not in _arr_vtk:
            _arr_vtk[t] = _extra[t]

    try:
        return _arr_vtk[numeric_array_type]
    except KeyError:
        for key in _arr_vtk:
            if numpy.issubdtype(numeric_array_type, key):
                return _arr_vtk[key]
    raise TypeError(
        "Couldn't translate array's type to VTK %s" % numeric_array_type
    )

def get_vtk_to_numeric_typemap():
    """Returns the VTK array type to numpy array type mapping."""
    _vtk_arr = {
        vtkConstants.VTK_BIT: numpy.bool_,
        vtkConstants.VTK_CHAR: numpy.int8,
        vtkConstants.VTK_SIGNED_CHAR: numpy.int8,
        vtkConstants.VTK_UNSIGNED_CHAR: numpy.uint8,
        vtkConstants.VTK_SHORT: numpy.int16,
        vtkConstants.VTK_UNSIGNED_SHORT: numpy.uint16,
        vtkConstants.VTK_INT: numpy.int32,
        vtkConstants.VTK_UNSIGNED_INT: numpy.uint32,
        vtkConstants.VTK_LONG: LONG_TYPE_CODE,
        vtkConstants.VTK_UNSIGNED_LONG: ULONG_TYPE_CODE,
        vtkConstants.VTK_LONG_LONG: numpy.int64,
        vtkConstants.VTK_ID_TYPE: ID_TYPE_CODE,
        vtkConstants.VTK_FLOAT: numpy.float32,
        vtkConstants.VTK_DOUBLE: numpy.float64
    }
    return _vtk_arr

def get_numeric_array_type(vtk_array_type):
    """Returns a numpy array typecode given a VTK array type."""
    return get_vtk_to_numeric_typemap()[vtk_array_type]

def get_sizeof_vtk_array(vtk_array_type):
    """Returns the size of a VTK array type."""
    _size_dict = {
        vtkConstants.VTK_BIT: 1,
        vtkConstants.VTK_CHAR: 1,
        vtkConstants.VTK_SIGNED_CHAR: 1,
        vtkConstants.VTK_UNSIGNED_CHAR: 1,
        vtkConstants.VTK_SHORT: 2,
        vtkConstants.VTK_UNSIGNED_SHORT: 2,
        vtkConstants.VTK_INT: 4,
        vtkConstants.VTK_UNSIGNED_INT: 4,
        vtkConstants.VTK_LONG: VTK_LONG_TYPE_SIZE,
        vtkConstants.VTK_UNSIGNED_LONG: VTK_LONG_TYPE_SIZE,
        vtkConstants.VTK_LONG_LONG: 8,
        vtkConstants.VTK_ID_TYPE: VTK_ID_TYPE_SIZE,
        vtkConstants.VTK_FLOAT: 4,
        vtkConstants.VTK_DOUBLE: 8
    }
    return _size_dict[vtk_array_type]

def create_vtk_array(vtk_arr_type):
    """Internal function used to create a VTK data array from another
    VTK array given the VTK array type.
    """
    tmp = vtk.vtkDataArray.CreateDataArray(vtk_arr_type)
    # CreateDataArray sets the refcount to 3 and this causes a severe
    # memory leak.
    tmp.SetReferenceCount(BASE_REFERENCE_COUNT)
    return tmp

def array2vtk(num_array, vtk_array=None):
    """Converts a real numpy Array (or a Python list) to a VTK array
    object.

    This function only works for real arrays.  Complex arrays are NOT
    handled.  It also works for multi-component arrays.  However, only
    1, and 2 dimensional arrays are supported.  This function is very
    efficient, so large arrays should not be a problem.

    Even in cases when no copy of the numpy array data is performed,
    a reference to the array is cached.  The passed array can
    therefore be deleted safely in all circumstances.

    Parameters
    ----------

    - num_array : numpy array or Python list/tuple

      The input array must be 1 or 2D.  A copy of the numeric array
      data passed is made in the following circumstances:

       1. A Python list/tuple was passed.
       2. A non-contiguous numpy array was passed.
       3. A `vtkBitArray` instance was passed as the second argument.
       4. The types of the `vtk_array` and the `num_array` are not
          equivalent to each other.  For example if one is an integer
          array and the other a float.

    - vtk_array : `vtkDataArray` (default: `None`)

      If an optional `vtkDataArray` instance, is passed as an argument
      then a new array is not created and returned.  The passed array
      is itself returned.

    """

    z = numpy.asarray(num_array)

    shape = z.shape
    assert len(shape) < 3, \
        "Only arrays of dimensionality 2 or lower are allowed!"
    assert not numpy.issubdtype(z.dtype, numpy.complexfloating), \
        "Complex numpy arrays cannot be converted to vtk arrays."\
        "Use real() or imag() to get a component of the array before"\
        " passing it to vtk."

    # First create an array of the right type by using the typecode.
    # Bit arrays need special casing.
    bit_array = False
    if vtk_array is None:
        vtk_typecode = get_vtk_array_type(z.dtype)
        result_array = create_vtk_array(vtk_typecode)
    elif vtk_array.GetDataType() == vtkConstants.VTK_BIT:
        vtk_typecode = vtkConstants.VTK_CHAR
        result_array = create_vtk_array(vtkConstants.VTK_CHAR)
        bit_array = True
    else:
        vtk_typecode = vtk_array.GetDataType()
        result_array = vtk_array

    # Find the shape and set number of components.
    if len(shape) == 1:
        result_array.SetNumberOfComponents(1)
    else:
        result_array.SetNumberOfComponents(shape[1])

    result_array.SetNumberOfTuples(shape[0])

    # Ravel the array appropriately.
    arr_dtype = get_numeric_array_type(vtk_typecode)
    if numpy.issubdtype(z.dtype, arr_dtype):
        z_flat = numpy.ravel(z)
    else:
        z_flat = numpy.ravel(z).astype(arr_dtype)

    # Point the VTK array to the numpy data.  The last argument (1)
    # tells the array not to deallocate.
    result_array.SetVoidArray(getbuffer(z_flat), len(z_flat), 1)

    if bit_array:
        # Handle bit arrays -- they have to be copied.  Note that bit
        # arrays are used ONLY when the user has passed one as an
        # argument to this function.
        vtk_array.SetNumberOfTuples(result_array.GetNumberOfTuples())
        vtk_array.SetNumberOfComponents(result_array.GetNumberOfComponents())
        for i in range(result_array.GetNumberOfComponents()):
            vtk_array.CopyComponent(i, result_array, i)
        result_array = vtk_array
    else:
        # Save a reference to the flatted array in the array cache.
        # This prevents the user from deleting or resizing the array
        # and getting into serious trouble.  This is only done for
        # non-bit array cases where the data is not copied.
        global _array_cache
        _array_cache.add(result_array, z_flat)

    return result_array

def vtk2array(vtk_array):
    """Converts a VTK data array to a numpy array.

    Given a subclass of vtkDataArray, this function returns an
    appropriate numpy array containing the same data.  The function
    is very efficient since it uses the VTK imaging pipeline to
    convert the data.  If a sufficiently new version of VTK (5.2) is
    installed then it actually uses the buffer interface to return a
    view of the VTK array in the returned numpy array.

    Parameters
    ----------

    - vtk_array : `vtkDataArray`

      The VTK data array to be converted.

    """
    typ = vtk_array.GetDataType()
    assert typ in get_vtk_to_numeric_typemap().keys(), \
        "Unsupported array type %s" % typ

    shape = (vtk_array.GetNumberOfTuples(),
             vtk_array.GetNumberOfComponents())
    if shape[0] == 0:
        dtype = get_numeric_array_type(typ)
        return numpy.array([], dtype)

    # First check if this array already has a numpy array cached,
    # if it does and the array size has not been changed, reshape
    # that and return it.
    if vtk_array in _array_cache:
        arr = _array_cache.get(vtk_array)
        if shape[1] == 1:
            shape = (shape[0], )
        if arr.size == numpy.prod(shape):
            arr = numpy.reshape(arr, shape)
            return arr

    # If VTK's new numpy support is available, use the buffer interface.
    if numpy_support is not None and typ != vtkConstants.VTK_BIT:
        dtype = get_numeric_array_type(typ)
        result = numpy.frombuffer(vtk_array, dtype=dtype)
        if shape[1] == 1:
            shape = (shape[0], )
        result.shape = shape
        return result

    # Setup an imaging pipeline to export the array.
    img_data = vtk.vtkImageData()
    img_data.SetDimensions(shape[0], 1, 1)
    if typ == vtkConstants.VTK_BIT:
        iarr = vtk.vtkCharArray()
        iarr.DeepCopy(vtk_array)
        img_data.GetPointData().SetScalars(iarr)
    elif typ == vtkConstants.VTK_ID_TYPE:
        # Needed since VTK_ID_TYPE does not work with VTK 4.5.
        iarr = vtk.vtkLongArray()
        iarr.SetNumberOfTuples(vtk_array.GetNumberOfTuples())
        nc = vtk_array.GetNumberOfComponents()
        iarr.SetNumberOfComponents(nc)
        for i in range(nc):
            iarr.CopyComponent(i, vtk_array, i)
        img_data.GetPointData().SetScalars(iarr)
    else:
        img_data.GetPointData().SetScalars(vtk_array)

    if typ == vtkConstants.VTK_ID_TYPE:
        r_dtype = get_numeric_array_type(vtkConstants.VTK_LONG)
    elif typ == vtkConstants.VTK_BIT:
        r_dtype = get_numeric_array_type(vtkConstants.VTK_CHAR)
    else:
        r_dtype = get_numeric_array_type(typ)
    img_data.Modified()

    exp = vtk.vtkImageExport()
    exp.SetInputData(img_data)

    # Create an array of the right size and export the image into it.
    im_arr = numpy.empty((shape[0]*shape[1],), r_dtype)
    exp.Export(im_arr)

    # Now reshape it.
    if shape[1] == 1:
        shape = (shape[0], )
    im_arr = numpy.reshape(im_arr, shape)
    return im_arr

def array2vtkCellArray(num_array, vtk_array=None):
    """Given a nested Python list or a numpy array, this method
    creates a vtkCellArray instance and returns it.

    A variety of input arguments are supported as described in the
    Parameter documentation.  If numpy arrays are given, this method
    is highly efficient.  This function is most efficient if the
    passed numpy arrays have a typecode `ID_TYPE_CODE`.  Otherwise a
    typecast is necessary and this involves an extra copy.  This
    method *always copies* the input data.

    An alternative and more efficient way to build the connectivity
    list is to create a vtkIdTypeArray having data of the form
    (npts,p0,p1,...p(npts-1), repeated for each cell) and then call
    <vtkCellArray_instance>.SetCells(n_cell, id_list).

    Parameters
    ----------

    - num_array : numpy array or Python list/tuple

      Valid values are:

        1. A Python list of 1D lists.  Each 1D list can contain one
           cell connectivity list.  This is very slow and is to be
           used only when efficiency is of no consequence.

        2. A 2D numpy array with the cell connectivity list.

        3. A Python list of 2D numpy arrays.  Each numeric array can
           have a different shape.  This makes it easy to generate a
           cell array having cells of different kinds.

    - vtk_array : `vtkCellArray` (default: `None`)

      If an optional `vtkCellArray` instance, is passed as an argument
      then a new array is not created and returned.  The passed array
      is itself modified and returned.

    Example
    -------

       >>> a = [[0], [1, 2], [3, 4, 5], [6, 7, 8, 9]]
       >>> cells = array_handler.array2vtkCellArray(a)
       >>> a = numpy.array([[0,1,2], [3,4,5], [6,7,8]], 'l')
       >>> cells = array_handler.array2vtkCellArray(a)
       >>> l_a = [a[:,:1], a[:2,:2], a]
       >>> cells = array_handler.array2vtkCellArray(l_a)

    """
    if vtk_array:
        cells = vtk_array
    else:
        cells = vtk.vtkCellArray()
    assert cells.GetClassName() == 'vtkCellArray', \
        'Second argument must be a `vtkCellArray` instance.'

    if len(num_array) == 0:
        return cells

    ########################################
    # Internal functions.
    def _slow_array2cells(z, cells):
        cells.Reset()
        vtk_ids = vtk.vtkIdList()
        for i in z:
            vtk_ids.Reset()
            for j in i:
                vtk_ids.InsertNextId(j)
            cells.InsertNextCell(vtk_ids)

    def _get_tmp_array(arr):
        try:
            tmp_arr = numpy.asarray(arr, ID_TYPE_CODE)
        except TypeError:
            tmp_arr = arr.astype(ID_TYPE_CODE)
        return tmp_arr

    def _set_cells(cells, n_cells, id_typ_arr):
        vtk_arr = vtk.vtkIdTypeArray()
        array2vtk(id_typ_arr, vtk_arr)
        cells.SetCells(n_cells, vtk_arr)
    ########################################

    msg = "Invalid argument.  Valid types are a Python list of lists,"\
          " a Python list of numpy arrays, or a numpy array."

    if issubclass(type(num_array), (list, tuple)):
        assert len(num_array[0]) > 0, "Input array must be 2D."
        tp = type(num_array[0])
        if issubclass(tp, list):  # Pure Python list.
            _slow_array2cells(num_array, cells)
            return cells
        elif issubclass(tp, numpy.ndarray):  # List of arrays.
            # Check shape of array and find total size.
            tot_size = 0
            n_cells = 0
            for arr in num_array:
                assert len(arr.shape) == 2, "Each array must be 2D"
                shp = arr.shape
                tot_size += shp[0]*(shp[1] + 1)
                n_cells += shp[0]
            # Create an empty array.
            id_typ_arr = numpy.empty((tot_size,), ID_TYPE_CODE)
            # Now populate it with the ids.
            count = 0
            for arr in num_array:
                tmp_arr = _get_tmp_array(arr)
                shp = arr.shape
                sz = shp[0]*(shp[1] + 1)
                set_id_type_array(tmp_arr, id_typ_arr[count:count+sz])
                count += sz
            # Now set them cells.
            _set_cells(cells, n_cells, id_typ_arr)
            return cells
        else:
            raise TypeError(msg)
    elif issubclass(type(num_array), numpy.ndarray):
        assert len(num_array.shape) == 2, "Input array must be 2D."
        tmp_arr = _get_tmp_array(num_array)
        shp = tmp_arr.shape
        id_typ_arr = numpy.empty((shp[0]*(shp[1] + 1),), ID_TYPE_CODE)
        set_id_type_array(tmp_arr, id_typ_arr)
        _set_cells(cells, shp[0], id_typ_arr)
        return cells
    else:
        raise TypeError(msg)

def array2vtkPoints(num_array, vtk_points=None):
    """Converts a numpy array/Python list to a vtkPoints object.

    Unless a Python list/tuple or a non-contiguous array is given, no
    copy of the data is made.  Thus the function is very efficient.

    Parameters
    ----------

    - num_array : numpy array or Python list/tuple

      The input array must be 2D with `shape[1] == 3`.

    - vtk_points : `vtkPoints` (default: `None`)

      If an optional `vtkPoints` instance, is passed as an argument
      then a new array is not created and returned.  The passed array
      is itself modified and returned.

    """
    if vtk_points:
        points = vtk_points
    else:
        points = vtk.vtkPoints()

    arr = numpy.asarray(num_array)
    assert len(arr.shape) == 2, "Points array must be 2 dimensional."
    assert arr.shape[1] == 3, "Incorrect shape: shape[1] must be 3."
    vtk_array = array2vtk(arr)
    points.SetData(vtk_array)
    return points

def array2vtkIdList(num_array, vtk_idlist=None):
    """Converts a numpy array/Python list to a vtkIdList object.

    Parameters
    ----------

    - num_array : numpy array or Python list/tuple

      The input array must be 2D with `shape[1] == 3`.

    - vtk_idlist : `vtkIdList` (default: `None`)

      If an optional `vtkIdList` instance, is passed as an argument
      then a new array is not created and returned.  The passed array
      is itself modified and returned.

    """
    if vtk_idlist:
        ids = vtk_idlist
    else:
        ids = vtk.vtkIdList()

    arr = numpy.asarray(num_array)
    assert len(arr.shape) == 1, "Array for vtkIdList must be 1D"
    ids.SetNumberOfIds(len(arr))
    for i, j in enumerate(arr):
        ids.SetId(i, j)
    return ids

######################################################################
# Array argument handling functions.
######################################################################

def is_array(arr):
    """Returns True if the passed `arr` is a numpy array or a List."""
    if issubclass(type(arr), (numpy.ndarray, list)):
        return True
    return False

def convert_array(arr, vtk_typ=None):
    """Convert the given array to the optional type specified by
    `vtk_typ`.

    Parameters
    ----------

    - arr : numpy array/list.
    - vtk_typ : `string` or `None`
      represents the type the array is to be converted to.

    """
    if vtk_typ:
        conv = {'vtkCellArray': array2vtkCellArray,
                'vtkPoints': array2vtkPoints,
                'vtkIdList': array2vtkIdList}
        if vtk_typ in conv.keys():
            vtk_arr = getattr(vtk, vtk_typ)()
            return conv[vtk_typ](arr, vtk_arr)
        elif vtk_typ.find('Array') > -1:
            try:
                vtk_arr = getattr(vtk, vtk_typ)()
            except TypeError:  # vtk_typ == 'vtkDataArray'
                return array2vtk(arr)
            else:
                return array2vtk(arr, vtk_arr)
        else:
            return arr
    else:
        return array2vtk(arr)

def is_array_sig(s):
    """Given a signature, return if the signature has an array."""
    if not isinstance(s, str):
        return False
    arr_types = ['Array', 'vtkPoints', 'vtkIdList']
    for i in arr_types:
        if s.find(i) > -1:
            return True
    return False

def is_array_or_vtkarray(arg):
    """Returns True if the argument is an array/Python list or if it
    is a vtk array."""

    if is_array(arg):
        return True
    else:
        if hasattr(arg, '_vtk_obj'):
            if is_array_sig(arg._vtk_obj.__class__.__name__):
                return True
    return False

def get_correct_sig(args, sigs):
    """Given a list of args and a collection of possible signatures,
    this function returns the most appropriate signature.  This
    function is only called by deref_array.  This implies that one of
    the signatures has an array type.

    """
    # First do the trivial cases.
    if sigs is None:
        return None
    if len(sigs) == 1:
        return sigs[0]
    else:
        # Non-trivial cases.
        la = len(args)
        candidate_sigs = [s for s in sigs if len(s) == la]
        count = len(candidate_sigs)
        if count == 0:
            # No sig has the right number of args.
            msg = "Insufficient number of arguments to method."\
                  "Valid arguments are:\n%s" % sigs
            raise TypeError(msg)
        elif count == 1:
            # If only one of the sigs has the right number of args,
            # return it.
            return candidate_sigs[0]
        else:
            # More than one sig has the same number of args.
            # Check if args need conversion at all.
            array_idx = [i for i, a in enumerate(args)
                         if is_array_or_vtkarray(a)]
            n_arr = len(array_idx)
            if n_arr == 0:
                # No conversion necessary so signature info is
                # useless.
                return None
            else:
                # Need to find the right sig.  This is done by finding
                # the first signature that matches all the arrays in
                # the argument.
                for sig in candidate_sigs:
                    array_in_sig = [is_array_sig(s) for s in sig]
                    if array_in_sig.count(True) != len(array_idx):
                        continue
                    bad = False
                    for i in array_idx:
                        if not array_in_sig[i]:
                            bad = True
                    if not bad:
                        return sig
                # Could not find any valid signature, so give up.
                return None

def deref_vtk(obj):
    """Dereferences the VTK object from the object if possible.  This
    is duplicated from `tvtk_base.py` because I'd like to keep this
    module independent of `tvtk_base.py`.
    """
    if hasattr(obj, '_vtk_obj'):
        return obj._vtk_obj
    else:
        return obj

def deref_array(args, sigs=None):
    """Given a bunch of arguments and optional signature information,
    this converts the arguments suitably.  If the argument is either a
    Python list or a numpy array it is converted to a suitable type
    based on the signature information.  If it is not an array, but a
    TVTK object the VTK object is dereferenced.  Otherwise nothing is
    done.  If no signature information is provided the arrays are
    automatically converted (this can sometimes go wrong).  The
    signature information is provided in the form of a list of lists.

    """
    ret = []
    sig = get_correct_sig(args, sigs)
    if sig:
        for a, s in zip(args, sig):
            if is_array(a) and is_array_sig(s):
                ret.append(convert_array(a, s))
            else:
                ret.append(deref_vtk(a))
    else:
        for a in args:
            if is_array(a):
                ret.append(convert_array(a))
            else:
                ret.append(deref_vtk(a))
    return ret


import matplotlib.pyplot as plt
import numpy as np

"""
Plot of success rate for a single NN with different control horizons
"""

# testing_result = [47.74, 52.76, 61.81, 63.82, 50.75]
# baseline_result = [44.72, 45.73, 51.25, 52.26, 46.73]

testing_result = [48, 53, 62, 64, 51]
baseline_result = [45, 46, 51, 52, 47]

# labels = ['1.5', '1', '0.5', '0.25', '0.15']
labels = ['0.67', '1', '2', '4', '6.67']
# x = np.arange(len(labels))
x_coordinate = np.asarray([1, 2, 3, 4, 5])

fig, ax = plt.subplots()
ax.set_ylabel('Success rate (%)', fontsize=12)
ax.set_xlabel('Replanning frequency (Hz)', fontsize=12)
ax.set_title('Success rate with different replanning frequencies', fontsize=12)
ax.set_xticks(x_coordinate)
ax.set_xticklabels(labels)
ax.set_yticks(np.arange(45, 65, 5))

plt.plot(x_coordinate, testing_result, 'o-', label='WayPtNav-ReachabilityCost')
plt.plot(x_coordinate, baseline_result, 's-', label='WayPtNav-HeuristicsCost')

# for i, value in enumerate(testing_result):
#     x = x_coordinate[i]
#     y = testing_result[i]
#     if i == 0:
#         scatter = ax.scatter(x, y, marker='x', color='red', label='Ours')
#     else:
#         scatter = ax.scatter(x, y, marker='x', color='red')
#     ax.text(x + 0.05, y + 0.05, value, fontsize=9)
#
#
# for i, value in enumerate(baseline_result):
#     x = x_coordinate[i]
#     y = baseline_result[i]
#     if i == 0:
#         ax.scatter(x, y, marker='o', color='blue', label='Baseline')
#     else:
#         ax.scatter(x, y, marker='o', color='blue')
#     ax.text(x + 0.05, y + 0.05, value, fontsize=9)

ax.legend(loc='lower right')

ax.set_aspect(aspect=0.2)

plt.show()

plot_path = '/home/anjianl/Desktop/project/WayPtNav_paper/plots/ctrlhorizon_success_rate.png'
fig.savefig(plot_path)


from django.urls import path

from . import views

app_name = "public"
urlpatterns = [
    path("", views.index, name="index"),
    path("about", views.about, name="about"),
    path("upload_dataset", views.upload_dataset, name="upload_dataset"),
    path("train_model", views.train_model, name="train_model"),
    path("test_model", views.test_model, name="test_model"),
]


import pandas as pd
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

#datamızı okuduk
df = pd.read_csv("C:\projects\intropattern\otu.csv")
#verinin tranzpozasını alarak left ve rightları column hale getirdik.
df=df.T
X = df.iloc[:,1:]
y = df.iloc[:,0]
#test ve veri setlerimizi ayırdık
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# AdaBoost modelimizi oluşturduk.n_estimators, dosyanın bir parçası olarak eğitilen zayıf modellerin sayısını ifade eder.
# Zayıf bir model, rastgele tahminden yalnızca biraz daha iyi olan ve tipik olarak tek bir güçlü modelden daha az doğru olan bir modeldir.
model = AdaBoostClassifier(n_estimators=100)

# Modeli oluşturduk
model.fit(X_train, y_train)

# Tahminde bulunuyoruz
predictions = model.predict(X_test)

accuracy = accuracy_score(y_test, predictions)
print("Accuracy:", accuracy)


import torch
from tqdm import tqdm

def evaluate(model, loader, device):
    """
    Evaluation function to calculate loss and accuracy on Val/test dataset
    Args:
        model (nn.Module): model to be evaluated on the give dataset
        loader (DataLoader): Validation/Test dataloader to evaluate the model on.
        device (torch.device): The device (CPU/GPU) to perform the evalutation on.
    """
    model.to(device)
    model.eval()
    correct = 0
    total = 0
    running_loss = 0.0
    criterion = torch.nn.CrossEntropyLoss()

    with torch.no_grad():
        for images, labels in loader:
            images, labels = images.to(device), labels.to(device)

            # Forward pass
            outputs = model(images)
            predicted = torch.argmax(outputs.data, 1)

            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            loss = criterion(outputs, labels)
            running_loss += loss.item()

    accuracy = 100 * correct / total
    avg_loss = running_loss / len(loader)
    model.train()
    return accuracy, avg_loss, correct


# -*- coding = utf-8 -*-
"""
    the :mod `dataset` module provides the dataset class
    and other subclasses which are used for managing datasets
"""
import pandas as pd
import numpy as np

class Dataset:
    """base class for loading datasets

    Note that you should never instantiate the class :class: `Dataset` class directly,
    and just use the below available methods for loading datasets.
    """

    def __init__(self,cfg):
        self._cfg = cfg

        self.movies = pd.read_csv(self._cfg.DATASET.MOVIE_SET)
        self.ratings_train = pd.read_csv(self._cfg.DATASET.TRAIN_SET)
        self.ratings_test = pd.read_csv(self._cfg.DATASET.TEST_SET)

        self.user_list_train = self.ratings_train['userId'].drop_duplicates().values.tolist()
        self.user_list_test = self.ratings_test['userId'].drop_duplicates().values.tolist()
        self.user_list = self.user_list_train + self.user_list_test
        self.user_list = list(set(self.user_list))

        self.movie_list = self.movies['movieId'].drop_duplicates().values.tolist()
        self.genre_list = self.movies['genres'].values.tolist()
        self.movie_type_list = self.get_movie_type_list(self.genre_list)

        self.user_map_train, self.user_map_reverse_train = self.get_list_index_map(self.user_list_train)
        self.user_map_test, self.user_map_reverse_test = self.get_list_index_map(self.user_list_test)
        self.type_map, self.type_map_reverse = self.get_list_index_map(self.movie_type_list)
        self.user_map, self.user_map_reverse = self.get_list_index_map(self.user_list)
        self.movie_map, self.movie_map_reverse = self.get_list_index_map(self.movie_list)

        self.movie_type_features = self.get_movie_type_features(self.movies)

    def get_movie_type_features(self,movies):
        """
        get the movie type features, tf-idf matrix
        """
        movie_type_features = np.zeros((len(self.movie_list),len(self.movie_type_list)))
        for row in self.movies.itertuples(index=True,name="Pandas"):
            movie_id = self.movie_map[getattr(row,'movieId')]
            movie_types = getattr(row,'genres').split('|')
            for movie_type in movie_types:
                if movie_type != '(no genres listed)':
                    movie_type_index = self.type_map[movie_type]
                    movie_type_features[movie_id,movie_type_index] = 1
        return movie_type_features # tfidf matri

    def get_list_index_map(self,list):
        """
        get the index map of a list
        """
        index_map = {}
        index_map_reverse = {}
        for i,item in enumerate(list):
            index_map[item] = i
            index_map_reverse[i] = item
        return index_map, index_map_reverse

    def get_movie_type_list(self,genres_list):
        """
        get the movie type list
        """
        movie_type_list = []
        for item in genres_list:
            movie_types = item.split('|')
            for movie_type in movie_types:
                if movie_type not in movie_type_list and movie_type != '(no genres listed)':
                    movie_type_list.append(movie_type)
        return movie_type_list

    def get_trainset(self):
        """
        get the trainset
        @return: (trainset,user_map,movie_map,type_map)
        """
        return (self.ratings_train,self.user_map,self.movie_map,self.movie_type_features)

    def get_testset(self):
        """
        get the testset
        @return: (testset,user_map,movie_map,type_map)
        """
        return (self.ratings_test,self.user_map,self.movie_map,self.movie_type_features)

    def get_movie_name_by_movie_id(self,movie_id):
        """
        get the movie name by movie id
        """
        return self.movies[self.movies['movieId'] == movie_id]['title'].values[0]

if __name__ == '__main__':
    from config import cfg
    dataset = Dataset(cfg)
    # print(dataset.user_list)
    # print(dataset.movie_list)
    # print(dataset.movie_type_list)
    print(dataset.type_map)
    print(dataset.type_map_reverse)
    # print(dataset.user_map)
    # print(dataset.user_map_reverse)
    # print(dataset.movie_map)
    # print(dataset.movie_map_reverse)

    # genres type list
    print(dataset.type_map.keys())
    # ['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy', 'Romance', 'Drama', 'Action', 'Crime', 'Thriller', 'Horror', 'Mystery', 'Sci-Fi', 'Documentary', 'IMAX', 'War', 'Musical', 'Western', 'Film-Noir']


# -*- coding: utf-8 -*-
from PyQt5 import QtWidgets
from PyQt5.QtWidgets import QMessageBox

class MyWindow(QtWidgets.QWidget):
    def __init__(self):
        super().__init__()
        self.myButton = QtWidgets.QPushButton(self)

        self.myButton.clicked.connect(self.msg)
        self.msg()
    def msg(self):
        reply = QMessageBox.information(self,  # 使用infomation信息框
                                        "标题",
                                        "消息",
                                        QMessageBox.Yes | QMessageBox.No)

if __name__ == "__main__":
    import sys

    app = QtWidgets.QApplication(sys.argv)
    myshow = MyWindow()
    myshow.show()
    sys.exit(app.exec_())


import shutil
import difflib
import filecmp

import logging
from prettyconf import config

BOOTSTRAP_TESTS = config("BOOTSTRAP_TESTS", default=False)

LOG = logging.getLogger(__name__)

def compare_files(*, got, expected):
    """Compares the contents of a test file against expected

    Args:
        got: file with temp data
        expected: file with expected data

    Note: setting the environment variable BOOTSTRAP_TESTS=1
    will overwrite the contents of the "expected" file with the
    "got" file (ie bootstrapping the test files).
    """
    if BOOTSTRAP_TESTS:
        LOG.warning("BOOTSTRAP_TESTS: copying '%s' to '%s'", got, expected)
        shutil.copy(got, expected)

    got = f"{got}"
    expected = f"{expected}"

    are_files_identical = filecmp.cmp(got, expected)

    if not are_files_identical:
        diff_result = list(
            difflib.unified_diff(
                open(got, "r").readlines(), open(expected, "r").readlines()
            )
        )
        LOG.warning(f"Difference between got ({got}) and expected ({expected}) ...")
        for diff_line in diff_result:
            LOG.warning(diff_line.strip())

    return are_files_identical


from flask import Flask, request, jsonify
import requests
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, BooleanType
import threading
import logging
import time

app = Flask(__name__)

# Create a SparkSession (Singleton)
spark = SparkSession.builder.appName("APIDataProcessing").getOrCreate()

# Define the schema for the DataFrame
schema = StructType([
    StructField("API", StringType(), True),
    StructField("Description", StringType(), True),
    StructField("Category", StringType(), True),
    StructField("Auth", StringType(), True),
    StructField("HTTPS", BooleanType(), True),
    StructField("Cors", StringType(), True),
    StructField("Link", StringType(), True),
])

# Function to fetch and filter data and create a Spark DataFrame
def fetch_and_filter_data(api_url):
    try:
        response = requests.get(api_url)
        response.raise_for_status()
        data = response.json()['entries']

        # Filter the data to include only HTTPS links
        filtered_data = [entry for entry in data if entry['HTTPS']]

        # Create a Spark DataFrame from the filtered data
        df = spark.createDataFrame(filtered_data, schema=schema)
        return df
    except requests.exceptions.RequestException as e:
        logging.error(f"Error fetching data from the API: {e}")
        return None

# Function to periodically fetch and save data
def periodic_data_fetch_and_save(api_url, interval_seconds, data_path):
    while True:
        df = fetch_and_filter_data(api_url)

        if df:
            # Save the DataFrame as a partitioned Parquet file
            df.write.partitionBy("Category").parquet(data_path, mode="append")
            logging.info(f"Data saved at {time.ctime()}")

        time.sleep(interval_seconds)

# Start a background thread to periodically fetch and save data
api_url = "https://api.publicapis.org/entries"
data_path = "api_data.parquet"
fetch_thread = threading.Thread(target=periodic_data_fetch_and_save, args=(api_url, 12*60*60, data_path))  # Fetch every 12 hours
fetch_thread.daemon = True
fetch_thread.start()

# Endpoint for /categories
@app.route('/categories', methods=['GET'])
def get_categories():
    # Load categories from the saved data file
    df = spark.read.parquet(data_path)
    categories = df.select("Category").distinct().rdd.flatMap(lambda x: x).collect()

    # Return categories as JSON
    return jsonify(categories)

# Endpoint for /data/<category>
@app.route('/data/<category>', methods=['GET'])
def get_data_by_category(category):
    # Load data from the saved data file
    df = spark.read.parquet(data_path)

    # Get the optional search query from the request
    search_query = request.args.get('search_query')

    # Filter data by category
    filtered_df = df.filter(df["Category"] == category)

    # Apply search filter if search_query is provided
    if search_query:
        search_query = search_query.lower()
        filtered_df = filtered_df.filter(
            (df["API"].rlike(search_query)) | (df["Description"].rlike(search_query))
        )

    # Convert the filtered DataFrame to a list of dictionaries
    filtered_data = filtered_df.rdd.map(lambda row: row.asDict()).collect()

    # Return filtered data as JSON
    return jsonify(filtered_data)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    app.run(debug=True)


from google.cloud import bigquery
import plotly.express as px

dataset_id = "salestest"

class DatasetManager(object):

    def __init__(self, dataset_id):
        self.dataset_id = dataset_id
        self.client = self._get_client()

    def print_listed_projeto(self):
        """INFORMA O NOME DO PROJETO DO SERVICE ACCOUNT"""
        projects = list(self.client.list_projects())
        print("Projects:")
        for project in projects:
            print(project.project_id)

    def data_set(self):
        """INFORMA O NOME DO DATASET DO SERVICE ACCOUNT"""
        datasets = list(self.client.list_datasets())
        print("Datasets:")
        for dataset in datasets:
            print(dataset.dataset_id)

    def tabelas(self):
        """INFORMA O NOME DAS TABELAS DO SERVICE ACCOUNT"""
        dataset_id = "sales_test"
        tables = list(self.client.list_tables(dataset_id))
        print("Tables:")
        for table in tables:
            print(table.table_id)

    def _get_client(self):
        return bigquery.Client.from_service_account_json('data/%s.json' % self.dataset_id)

    def query_dataset(self, query):
        return self.client.query(query).result().to_dataframe()

        # DÁ QUERY NA BIG QUERY DO SERVICE ACCOUNT

def get_vendas_por_marca_u6m(dataset_manager):
    query = f"""SELECT
                brand,
                SUM(value) as vendas_por_marca_u6m

                FROM salestest-373621.sales_test.fact_sales_product_day
                LEFT JOIN salestest-373621.sales_test.dim_product
                ON dim_product.product_id = fact_sales_product_day.product_id

                WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 24 MONTH)

                GROUP BY brand
                ORDER BY vendas_por_marca_u6m DESC
                """
    df = dataset_manager.query_dataset(query)
    return df.head()

def get_vendas_por_marca_por_dia_u6m(dataset_manager):
    projeto = "salestest-373621.sales_test"

    query = f"""SELECT
                brand,
                date,
                SUM(value) as vendas_por_marca_por_dia_u6m

                FROM {projeto}.fact_sales_product_day
                JOIN {projeto}.dim_product
                ON dim_product.product_id = fact_sales_product_day.product_id

                WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 6 MONTH)

                GROUP BY brand, date"""
    df = dataset_manager.query_dataset(query)
    return df

def plotar_vendas(dataset_manager):
    query = f"""SELECT
                brand,
                SUM(value) as vendas_por_marca_u6m

                FROM salestest-373621.sales_test.fact_sales_product_day
                LEFT JOIN salestest-373621.sales_test.dim_product
                ON dim_product.product_id = fact_sales_product_day.product_id

                WHERE date >= DATE_SUB(CURRENT_DATE(), INTERVAL 12 MONTH)

                GROUP BY brand
                ORDER BY vendas_por_marca_u6m DESC
                """
    df = dataset_manager.query_dataset(query)

    fig = px.bar(df,
                 title="VENDAS",
                 x="brand",
                 y="vendas_por_marca_u6m",
                 color="brand",
                 text_auto='.2s'

                 )
    fig.update_layout(paper_bgcolor="white",
                      plot_bgcolor="white",
                      yaxis_title='Faturamento'
                      )
    fig.update_traces(marker_color='darkgreen',
                      marker_line_color='rgb(8,48,107)',
                      marker_line_width=1.5,
                      opacity=0.9,
                      textfont_size=12,
                      textangle=0,
                      textposition="outside",
                      cliponaxis=False)
    return fig.show()

if __name__ == "__main__":
    query_manager = DatasetManager(dataset_id="salestest")
    plotar_vendas(query_manager)


import torch.nn as nn
from utils.config import config
import torch
import numpy as np
from model.bbox_tools import *
import torch.nn.functional as F
from torchvision.ops import nms

def nograd(f):
    def new_f(*args, **kwargs):
        with torch.no_grad():
            return f(*args, **kwargs)
    return new_f

class FasterRCNN(nn.Module):
    def __init__(self,
                 extractor,
                 rpn,
                 head,
                 offset_normalize_mean=(0., 0., 0., 0.),
                 offset_normalize_std=(0.1, 0.1, 0.2, 0.2),
                 ):
        super(FasterRCNN, self).__init__()
        self.extractor = extractor
        self.rpn = rpn
        self.head = head
        self.offset_normalize_mean = offset_normalize_mean
        self.offset_normalize_std = offset_normalize_std

    @property
    def n_class(self):
        return self.head.n_class

    def forward(self, x, scale):
        img_size = x.shape[2:]
        x = self.extractor(x)
        rois, rois_index, _, _, _ = self.rpn(
            x=x, img_size=img_size, scale=scale)
        roi_bbox_pred, roi_cls_scores = self.head(
            x=x, rois=rois, rois_index=rois_index)
        return rois, roi_bbox_pred, roi_cls_scores

    @nograd
    def predict(self, img, scale=1., eval=True):
        if eval:
            self.nms_thresh = 0.3
            self.score_thresh = 0.05
            # self.score_thresh = 0.65
        else:
            self.nms_thresh = 0.3
            self.score_thresh = 0.7

        _, _, H, W = img.shape
        img_size = (H, W)
        device = img.device

        self.eval()
        # with torch.no_grad():
        roi, roi_bbox_pred, roi_cls_scores = self(img, scale=scale)

        mean = torch.Tensor(self.offset_normalize_mean).to(
            device)[None, None, :]
        std = torch.Tensor(self.offset_normalize_std).to(device)[None, None, :]
        roi_bbox_pred = roi_bbox_pred.view(roi_bbox_pred.shape[0], -1, 4)
        roi_bbox_pred = (roi_bbox_pred * std) + mean

        roi = torch.FloatTensor(roi).to(
            device).view(-1, 1, 4).expand_as(roi_bbox_pred)
        pred_bbox = offset2bbox(roi.cpu().numpy().reshape((-1, 4)),
                                roi_bbox_pred.cpu().numpy().reshape((-1, 4)))
        pred_bbox = torch.FloatTensor(pred_bbox).to(device)
        pred_bbox = pred_bbox.view(-1, self.n_class * 4)

        pred_bbox[:, 0::2] = (pred_bbox[:, 0::2]).clamp(min=0, max=img_size[0])
        pred_bbox[:, 1::2] = (pred_bbox[:, 1::2]).clamp(min=0, max=img_size[1])

        prob = F.softmax(roi_cls_scores, dim=1)

        bbox, label, score = self.suppress(pred_bbox, prob)

        self.train()
        return bbox, label, score

    def suppress(self, pred_bbox, prob):
        bbox = list()
        label = list()
        score = list()

        for i in range(1, self.n_class):
            pred_bbox_i = pred_bbox.view(-1, self.n_class, 4)[:, i, :]
            prob_i = prob[:, i]
            mask = (prob_i > self.score_thresh)
            pred_bbox_i = pred_bbox_i[mask, :]
            prob_i = prob_i[mask]
            index_keep = nms(pred_bbox_i, prob_i, self.nms_thresh)
            bbox.append(pred_bbox_i[index_keep].cpu().numpy())
            label.append((i - 1) * np.ones((len(index_keep),)))
            score.append(prob_i[index_keep].cpu().numpy())
        bbox = np.concatenate(bbox, axis=0).astype(np.float32)
        label = np.concatenate(label, axis=0).astype(np.int32)
        score = np.concatenate(score, axis=0).astype(np.float32)

        return bbox, label, score

    def get_optimizer(self):
        self.optimizer = \
            torch.optim.SGD(self.parameters(), lr=config.lr,
                            momentum=0.9, weight_decay=config.weight_decay)
        return self.optimizer

    def scale_lr(self, decay=0.1):
        for param_group in self.optimizer.param_groups:
            param_group['lr'] *= decay
        return self.optimizer


from dronekit import Vehicle, connect, VehicleMode, Command
import time
from pymavlink.dialects.v20 import ardupilotmega
from pymavlink import mavutil

class DKVehicle(Vehicle):

    def __init__(self, connection):
        print ("Connecting to vehicle on: %s" % connection)
        self.vehicle = connect(connection, baud=57600, wait_ready=True)
        print ("Connected to vehicle on: %s" % connection)
        self.connection = connection

        # Force Dronekit to use Mavlink v2.0
        self.vehicle._master.first_byte = True

        self.servo_output = []
        for x in range(17):
            self.servo_output.append(0)

        self.servo_func = {
            '1': 4,             # Aileron
            '2': 19,            # Elevator
            '3': 70,            # Throttle
            '4': 21,            # Rudder
            '5': 33,            # Motor1
            '6': 34,            # Motor2
            '7': 35,            # Motor3
            '8': 36,            # Motor4
            '9': 0,             # Disabled
            '10': 0,            # Disabled
            '11': 0,            # Disabled
            '12': 0,            # Disabled
            '13': 0,            # Disabled
            '14': 0,            # Disabled
            '15': 0,            # Disabled
            '16': 0,            # Disabled
            }

    def printstats(self):
        print ("Vehicle: %s" % self.vehicle.version)
        print (" Connected on: %s" % self.connection)
        print (" GPS: %s" % self.vehicle.gps_0)
        print (" Battery: %s" % self.vehicle.battery)
        print (" Last Heartbeat: %s" % self.vehicle.last_heartbeat)
        print (" Is Armable?: %s" % self.vehicle.is_armable)
        print (" System status: %s" % self.vehicle.system_status.state)
        print (" Mode: %s" % self.vehicle.mode.name)

    def print_servos(self):
        for x in range(0,16):
            print("Servo%s:  %s" % (x, self.servo_output[x]))

    def print_servo(self, servo):
        print("Servo%s:  %s" % (servo, self.servo_output[servo]))

    def print_channels(self):
        print(" Ch1: %s" % self.vehicle.channels['1'])
        print(" Ch2: %s" % self.vehicle.channels['2'])
        print(" Ch3: %s" % self.vehicle.channels['3'])
        print(" Ch4: %s" % self.vehicle.channels['4'])
        print(" Ch5: %s" % self.vehicle.channels['5'])
        print(" Ch6: %s" % self.vehicle.channels['6'])
        print(" Ch7: %s" % self.vehicle.channels['7'])
        print(" Ch8: %s" % self.vehicle.channels['8'])

    def override_servo(self, servo,val):
        servo_string = 'SERVO' + str(servo) + '_FUNCTION'
        self.vehicle.parameters[servo_string]=0

        msg = self.vehicle.message_factory.command_long_encode(0, 0, mavutil.mavlink.MAV_CMD_DO_SET_SERVO, 0, servo, val, 0, 0, 0, 0, 0)
        self.vehicle.send_mavlink(msg)

        self.vehicle.flush()

    def disable_servo(self, servo):
        servo_string = 'SERVO' + str(servo) + '_FUNCTION'
        servo_trim = 'SERVO' + str(servo) + '_TRIM'
        self.vehicle.parameters[servo_string]=0
        val = self.vehicle.parameters[servo_trim]

        msg = self.vehicle.message_factory.command_long_encode(0, 0, mavutil.mavlink.MAV_CMD_DO_SET_SERVO, 0, servo, val, 0, 0, 0, 0, 0)
        self.vehicle.send_mavlink(msg)

        self.vehicle.flush()

    def enable_servo(self, servo):
        servo_string = 'SERVO' + str(servo) + '_FUNCTION'
        servo_trim = 'SERVO' + str(servo) + '_TRIM'

        val = self.vehicle.parameters[servo_trim]
        if self.servo_func[str(servo)] == 0:
            val = 0

        msg = self.vehicle.message_factory.command_long_encode(0, 0, mavutil.mavlink.MAV_CMD_DO_SET_SERVO, 0, servo, val, 0, 0, 0, 0, 0)
        self.vehicle.send_mavlink(msg)

        self.vehicle.parameters[servo_string]=self.servo_func[str(servo)]

        self.vehicle.flush()

    def print_servo_functions(self):
        for servo in range(1,17):
            servo_string = 'SERVO' + str(servo) + '_FUNCTION'
            print(servo_string + ': ' + str(self.vehicle.parameters[servo_string]))

    def print_frame_type(self):
        print("Q_ENABLE: " + str(self.vehicle.parameters['Q_ENABLE']))
        print("Q_FRAME_TYPE: " + str(self.vehicle.parameters['Q_FRAME_TYPE']))

    def set_frame_type(self,frame):
        self.vehicle.parameters['Q_FRAME_TYPE'] = frame


from Transformer import Transformer
from MultiHeadAttention import MultiHeadAttention
from tqdm import tqdm
from Metrics import grad
from Metrics import loss_function
from Metrics import loss_function2
from Metrics import accuracy_function
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score

from einops import rearrange

import tensorflow as tf
import os
import numpy as np
import joblib
import pandas as pd
import logging
import sqlite3 as sql
import time

num_layers = int(os.environ["TRANSFORMER_LAYERS"])
d_model = int(os.environ["W2V_EMBED_SIZE"])
dff = int(os.environ["TRANSFORMER_DFF"])
num_heads = int(os.environ["TRANSFORMER_HEADS"])
batch_size = int(os.environ["BATCH_SIZE"])
training = bool(int(os.environ["TRAINING"]))
epochs = int(os.environ["EPOCHS"])
max_seq_len = 200

class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):

    def __init__(self, d_model: int, warmup_steps=4000):
        super(CustomSchedule, self).__init__()

        self.d_model = d_model
        self.d_model = tf.cast(self.d_model, tf.float32)
        self.warmup_steps = warmup_steps

    def __call__(self, step):
        arg1 = tf.math.rsqrt(step)
        arg2 = step * (self.warmup_steps ** -1.5)

        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)

learning_rate = CustomSchedule(d_model)

optimus_prime = None
adm_optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)

epoch_loss = tf.keras.metrics.Mean(name='train_loss')
epoch_accuracy = tf.keras.metrics.Mean(name='train_accuracy')

train_step_signature = [
    tf.TensorSpec(shape=([batch_size, None]), dtype=tf.float32),
    tf.TensorSpec(shape=([batch_size]), dtype=tf.float32)
]

add_att_layer = tf.keras.layers.AdditiveAttention()
softmax = tf.keras.layers.Softmax()
lr = LogisticRegression()

s1 = tf.keras.Sequential([
    tf.keras.layers.Dense(512),
    tf.keras.layers.Dense(4),
    tf.keras.layers.Softmax()
])

@tf.function(input_signature=train_step_signature)
def train_step(log_batch: tf.Tensor, labels: tf.Tensor):

    transformer_input = tf.tuple([
        log_batch,  # <tf.Tensor: shape=(batch_size, max_seq_len), dtype=float32>
        labels  # <tf.Tensor: shape=(batch_size, num_classes), dtype=float32>
    ])

    with tf.GradientTape() as tape:
        Rs, _ = optimus_prime.call(transformer_input)

        # a_s = add_att_layer([Rs, Rs])
        # y = softmax(a_s * Rs)

        y = Rs

        loss = tf.py_function(loss_function, [labels, y], tf.float32)

        pred = s1(y)
        labels = tf.cast(labels, tf.int64)

    # Optimize the model
    grads = tape.gradient(loss, optimus_prime.trainable_variables)
    adm_optimizer.apply_gradients(zip(grads, optimus_prime.trainable_variables))

    # Tracking Progress
    epoch_loss.update_state(loss)  # Adding Batch Loss
    epoch_accuracy.update_state(accuracy_function(labels, pred))

logging.basicConfig(format='%(asctime)s %(levelname)s | %(message)s',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

def database_builder(path: str) -> pd.DataFrame():
    logger.info('Building DataFrame ...')
    (_, _, files) = next(os.walk(path))
    sql_query = 'SELECT * FROM logs'
    data = []
    for f in files:
        if '.db' in f:
            conn = create_connection(path + f)
            d = pd.read_sql_query(sql_query, conn)
            data.append(d)
    logger.info('...complete!')
    return pd.concat(data)

def create_connection(path: str) -> sql.Connection:
    """
    Creates a database connection
    :param path: str
        path to database object
    :return sql.Connection
        a connection to the database
    """
    try:
        conn = sql.connect(path)
        logger.info('Connected to database ' + path)
        return conn
    except sql.Error as e:
        logger.warning(e)

def get_max_length_(dataset: pd.DataFrame, buffer_size: float) -> int:
    return int((1 + buffer_size) * dataset['log'].str.len().max())

def process_batch(dataset: pd.DataFrame,
                  vocabulary: dict,
                  max_seq_len: int,
                  idx: int,
                  labels: dict) -> tuple:
    logs = np.zeros((batch_size, max_seq_len))
    y_true = np.empty((batch_size,))

    start_window = idx * batch_size
    end_window = (idx + 1) * batch_size
    for log_idx, log in enumerate(dataset['log'][start_window:end_window]):
        for seq_idx, word in enumerate(log.split()):
            if seq_idx >= max_seq_len:
                break
            logs[log_idx, seq_idx] = vocabulary[word] if word in vocabulary.keys() else 0
        y_true[log_idx] = labels[dataset['label'][log_idx]]

    return tf.convert_to_tensor(logs, dtype=tf.float32), tf.convert_to_tensor(y_true, dtype=tf.float32)

if __name__ == '__main__':

    logging.info('Loading assets')
    word_embedding_matrix = joblib.load("/results/w2v_weights.joblib")
    vocabulary = joblib.load("/results/vocab_dict.joblib")
    dataset = database_builder('/database/')
    dataset = dataset.sample(frac=1).reset_index(drop=True)
    max_seq_len = 200  # get_max_length_(dataset, 0.0)
    vocab_size = len(vocabulary)

    logging.info('Processing logs for training')
    label_unique = dataset['label'].unique()
    lbp = LabelEncoder().fit(label_unique)
    binary_labels = lbp.transform(label_unique)

    log_labels = {}
    for idx, label in enumerate(label_unique):
        log_labels.update({
            label: binary_labels[idx]
        })

    n_logs = len(dataset.index)
    n_iter = n_logs // batch_size
    remainder = n_logs % batch_size
    attns = []

    optimus_prime = Transformer(
        num_layers,
        d_model,
        num_heads,
        dff,
        vocab_size,
        word_embedding_matrix,
        max_seq_len,
        rate=0.1)

    optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)

    checkpoint_path = "./checkpoints/train"
    checkpoint = tf.train.Checkpoint(step=tf.Variable(1), transformer=optimus_prime, optimizer=optimizer)
    checkpoint_manager = tf.train.CheckpointManager(checkpoint, checkpoint_path, max_to_keep=5)

    # if a checkpoint exists, restore the latest checkpoint.
    if checkpoint_manager.latest_checkpoint:
        checkpoint.restore(checkpoint_manager.latest_checkpoint)
        print('Latest checkpoint restored!!')

    for epoch in tqdm(range(epochs)):

        start = time.time()

        epoch_loss.reset_states()
        epoch_accuracy.reset_states()

        for idx in range(n_iter):
            log_batch, labels = process_batch(dataset, vocabulary, max_seq_len, idx, log_labels)

            # Returns Eager Tensor for Predictions
            train_step(log_batch, labels)
            checkpoint.step.assign_add(1)

            if int(checkpoint.step) % 10 == 0:
                save_path = checkpoint_manager.save()
                print(f'Saved checkpoint for step {int(checkpoint.step)}: {save_path}')
                print(f'Loss {epoch_loss.result():.3f}, Accuracy: {epoch_accuracy.result():.3%}')

        print("Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}".format(epoch,
                                                                    epoch_loss.result(),
                                                                    epoch_accuracy.result()))

        print(f'Time taken for 1 epoch: {time.time() - start:.2f} secs\n')


import sys
from pyspark import SparkConf, SparkContext

import re

from bwt import reverseBwt
from radix import radixSort
from segment import segSort
from default import defaultSort
from partition import partitionSort

# config spark context, set master, name and memory size
def getSC(master, name):
    conf = (SparkConf()
             .setMaster(master)
             .setAppName(name)
             #.set("spark.executor.memory", "1g")
             .set("spark.akka.frameSize", "512")
             )
    sc = SparkContext(conf = conf)

    sc.addPyFile('default.py')
    sc.addPyFile('segment.py')
    sc.addPyFile('radix.py')
    sc.addPyFile('partition.py')
    sc.addPyFile('bwt.py')

    return sc

# select a sort method
def sort(sort_name, reads, threads_number, output_path, prefixes):
    if (sort_name=='radix'):
        bwt = radixSort(reads)
    elif (sort_name=='segment'):
        bwt = segSort(reads)
    elif (sort_name=='partition'):
        bwt = partitionSort(reads, threads_number, output_path, prefixes)
    else:
        bwt = defaultSort(reads, threads_number, output_path, prefixes)
    return bwt

# RDD does not support communications among lines,
# because each line is independent during processing.
# Thus we first collect RDD (RDD->List), then parallelize List (List->RDD)
def collectReads(lines, file_type):
    if file_type == 'fasta' :
        reads = []
        read = ''
        lines = lines.collect()
        #concatinate lines begin with '>'
        for line in lines :
            if '>' not in line:
                read += line
            else :
                if len(read)>0: reads.append(read)
                read = ''
        if len(read)>0: reads.append(read)
    elif file_type == 'fastq' :
        #choose the second line of every four lines
        reads = lines.collect()[1::4]
    else :
        reads = lines.collect()
    return reads

def filterReads(lines, file_type):
    if file_type == 'fasta' :
        reads = lines.filter(lambda line: '>' not in line)
    elif file_type == 'fastq' :
        reads = lines.filter(lambda line: re.match('^[ACGTN]*$', line))
    else :
        reads = lines
    return reads

def getReads(lines, file_type, collect, reads_output_path):
    if collect:
        # collect RDD (RDD->List)
        reads = collectReads(lines,file_type)
        # parallelize List (List->RDD)
        reads = sc.parallelize(reads,int(threads_number))
    else :
        reads = filterReads(lines,file_type)

    # output reads
    # reads.saveAsTextFile(reads_output_path)
    return reads

if __name__ == "__main__":
    if len(sys.argv) < 7:
        print >> sys.stderr, "Usage: <sort> <master> <threads_num> <file_type> <input> <bwt_output_path>"
        exit(-1)

    sort_method = sys.argv[1]
    master_address = sys.argv[2]
    threads_number = sys.argv[3]
    file_type = sys.argv[4]
    input_path = sys.argv[5]
    reads_output_path = ""
    bwt_output_path = sys.argv[6]

    sc = getSC(master_address, sort_method+threads_number+input_path)

    lines = sc.textFile(input_path,int(threads_number))

    reads = getReads(lines,file_type, False, reads_output_path).cache()

    prefixes='$ACGNT'
    #prefixes = ['$','AA','CA','GA','NA','TA','AC','CC','GC','NC','TC','AG','CG','GG','NG','TG','AN','CN','GN','NN','TN','AT','CT','GT','NT','TT']
    # sort suffixes
    bwt = sort(sort_method,reads, int(threads_number), bwt_output_path, prefixes)


import logging
import os
import pwd
import sys

from aiohttp import web
from aiomisc.utils import bind_socket
from configargparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from setproctitle import setproctitle
from yarl import URL

from megamarket.api.app import create_app
from megamarket.utils.argparse import positive_int
from megamarket.utils.pg import DEFAULT_PG_URL

ENV_VAR_PREFIX = 'MEGAMARKET_'

logging.basicConfig(level=logging.DEBUG)

parser = ArgumentParser(
    auto_env_var_prefix=ENV_VAR_PREFIX,
    formatter_class=ArgumentDefaultsHelpFormatter
)

parser.add_argument('--user', required=False, type=pwd.getpwnam,
                    help='Changes UID')

group = parser.add_argument_group('API Options')
group.add_argument('--api-address', default='0.0.0.0',
                   help='IPv4/IPv6 address API server should listen on')
group.add_argument('--api-port', type=positive_int, default=8081,
                   help='TCP port API server should listen on')

group = parser.add_argument_group('PostgreSQL Options')
group.add_argument('--pg-url', type=URL, default=URL(DEFAULT_PG_URL),
                   help='URL to use to connect to the database')

group = parser.add_argument_group('Logging Options')
group.add_argument('--log-level', default='INFO',
                   choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])

def main():
    args = parser.parse_args()

    # clear_environ(lambda k: k.startswith(ENV_VAR_PREFIX))

    logging.basicConfig(level=args.log_level, format='color')

    sock = bind_socket(address=args.api_address, port=args.api_port,
                       proto_name='http')

    if args.user is not None:
        logging.info('Changing user to %r', args.user.pw_name)
        os.setgid(args.user.pw_gid)
        os.setuid(args.user.pw_uid)

    setproctitle(os.path.basename(sys.argv[0]))

    app = create_app(args)
    web.run_app(app, sock=sock)

if __name__ == '__main__':
    main()


from __future__ import unicode_literals, absolute_import, print_function, division

import datetime
import time

from sopel.module import commands, rule, priority, thread
from sopel.tools import Identifier
from sopel.tools.time import seconds_to_human

@commands('seen')
def seen(bot, trigger):
    """Reports when and where the user was last seen."""
    if not trigger.group(2):
        bot.say(".seen <nick> - Reports when <nick> was last seen.")
        return
    nick = trigger.group(2).strip()
    if nick == bot.nick:
        bot.reply("I'm right here!")
        return
    timestamp = bot.db.get_nick_value(nick, 'seen_timestamp')
    if timestamp:
        channel = bot.db.get_nick_value(nick, 'seen_channel')
        message = bot.db.get_nick_value(nick, 'seen_message')
        action = bot.db.get_nick_value(nick, 'seen_action')

        saw = datetime.datetime.utcfromtimestamp(timestamp)
        delta = seconds_to_human((trigger.time - saw).total_seconds())

        msg = "I last saw " + nick
        if Identifier(channel) == trigger.sender:
            if action:
                msg += " in here {since}, doing: {nick} {action}".format(
                    since=delta,
                    nick=nick,
                    action=message)
            else:
                msg += " in here {since}, saying: {message}".format(
                    since=delta,
                    message=message)
        else:
            msg += " in another channel {since}.".format(since=delta)
        bot.reply(msg)
    else:
        bot.say("Sorry, I haven't seen {nick} around.".format(nick=nick))

@thread(False)
@rule('(.*)')
@priority('low')
def note(bot, trigger):
    if not trigger.is_privmsg:
        bot.db.set_nick_value(trigger.nick, 'seen_timestamp', time.time())
        bot.db.set_nick_value(trigger.nick, 'seen_channel', trigger.sender)
        bot.db.set_nick_value(trigger.nick, 'seen_message', trigger)
        bot.db.set_nick_value(trigger.nick, 'seen_action', 'intent' in trigger.tags)


# coding=utf-8
import tensorflow as tf
import numpy as np
from data_helper import *
import gensim
import os
import time
import datetime
import csv

# TF log level
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
# Parameters
# ==================================================
flags = tf.flags
logging = tf.logging
# Data Parameters
flags.DEFINE_string("test_data_x", "./pan11-corpus-test/LargeTest.xml", "Data source for the X of test data")
flags.DEFINE_string("test_data_y", "./pan11-corpus-test/GroundTruthLargeTest.xml", "Data source for the Y of test data")

flags.DEFINE_string("lda_path", "./lda_model/model", "LDA model file path")
flags.DEFINE_string("word2vec", "./dict_data/word_embedding_dic.json", "Data source for prepared word2vec dict")
flags.DEFINE_string("author_dict", "./dict_data/author_dict.json", "Data source for author dict")
flags.DEFINE_string("char_dict", "./dict_data/char_dict.json", "Data source for char dict")
flags.DEFINE_string("n_grams_dict", "./dict_data/n_grams_dict.json", "Data source for n-grams dict (default: 2-grams)")

flags.DEFINE_integer("max_len_char", 1000, "Number of characters in a sequence (default: 1000 >> 140)")
flags.DEFINE_integer("max_len_word", 10, "Number of words in a sequence (default: 10)")
flags.DEFINE_integer("num_topics", 200, "Number of LDA topics")

# Eval Parameters
tf.flags.DEFINE_string("checkpoint_dir", "", "Checkpoint directory from training run")
tf.flags.DEFINE_boolean("eval_model", False, "Evaluate on all test data")

# Misc Parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")

FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
    print("{}={}".format(attr.upper(), value))
print("")

# word2vec = KeyedVectors.load_word2vec_format("GoogleNews-vectors-negative300.bin", binary=True)
word2vec = get_json(FLAGS.word2vec)
print("word_vectors loaded")

lda_model = gensim.models.LdaModel.load(FLAGS.lda_path + str(FLAGS.num_topics), mmap="r")
print("lda model loaded")

author_dict = get_json(FLAGS.author_dict)
print("author_dict has {} keys".format(len(author_dict)))

grams_dict = get_json("./dict_data/n_grams_dict.json")
print("char_dict has {}+1 keys, 1 means unk".format(len(grams_dict)))

x_dev, y_dev = get_dev_data(FLAGS.test_data_x,
                            FLAGS.test_data_y)
print("test data loaded, which have {} items".format(len(y_dev)))

# CHANGE THIS: Load data. Load your own data here
if not FLAGS.eval_model:
    x_dev = ["Please let me know if you have any questions or need anything else."]
    y_dev = ["x9971451464197140"]
    FLAGS.max_len_char = 20
    FLAGS.max_len_word = 20

dev_data_char = gen_char_batch(texts=x_dev,
                               authors=y_dev,
                               author_dict=author_dict,
                               n_grams_dict=grams_dict,
                               batch_size=len(y_dev),
                               max_len_char=FLAGS.max_len_char,
                               )
dev_data_word = gen_word_batch(texts=x_dev,
                               authors=y_dev,
                               word_vectors=word2vec,
                               author_dict=author_dict,
                               batch_size=len(y_dev),
                               max_len_word=FLAGS.max_len_word)
dev_data_topic = gen_topic_batch(texts=x_dev,
                                 authors=y_dev,
                                 author_dict=author_dict,
                                 lda_model=lda_model,
                                 batch_size=len(y_dev))

# Evaluation
# ==================================================
print("\nEvaluating...\n")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
    session_conf = tf.ConfigProto(
        allow_soft_placement=FLAGS.allow_soft_placement,
        log_device_placement=FLAGS.log_device_placement)
    sess = tf.Session(config=session_conf)
    with sess.as_default():
        # Load the saved meta graph and restore variables
        saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
        saver.restore(sess, checkpoint_file)

        # Get the placeholders from the graph by name
        x_char = graph.get_operation_by_name("x_char").outputs[0]
        x_word = graph.get_operation_by_name("x_word").outputs[0]
        x_topic = graph.get_operation_by_name("x_topic").outputs[0]
        y = graph.get_operation_by_name("y").outputs[0]
        char_dropout_keep = graph.get_operation_by_name("char_dropout_keep").outputs[0]
        word_dropout_keep = graph.get_operation_by_name("word_dropout_keep").outputs[0]
        topic_dropout_keep = graph.get_operation_by_name("topic_dropout_keep").outputs[0]
        dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]

        # Tensors we want to evaluate
        accuracy = graph.get_operation_by_name("accuracy/accuracy").outputs[0]

        # Generate batches for one epoch
        x_char_dev, y_char_dev = dev_data_char.__next__()
        x_word_dev, y_word_dev = dev_data_word.__next__()
        x_topic_dev, y_topic_dev = dev_data_topic.__next__()
        assert np.all(y_char_dev == y_word_dev) and np.all(y_word_dev == y_topic_dev), ""
        y_dev = y_char_dev

        # Collect the predictions here
        all_predictions = []

        accuracy = sess.run(accuracy, {x_char: x_char_dev, x_word: x_word_dev, x_topic: x_topic_dev, y: y_dev,
                                       topic_dropout_keep: 1.0, char_dropout_keep: 1.0, word_dropout_keep: 1.0,
                                       dropout_keep_prob: 1.0})
        print(accuracy)


# coding=utf-8
import streamlit as st

class selectValues():
    def __init__(self):
        self.points = ['腕', '肘', '膝', '頭', '投げ手', '足']
        self.eval_kinds = ['パフォーマンス観点', '怪我観点']
        self.timings = ['投げ始め', 'リリース時', '投げ終わり']
        self.evaluates = ['○', '×']
        self.total_evaluates = ['未評価', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10']

def set_slider():
    DEFAULT_WIDTH = 60
    width = st.slider(
        label="", min_value=0, max_value=100, value=DEFAULT_WIDTH, format="%d%%"
    )

    width = max(width, 0.01)
    side = max((100 - width) / 2, 0.01)
    _, container, _ = st.columns([side, width, side])
    return container


"""
Example:
words = ['cat', 'baby', 'dog', 'bird', 'car', 'ax']
string1 = 'tabncihjs'
find_embedded_word(words, string1) -> cat (the letters do not have to be in order)
"""

import collections
from typing import List
import unittest

# Using sorting
# words = ['cat', 'baby', 'dog', 'bird', 'car', 'ax'] -> act, abbbdyx, dgo, bdir, acr, ax
# string1 = 'tabncihjs' -> abbcccccdhijnsty'
def find_embedded_word_sort(words: List[str], string1: str) -> str:
    sorted_string1 = sorted(string1)
    for word in words:
        sorted_word = sorted(word)
        sorted_string1_ptr = 0
        for ch in sorted_word:
            while sorted_string1_ptr < len(sorted_string1) and sorted_string1[sorted_string1_ptr] != ch:
                sorted_string1_ptr += 1
            sorted_string1_ptr += 1
        if sorted_string1_ptr <= len(sorted_string1):
            return word
    return None

# Using Counter(), with count
def find_embedded_word_counter(words: List[str], string1: str) -> str:
    dict_string1 = collections.Counter(string1)
    for word in words:
        dict_word = collections.Counter(word)
        count = 0
        for key in dict_word:
            if dict_word[key] <= dict_string1[key]:
                count += 1
        if count == len(dict_word):
            return word
    return None

# Using Counter(), with boolean
def find_embedded_word_bool(words: List[str], string1: str) -> str:
    dict_string1 = collections.Counter(string1)
    for word in words:
        dict_word = collections.Counter(word)
        word_present = True
        for key in dict_word:
            if dict_word[key] > dict_string1[key]:
                word_present = False
                break
        if word_present:
            return word
    return None

class TestProblems(unittest.TestCase):

    def test_embedded_words(self):
        actual = find_embedded_word_sort(['cat', 'baby', 'dog', 'bird', 'car', 'ax'], 'tabncihjs')
        expected = 'cat'
        self.assertTrue(actual, expected)

        actual = find_embedded_word_bool(['cat', 'baby', 'dog', 'bird', 'car', 'ax'], 'tabncihjs')
        expected = 'cat'
        self.assertTrue(actual, expected)

        actual = find_embedded_word_counter(['cat', 'baby', 'dog', 'bird', 'car', 'ax'], 'tabncihjs')
        expected = 'cat'
        self.assertTrue(actual, expected)

if __name__ == '__main__':
    unittest.main()


from collections import defaultdict

from src.data import data_manager
from src.data.neuron_info import ntype
from src.data.dataset_info import all_datasets, datasets_with_adj, timepoint

from src.plotting import plotter

class Figure(object):

    def __init__(self, output_path, page_size=7.20472):
        self.plt = plotter.Plotter(output_path=output_path, page_size=page_size)

    def _feed_type(self, edge, only_sensory=False, with_muscle=False):
        edge_type = (ntype(edge[0]), ntype(edge[1]))
        if only_sensory:
            if edge_type in (('sensory', 'inter'), ('sensory', 'motor'), ('sensory', 'modulatory')):
                return 'Feed-forward'
            if edge_type in (('inter', 'sensory'), ('motor', 'sensory'), ('modulatory', 'sensory')):
                return 'Feed-back'
            if edge_type in (('sensory', 'sensory'), ):
                return 'Recurrent'
            return None
        if edge_type in (('sensory', 'inter'), ('inter', 'motor'), ('sensory', 'motor'), ('modulatory', 'inter'), ('sensory', 'modulatory'), ('modulatory', 'motor')):
            return 'Feed-forward'
        if edge_type in (('inter', 'sensory'), ('motor', 'inter'), ('motor', 'sensory'), ('inter', 'modulatory'), ('modulatory', 'sensory')):
            return 'Feed-back'
        if edge_type in (('sensory', 'sensory'), ('inter', 'inter'), ('motor', 'motor'), ('modulatory', 'modulatory')):
            return 'Recurrent'
        if with_muscle and edge_type[1] == 'muscle':
            return 'Feed-forward'
        return None

    def feedforward_stable_increase(self, f, edge_classifications, use_size=False):

        G = data_manager.get_connections()['size' if use_size else 'count'].copy()
        G = G[G.sum(axis=1) > 0]  # remove edges without size if need be
        G = data_manager.remove_postemb(G)

        edge_classifications = edge_classifications.copy()

        edges = [e for e in G.index if edge_classifications[e] == 'stable']

        feed_types = ['Feed-forward', 'Recurrent', 'Feed-back']
        feed_colors = {'Feed-forward': '#C7EAE4', 'Feed-back': '#EAC9C1', 'Recurrent': 'white'}
        syn_increases = {ft: [] for ft in feed_types}

        for edge in edges:

            feed_type = self._feed_type(edge)
            if not feed_type:
                continue

            syns = G.loc[edge]
            if syns[0] == 0 and syns[1] == 0:
                continue

            syn_increase_relative = syns[['Dataset7', 'Dataset8']].mean() / syns[['Dataset1', 'Dataset2']].mean()
            syn_increases[feed_type].append(syn_increase_relative)

        data, c, l = [], [], []
        for ft in feed_types:
            data.append(syn_increases[ft])
            l.append(ft)
            c.append(feed_colors[ft])

        if use_size:
            ylim = (0, 15)
            yticks = (0, 5, 10, 15)
            y_label = 'Relative synapse volume increase'
            size = (0.15, 0.15)
        else:
            ylim = (0, 12)
            yticks = (0, 4, 8, 12)
            y_label = 'Relative synapse addition'
            size = (0.15, 0.15)

        self.plt.plot(
            'box_plot', data, size=size,
            margin={'left': 0.04, 'right': 0.01, 'top': 0.05, 'bottom': 0.04},
            colors=c, xticklabels=l, xtickpad=3, xpad=5, ylim=ylim, yticks=yticks,
            y_label=y_label, x_label='Stable connection directionality',
            show_outliers=False, stats=((2, 3), (1, 2), (1, 3)),
            save=f+'_feedforward_stable_increase' + ('_size' if use_size else '')
        )

    def feedforward_edge_proportion(self, f, edge_classifications, use_size=False):

        G = data_manager.get_connections()['size' if use_size else 'count'].copy()
        G = G[G.sum(axis=1) > 0] # remove edges without size if need be
        G = data_manager.remove_postemb(G)

        edge_classifications = edge_classifications.copy()

        feed_types = ['Feed-back', 'Recurrent', 'Feed-forward']
        edge_types = ('stable', 'increase', 'decrease')
        feed_colors = {'Feed-forward': '#C7EAE4', 'Feed-back': '#EAC9C1', 'Recurrent': 'white'}

        connections = defaultdict(lambda: {ft: 0 for ft in feed_types})

        edges_per_type = {
            'stable': [e for e in G.index if edge_classifications[e] == 'stable'],
            'increase': [e for e in G.index if edge_classifications[e] == 'increase'],
            'decrease': [e for e in G.index if edge_classifications[e] == 'decrease'],
#            'Variable': [e for e in G if edge_classifications[(npair(e[0]), npair(e[1]))] in ('remainder', 'noise')]
        }

        xlabels = (
            'Stable',
            'Strengthened',
            'Weakened',
        )

        for edge_type, edges in edges_per_type.items():
            for edge in edges:
                feed_type = self._feed_type(edge)
                if not feed_type:
                    continue
                connections[edge_type][feed_type] += 1

        data = tuple((ft, [connections[et][ft] for et in edge_types]) for ft in feed_types)
        print(data)

        self.plt.plot(
            'stacked_bar_graph', data, stats=((1, 2), (1, 3)), size=(0.15, 0.15),
            margin={'left': 0.04, 'right': 0.08, 'top': 0.05, 'bottom': 0.04},
            y_label='Proportion of connections',
            colors=feed_colors, xlabels=xlabels, x_label='Connection classification',
            xtickpad=3, xpad=5,
            legendpos='right', legendcol=1, legendreverse=True, width=0.5,
            save=f+'_feedforward_edge_proportion'
        )

    def feedforward_global_shift(self, f, only_sensory=False, use_size=False):

        datasets = list(datasets_with_adj if use_size else all_datasets)

        G = data_manager.get_connections()['size' if use_size else 'count'].copy()
        G = G[G.sum(axis=1) > 0] # remove edges without size if need be
        G = G[datasets]
        G = data_manager.remove_postemb(G)

        y_label = 'Proportion of synapses'
        ylim = (0, 0.6)

        if use_size:
            y_label = 'Proportion of synapse volume'

        if only_sensory:
            y_label += ' to\nor from sensory neurons'
            ylim = (0, 0.8)

        feed_types = ['Feed-back', 'Feed-forward', 'Recurrent']

        G['feed_types'] = G.index.map(self._feed_type)
        feed_type_counts = G.groupby('feed_types').sum()
        feed_type_counts = feed_type_counts / feed_type_counts.sum()

        xs = [timepoint[d] for d in datasets]
        colors = {
            'Feed-forward': '#C7EAE4', 'Feed-back': '#EAC9C1', 'Recurrent': 'white',
            'Feed-forward_edge': '#7ccfc1', 'Feed-back_edge': '#d18876', 'Recurrent_edge': '#999999'
        }

        data = (xs, [(ft, feed_type_counts.loc[ft]) for ft in feed_types])

        self.plt.plot(
            'xy_graph', data, size=(0.08, 0.16),
            margin={'left': 0.04, 'right': 0.10, 'top': 0.01, 'bottom': 0.04},
            y_label=y_label, ylim=ylim, stats='spearmanr', colors=colors,
            x_label='Developmental age',
            legendpos='right', rev_legend=True, legend_shift_top=0.03, legend_shift_right=0.05,
            save=f+'_feedforward_global_shift', linkpoints=False,
#            hlines=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6)
        )


# encoding: utf-8
"""

"""
__author__ = 'Richard Smith'
__date__ = '31 Jul 2020'
__copyright__ = 'Copyright 2018 United Kingdom Research and Innovation'
__license__ = 'BSD - see LICENSE file in top-level package directory'
__contact__ = 'richard.d.smith@stfc.ac.uk'

from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from cci_tagger.facets import Facets
import json

class Command(BaseCommand):
    help = 'Downloads vocabs from vocab server to json file'

    def handle(self, *args, **options):

        facets = Facets()
        with open(settings.VOCAB_CACHE_FILE,'w') as writer:
            json.dump(facets.to_json(), writer)


import json
import glob
import os
import re
import collections

import yaml
from yaml.parser import ParserError, ScannerError

from saddlebags import exceptions

SUPPORTED_FILE_TYPES = ['json', 'yaml', 'yml']

class Saddlebag(collections.MutableMapping):
    """
    Provides access to the contents of JSON/YAML configuration
    files using standard dictionary style syntax.
    """

    def __init__(self,
                 configuration_locations: list=None,
                 strict: str=True):
        """
        The constructor creates an top-level key for each
        configuration file found in the directories specified by
        a list of environment variables.

        Additionally read/write access to environment variables
        is available via the `env` object attribute.

        Args:
            configuration_locations: List of environment variables
                which point to directories containing configuration files.
            strict: If True, instances will raise an
                exception if requested data is not present.
        """
        self.strict = strict

        self.env = os.environ
        self._data = dict()
        if not configuration_locations:
            configuration_locations = []

        # Obtain list of all support configuration files.
        configuration_files = (
            self._configuration_files(configuration_locations))

        for configuration_file in configuration_files:
            key_name = re.search(
                r"([-_A-Za-z0-9]+)\.(json|yaml|yml|conf)",
                configuration_file).group(1)

            self._check_for_name_collision(key_name)
            self._load_configuration_file(key_name, configuration_file)

    def __getitem__(self, key: str):
        if self.strict:
            try:
                return self._data[key.lower()]
            except KeyError:
                raise KeyError(
                    "The requested key '{}' does not exist. This most likely "
                    "indicates that you anticipated a configuration file "
                    "being loaded that actually hasn't been.".format(key))

        return self._data.get(key.lower())

    def __setitem__(self, key: str, value):
        self._data[key.lower()] = value

    def __delitem__(self, key):
        del self._data[key.lower()]

    def __iter__(self):
        return iter(self._data)

    def __len__(self):
        return len(self._data)

    def __repr__(self):
        return str(self._data)

    def _configuration_files(self, config_files_locations):
        """
        Identify all configuration files in a given location.

        Returns:
            A list containing paths to configuration files.

        Raises:
            ValueError: When a non-existent ENV_VAR is referenced.
        """
        configuration_files = list()
        for location in config_files_locations:
            try:
                configuration_files.extend(
                    [file for file in glob.glob(os.environ[location] + '/*')
                     if file.rpartition('.')[2] in SUPPORTED_FILE_TYPES])
            except KeyError:
                raise ValueError(
                    'The environment variable specified '
                    'by the client ({}) for use by '
                    'the constructor does not exist '
                    'on the system.'.format(location))
        return configuration_files

    def _check_for_name_collision(self, key):
        """
        Ensure that a given element key is not already present on the
        object.

        Args:
            key: The key to evaluate.

        Raises:
             DuplicationConfigurationFile: If another configuration file
                of the same name has already been loaded onto the file.
        """
        try:
            existing_key = self[key]
        except KeyError:
            existing_key = None

        if existing_key:
            raise exceptions.DuplicateConfigurationFile(
                "Two configuration files share the following name "
                "{}.  This is not allowed.".format(key))

    def _load_configuration_file(self, attribute_name, configuration_file):
        with open(configuration_file) as configuration_data:
            file_extension = configuration_file.partition('.')[2].lower()

            # JSON Loading
            if file_extension == 'json':
                try:
                    self.update(
                        {attribute_name: json.load(configuration_data)})
                except ValueError:
                    raise exceptions.MalformedConfigurationFile(
                        "The configuration file, {}, contains "
                        "syntax errors.".format(configuration_file))

            # YAML Loading
            elif file_extension in ['yaml', 'yml']:
                try:
                    results = list(yaml.load_all(configuration_data))
                except (ParserError, ScannerError):
                    raise exceptions.MalformedConfigurationFile(
                        "The configuration file, {}, contains "
                        "syntax errors.".format(configuration_file))
                else:
                    if len(results) > 1:
                        self.update({attribute_name: results})
                    else:
                        self.update({attribute_name: results[0]})


import datetime
import ipaddress
import unittest
from typing import Any, Optional

from dataclasses import dataclass

from podman import api

class ParseUtilsTestCase(unittest.TestCase):
    def test_parse_repository(self):
        @dataclass
        class TestCase:
            name: str
            input: Any
            expected: Optional[str]

        cases = [
            TestCase(name="empty str", input="", expected=("", None)),
            TestCase(
                name="name",
                input="quay.io/libpod/testimage",
                expected=("quay.io/libpod/testimage", None),
            ),
            TestCase(
                name="@digest",
                input="quay.io/libpod/testimage@71f1b47263fc",
                expected=("quay.io/libpod/testimage", "71f1b47263fc"),
            ),
            TestCase(
                name=":tag",
                input="quay.io/libpod/testimage:latest",
                expected=("quay.io/libpod/testimage", "latest"),
            ),
        ]

        for case in cases:
            actual = api.parse_repository(case.input)
            self.assertEqual(
                case.expected,
                actual,
                f"failed test {case.name} expected {case.expected}, actual {actual}",
            )

    def test_decode_header(self):
        actual = api.decode_header("eyJIZWFkZXIiOiJ1bml0dGVzdCJ9")
        self.assertDictEqual(actual, {"Header": "unittest"})

        self.assertDictEqual(api.decode_header(None), {})

    def test_prepare_timestamp(self):
        time = datetime.datetime(2022, 1, 24, 12, 0, 0)
        self.assertEqual(api.prepare_timestamp(time), 1643025600)
        self.assertEqual(api.prepare_timestamp(2), 2)

        self.assertEqual(api.prepare_timestamp(None), None)
        with self.assertRaises(ValueError):
            api.prepare_timestamp("bad input")

    def test_prepare_cidr(self):
        net = ipaddress.IPv4Network("127.0.0.0/24")
        self.assertEqual(api.prepare_cidr(net), ("127.0.0.0", "////AA=="))

if __name__ == '__main__':
    unittest.main()


import cv2
import numpy as np
import os
import zipfile
from show import blob_imagem,alturaXlargura
from work import deteccoes, funcoes_imagem
from drive import driveFile

if not os.path.exists("modelo.zip"):
  """
  Verifica se o modelo já se encontra no diretório
  se não se encontra no diretória, então baixa o mesmo do link a baixo
  se o modelo já estiver baixado, não faz nada
  """

  yolov4URL = "https://drive.google.com/u/0/uc?id=1kPKs0ZlEK5O_WbbTGSbiI1A3JI8C6UHc&export=download"

  driveFile(yolov4URL,"modelo.zip")

  zip_object = zipfile.ZipFile(file="modelo.zip", mode='r')
  zip_object.extractall('./')
  zip_object.close()

  print("Terminei de extrair")

try:
  """
  Tenta realizar o processamento da imagem, para tanto segue os seguintes passosa

  C

  """

  labelsPath = os.path.sep.join(["cfg", "coco.names"])

  LABELS = open(labelsPath).read().strip().split("\n")

  configPath = os.path.sep.join(["cfg", "yolov4.cfg"])

  weightsPath = "yolov4.weights"

  net = cv2.dnn.readNet(configPath, weightsPath)

  np.random.seed(42)
  COLORS = np.random.randint(0, 255, size=(len(LABELS), 3), dtype="uint8")

  ln = net.getLayerNames()
  #print("Todas as camadas (layers):")
  #print(ln)
  #print("Total: "+ str(len(ln)))
  #print("Camadas de saída: ")
  #print(net.getUnconnectedOutLayers())
  ln = [ln[i - 1] for i in net.getUnconnectedOutLayers()]
  #print(ln)

  imagePath = os.path.sep.join(['imagens', "cachorros02.jpg"])

  imagem = cv2.imread(imagePath)

  net, imagem, layerOutputs = blob_imagem(net, imagem,ln)

  threshold = 0.5
  threshold_NMS = 0.3
  caixas = []
  confiancas = []
  IDclasses = []

  H,W = alturaXlargura(imagem)

  for output in layerOutputs:
      for detection in output:
          caixas, confiancas, IDclasses = deteccoes(detection, threshold, caixas, confiancas, IDclasses,H,W)

  objs = cv2.dnn.NMSBoxes(caixas, confiancas, threshold, threshold_NMS)

  print("\nObjetos detectados: " + str(len(objs)))

  if len(objs) > 0:
    for i in objs.flatten():
      imagem, x, y, w, h = funcoes_imagem(imagem, i, confiancas, caixas, COLORS, LABELS,IDclasses)
      objeto = imagem[y:y + h, x:x + w]

  name_img = os.path.sep.join(["resultados","Teste1.jpg"])
  print(name_img)
  #name_img = 'teste1.jpg'
  cv2.imwrite(name_img, imagem)
except:
  print("Deu erro")