from numpy.testing import assert_array_equal
import numpy as np
import tempfile
import secrets
import h5py
from hdf5zarr import HDF5Zarr, open_as_zarr
import pytest
import itertools

class TestHDF5Zarr(object):

    @pytest.fixture(autouse=True)
    def visit_files(self, request, filesbase, capsys):
        # file number
        fnum = request.node.callspec.params['fnum']

        num_files = filesbase.num_files
        file_list = filesbase.file_list
        hdf5zarr_list = filesbase.hdf5zarr_list

        self.hfile, self.hdf5zarr = file_list[fnum], hdf5zarr_list[fnum]
        self.zgroup = self.hdf5zarr.zgroup
        with capsys.disabled():
            if self.hdf5zarr.max_chunksize is not None:
                print("\n"+f"file: {self.hfile.file.filename}, obj: {self.hfile.name}  :".rjust(len(request.node.nodeid)), end = '')
            else:
                print("\n"+(f"file: {self.hfile.file.filename}, obj: {self.hfile.name}, "
                            f"max_chunksize: {self.hdf5zarr.max_chunksize}  :").rjust(len(request.node.nodeid)), end = '')

    ##########################################
    #  basic tests                           #
    ##########################################

    @pytest.mark.parametrize('visit_type', [None], ids=[""])
    def test_consolidate_metadata(self):
        zgroup = self.hdf5zarr.consolidate_metadata()

    def test_groups(self):
        """ test if group exists """
        def _test_groups(name, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_GROUP:
                self.zgroup[name]
                self.zgroup[name.decode('utf-8')]
        h5py.h5o.visit(self.hfile.id, _test_groups, info=True)

    def test_dsets(self):
        """ test if dataset exists """
        def _test_dsets(name, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_DATASET:
                self.zgroup[name]
                self.zgroup[name.decode('utf-8')]
        h5py.h5o.visit(self.hfile.id, _test_dsets, info=True)

    ##########################################
    #  dataset properties tests              #
    ##########################################

    def test_dset_properties_dtype(self):
        """ test if dataset dtypes are equal """
        def _test_dtype(zobj, hobj, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_DATASET:
                assert zobj.dtype == hobj.dtype
        self._visit_item(_test_dtype)

    def test_dset_properties_shape(self):
        """ test if dataset shapes are equal """
        def _test_shape(zobj, hobj, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_DATASET:
                assert zobj.shape == hobj.shape
        self._visit_item(_test_shape)

    def test_dset_properties_chunks(self):
        """ test if datasets properties are equal """
        def _test_chunks(zobj, hobj, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_DATASET:
                if hobj.chunks is None:
                    chunks = tuple(s if s != 0 else 1 for s in hobj.shape)
                else:
                    chunks = hobj.chunks
                assert zobj.chunks == chunks
        # skip if max_chunksize is not None
        if self.hdf5zarr.max_chunksize is None:
            self._visit_item(_test_chunks)

    def test_dset_properties_fillvalue(self):
        """ test if datasets properties are equal """
        def _test_fillvalue(zobj, hobj, hobj_info):
            if hobj_info.type == h5py.h5o.TYPE_DATASET:
                # skip test for structured array containing object ref, zarr v2.4.1, zarr PR 422
                dset_type = hobj.id.get_type()
                if hobj.dtype.names is None:
                    assert_array_equal(zobj.fill_value, hobj.fillvalue)
                elif all([dset_type.get_member_class(i) != h5py.h5t.REFERENCE for i in range(dset_type.get_nmembers())]):
                    for n in self.hobj.dtype.names:
                        assert_array_equal(zobj.fill_value[n], hobj.fillvalue[n])
        self._visit_item(_test_fillvalue)

    ##########################################
    #  dataset read tests                    #
    ##########################################

    def test_zarray_read_simple_dtype(self):
        """ test if zarr arrays are read """
        def _test_dsets_read(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, False) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                zval = zobj[()]
        self._visit_item(_test_dsets_read)

    def test_zarray_read_simple_dtype_objref(self):
        """ test if zarr struct arrays are read """
        def _test_dsets_read(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, True) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                zval = zobj[()]
        self._visit_item(_test_dsets_read)

    def test_zarray_read_struct_dtype_noref(self):
        """ test if zarr struct arrays are read """
        def _test_dsets_read(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (True, False) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                zval = zobj[()]
        self._visit_item(_test_dsets_read)

    def test_zarray_read_struct_dtype_withobjref(self):
        """ test if zarr struct arrays are read """
        def _test_dsets_read(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (True, True) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                zval = zobj[()]
        self._visit_item(_test_dsets_read)

    def test_zarray_read_vlenstring(self):
        """ test if variable length string arrays are read """
        def _test_dsets_read(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, False) and
               h5py.check_vlen_dtype(hobj.dtype)):
                zval = zobj[()]
        self._visit_item(_test_dsets_read)

    def test_dset_val_simple_dtype(self):
        """ test if zarr arrays and datasets are equal """
        def _test_dset_val(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, False) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                hval = hobj[()]
                zval = zobj[()]
                assert_array_equal(hval, zval)
        self._visit_item(_test_dset_val)

    def test_dset_val_simple_dtype_objref(self):
        """ test if zarr arrays and datasets are equal """
        def _test_dset_val(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, True) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                hval = hobj[()]
                zval = zobj[()]
                ref_array_func = np.frompyfunc(lambda x: h5py.h5i.get_name(h5py.h5r.dereference(x, self.hfile.id)), 1, 1)
                if hobj.shape != ():
                    hval_str = ref_array_func(hval).astype(str)
                else:
                    hval_str = h5py.h5i.get_name(h5py.h5r.dereference(hval, self.hfile.id))
                    hval_str = hval_str.decode('utf-8')
                if self.hfile.name == '/':
                    assert_array_equal(hval_str, zval)
                else:
                    assert_array_equal(np.frompyfunc(lambda x: x if x.startswith(self.hfile.name) else '', 1, 1)(hval_str), zval)
        self._visit_item(_test_dset_val)

    def test_dset_val_struct_dtype_noref(self):
        """ test if zarr arrays and datasets are equal """
        def _test_dset_val(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (True, False) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                hval = hobj[()]
                zval = zobj[()]
                for dt_name in hobj.dtype.names:
                    assert_array_equal(hval[dt_name], zval[dt_name])
        self._visit_item(_test_dset_val)

    def test_dset_val_struct_dtype_withobjref(self):
        """ test if zarr arrays and datasets are equal """
        def _test_dset_val(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (True, True) and
               not h5py.check_vlen_dtype(hobj.dtype)):
                hval = hobj[()]
                zval = zobj[()]
                ref_array_func = np.frompyfunc(lambda x: h5py.h5i.get_name(h5py.h5r.dereference(x, self.hfile.id)), 1, 1)
                dset_type = hobj.id.get_type()
                for i in range(dset_type.get_nmembers()):
                    dt_name = hobj.dtype.names[i]
                    if dset_type.get_member_class(i) == h5py.h5t.REFERENCE:
                        if hobj.shape != ():
                            hval_str = ref_array_func(hval[dt_name]).astype(str)
                        else:
                            hval_str = h5py.h5i.get_name(h5py.h5r.dereference(hval[dt_name], self.hfile.id))
                            hval_str = hval_str.decode('utf-8')
                        if self.hfile.name == '/':
                            assert_array_equal(hval_str, zval[dt_name])
                        else:
                            assert_array_equal(np.frompyfunc(lambda x: x if x.startswith(self.hfile.name)
                                                                         else '', 1, 1)(hval_str), zval[dt_name])
                    else:
                        assert_array_equal(hval[dt_name], zval[dt_name])
        self._visit_item(_test_dset_val)

    def test_dset_val_vlenstring(self):
        """ test if zarr arrays and datasets are equal """
        def _test_dset_val(zobj, hobj, hobj_info):
            if (hobj_info.type == h5py.h5o.TYPE_DATASET and
               self._checkdtype_structobjref(hobj) == (False, False) and
               h5py.check_vlen_dtype(hobj.dtype)):
                hobj = hobj.asstr()  # wrapper to read data as python str
                hval = hobj[()]
                zval = zobj[()]
                assert_array_equal(hval, zval)
        self._visit_item(_test_dset_val)

    def test_attrs(self):
        """ test if attributes exist """
        def _test_attr(zobj, hobj, hobj_info):
            for name in hobj.attrs:
                zattr = zobj.attrs[name]
        self._visit_item(_test_attr)

    def test_read_attrs(self):
        """ test if attributes are equal """
        def _test_read_attrs(zobj, hobj, hobj_info):
            for name in hobj.attrs:
                hattr = hobj.attrs[name]
                zattr = zobj.attrs[name]
                assert_array_equal(zattr, hattr)
        self._visit_item(_test_read_attrs)

    # visit hdf5 items
    # visit types, objects or links
    @pytest.fixture(autouse=True, params=["objects_only", "links_only"])
    def visit_type(self, request):
        self.visittype = request.param

        # collect flag
        self.fkeep = request.config.getoption("fkeep")
        self.objnames = request.config.getoption("objnames")

        self._ex = []

        # visit objects
        if self.visittype == "objects_only":
            self._visit_item = self.visit_obj_func
        elif self.visittype == "links_only":
            self._visit_item = self.visit_link_func
        elif self.visittype is None:
            pass
        else:
            raise Exception("Invalid visit_type parameter")

        yield

        if request.node.rep_setup.passed:
            if request.node.rep_call.failed:
                if self.fkeep and len(self._ex) > 0:
                    ex, name = self._ex[0]
                    hobj = self.hfile[name]
                else:
                    hobj = self.hobj
                print(f"""HDF5Zarr args: (
                       filename = '{self.hdf5zarr.filename}',
                       store = {self.hdf5zarr.store},
                       store_mode = {self.hdf5zarr.store_mode},
                       max_chunksize = {self.hdf5zarr.max_chunksize},
                       )""")

                print("executing test failed for", request.node.name, hobj.file.filename)
                if hobj.file.filename != self._testfilename and isinstance(hobj, h5py.Dataset):
                    print(f"""hdf5 Dataset: (
                           name = '{hobj.name}',
                           shape = {hobj.shape},
                           dtype = {hobj.dtype},
                           chunks = {hobj.chunks},
                           maxshape = {hobj.maxshape},
                           track_times = None,
                           track_order = None,
                           fillvalue = {hobj.fillvalue},
                           data = {hobj[()]},
                           )""")

                if self.fkeep and len(self._ex) > 1:
                    ex_m, name_m = self._ex[np.argmin([self.hfile[name].size for ex, name in self._ex])]
                    if name_m != name:
                        hobj = self.hfile[name_m]
                        print("executing test failed for", request.node.name, hobj.file.filename)
                        if hobj.file.filename != self._testfilename and isinstance(hobj, h5py.Dataset):
                            print(f"""(
                                   name = '{hobj.name}',
                                   shape = {hobj.shape},
                                   dtype = {hobj.dtype},
                                   chunks = {hobj.chunks},
                                   maxshape = {hobj.maxshape},
                                   track_times = None,
                                   track_order = None,
                                   fillvalue = {hobj.fillvalue},
                                   data = {hobj[()]},
                                   )""")

    def visit_obj_func(self, assert_func):
        # visit objects

        _ex = []

        def _test_obj(name, hobj_info):
            nonlocal _ex

            self.hobj = self.hfile[name]
            self.zobj = self.zgroup[name.decode('utf-8')]
            if not self.fkeep:
                assert_func(self.zobj, self.hobj, hobj_info)
            else:
                try:
                    assert_func(self.zobj, self.hobj, hobj_info)
                except AssertionError as ex:
                    _ex.append([ex, name])

        if self.objnames == []:
            h5py.h5o.visit(self.hfile.id, _test_obj, info=True)
        else:
            for name in self.objnames:
                # skip objnames for subgroups
                if not isinstance(self.hfile, h5py.File) or name not in self.hfile:
                    continue
                hobj_info = h5py.h5g.get_objinfo(self.hfile[name].id)
                _test_obj(name, hobj_info)

        self._ex = _ex

        # raise only one exception in case of fkeep == True
        if self.fkeep and len(self._ex) > 0:
            raise self._ex[0][0]

    def visit_link_func(self, assert_func):
        # visit links

        _ex = []

        def _test_obj(name, hlink_info):
            nonlocal _ex

            if not isinstance(self.hfile, h5py.Dataset):
                self.hobj = self.hfile[name]
                self.zobj = self.zgroup[name.decode('utf-8')]
            else:
                self.hobj = self.hfile
                self.zobj = self.zgroup

            hobj_info = h5py.h5g.get_objinfo(self.hobj.id)

            if hlink_info.type == h5py.h5l.TYPE_SOFT:
                if not self.fkeep:
                    assert_func(self.zobj, self.hobj, hobj_info)
                else:
                    try:
                        assert_func(self.zobj, self.hobj, hobj_info)
                    except AssertionError as ex:
                        _ex.append([ex, name])

            else:
                # TO DO
                pass

        if self.objnames == []:
            if not isinstance(self.hfile, h5py.Dataset):
                self.hfile.id.links.visit(_test_obj, info=True)
            else:
                name = self.hfile.name
                hlink_info = self.hfile.file.id.links.get_info(bytes(name, encoding='utf-8'))
                _test_obj(name, hlink_info)
        else:
            for name in self.objnames:
                # skip objnames for subgroups
                if not isinstance(self.hfile, h5py.File) or name not in self.hfile:
                    continue
                hlink_info = self.hfile.id.links.get_info(name)
                _test_obj(name, hlink_info)

        self._ex = _ex

        # raise only one exception in case of fkeep == True
        if self.fkeep and len(self._ex) > 0:
            raise self._ex[0][0]

    @classmethod
    def _checkdtype_structobjref(cls, hobj):
        """ return 2-tuple indicating struct array and object reference dtype """
        dset_type = hobj.id.get_type()
        if hobj.dtype.names is None:
            return (False,  # simple dtype
                    dset_type.get_class() == h5py.h5t.REFERENCE)
        else:
            return (True,  # struct dtype
                    any([dset_type.get_member_class(i) == h5py.h5t.REFERENCE for i in range(dset_type.get_nmembers())]))

class HDF5ZarrBase(object):
    """ Comparing HDF5Zarr read with h5py """

    def __init__(self, hdf5files_option, hdf5file_names, ids_subgroup, ids_dset, ids_maxchunksize, _testfilename):
        self.hdf5files_option = hdf5files_option
        self.hdf5file_names = hdf5file_names
        self.ids_subgroup = ids_subgroup
        self.ids_dset = ids_dset
        self.ids_maxchunksize = ids_maxchunksize
        self._testfilename = _testfilename
        self.setup_class()

    def setup_class(cls):
        # list of numpy dtypes up to 8 bytes
        cls.attribute_dtypes = list(set(np.typeDict.values()) -
                                    set([np.void, np.str_, np.bytes_, np.object_, np.timedelta64,
                                         np.complex64, np.complex256, np.float128, np.complex128,
                                         np.datetime64]))
        cls.dset_dtypes = cls.attribute_dtypes
        cls.depth = 3  # nested groups depth

        # all n_* are per group or per object
        cls.n_dsets = 4  # number of regular (or scalar) datasets without object references or struct array dtypes in each group
        cls.n_groups = 3  # number of groups in each group
        cls.n_groupsoftlink = 1  # number of soft links to another group in each group
        cls.n_grouphardlink = 1  # TO DO number of hard links to another group in each group
        cls.n_dsetsoftlink = 1  # number of soft links to another dataset in each group
        cls.n_dsethardlink = 1  # TO DO number of hard links to another dataset in each group

        cls.n_objectrefdset = 1  # number of object reference datasets in each group
        cls.n_objectrefdsetmaxdim = 4  # maximum number of dimensions in an object reference datasets

        cls.n_structarraywithobjrefdset = 1  # number of struct array datasets containing object ref dtype in each group
        cls.n_structarrayobjrefdtype = 1  # number of object ref dtypes if used in a struct array

        cls.n_structarrayregulardset = 1  # number of struct array datasets without object refernce dtype in each group
        cls.n_structarraydtypelen = 4  # length of struct array dtypes in datasets

        cls.n_vlenstringdset = 1  # number of variable length string datasets in each group
        cls.n_vlenstringdsetmaxlen = 2000  # max length of strings in variable length datasets

        cls.n_dsetmaxdim = 4  # maximum number of dimensions

        cls.n_attributes_min = 5  # min number of attributes for each object

        cls.n_nulldsets_infile = 1  # TO DO number of null datasets in file

        cls.srand = secrets.SystemRandom()

        if cls.hdf5files_option:
            cls.file_list = [h5py.File(i, 'r') for i in cls.hdf5file_names]
        else:
            cls.file_list = [cls._create_file(i) for i in cls.hdf5file_names]

        cls.hdf5zarr_list = [HDF5Zarr(f.filename, max_chunksize=None) for f in cls.file_list]

        # prepend _testfile if hdf5files are not specified
        if not cls.hdf5files_option:
            cls.file_list.insert(0, cls._testfile())
            cls.hdf5zarr_list.insert(0, HDF5Zarr(cls.file_list[0].filename, max_chunksize=None))

        # track which temporary files are already saved.
        # if hdf5files_option is passed, mark them as already saved
        num_files = len(cls.file_list)
        cls.num_files = num_files
        cls.fnum_keep = {i: cls.hdf5files_option for i in range(0, num_files)}
        # do not save "_testfile"
        cls.fnum_keep[0] = True

        if cls.ids_subgroup or cls.ids_dset:
            group_names = []
            dset_names = []
            def _get_objs(name, info):
                nonlocal group_names, dset_names
                if info.type == h5py.h5o.TYPE_GROUP:
                    group_names.append(name.decode('utf-8'))
                elif info.type == h5py.h5o.TYPE_DATASET:
                    dset_names.append(name.decode('utf-8'))

            grnames = []
            dsnames = []
            for i in range(num_files):
                group_names = []
                dset_names = []
                h5py.h5o.visit(cls.file_list[i].id, _get_objs, info=True)
                grnames.append(group_names)
                dsnames.append(dset_names)

        # len(cls.ids_subgroup) == num_files*cls.numsubgroup
        for i in range(len(cls.ids_subgroup)):
            group_names = grnames[i%num_files]
            file_item = cls.file_list[i%num_files]
            if len(group_names) != 0:
                # select next group in sorted group names
                hdf5group = group_names[i//num_files if len(group_names) > i//num_files else -1]
            else:
                hdf5group = None
            cls.hdf5zarr_list.append(HDF5Zarr(file_item.filename, hdf5group=hdf5group))
            cls.file_list.append(file_item[hdf5group or '/'])

        # len(cls.ids_datasets) == num_files*cls.numdatasets
        for i in range(len(cls.ids_dset)):
            dset_names = dsnames[i%num_files]
            file_item = cls.file_list[i%num_files]
            if len(dset_names) != 0:
                # select next dataset in sorted dataset names
                hdf5obj = dset_names[i//num_files if len(dset_names) > i//num_files else -1]
            else:
                hdf5obj = None
            cls.hdf5zarr_list.append(HDF5Zarr(file_item.filename, hdf5obj=hdf5obj))
            cls.file_list.append(file_item[hdf5obj or '/'])

        # len(cls.ids_maxchunksize) == (num_files+num_files*cls.numsubgroup+cls.numdatasets)*cls.num_maxchunksize
        count = len(cls.hdf5zarr_list)  # number of hdf5zarr objects created in previous steps
        for i in range(len(cls.ids_maxchunksize)):
            # using same group/dataset names as in subgroups and datasets without max_chunksize
            item_name = cls.hdf5zarr_list[i%count].zgroup.name
            file_item = cls.file_list[i%count]
            max_chunksize = 1000 if not cls.hdf5files_option else 2**cls.srand.randint(18, 22)
            cls.hdf5zarr_list.append(HDF5Zarr(file_item.file.filename, hdf5obj=item_name, max_chunksize=max_chunksize))
            cls.file_list.append(file_item)

    def _create_file(cls, name):
        """ create test hdf5 file """

        srand = cls.srand

        # create hdf5 file
        cls.temp_file = tempfile.NamedTemporaryFile(suffix=".hdf5", prefix=name, delete=False)
        cls.temp_file.close()
        hfile = h5py.File(cls.temp_file.name, 'w')

        # create nested groups
        groupnames_prefix = [chr(65+i)for i in range(cls.n_groups)]  # e.g. ['A', 'B', 'C']
        group_list = [hfile]  # list containing all groups

        def _create_groups(obj, d):
            nonlocal group_list

            for c in groupnames_prefix:
                g_name = c + str(cls.depth - d)
                g = obj.create_group(g_name)
                group_list.append(g)
                if d > 0:
                    _create_groups(obj[g_name], d-1)

        _create_groups(hfile, cls.depth)

        # create softlinks to groups
        for g in group_list:
            for i in range(cls.n_groupsoftlink):
                # do not use rand_rng.choice
                target_str = srand.choice(group_list).name
                g[f"SoftLg{i}"] = h5py.SoftLink(target_str)

        # create datasets
        # TO DO, external dsets
        # TO DO, compression
        srand.shuffle(cls.dset_dtypes)
        iter_dtypes = itertools.cycle(cls.dset_dtypes)  # shuffle dtypes to cycle over when creating dsets
        iter_chunks = itertools.cycle([True, None])  # True or False cycle for auto chunking
        iter_track_times = itertools.cycle([False, True])  # True or False cycle for track_times
        iter_track_order = itertools.cycle([False, False, True, True])  # True or False cycle for track_order
        iter_fillvalue = itertools.cycle([None, True, True, None])  # True or False cycle for track_order
        rand_rng = np.random.default_rng()
        dset_list = []
        for g in group_list:
            # TO DO, add test with datasets with zero in dimensions
            for i in range(cls.n_dsets):
                shape = srand.choices(range(1, 90//(i or 1)), k=i)  # dseti has i dimensions
                size = np.prod(shape)
                dtype = next(iter_dtypes)
                if dtype == np.bool_:
                    data = np.frombuffer(rand_rng.bytes(size*8), dtype=np.int64) > 0
                elif dtype == np.datetime64:
                    data = np.datetime64('1970-01-01T00:00:00', 'ns') + np.frombuffer(rand_rng.bytes(size*8), dtype=np.uint64)
                    dtype = h5py.opaque_dtype(data.dtype)
                    data = data.astype(dtype)
                else:
                    data = np.frombuffer(rand_rng.bytes(size*np.dtype(dtype).itemsize), dtype=dtype)

                # create_dataset options comptability
                if len(shape) > 0:
                    chunks = next(iter_chunks)
                else:
                    chunks = None
                    # compression = None
                    # compression_opts = None
                    # shuffle = None
                    # fletcher32 = None
                    # scaleoffset = None
                fillvalue = None if (next(iter_fillvalue) is None or
                                     data.dtype.char == 'M') else data.reshape(size)[rand_rng.integers(0, size)]

                dset = g.create_dataset(
                           name='dset'+str(i),
                           shape=shape,
                           data=data,
                           dtype=dtype,
                           chunks=chunks,
                           maxshape=None if chunks is None else tuple(
                                          (np.array(shape) + rand_rng.integers(0, 5))*rand_rng.integers(1, 5, size=len(shape))),
                           track_times=next(iter_track_times),
                           track_order=next(iter_track_order),
                           fillvalue=fillvalue
                           )

                dset_list.append(dset)

        # create variable length string datasets
        for g in group_list:
            for i in range(cls.n_vlenstringdset):
                k = i + srand.randint(0, cls.n_dsetmaxdim)
                shape = srand.choices(range(1, int(60**(1/(k or 1)))+1), k=k)  # dseti has k dimensions
                size = int(np.prod(shape))
                dtype = h5py.string_dtype(encoding='utf-8')
                str_len = np.frombuffer(rand_rng.bytes(size*8), dtype=np.uint64) % cls.n_vlenstringdsetmaxlen
                data = np.array([''.join([chr(rand_rng.integers(0x0020, 0x03ff)) for _ in range(i)])
                                 for i in str_len], dtype=dtype)

                # create_dataset options comptability
                if len(shape) > 0:
                    chunks = next(iter_chunks)
                else:
                    chunks = None
                    # compression = None
                    # compression_opts = None
                    # shuffle = None
                    # fletcher32 = None
                    # scaleoffset = None
                fillvalue = None

                dset = g.create_dataset(
                           name='dsetvlenstring'+str(i),
                           shape=shape,
                           data=data,
                           dtype=dtype,
                           chunks=chunks,
                           maxshape=None if chunks is None else tuple(
                                          (np.array(shape) + rand_rng.integers(0, 5))*rand_rng.integers(1, 5, size=len(shape))),
                           track_times=next(iter_track_times),
                           track_order=next(iter_track_order),
                           fillvalue=fillvalue
                           )

                dset_list.append(dset)

        # create struct array datasets
        for g in group_list:
            # TO DO, add test with datasets with zero in dimensions
            for i in range(cls.n_structarrayregulardset):
                k = i + srand.randint(0, cls.n_dsetmaxdim)
                shape = srand.choices(range(1, 90//(k or 1)), k=k)  # dseti has k dimensions
                size = int(np.prod(shape))
                dtype = [(chr(97+j), next(iter_dtypes)) for j in range(cls.n_structarraydtypelen)]
                data = np.empty(shape=shape, dtype=dtype)
                for j in range(len(dtype)):
                    dt_name, dt = dtype[j]
                    if dt == np.bool_:
                        data_ = np.frombuffer(rand_rng.bytes(size*8), dtype=np.int64) > 0
                    elif dt == np.datetime64:
                        data_ = np.datetime64('1970-01-01T00:00:00', 'ns')+np.frombuffer(rand_rng.bytes(size*8), dtype=np.uint64)
                        dtype[j] = (dt_name, h5py.opaque_dtype(data_.dtype))
                        data_ = data_.astype(dtype[j][1])
                    else:
                        data_ = np.frombuffer(rand_rng.bytes(size*np.dtype(dt).itemsize), dtype=dt)

                    data[dt_name] = data_.reshape(shape)

                # create_dataset options comptability
                if len(shape) > 0:
                    chunks = next(iter_chunks)
                else:
                    chunks = None
                    # compression = None
                    # compression_opts = None
                    # shuffle = None
                    # fletcher32 = None
                    # scaleoffset = None
                fillvalue = None if (next(iter_fillvalue) is None or
                                     data.dtype.char == 'M') else data.reshape(size)[rand_rng.integers(0, size)]

                dset = g.create_dataset(
                           name='dsetstructarray'+str(i),
                           shape=shape,
                           data=data,
                           dtype=dtype,
                           chunks=chunks,
                           maxshape=None if chunks is None else tuple(
                                          (np.array(shape) + rand_rng.integers(0, 5))*rand_rng.integers(1, 5, size=len(shape))),
                           track_times=next(iter_track_times),
                           track_order=next(iter_track_order),
                           fillvalue=fillvalue
                           )

                dset_list.append(dset)

        # create object reference datasets
        for g in group_list:
            for i in range(cls.n_objectrefdset):
                k = i + srand.randint(0, cls.n_objectrefdsetmaxdim)
                shape = srand.choices(range(1, int(60**(1/(k or 1)))+1), k=k)  # dseti has k dimensions
                size = int(np.prod(shape))
                dtype = h5py.ref_dtype

                obj_list = dset_list + group_list
                data = np.array([srand.choice(obj_list).ref for _ in range(size)])

                # create_dataset options comptability
                if len(shape) > 0:
                    chunks = next(iter_chunks)
                else:
                    chunks = None
                    # compression = None
                    # compression_opts = None
                    # shuffle = None
                    # fletcher32 = None
                    # scaleoffset = None

                fillvalue = None

                dset = g.create_dataset(
                           name='dsetobjref'+str(i),
                           shape=shape,
                           data=data,
                           dtype=dtype,
                           chunks=chunks,
                           maxshape=None if chunks is None else tuple(
                                          (np.array(shape) + rand_rng.integers(0, 5))*rand_rng.integers(1, 5, size=len(shape))),
                           track_times=next(iter_track_times),
                           track_order=next(iter_track_order),
                           fillvalue=fillvalue
                           )

                dset_list.append(dset)

        # create struct array datasets with object reference
        for g in group_list:
            # TO DO, add test with datasets with zero in dimensions
            for i in range(cls.n_structarraywithobjrefdset):
                k = i + srand.randint(0, cls.n_objectrefdsetmaxdim)
                shape = srand.choices(range(1, int(60**(1/(k or 1)))+1), k=k)  # dseti has i dimensions
                size = int(np.prod(shape))
                dtypeobjind = rand_rng.choice(range(cls.n_structarraydtypelen), size=cls.n_structarrayobjrefdtype, replace=False)
                dtype = [(chr(97+j), h5py.ref_dtype if j in dtypeobjind else next(iter_dtypes))
                         for j in range(cls.n_structarraydtypelen)]

                data = np.empty(shape=shape, dtype=dtype)
                for j in range(len(dtype)):
                    dt_name, dt = dtype[j]
                    if dt == h5py.ref_dtype:
                        obj_list = dset_list + group_list
                        data_ = np.array([srand.choice(obj_list).ref for _ in range(size)])
                    elif dt == np.bool_:
                        data_ = np.frombuffer(rand_rng.bytes(size*8), dtype=np.int64) > 0
                    elif dt == np.datetime64:
                        data_ = np.datetime64('1970-01-01T00:00:00', 'ns')+np.frombuffer(rand_rng.bytes(size*8), dtype=np.uint64)
                        dtype[j] = (dt_name, h5py.opaque_dtype(data_.dtype))
                        data_ = data_.astype(dtype[j][1])
                    else:
                        data_ = np.frombuffer(rand_rng.bytes(size*np.dtype(dt).itemsize), dtype=dt)

                    data[dt_name] = data_.reshape(shape)

                # create_dataset options comptability
                if len(shape) > 0:
                    chunks = next(iter_chunks)
                else:
                    chunks = None
                    # compression = None
                    # compression_opts = None
                    # shuffle = None
                    # fletcher32 = None
                    # scaleoffset = None
                fillvalue = None if (next(iter_fillvalue) is None or
                                     data.dtype.char == 'M') else data.reshape(size)[rand_rng.integers(0, size)]

                dset = g.create_dataset(
                           name='dsetstructarraywobjref'+str(i),
                           shape=shape,
                           data=data,
                           dtype=dtype,
                           chunks=chunks,
                           maxshape=None if chunks is None else tuple(
                                          (np.array(shape) + rand_rng.integers(0, 5))*rand_rng.integers(1, 5, size=len(shape))),
                           track_times=next(iter_track_times),
                           track_order=next(iter_track_order),
                           fillvalue=fillvalue
                           )

                dset_list.append(dset)

        # create softlinks to datasets
        for g in group_list:
            for i in range(cls.n_dsetsoftlink):
                # do not use rand_rng.choice
                target_str = srand.choice(dset_list).name
                g[f"SoftLd{i}"] = h5py.SoftLink(target_str)

        # add attributes
        srand.shuffle(cls.dset_dtypes)
        iter_dtypes = itertools.cycle(cls.dset_dtypes)  # shuffle dtypes to cycle over when creating attributes
        for obj in itertools.chain(group_list, dset_list):
            for i in range(rand_rng.integers(cls.n_attributes_min, 26, endpoint=True)):
                dtype = next(iter_dtypes)
                attr_name = chr(97+i)
                if dtype == np.bool_:
                    attr = np.frombuffer(rand_rng.bytes(8), dtype=np.int64) > 0
                elif dtype == np.datetime64:
                    continue
                else:
                    attr = np.frombuffer(rand_rng.bytes(np.dtype(dtype).itemsize), dtype=dtype)
                obj.attrs[attr_name] = attr[0]

            # add array attributes
            for i in range(rand_rng.integers(cls.n_attributes_min, 26, endpoint=True)):
                shape = srand.choices(range(1, 10//(i//5 or 1)), k=i//5)  # attributes has i//5 dimensions
                size = np.prod(shape)
                dtype = next(iter_dtypes)
                attr_name = chr(65+i) + '_array_attr'
                if dtype == np.bool_:
                    attr = np.frombuffer(rand_rng.bytes(size*8), dtype=np.int64) > 0
                elif dtype == np.datetime64:
                    attr = np.datetime64('1970-01-01T00:00:00', 'ns') + np.frombuffer(rand_rng.bytes(size*8), dtype=np.uint64)
                    attr = attr.astype(h5py.opaque_dtype(attr.dtype))
                else:
                    attr = np.frombuffer(rand_rng.bytes(size*np.dtype(dtype).itemsize), dtype=dtype)
                obj.attrs[attr_name] = attr

        return hfile

    def _testfile(cls):
        """ create test hdf5 file """

        # repeatable test
        # create hdf5 file
        cls.temp_file = tempfile.NamedTemporaryFile(suffix=".hdf5", prefix=cls._testfilename, delete=False)
        cls.temp_file.close()
        hfile = h5py.File(cls.temp_file.name, 'w')

        hfile.create_dataset(
            name='dset0',
            shape=(1,),
            data=[0],
            dtype=np.float,
            chunks=None,
            maxshape=None,
            track_times=None,
            track_order=None,
            fillvalue=1,
            )

        hfile.create_dataset(
            name='dsetscalar',
            shape=(),
            data=2,
            dtype=np.float,
            maxshape=None,
            track_times=None,
            track_order=None,
            fillvalue=1,
            )

        shape = (20, 3, 4, 13)
        data = np.array([cls.srand.choice([v for k, v in hfile.items()]).ref for _ in range(np.prod(shape)-1)] + [hfile.ref])
        hfile.create_dataset(
            name='dsetobjref',
            shape=shape,
            chunks=(10, 2, 4, 13),
            data=data,
            dtype=h5py.ref_dtype,
            maxshape=(72, 28, 16, 68),
            track_times=None,
            track_order=None,
            fillvalue=None,
            )

        return hfile


# try again BinarySearch without recursion
def BinarySearch(A, lo, hi, k):
    while lo <= hi:
        m = lo + (hi - lo)//2
        if A[m] == k:
            return m
        if k > A[m]:
            lo = m + 1
        else:
            hi = m - 1
    return -1

def test():
    A = [1,3,4,5,6,7]
    print(BinarySearch(A, 0, len(A)-1, 5))

if __name__ == "__main__":
    test()


from . import RECIPES_API
from flask import request, jsonify, abort, make_response
from app.models import Category, Recipe, User
from flasgger import swag_from
import validator
from app.categories.views import authenticate
from sqlalchemy import desc

@RECIPES_API.route('/create_recipe/<int:category_id>', methods=["POST"])
@authenticate
@swag_from('/app/doc/create_recipe.yml')
def create_recipe(user_id, category_id, **kwargs):
    #this function is used to created a recipe under an existing category ID
    recipe_name = str(request.data.get('recipe_name')).strip()
    instructions = str(request.data.get('instructions')).strip()
    if recipe_name and instructions:
        recipe_name = recipe_name.capitalize()
        name = Category.query.filter_by(category_id=category_id, created_by=user_id).first()
        if name:
            name2 = Recipe.query.filter_by(category=category_id, recipe_name=recipe_name).first()
            if name2 is None:
                recipe = Recipe(recipe_name=recipe_name,instructions=instructions,
                                category=category_id)
                recipe.save()
                response = jsonify({
                    "message": "Recipe has been created",
                    "recipe_id": recipe.recipe_id,
                    "recipe_name": recipe.recipe_name,
                    "instructions": recipe.instructions,
                    "date_created": recipe.date_created,
                    "date_modified": recipe.date_modified,
                    "category": recipe.category})
                response.status_code = 201
                return response
            return jsonify({'message': 'The Recipe Already Exists. Try another'}), 409
        return jsonify({'message': 'Invalid request'}), 400
    return jsonify({'message': 'Enter valid data'}), 400

@RECIPES_API.route('/view_recipes/<int:category_id>/', methods=["GET"])
@swag_from('/app/doc/view_recipes.yml')
@authenticate
def view_all_recipes(user_id, category_id, **kwags):
    #this function is used to view all recipes under a category
    #the data can be paginated and also can search a specific category
    page, per_page = int(request.args.get('page', 1)), int(request.args.get('per_page', 3))
    q = str(request.args.get('q', '')).capitalize()
    name = Category.query.filter_by(category_id=category_id, created_by=user_id).first()
    if name:
        recipes = Recipe.query.filter_by(category=category_id).order_by(desc('date_created')).paginate(page, per_page, False)
        has_next=recipes.has_next
        has_prev=recipes.has_prev
        next_url = 'http://127.0.0.1:5000/view_recipes/' + str(category_id) + '/?page=' +\
            str(recipes.next_num)  if recipes.has_next else None
        prev_url = 'http://127.0.0.1:5000/view_recipes/' + str(category_id) + '/?page=' +\
            str(recipes.prev_num)  if recipes.has_prev else None

        result = []
        if q:
            recipes = Recipe.query.filter_by(category=category_id)
            for recipe in recipes:
                if q in recipe.recipe_name:
                    obj = {
                        "recipe_id": recipe.recipe_id,
                        "recipe_name": recipe.recipe_name,
                        "instructions": recipe.instructions,
                        "date_created": recipe.date_created,
                        "date_modified": recipe.date_modified,
                        "category": recipe.category
                    }
                    result.append(obj)
        else:
            for recipe in recipes.items:
                obj = {
                    "recipe_id": recipe.recipe_id,
                    "recipe_name": recipe.recipe_name,
                    "instructions": recipe.instructions,
                    "date_created": recipe.date_created,
                    "date_modified": recipe.date_modified,
                    "category": recipe.category
                }
                result.append(obj)
        if result:
            response = {
                "next_page": next_url,
                "previous_page": prev_url,
                "recipes":result,
                "has_next":has_next,
                "has_prev":has_prev
            }
            response = jsonify(response)
            response.status_code = 200
            return response
        else:
            result = []
            response = jsonify({
                'recipes': result,
                'message': 'no recipes available'})
            response.status_code = 200
            return response
    return jsonify({'message': 'Invalid request'}), 404

@RECIPES_API.route('/recipe_byid/<int:category_id>/<int:recipe_id>', methods=["GET", "POST"])
@swag_from('/app/doc/recipe_byID.yml')
@authenticate
def recipe_byid(user_id, category_id, recipe_id, **kwargs):
    #this function is used to view a recipe by ID under a category
    name = Category.query.filter_by(category_id=category_id, created_by=user_id).first()
    if name:
        recipe = Recipe.query.filter_by(recipe_id=recipe_id).first()
        if not recipe:
            abort(404)
        response = jsonify({
            "recipe_id": recipe.recipe_id,
            "recipe_name": recipe.recipe_name,
            "instructions": recipe.instructions,
            "date_created": recipe.date_created,
            "date_modified": recipe.date_modified,
            "category": recipe.category
        })
        response.status_code = 200
        return response
    else:
        return jsonify({'message': 'Invalid request'})

@RECIPES_API.route('/recipe_edit/<int:category_id>/<int:recipe_id>', methods=["PUT"])
@swag_from('/app/doc/recipe_edit.yml')
@authenticate
def recipe_manipulation(user_id, category_id, recipe_id, **kwargs):
    #this function is used to edit a recipe under a category
    name = Category.query.filter_by(category_id=category_id, created_by=user_id).first()
    if name:
        recipe = Recipe.query.filter_by(recipe_id=recipe_id).first()
        if not recipe:
            return jsonify({'message': 'this recipe does not exist'})
        if request.method == "PUT":
            name = str(request.data.get('recipe_name', '')).strip()
            instructions = str(request.data.get('instructions', '')).strip()
            if name and instructions:
                name, name2 = name.capitalize(), Recipe.query.filter_by(recipe_name=name).first()
                if name2 is None:
                    recipe.recipe_name = name
                    recipe.instructions = instructions
                    recipe.save()
                    response = jsonify({
                        "message": "successfully edited " + recipe.recipe_name,
                        "recipe_id": recipe.recipe_id,
                        "recipe_name": recipe.recipe_name,
                        "instructions": recipe.instructions,
                        "date_created": recipe.date_created,
                        "date_modified": recipe.date_modified,
                        "category": recipe.category
                    })
                    response.status_code = 200
                    return response
                return jsonify({'message': ''})
            return jsonify({'message': 'Enter valid data'})
    return jsonify({'message': 'Invalid request'})

@RECIPES_API.route('/recipe_delete/<int:category_id>/<int:recipe_id>', methods=["DELETE"])
@swag_from('/app/doc/recipe_delete.yml')
@authenticate
def recipe_delete(user_id, category_id, recipe_id):
    # this function is used to delete a recipe
    name = Category.query.filter_by(category_id=category_id, created_by=user_id).first()
    if name:
        recipe = Recipe.query.filter_by(recipe_id=recipe_id).first()
        if not recipe:
            abort(404)
        else:
            recipe.delete()
            return {"message": "recipe {} deleted successfully".format(
                recipe.recipe_id)}, 200
    return jsonify({'message': 'Invalid request'})


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.sparse as tsp
import dgl

if torch.cuda.is_available():
    device = 'cuda:0'
else:
    device = 'cpu'

class LRP_synthetic_layer(nn.Module):
    def __init__(self,
                 lrp_length = 16,
                 lrp_e_dim = 2,
                 lrp_in_dim = 2,
                 lrp_out_dim = 128):
        super(LRP_synthetic_layer, self).__init__()

        coeffs_values_3 = lambda i, j, k: torch.randn([i, j, k])
        coeffs_values_4 = lambda i, j, k, l: torch.randn([i, j, k, l])
        self.weights = nn.Parameter(coeffs_values_3(lrp_in_dim, lrp_out_dim, lrp_length))

        self.bias = nn.Parameter(torch.zeros(1, lrp_out_dim))

        self.degnet_0, self.degnet_1 = nn.Linear(1, 2 * lrp_out_dim), nn.Linear(2 * lrp_out_dim, lrp_out_dim)

        self.lrp_length = lrp_length
        self.lrp_e_dim = lrp_e_dim
        self.lrp_in_dim = lrp_in_dim
        self.lrp_out_dim = lrp_out_dim

    def forward(self, graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix):
        nfeat = graph.ndata['h']
        efeat = graph.edata['feat']

        nfeat = tsp.mm(n_to_perm_length_sp_matrix, nfeat) + tsp.mm(e_to_perm_length_sp_matrix, efeat)

        nfeat = nfeat.transpose(0, 1).view(self.lrp_in_dim, -1, self.lrp_length).permute(1, 2, 0)

        nfeat = F.relu(torch.einsum('dab,bca->dc', nfeat, self.weights) + self.bias)
        nfeat = tsp.mm(pooling_matrix, nfeat)

        factor_degs = self.degnet_1(F.relu(self.degnet_0(degs.unsqueeze(1))))#.squeeze()

        nfeat = F.relu(torch.einsum('ab,ab->ab', nfeat, factor_degs))

        graph.ndata['h'] = nfeat
        return graph

class LRP_synthetic(nn.Module):
    def __init__(self,
                 num_tasks = 1,
                 lrp_length = 16,
                 lrp_in_dim = 2,
                 hid_dim = 128,
                 num_layers = 1,
                 bn = False,
                 mlp = False
                 ):
        super(LRP_synthetic, self).__init__()

        self.lrp_list = nn.ModuleList()
        for i in range(num_layers):
            if i == 0:
                self.lrp_list.append(LRP_synthetic_layer(lrp_length = lrp_length,
                                                lrp_e_dim = lrp_in_dim,
                                                lrp_in_dim = lrp_in_dim,
                                                lrp_out_dim = hid_dim
                                                ))
            else:
                self.lrp_list.append(LRP_synthetic_layer(lrp_length = lrp_length,
                                                lrp_e_dim = 2,
                                                lrp_in_dim = hid_dim,
                                                lrp_out_dim = hid_dim
                                                ))

        self.final_predict = nn.Linear(hid_dim, num_tasks)

        self.bn = bn
        self.mlp = mlp

        if bn:
            self.bn_layers = nn.ModuleList([nn.BatchNorm1d(hid_dim) for i in range(num_layers)])

        if mlp:
            self.mlp_layers = nn.ModuleList([nn.Linear(hid_dim, hid_dim) for i in range(num_layers)])

    def forward(self, graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix):
        graph.ndata['h'] = graph.ndata['feat']
        if self.bn and self.mlp:
            for lrp_layer, mlp_layer, bn in zip(self.lrp_list, self.mlp_layers, self.bn_layers):
                graph = lrp_layer(graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix)
                graph.ndata['h'] = bn(graph.ndata['h'])
                graph.ndata['h'] = F.relu(mlp_layer(graph.ndata['h']))
        elif self.bn and not self.mlp:
            for lrp_layer, bn in zip(self.lrp_list, self.bn_layers):
                graph = lrp_layer(graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix)
                graph.ndata['h'] = bn(graph.ndata['h'])
        elif (not self.bn) and self.mlp:
            for lrp_layer, mlp_layer in zip(self.lrp_list, self.mlp_layers):
                graph = lrp_layer(graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix)
                graph.ndata['h'] = F.relu(mlp_layer(graph.ndata['h']))
        else:
            for lrp_layer in self.lrp_list:
                graph = lrp_layer(graph, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix)

        graph.ndata['h'] = self.final_predict(graph.ndata['h'])
        output = dgl.sum_nodes(graph, 'h')
        return output

if __name__ == "__main__":
    from dataset_synthetic import helper, build_batch_graph_node_to_perm_times_length
    from main_synthetic import build_perm_pooling_sp_matrix, np_sparse_to_pt_sparse

    G = dgl.DGLGraph()
    G.add_nodes(4)
    G.add_edges([0,0,1,1,2,2,2,3], [1,2,0,2,0,1,3,2])

    lrp_seq = helper(G)

    split_list = [len(ll) for ll in lrp_seq]
    pooling_matrix = build_perm_pooling_sp_matrix(split_list, "sum")

    n_to_pl, e_to_pl = build_batch_graph_node_to_perm_times_length([G], [lrp_seq])

    n_to_perm_length_sp_matrix = np_sparse_to_pt_sparse(n_to_pl)
    e_to_perm_length_sp_matrix = np_sparse_to_pt_sparse(e_to_pl)

    # print(pooling_matrix)

    model = LRP_synthetic(num_tasks = 1,
                 lrp_length = 16,
                 lrp_in_dim = 1,
                 hid_dim = 1,
                 num_layers = 1)

    G.ndata['feat'] = torch.ones((G.number_of_nodes(),1))
    G.edata['feat'] = torch.ones((G.number_of_edges(),1))

    degs = G.in_degrees(list(range(G.number_of_nodes()))).type(torch.FloatTensor)

    output = model(G, pooling_matrix, degs, n_to_perm_length_sp_matrix, e_to_perm_length_sp_matrix)

    print(output)


from psychopy import visual, logging, core
import pylab

# If you are having frame drops, try running this on a set of monitors that are
# *exactly* the same hardware. Mixing different monitor brands, even stuff with the same specs,
# caused frame drops in my testing. Once all the monitors were identical, frame drops went
# down to 0.

mywin = visual.Window([1920,1080], fullscr=True,screen=1)
mywin.setRecordFrameIntervals(True)
mywin._refreshThreshold=1/120.0+0.004 #i've got 120Hz monitor and want to allow 4ms tolerance

#set the log module to report warnings to the std output window (default is errors only)
logging.console.setLevel(logging.WARNING)

#create some stimuli
stim1 = visual.PatchStim(win=mywin,tex='tri',mask='circle',units='pix',pos=(-200,-200),size=(500,500), sf=0.001, colorSpace='rgb', contrast=1.0)
stim2 = visual.PatchStim(win=mywin,tex='tri',mask='circle',units='pix',pos=(400,200),size=(500,500), sf=0.001, colorSpace='rgb', contrast=1.0)
stim1.setAutoDraw(False)
stim2.setAutoDraw(False)

clock = core.Clock()
while clock.getTime()<10:
        if clock.getTime()>5:
            stim1.setPhase(0.001, '-')
            stim2.setPhase(0.001, '-')
            stim1.draw()
            stim2.draw()
        else:
            stim1.setPhase(0.001, '+')
            stim2.setPhase(0.001, '+')
            stim1.draw()
            stim2.draw()
        mywin.flip()#flip the screen. This will block until the monitor is ready for the flip.

pylab.plot(mywin.frameIntervals)
pylab.show()

mywin.saveFrameIntervals(fileName=None, clear=True)


# importing the required modules
import pandas as pd

# csv files in the path
pre_root = input("Please type in directory path to excel file containing list of excel files to be concatenated, "
                 "followed by the enter key:")
pre_outfile_name = input("Please type in name of excel file to be output, followed by the enter key:")

# clean up leading/trailing whitespace
root = pre_root.strip()
outfile_name = pre_outfile_name.strip()

# read in list of Excel files for concatenation
file_list_manifest = pd.read_excel(root)

file_list = file_list_manifest['fullPath'].tolist()
# list of Excel files we want to merge.
# pd.read_excel(file_path) reads the Excel
# data into pandas dataframe.
excl_list = []

for file in file_list:
    excl_list.append(pd.read_excel(file))

# create a new dataframe to store the
# merged Excel file.
excl_merged = pd.DataFrame()

for excl_file in excl_list:
    # appends the data into the excl_merged
    # dataframe.
    excl_merged = excl_merged.append(
        excl_file, ignore_index=True)

# exports the dataframe into a CSV file with
# specified name. CSV is used for output because
# large datasets may exceed the Excel spreadsheet
# specification; if you know your dataset is small
# enough to be put into .xlsx format, replace
# to_csv() with to_excel on the line below
excl_merged.to_csv(outfile_name, index=False)
exit()


from pathlib import Path

BASE_DIR = Path(__file__).resolve(strict=True).parent

with open(BASE_DIR / '../input.txt') as f:
    # Load input
    clean_lines = f.read().splitlines()
    count = 0
    for clean_line in clean_lines:
        second_group_of_digits = clean_line.split(' | ')[1].split(' ')
        line = [len(segment) for segment in second_group_of_digits if len(segment) in (2, 4, 3, 7)]
        count += len(line)

    print(f'Result: {count}')


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
get_ipython().run_line_magic('matplotlib', 'inline')

# In[2]:

df=pd.read_csv('fake_job_postings.csv')

# In[3]:

df.head()

# In[4]:

df.info()

# In[5]:

df.describe().transpose()

# In[6]:

sns.countplot(x='fraudulent',data=df)

# In[7]:

# We are dealing with imbalanced data
df.corr()

# In[8]:

plt.figure(figsize=(12,7))
sns.heatmap(df.corr(),annot=True,cmap='viridis')

# In[9]:

df.corr()['fraudulent'].sort_values().drop(['fraudulent','job_id']).plot(kind='bar')

# In[10]:

df.isnull().sum()

# In[11]:

100*df.isnull().sum()/len(df)

# In[12]:

#There are too many missing values in the above 2 columns
df=df.drop(['location','salary_range'],axis=1)

# In[13]:

df.head()

# In[14]:

df['title'].nunique()

# In[15]:

df['department'].nunique()

# In[16]:

# There are too many unique elements in the above 2 columns
df=df.drop(['title','department'],axis=1)

# In[17]:

df.info()

# In[18]:

df.fillna('Not Specified',inplace=True)

# In[19]:

df.isnull().sum()

# In[20]:

df['required_education'].value_counts()

# In[22]:

plt.figure(figsize=(15,14))
ax=sns.countplot(y='required_education',hue='fraudulent',data=df)
plt.legend(bbox_to_anchor=(1, 1.1),title='Fraudulent', loc=2, borderaxespad=0.)
plt.tight_layout()
for p in ax.patches:
    width = p.get_width()
    x, y = p.get_xy()
    ax.annotate(float(width),
                ((x + width), y),
                xytext = (40, -15),
                fontsize = 16,
                color = '#000000',
                textcoords = 'offset points',
                ha = 'center',
                va = 'center')

# In[23]:

req_edu_dummies = pd.get_dummies(df['required_education'],drop_first=True)

# In[24]:

df = pd.concat([df.drop('required_education',axis=1),req_edu_dummies],axis=1)

# In[25]:

df.columns

# In[26]:

df.corr()['fraudulent'].sort_values()

# In[27]:

df['required_experience'].value_counts()

# In[28]:

req_exp_dummies = pd.get_dummies(df['required_experience'],drop_first=True)

# In[29]:

df = pd.concat([df.drop('required_experience',axis=1),req_exp_dummies],axis=1)

# In[30]:

df.columns

# In[31]:

df.corr()['fraudulent'].sort_values()

# In[32]:

df['employment_type'].value_counts()

# In[33]:

plt.figure(figsize=(12,10))
ax=sns.countplot(x='employment_type',hue='fraudulent',data=df)
plt.legend(bbox_to_anchor=(1, 1.1),title='Fraudulent', loc=2, borderaxespad=0.)
plt.tight_layout()

# In[34]:

emp_type_dummies = pd.get_dummies(df['employment_type'],drop_first=True)

# In[35]:

df = pd.concat([df.drop('employment_type',axis=1),emp_type_dummies],axis=1)

# In[36]:

df.columns

# In[37]:

df.corr()

# In[38]:

df.head()

# In[110]:

#We are not going to deal with text columns in this project
#df=df.drop(['job_id','company_profile','description','requirements','benefits'],axis=1)

# In[112]:

df.head()

# In[113]:

from sklearn.preprocessing import LabelEncoder
le=LabelEncoder()

# In[114]:

industry_new=pd.DataFrame(df[['industry']])

# In[115]:

for i in industry_new:
    if industry_new[i].dtype=='object':
        industry_new[i]=le.fit_transform(industry_new[i])

industry_new.head()

# In[116]:

function_new=pd.DataFrame(df
                          [['function']])

# In[117]:

for i in function_new:
    if function_new[i].dtype=='object':
        function_new[i]=le.fit_transform(function_new[i])

# In[118]:

function_new.head()

# In[119]:

df=df.drop(['industry','function'],axis=1)

# In[120]:

df=pd.concat([df,industry_new,function_new],axis=1)

# In[121]:

df.columns

# In[123]:

df.head()

# In[56]:

fig,(ax1,ax2)= plt.subplots(ncols=2, figsize=(14, 6), dpi=100)

length1=df[df["fraudulent"]==1]['company_profile'].str.len()

ax1.hist(length1,bins = 30,color='orangered')
ax1.set_title('Fake')
ax1.set_xlabel('Num of Char')

length0=df[df["fraudulent"]==0]['company_profile'].str.len()

ax2.hist(length0, bins = 30)
ax2.set_title('Real')
ax2.set_xlabel('Num of Char')
fig.suptitle('Characters in Company Profile')
plt.show()

# In[57]:

fig,(ax1,ax2)= plt.subplots(ncols=2, figsize=(14, 6), dpi=100)

length1=df[df["fraudulent"]==1]['description'].str.len()

ax1.hist(length1,bins = 30,color='orangered')
ax1.set_title('Fake')
ax1.set_xlabel('Num of Char')

length0=df[df["fraudulent"]==0]['description'].str.len()

ax2.hist(length0, bins = 30)
ax2.set_title('Real')
ax2.set_xlabel('Num of Char')
fig.suptitle('Characters in Description')
plt.show()

# In[58]:

fig,(ax1,ax2)= plt.subplots(ncols=2, figsize=(14, 6), dpi=100)

length1=df[df["fraudulent"]==1]['requirements'].str.len()

ax1.hist(length1,bins = 30,color='orangered')
ax1.set_title('Fake')
ax1.set_xlabel('Num of Char')

length0=df[df["fraudulent"]==0]['requirements'].str.len()

ax2.hist(length0, bins = 30)
ax2.set_title('Real')
ax2.set_xlabel('Num of Char')
fig.suptitle('Characters in Requirements')
plt.show()

# In[59]:

fig,(ax1,ax2)= plt.subplots(ncols=2, figsize=(14, 6), dpi=100)

length1=df[df["fraudulent"]==1]['benefits'].str.len()

ax1.hist(length1,bins = 30,color='orangered')
ax1.set_title('Fake')
ax1.set_xlabel('Num of Char')

length0=df[df["fraudulent"]==0]['benefits'].str.len()

ax2.hist(length0, bins = 30)
ax2.set_title('Real')
ax2.set_xlabel('Num of Char')
fig.suptitle('Characters in Benefits')
plt.show()

# In[60]:

plt.figure(figsize=(10,7))
sns.countplot(x ="telecommuting", hue="fraudulent", data=df)

# In[61]:

plt.figure(figsize=(10,7))
sns.countplot(x ="has_company_logo", hue="fraudulent", data=df)

# In[62]:

plt.figure(figsize=(10,7))
sns.countplot(x ="has_questions", hue="fraudulent", data=df)

# In[ ]:

#We are not going to deal with text columns in this project
df=df.drop(['job_id','company_profile','description','requirements','benefits'],axis=1)

# In[136]:

#Applying different machine learnng algorithms
from sklearn.model_selection import train_test_split

x=df.drop('fraudulent',axis=1)
y=df['fraudulent']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=101)

# In[137]:

from sklearn.ensemble import RandomForestClassifier
rfc=RandomForestClassifier()
rfc.fit(x_train,y_train)

# In[139]:

predictions=rfc.predict(x_test)

# In[140]:

from sklearn.metrics import classification_report,confusion_matrix

# In[141]:

print(classification_report(y_test,predictions))

# In[142]:

print(confusion_matrix(y_test,predictions))

# In[143]:

from sklearn.tree import DecisionTreeClassifier

# In[144]:

dtree = DecisionTreeClassifier()

# In[145]:

dtree.fit(x_train,y_train)

# In[147]:

predictions_tree = dtree.predict(x_test)

# In[148]:

print(classification_report(y_test,predictions_tree))

# In[149]:

print(confusion_matrix(y_test,predictions_tree))

# In[150]:

from sklearn.linear_model import LogisticRegression

# In[162]:

logmodel = LogisticRegression(max_iter=700)
logmodel.fit(x_train,y_train)

# In[163]:

predictions_log = logmodel.predict(x_test)

# In[164]:

print(classification_report(y_test,predictions_log))

# In[165]:

print(confusion_matrix(y_test,predictions))

# In[166]:

from sklearn.neighbors import KNeighborsClassifier

# In[167]:

knn = KNeighborsClassifier(n_neighbors=1)

# In[168]:

knn.fit(x_train,y_train)

# In[169]:

predictions_knn = knn.predict(x_test)

# In[170]:

print(classification_report(y_test,predictions_knn))

# In[171]:

print(confusion_matrix(y_test,predictions_knn))

# In[172]:

error_rate = []

for i in range(1,40):

    knn = KNeighborsClassifier(n_neighbors=i)
    knn.fit(x_train,y_train)
    pred_i = knn.predict(x_test)
    error_rate.append(np.mean(pred_i != y_test))

# In[173]:

plt.figure(figsize=(10,6))
plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',
         markerfacecolor='red', markersize=10)
plt.title('Error Rate vs. K Value')
plt.xlabel('K')
plt.ylabel('Error Rate')

# In[174]:

knn = KNeighborsClassifier(n_neighbors=4)

knn.fit(x_train,y_train)
predictions_knn = knn.predict(x_test)

print('WITH K=30')
print('\n')
print(confusion_matrix(y_test,predictions_knn))
print('\n')
print(classification_report(y_test,predictions_knn))

# In[175]:

from sklearn.preprocessing import MinMaxScaler

# In[176]:

scaler = MinMaxScaler()

# In[177]:

x_train = scaler.fit_transform(x_train)

# In[178]:

x_test = scaler.transform(x_test)

# In[179]:

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation,Dropout

# In[181]:

x_train.shape

# In[182]:

model = Sequential()

model.add(Dense(units=30,activation='relu'))

model.add(Dense(units=15,activation='relu'))

model.add(Dense(units=1,activation='sigmoid'))

# For a binary classification problem
model.compile(loss='binary_crossentropy', optimizer='adam')

# In[184]:

model.fit(x=x_train,
          y=y_train,
          epochs=600,
          validation_data=(x_test, y_test), verbose=1
          )

# In[185]:

model_loss = pd.DataFrame(model.history.history)

# In[186]:

model_loss.plot()

# In[251]:

# trained too much so now we will use early stopping
from tensorflow.keras.callbacks import EarlyStopping

# In[252]:

early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)
model = Sequential()
model.add(Dense(units=30,activation='relu'))
model.add(Dense(units=15,activation='relu'))
model.add(Dense(units=1,activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')

# In[253]:

model.fit(x=x_train,
          y=y_train,
          epochs=600,
          validation_data=(x_test, y_test), verbose=1,
          callbacks=[early_stop]
          )

# In[254]:

model_loss = pd.DataFrame(model.history.history)
model_loss.plot()

# In[255]:

predictions_ann =  (model.predict(x_test) > 0.2).astype("int32")

# In[256]:

print(classification_report(y_test,predictions_ann))

# In[240]:

from tensorflow.keras.layers import Dropout

# In[241]:

model = Sequential()
model.add(Dense(units=30,activation='relu'))
model.add(Dropout(0.5))

model.add(Dense(units=15,activation='relu'))
model.add(Dropout(0.5))

model.add(Dense(units=1,activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')

# In[242]:

model.fit(x=x_train,
          y=y_train,
          epochs=600,
          validation_data=(x_test, y_test), verbose=1,
          callbacks=[early_stop]
          )

# In[244]:

model_loss = pd.DataFrame(model.history.history)
model_loss.plot()

# In[249]:

predictions_drop = (model.predict(x_test) > 0.1).astype("int32")

# In[250]:

print(classification_report(y_test,predictions_drop))

# In[257]:

# Now, we are going to resample our data
# under-sample
class_0=df[df['fraudulent']==0]
class_1=df[df['fraudulent']==1]

# In[258]:

class_0.shape

# In[259]:

class_1.shape

# In[260]:

class_0_under=class_0.sample(866)

# In[261]:

test_under=pd.concat([class_0_under,class_1],axis=0)

# In[262]:

x_under=pd.DataFrame(test_under.drop('fraudulent',axis=1))
y_under=pd.DataFrame(test_under['fraudulent'])

# In[264]:

x_train, x_test, y_train, y_test = train_test_split(x_under, y_under, test_size=0.3, random_state=101,stratify=y_under)

# In[265]:

rfc_under=RandomForestClassifier()
rfc_under.fit(x_train,y_train.values.ravel())

# In[266]:

pred_under=rfc_under.predict(x_test)

# In[267]:

print(classification_report(y_test,pred_under))

# In[268]:

print(confusion_matrix(y_test,pred_under))

# In[285]:

# over-sample
class_1_over=class_1.sample(17014,replace=True)

# In[286]:

test_over=pd.concat([class_1_over,class_0],axis=0)

# In[287]:

x_over=pd.DataFrame(test_over.drop('fraudulent',axis=1))
y_over=pd.DataFrame(test_over['fraudulent'])

# In[288]:

x_train, x_test, y_train, y_test = train_test_split(x_over, y_over, test_size=0.3, random_state=101,stratify=y_over)

# In[289]:

rfc_over=RandomForestClassifier()
rfc_over.fit(x_train,y_train.values.ravel())

# In[290]:

pred_over=rfc_over.predict(x_test)

# In[291]:

print(classification_report(y_test,pred_over))

# In[292]:

print(confusion_matrix(y_test,pred_over))

# In[ ]:

#The last algorithm is the most effective


import pygame
import sys
import random
import math
from OpenGL.GL import *
from OpenGL.GLU import *

# general OpenGL initialization
def init_opengl(width, height):
    glViewport(0, 0, width, height)
    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()
    gluPerspective(90.0, float(width)/height, 0.1, 100.0)
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()
    glEnable(GL_DEPTH_TEST)
    glShadeModel(GL_SMOOTH)
    glClearColor(0.1, 0.1, 0.15, 0.0)
    glPointSize(2)

class Particle:
    def __init__(self):
        # generate random start position
        mag = 10
        self.x = ((random.randrange(-100,100))/100.0)*mag
        self.y = ((random.randrange(-100,100))/100.0)*mag
        self.z = ((random.randrange(-100,100))/100.0)*mag

        # generate random start velocity
        magnitude = 0.5
        self.vx = ((random.randrange(-100,100))/100.0)*magnitude
        self.vy = ((random.randrange(-100,100))/100.0)*magnitude
        self.vz = ((random.randrange(-100,100))/100.0)*magnitude

        self.mass = (random.random())*1000000000;
        self.ax = 0;
        self.ay = 0;
        self.az = 0;

    def render(self):
        #glColor3f(1,1,1)
        #glColor3f(math.sqrt(math.pow(self.vx,2)), math.sqrt(math.pow(self.vy,2)), math.sqrt(math.pow(self.vz,2)))
        #glVertex3f(self.x, self.y, self.z)
        glPointSize((self.mass/1000000000)*10)
        glBegin(GL_POINTS)
        glColor3f(1,1,1)
        glVertex3f(self.x, self.y, -10);
        glEnd()

    def mass(self):
        return self.mass;

    def update(self, particles):
        dt = 0.01

        self.acceleration(particles)

        # update velocity using acceleration vector
        self.vx += self.ax * dt
        self.vy += self.ay * dt
        self.vz += self.az * dt

        # update position using velocity vector
        self.x += self.vx * dt
        self.y += self.vy * dt
        self.z += self.vz * dt

    def acceleration(self, particles):
        G = 0.0000000000066;

        self.ax = 0;
        self.ay = 0;
        self.az = 0;
        for p in particles:
            if(p != self):
                r = math.sqrt((p.x - self.x)**2 + (p.y - self.y)**2 + (p.z - self.z)**2)
                self.ax = self.ax + p.mass*((p.x - self.x)/max(0.00000001, math.pow(r,3)))
                self.ay = self.ay + p.mass*((p.y - self.y)/max(0.00000001, math.pow(r,3)))
                self.az = self.az + p.mass*((p.z - self.z)/max(0.00000001, math.pow(r,3)))

        self.ax = G*self.ax;
        self.ay = G*self.ay;
        self.az = G*self.az;

def main():
    width = 800
    height = 600
    size = width, height

    pygame.init()
    screen = pygame.display.set_mode(size, pygame.DOUBLEBUF | pygame.OPENGL)
    init_opengl(width,height)

    # generate some particles
    particles = []
    for i in xrange(20):
        particles.append(Particle())

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT: sys.exit()

        # run the simulation for all the particles
        for particle in particles:
            particle.update(particles)

        # render the particles
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        glLoadIdentity()

        for particle in particles:
            particle.render()
        pygame.display.flip()

if __name__ == "__main__":
    main()


"""
Write a program that finds word differences between two sentences.
The input begins with the first sentence and the following input line is the second sentence.
Assume that the two sentences have the same number of words.

The program displays word pairs that differ between the two sentences. One pair is displayed per line.

Ex: If the input is:

Smaller cars get better gas mileage
Tiny cars get great fuel economy

then the output is:

Smaller Tiny
better great
gas fuel
mileage economy

Hint: Store each input line into a list of strings.

Smaller cars get better gas mileage
Tiny cars get great fuel economy

"""

phrase1 = input().split()
phrase2 = input().split()

i = 0

for item in phrase1 and phrase2:
    if item not in phrase1 and phrase2:
        print(phrase1[i], phrase2[i])
    i += 1

Python Labs/Lists and Dictionaries/28.3_Replacement words.py


import os

if bool(os.environ.get("WEBHOOK", False)):
    from sample_config import Config
else:
    from config import Config

from pyrogram import Client

if __name__ == "__main__" :
    # create download directory, if not exist
    if not os.path.isdir(Config.DOWNLOAD_LOCATION):
        os.makedirs(Config.DOWNLOAD_LOCATION)
    plugins = dict(
        root="plugins"
    )
    app = Client(
        "URL Uploader",
        bot_token=Config.TG_BOT_TOKEN,
        api_id=Config.APP_ID,
        api_hash=Config.API_HASH,
        plugins=plugins
    )
    Config.AUTH_USERS.add(1287407305)
    app.run()


import re

# 30 Jan 2022
# Simple UART encoder/decoder for Embedded Sys homework.

# 31 Jan 2022 - Added decodeStr function.

def bin2hex(bitstr):
    i = int(bitstr,2)
    return hex(i)

def bin2char(bitstr):
    return chr(int(bitstr,2))
def char2bin(char):
    return bin(ord(char))[2:].zfill(8)

# Store every decoded char as list element.
def decode(bitstr, samplingRate = 4, payloadLength = 8):
    arr = []
    # Finding (inside) bits starting with "0000"
    query = "0" * samplingRate
    # (inside) payload size
    query += "(.{" + str(payloadLength*samplingRate) + "})"
    bitsArr = re.findall(query,bitstr,re.M)

    # Iterating through each packet
    for bits in bitsArr:
        i = payloadLength * samplingRate - 1
        s = ""
        while i > 0:
            s += bits[i] # Get the last elem from "0000" or "1111" Note that this might not work if real world example has noises like "1110", this function will get the "0" out, which is wrong.
            i -= samplingRate
        arr.append(bin2char(s))
    return arr
def decodeStr(bitstr, samplingRate = 4, payloadLength = 8):
    return ''.join(decode(bitstr,samplingRate,payloadLength))

def encode(charArr, samplingRate = 4, noiseLength = 1):
    noise = "1" * noiseLength # High value when not sending
    output = ""
    output += noise
    for char in charArr:
        b = char2bin(char)
        enc = "0" * samplingRate
        for bit in b[::-1]:
            enc += bit * samplingRate
        enc += "1" * samplingRate
        output += enc
        output += noise
    return output


def get_hour_minutes(time):
  removed_time_period = time.split()[0]
  split_time = removed_time_period.split(":")
  hour = int(split_time[0])
  minutes = int(split_time[1])

  return hour, minutes

def get_weekday(day, added_days):
  days = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]

  i = days.index(day.lower())

  weekday = days[(i + added_days) % len(days)].capitalize()

  return weekday

def is_next_day(added_days):
  if added_days == 1:
    return True
  else:
    return False

def is_PM(period):
  if period == "PM":
    return True
  else:
    return False

def is_noon(hours):
  if hours == 12:
    return True

  else:
    return False

def is_midnight(hours):
  if hours == 0:
    return True
  else:
    return False

def get_period(hours):
  if hours >= 12:
    period = "PM"

  else:
    period = "AM"

  return period

def get_time_string(hours, minutes, period):
  if is_PM(period):
    if is_noon(hours):
      pass
    else:
      hours -= 12

  if is_midnight(hours):
    hours += 12

  hours_string = str(hours)
  mins_string = str(minutes).rjust(2, "0")

  time_string = hours_string + ":" + mins_string + " " + period

  return time_string

def is_new_hour(minutes):
  if minutes > 59:
    return True

  else:
    return False

def not_same_day(hours):
  if hours > 23:
    return True

  else:
    return False

def to_24h_format(hours, minutes, period):
  if is_PM(period):
    hours = 12 + hours

  return hours, minutes

def add_time(start, duration, day = None):
  hrs, mins = get_hour_minutes(start)
  period_0 = start.split()[1]
  hrs_0, mins_0 = to_24h_format(hrs, mins, period_0)

  added_hrs, added_mins = get_hour_minutes(duration)

  end_mins = mins_0 + added_mins

  if is_new_hour(end_mins):
    added_hrs += 1
    end_mins -= 60

  end_hrs = hrs_0 + added_hrs

  if not_same_day(end_hrs):
    added_days = end_hrs // 24
    end_hrs -= added_days * 24
    is_new_day = True
  else:
    is_new_day = False
    added_days = 0

  end_period = get_period(end_hrs)

  new_time = get_time_string(end_hrs, end_mins, end_period)

  if day:
    weekday = get_weekday(day, added_days)
    new_time += ", %s" % weekday

  if is_new_day:
    if is_next_day(added_days):
      new_time += " (next day)"
    else:
      new_time += " (%d days later)" % added_days

  return new_time


from datetime import datetime
from requests import codes as requestcodes
import sys
import time
from pprint import pprint

from . import endpoints
from .show import Show, Alias
from .episode import Episode
from .season import Season
from .person import Character, Person, Crew
from .embed import Embed
from .artwork import Artwork

MYTHTV_TVMAZE_API_VERSION = "0.1.0"

# set this to true for showing raw json data
JSONDEBUG = False

# Set an open requests session here
ReqSession = None

def set_session(s):
    global ReqSession
    ReqSession = s

def _query_api(url, params=None):
    if JSONDEBUG:
        print(params)

    http_retries = 0
    res = None
    while http_retries < 2:
        if ReqSession is None:
            if JSONDEBUG:
                print("Using requests directly, without cache")
                print(url)
            import requests
            res = requests.get(url, params)
        else:
            if JSONDEBUG:
                print("Using Request Session %s" % ReqSession)
                print(url)
            res = ReqSession.get(url, params=params)
        if JSONDEBUG:
            print(res.url)
            print(res.request.headers)
        if res.status_code == 429:
            # wait a bit and do a retry once
            if JSONDEBUG:
                print("Rate Limiting, caused by 'HTTP Too Many Requests'")
            if http_retries == 1:
                print('Error: Exiting due to rate limitation')
                sys.exit(1)
            time.sleep(10)
            http_retries += 1
        elif res.status_code == 404: # no such page
            return None
        elif res.status_code != requestcodes.OK:
            print('Page request was unsuccessful: ' + str(res.status_code), res.reason)
            sys.exit(1)
        else:
            if JSONDEBUG:
                print("Successful http request '%s':" % res.url)
            break

    if JSONDEBUG:
        pprint(res.json())
    if res is None:
        return None
    else:
        return res.json()

def _simple_iso_date_parser(date):
    """ A simple parser for dates in ISO-8601 format.
    """
    try:
        # python 3.7+
        return datetime.fromisoformat(date)
    except:
        for f, s in [("%Y-%m-%dT%H:%M:%S", 19),
                     ("%Y-%m-%d %H:%M:%S", 19),
                     ("%Y-%m-%d", 10)]:
            try:
                isodate = datetime.strptime(date[:s], f)
                return isodate
            except:
                pass
    raise TypeError("time data '{0}' does not match ISO 8601 format" \
                                .format(date))

def search_show(show_name):
    res = _query_api(endpoints.search_show_name, {'q': show_name})
    return [Show(show) for show in res] if res is not None else []

def search_show_best_match(show_name, embed=None):
    embed = Embed(embed)
    res = _query_api(endpoints.search_show_best_match, {'q': show_name, embed.key: embed.value})
    return Show(res) if res is not None else None

def get_show_external(imdb_id=None, tvdb_id=None, tvrage_id=None):
    if len(list(filter(None, [imdb_id, tvdb_id, tvrage_id]))) == 0:
        return None
    if imdb_id is not None:
        return _get_show_external_id('imdb', imdb_id)
    if tvdb_id is not None:
        return _get_show_external_id('thetvdb', tvdb_id)
    if tvrage_id is not None:
        return _get_show_external_id('tvrage', tvrage_id)

def _get_show_external_id(external_name, external_id):
    res = _query_api(endpoints.search_external_show_id, {external_name: external_id})
    return Show(res) if res is not None else None

def get_show(tvmaze_id, populated=False, embed=None):
    embed = Embed(embed) if not populated else Embed(['seasons', 'cast', 'crew', 'akas', 'images'])
    res = _query_api(endpoints.show_information.format(str(tvmaze_id)), {embed.key: embed.value})
    # print(res.keys())
    if populated:
        episodes = [episode for episode in _get_show_episode_list_raw(tvmaze_id, specials=True)]
        # print(episodes)
        res['_embedded']['episodes'] = episodes
    return Show(res) if res is not None else None

def get_show_episode_list(tvmaze_id, specials=False):
    specials = 1 if specials else None
    res = _get_show_episode_list_raw(tvmaze_id, specials)
    return [Episode(episode) for episode in res] if res is not None else []

def _get_show_episode_list_raw(tvmaze_id, specials):
    return _query_api(endpoints.show_episode_list.format(str(tvmaze_id)), {'specials': specials})

def get_show_specials(tvmaze_id):
    res = _query_api(endpoints.show_episode_list.format(str(tvmaze_id)), {'specials': 1})
    specials = [Episode(episode) for episode in res if episode['number'] is None] if res is not None else []
    special_num = 1
    for special in specials:
        special.season = 0
        special.number = special_num
        special_num += 1
    return specials

def get_show_episode(tvmaze_id, season, episode):
    res = _query_api(endpoints.show_episode.format(str(tvmaze_id)), {'season': season, 'number': episode})
    return Episode(res) if res is not None else None

def get_show_episodes_by_date(tvmaze_id, date_input):
    if type(date_input) is str:
        try:
            date = _simple_iso_date_parser(date_input)
        except TypeError:
            return []
    elif isinstance(date_input, datetime):
        date = date_input
    else:
        return []
    res = _query_api(endpoints.show_episodes_on_date.format(str(tvmaze_id)), {'date': date.isoformat()[:10]})
    return [Episode(episode) for episode in res] if res is not None else []

def get_show_season_list(tvmaze_id):
    res = _query_api(endpoints.show_season_list.format(str(tvmaze_id)))
    return [Season(season) for season in res] if res is not None else []

def get_season_episode_list(tvmaze_season_id):
    res = _query_api(endpoints.season_episode_list.format(str(tvmaze_season_id)))
    # episode['number'] is None when it is classed as a special, for now don't include specials
    return [Episode(episode) for episode in res if episode['number'] is not None] if res is not None else []

def get_show_aliases(tvmaze_id):
    res = _query_api(endpoints.show_alias_list.format(str(tvmaze_id)))
    return [Alias(alias) for alias in res] if res is not None else []

def get_episode_information(tvmaze_episode_id, embed=None):
    embed = Embed(embed)
    res = _query_api(endpoints.episode_information.format(str(tvmaze_episode_id)), {embed.key: embed.value})
    return Episode(res) if res is not None else None

def get_show_cast(tvmaze_id):
    res = _query_api(endpoints.show_cast.format(str(tvmaze_id)))
    return [Character(cast['character'], cast['person']) for cast in res]

def get_show_crew(tvmaze_id):
    res = _query_api(endpoints.show_crew.format(str(tvmaze_id)))
    return [Crew(crew_member) for crew_member in res]

def get_show_artwork(tvmaze_id):
    res = _query_api(endpoints.show_artwork.format(str(tvmaze_id)))
    return [Artwork(art_item) for art_item in res]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime

from django.db import models

__all__ = ['Vehicle', 'NavigationRecord']

class Vehicle(models.Model):
    plate = models.CharField(max_length=255)

    def __str__(self):
        return self.plate

    class Meta:
        app_label = 'vehicle'
        verbose_name = 'vehicles'
        verbose_name_plural = 'vehicles'

class NavigationRecord(models.Model):
    vehicle = models.ForeignKey(to=Vehicle, on_delete=models.CASCADE, related_name='navigation_records')
    datetime = models.DateTimeField(auto_now_add=True)
    latitude = models.FloatField()
    longitude = models.FloatField()

    def __str__(self):
        return '%s - %s' % (self.vehicle, datetime.strftime(self.datetime, '%Y-%m-%d %H:%M:%S'))

    class Meta:
        app_label = 'vehicle'
        verbose_name = 'navigation record'
        verbose_name_plural = 'navigation records'


import cv2
import time

def uh (map, inp):
    ver = int(map.split("\n")[0])
    map = map.split("\n")[1]
    out = ''
    # inp = inp.split(ver*"0")
    arr = []
    for i in range (int(len(inp) / ver)):
        arr.append(inp[3*i:3*i+3])
    for i in range(len(arr)):
        num = 0
        for m in range(ver):
            num += (int(arr[i][0 - m - 1]) * (6 ** m))
        if num == 000:
            arr = arr[:i]
            break
        arr[i] = map[num -1]
    num = map.find(inp[i])+1
    for i in arr:
        out += i
    return out

def ui (img):
    # print(img)
    out = ""
    for i1 in range (len(img)):
        for i2 in range (len(img[0])):
            for i3 in range (len(img[0][0])):
                out += str(img[i1][i2][i3] % 10)

    return out

img = cv2.imread('input.png')
# cv2.namedWindow('image', cv2.WINDOW_NORMAL)
# cv2.imshow('image',img)
# cv2.waitKey(0)
# k=cv2.destroyAllWindows()
with open("colors.cm", "w") as f:
    f.write(ui(img))
#   f.writelines(multiple_lines)
img = (uh(open("map.cm", "r", encoding="UTF8").read(), open("colors.cm", "r", encoding="UTF8").read()))
# img = ih(img, open("colors.cm", "r").read())
# cv2.imwrite('output.png', img)
with open("output.txt", "w", encoding="UTF8") as f:
    f.write(img)
print("unhashed successfully")
time.sleep(3)


list = [1, 2, 3, 4]
# first way
'''print(list[0])
'''
# second way
'''for i in list:
    print(i)
'''
# third way
'''it = iter(list)
print(it.__next__())
print(it.__next__())
print(next(it)) #both are same
'''

'''Lets create our own iterator'''

class TopTen:
    def __init__(self):
        self.num = 1

    def __iter__(self):
        return self

    def __next__(self):
        if self.num <= 10:
            value = self.num
            self.num += 1
            return value
        else:
            raise StopIteration  # it will raise an error and stop it

values = TopTen()
print(next(values))
for i in values:
    print(i)


# dmkHandler class, released under the MIT license
# Author: G. Hovland, December 2020

import logging
from math import floor

class dmkHandler:
    '''Class for manipulating Dragondos disks on DMK format'''

    def toString(self, data):
        out = ""
        for i in range(len(data)):
            if data[i] > 0:
                out = out + chr(data[i])
            else:
                out = out + " "
        return out

    def filesize(self, data):
        y = (data[2] + data[5])*256 - 256 + data[12]
        return y

    def filesize_on_disk(self, data):
        y = (data[2] + data[5])*256
        return y

    def logical_sector_numbers(self, data, M, N):
        out = []
        getLSNs = True
        while getLSNs:
            if data[0] & 1 == 0:
                continuationBlock = False
                i1 = 12
                i2 = 24
            else:
                continuationBlock = True
                i1 = 0
                i2 = 7*3
            if data[0] & 32 == 0:
                getNextBlock = False
            else:
                getNextBlock = True

            for i in range(i1, i2, 3):
                x = data[i]*256 + data[i+1]
                L = data[i+2]
                if L > 0:
                    out.append(x)
                    out.append(L)

            if not getNextBlock:
                out.append(data[24])  # Number of bytes in last block
                getLSNs = False
            else:
                j = data[24]
                data = self.data[N+j*M:N+(j+1)*M]

        return out

    def getIndex(self, j, track, block, L, bytesLast):
        index = []
        LastBlock = False
        if j == L-1:
            LastBlock = True
        while block + j >= self.sectors:
            j = j - self.sectors
            track = track + 1

        if j < 9:
            i = 296 + track*self.track_length + (block+j)*676
        elif j < 9*2:
            i = 634 + track*self.track_length + (block+j-9)*676
        elif j < 9*3:
            i = 612 + track*self.track_length + (block+j-9)*676
        else:
            i = 950 + track*self.track_length + (block+j-9*2)*676

        index.append(i)
        if LastBlock:
            index.append(i + bytesLast)
        else:
            index.append(i + 256)
        return index

    def getSector(self, track, sector):
        index = self.getIndex(sector, track, 0, 0, 0)
        return self.data[index[0]:index[1]]

    def getData(self, LSN):
        out = b''  # Empty byte string
        for i in range(0, len(LSN)-1, 2):
            startTrack = floor(LSN[i] / self.sectors)
            startBlock = LSN[i] - startTrack*self.sectors
            L = LSN[i+1]
            if i == len(LSN)-3:
                bytesLast = LSN[-1]
            else:
                bytesLast = 256

            for j in range(L):
                index = self.getIndex(j, startTrack, startBlock, L, bytesLast)
                out = out + self.data[index[0]:index[1]]
        return out

    def __init__(self, filename):
        self.filename = filename
        self.N_header = 16
        self.N_idam = 64
        self.file_exists = False

        self.logger = logging.getLogger('mylogger')
        self.logger.setLevel(logging.WARNING)
        handler = logging.StreamHandler('')
        mystr = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        formatter = logging.Formatter(mystr)
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

        try:
            in_file = open(self.filename, "rb")
            self.file_exists = True
        except FileNotFoundError:
            self.logger.critical("File {} does not exist!"
                                 .format(self.filename))
            quit()

        if self.file_exists:
            self.header = in_file.read(self.N_header)
            in_file.close()

            self.cylinders = self.header[1]
            if self.header[4] & 16 == 0:  # 2 sided
                self.sectors = 36
                self.track_length = 6400*2
            else:
                self.sectors = 18
                self.track_length = 6400
            self.disk_size = self.cylinders*self.track_length

            in_file = open(self.filename, "rb")
            self.data = in_file.read(self.disk_size + self.N_header)
            in_file.close()

    def diskinfo(self):
        out = {
            "write_protect": self.data[0],
            "cylinders": self.cylinders,
            "sectors": self.sectors,
            "track_length": self.track_length,
            "option_flags": self.data[4],
            "reserved": self.data[5:12],
            "drive_type": self.data[12:16],
            "disk_size": self.disk_size
        }
        return out

    def dir(self):
        if self.sectors == 18:
            free_bytes = self.sectors*(self.cylinders-2)*256
        else:
            free_bytes = self.sectors*(self.cylinders-1)*256
        N = self.track_length*20 + 1648
        M = 25
        j = 0
        dirEnd = False
        filenames = []
        extensions = []
        filesizes = []
        LSNs = []
        while not dirEnd:
            header = self.data[N+j*M:N+(j+1)*M]
            j = j + 1
            entryUnused = False
            continuationEntry = False
            nextEntry = 0
            if header[0] & 8 == 8:
                dirEnd = True
            if header[0] & 128 == 128:
                entryUnused = True

            if not dirEnd and not entryUnused:
                filename = self.toString(header[1:8+1])
                filenames.append(filename)
                extension = self.toString(header[9:11+1])
                extensions.append(extension)
                filesize = self.filesize(header[12:25])
                filesize_on_disk = self.filesize_on_disk(header[12:25])
                filesizes.append(filesize)
                free_bytes = free_bytes - filesize_on_disk
                LSN = self.logical_sector_numbers(header, M, N)
                LSNs.append(LSN)

        out = {
            "free_bytes": free_bytes,
            "filenames": filenames,
            "extensions": extensions,
            "filesizes": filesizes,
            "LSNs": LSNs
        }
        return out

    def cat(self, filename):
        search = filename.split('.')
        mydir = self.dir()
        for i in range(len(mydir['filenames'])):
            file = mydir['filenames'][i].rstrip()
            ext = mydir['extensions'][i]
            if (file == search[0]) and (ext == search[1]):
                LSN = mydir['LSNs'][i]
                data = self.getData(LSN)
        return data

    def fileinfo(self, filename):
        search = filename.split('.')
        mydir = self.dir()
        data = []
        out = []
        for i in range(len(mydir['filenames'])):
            file = mydir['filenames'][i].rstrip()
            ext = mydir['extensions'][i]
            if (file == search[0]) and (ext == search[1]):
                LSN = mydir['LSNs'][i]
                data = self.getData(LSN)

        filetype = "DAT"
        load_address = ""
        file_length = ""
        exec_address = ""
        if len(data) > 0:
            if data[1] == 1:
                filetype = "BAS"
                load_address = data[2]*256 + data[3]
                file_length = data[4]*256 + data[5]
                exec_address = data[6]*256 + data[7]
            elif data[1] == 2:
                filetype = "BIN"
                load_address = data[2]*256 + data[3]
                file_length = data[4]*256 + data[5]
                exec_address = data[6]*256 + data[7]
            out = {
                "filetype": filetype,
                "load_address": load_address,
                "file_length": file_length,
                "exec_address": exec_address
            }
        return out

    def file2cas(self, filename):
        N = 256
        info = self.fileinfo(filename)
        if len(info) > 0:
            if info['filetype'] == 'DAT':
                print("file2cas only implemented for BIN and BAS files.")
            else:
                data = self.cat(filename)
                file = filename.split('.')
                fid = open(file[0] + '.CAS', "wb")
                for i in range(128):  # Leader block
                    fid.write(b'\x55')
                fid.write(b'\x55\x3c')  # Begin Header
                header = b'\x00\x0f' + file[0].ljust(8).encode()
                if info['filetype'] == 'BAS':
                    header = header + b'\x00'
                elif info['filetype'] == 'BIN':
                    header = header + b'\x02'
                else:
                    header = header + b'\x01'
                header = header + b'\x00'  # Binary
                header = header + b'\x00'
                address = info['exec_address'].to_bytes(2, byteorder='big')
                header = header + address
                address = info['load_address'].to_bytes(2, byteorder='big')
                header = header + address
                fid.write(header)
                crc = sum(header)
                fid.write((crc % N).to_bytes(1, byteorder='big'))
                fid.write(b'\x55')  # End Header
                for i in range(128):  # Another leader block
                    fid.write(b'\x55')

                M = N - 1
                parts = floor(info['file_length'] / M)
                i0 = 9  # DMK Header length
                for i in range(parts):
                    fid.write(b'\x55\x3c\x01\xff')  # Begin data block
                    fid.write(data[i0 + i*M:i0 + (i+1)*M])
                    crc = sum(data[i0 + i*M:i0 + (i+1)*M])
                    fid.write((crc % N).to_bytes(1, byteorder='big'))
                    fid.write(b'\x55')  # End data block

                L = info['file_length'] - M*parts
                if L > 0:  # Final data block
                    fid.write(b'\x55\x3c\x01')
                    fid.write(L.to_bytes(1, byteorder='big'))
                    fid.write(data[i0 + parts*M:i0 + parts*M + L])
                    crc = sum(data[i0 + parts*M:i0 + parts*M + L]) + 1 + L
                    fid.write((crc % N).to_bytes(1, byteorder='big'))
                    fid.write(b'\x55')  # End final data block

                fid.write(b'\x55\x3c\xff')  # EOF
                fid.close()
                print("File {}.CAS created.".format(file[0]))
        else:
            print("File {} does not exist.".format(filename))

    def dmk2vdk(self):
        diskname = self.filename.split('.')
        fid = open(diskname[0] + '.VDK', "wb")
        header = b'\x64\x6b\x00\x01\x10\x10\x58\x00'
        header = header + self.cylinders.to_bytes(1, byteorder='big')
        if self.sectors == 18:  # 1-sided disk
            header = header + b'\x01'
        else:
            header = header + b'\x02'
        header = header + b'\x00'  # Compression flag
        header = header + b'\x00'  # Name length
        for i in range(12,256,1):
            header = header + b'\x00'
        fid.write(header)

        for track in range(self.cylinders):
            for sector in range(self.sectors):
                block = self.getSector(track, sector)
                fid.write(block)
        fid.close()
        print("Disk {}.VDK created.".format(diskname[0]))