import unittest
import tkinter
from test import support
from tkinter.test.support import AbstractTkTest

support.requires('gui')

class MiscTest(AbstractTkTest, unittest.TestCase):

    def test_repr(self):
        t = tkinter.Toplevel(self.root, name='top')
        f = tkinter.Frame(t, name='child')
        self.assertEqual(repr(f), '<tkinter.Frame object .top.child>')

    def test_generated_names(self):
        t = tkinter.Toplevel(self.root)
        f = tkinter.Frame(t)
        f2 = tkinter.Frame(t)
        b = tkinter.Button(f2)
        for name in str(b).split('.'):
            self.assertFalse(name.isidentifier(), msg=repr(name))

    def test_tk_setPalette(self):
        root = self.root
        root.tk_setPalette('black')
        self.assertEqual(root['background'], 'black')
        root.tk_setPalette('white')
        self.assertEqual(root['background'], 'white')
        self.assertRaisesRegex(tkinter.TclError,
                '^unknown color name "spam"$',
                root.tk_setPalette, 'spam')

        root.tk_setPalette(background='black')
        self.assertEqual(root['background'], 'black')
        root.tk_setPalette(background='blue', highlightColor='yellow')
        self.assertEqual(root['background'], 'blue')
        self.assertEqual(root['highlightcolor'], 'yellow')
        root.tk_setPalette(background='yellow', highlightColor='blue')
        self.assertEqual(root['background'], 'yellow')
        self.assertEqual(root['highlightcolor'], 'blue')
        self.assertRaisesRegex(tkinter.TclError,
                '^unknown color name "spam"$',
                root.tk_setPalette, background='spam')
        self.assertRaisesRegex(tkinter.TclError,
                '^must specify a background color$',
                root.tk_setPalette, spam='white')
        self.assertRaisesRegex(tkinter.TclError,
                '^must specify a background color$',
                root.tk_setPalette, highlightColor='blue')

    def test_after(self):
        root = self.root

        def callback(start=0, step=1):
            nonlocal count
            count = start + step

        # Without function, sleeps for ms.
        self.assertIsNone(root.after(1))

        # Set up with callback with no args.
        count = 0
        timer1 = root.after(0, callback)
        self.assertIn(timer1, root.tk.call('after', 'info'))
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1))
        root.update()  # Process all pending events.
        self.assertEqual(count, 1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)

        # Set up with callback with args.
        count = 0
        timer1 = root.after(0, callback, 42, 11)
        root.update()  # Process all pending events.
        self.assertEqual(count, 53)

        # Cancel before called.
        timer1 = root.after(1000, callback)
        self.assertIn(timer1, root.tk.call('after', 'info'))
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1))
        root.after_cancel(timer1)  # Cancel this event.
        self.assertEqual(count, 53)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)

    def test_after_idle(self):
        root = self.root

        def callback(start=0, step=1):
            nonlocal count
            count = start + step

        # Set up with callback with no args.
        count = 0
        idle1 = root.after_idle(callback)
        self.assertIn(idle1, root.tk.call('after', 'info'))
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1))
        root.update_idletasks()  # Process all pending events.
        self.assertEqual(count, 1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)

        # Set up with callback with args.
        count = 0
        idle1 = root.after_idle(callback, 42, 11)
        root.update_idletasks()  # Process all pending events.
        self.assertEqual(count, 53)

        # Cancel before called.
        idle1 = root.after_idle(callback)
        self.assertIn(idle1, root.tk.call('after', 'info'))
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1))
        root.after_cancel(idle1)  # Cancel this event.
        self.assertEqual(count, 53)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)

    def test_after_cancel(self):
        root = self.root

        def callback():
            nonlocal count
            count += 1

        timer1 = root.after(5000, callback)
        idle1 = root.after_idle(callback)

        # No value for id raises a ValueError.
        with self.assertRaises(ValueError):
            root.after_cancel(None)

        # Cancel timer event.
        count = 0
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', timer1))
        root.tk.call(script)
        self.assertEqual(count, 1)
        root.after_cancel(timer1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)
        self.assertEqual(count, 1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call('after', 'info', timer1)

        # Cancel same event - nothing happens.
        root.after_cancel(timer1)

        # Cancel idle event.
        count = 0
        (script, _) = root.tk.splitlist(root.tk.call('after', 'info', idle1))
        root.tk.call(script)
        self.assertEqual(count, 1)
        root.after_cancel(idle1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call(script)
        self.assertEqual(count, 1)
        with self.assertRaises(tkinter.TclError):
            root.tk.call('after', 'info', idle1)

tests_gui = (MiscTest, )

if __name__ == "__main__":
    support.run_unittest(*tests_gui)


import uproot
import awkward #need for flatten or other operations to nested arrays
import numpy as np

def access_track_branches(simtree, prefix="MCTrack.f", entry_start=0, entry_stop = -1):
 '''reading MCTracks from entry_start to lastentry'''
 if (entry_stop < 0):
        lastentry = simtree.num_entries
 else:
        lastentry = entry_stop
 #what branches do we want to read
 suffixes = ["MotherId","PdgCode","StartX","StartY","StartZ","Px","Py","Pz"]
 branchnames = [prefix + suff for suff in suffixes]
 print(branchnames)
 #reading the branches
 tracks = simtree.arrays(branchnames,entry_start,lastentry)
 #returning the produced array
 return tracks

def access_mcpoint_branches(simtree, prefix = "ScifiPoint.f", entry_start=0, entry_stop = -1 ):
 '''reading MCTracks from entry_start to lastentry, to be launched after access_track_branches'''
 if (entry_stop < 0):
        lastentry = simtree.num_entries
 else:
        lastentry = entry_stop

 suffixes = ["PdgCode","TrackID","X","Y","Z","Px","Py","Pz"]
 branchnames = [prefix + suff for suff in suffixes]
 print(branchnames)
 #reading the branches
 mcpoints = simtree.arrays(branchnames[0:2],entry_start = entry_start, entry_stop = lastentry)
 #adding float interpretation manually
 floatinterpretation = simtree["MCTrack.fStartX"].interpretation
 for name in branchnames[2:]:
  mcpoints[name] = simtree[name].array(floatinterpretation)
 #returning the produced array
 return mcpoints

def eventIDs(simtree):
 '''return list of entries'''
 nevents = simtree.num_entries
 events = np.arange(0,nevents)
 return events


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('localore_admin', '0005_auto_20160331_1015'),
        ('blog', '0030_auto_20160404_1610'),
    ]

    operations = [
        migrations.AddField(
            model_name='blogpage',
            name='tile_image',
            field=models.ForeignKey(blank=True, related_name='+', help_text='Optional: The image to use on the connections index page. Will use poster image if not set.', on_delete=django.db.models.deletion.SET_NULL, null=True, to='localore_admin.LocaloreImage'),
        ),
    ]


from __future__ import print_function
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.optimizers import SGD
from keras.utils import np_utils
import numpy as np
import matplotlib.pyplot as plt
import random

Epoch = 30
Number_Hidden = 128
Speed_Learning = 0.01
Opt = SGD(lr=Speed_Learning)
Proportion_Validation = 0.2
Size_Batch = 1024
Verbose = 1
Number_Classes = 10
Length_Unit = 784
np.random.seed(461)

(input_X_train, output_Y_train), (input_X_test, output_Y_test) = mnist.load_data()
print("Training data input shape: ", input_X_train.shape)
print("Training data output shape: ", output_Y_train.shape)
print("Test data input shape: ", input_X_test.shape)
print("Test data output shape: ", output_Y_test.shape)

image = input_X_train[10]
print("A sample image is as an array: ")
print(str(image))
# plt.imshow(image, cmap='gray')
# plt.show()

input_X_train = input_X_train.reshape(60000, Length_Unit)
input_X_test = input_X_test.reshape(10000, Length_Unit)
input_X_train = input_X_train.astype('float32')
input_X_test = input_X_test.astype('float32')

input_X_train /= 255
input_X_test /= 255
output_Y_train = np_utils.to_categorical(output_Y_train, Number_Classes)
output_Y_test = np_utils.to_categorical(output_Y_test, Number_Classes)

model = Sequential()

model.add(Dense(Number_Hidden, input_shape=(Length_Unit,)))
model.add(Activation('relu'))

model.add(Dense(Number_Hidden))
model.add(Activation('relu'))

model.add(Dense(Number_Hidden))
model.add(Activation('relu'))

model.add(Dense(Number_Hidden))
model.add(Activation('relu'))

model.add(Dense(Number_Hidden))
model.add(Activation('relu'))

# model.add(Dense(Number_Hidden))
# model.add(Activation('relu'))

# model.add(Dense(Number_Classes))
# model.add(Activation('relu'))

# model.add(Dense(Number_Classes))
# model.add(Activation('relu'))

# model.add(Dense(Number_Classes))
# model.add(Activation('relu'))

# model.add(Dense(Number_Classes))
# model.add(Activation('relu'))

model.add(Dense(Number_Classes))
model.add(Activation('softmax'))

model.summary()
model.compile(loss='categorical_crossentropy', optimizer=Opt, metrics=['accuracy'])

fitted_model = model.fit(input_X_train, output_Y_train, batch_size=Size_Batch, epochs=Epoch, verbose=Verbose, validation_split=Proportion_Validation)

score = model.evaluate(input_X_test, output_Y_test, verbose=Verbose)
print("Test score/loss:", score[0])
print("Test accuracy:", score[1])

plt.plot(fitted_model.history['acc'])
plt.plot(fitted_model.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()

plt.plot(fitted_model.history['loss'])
plt.plot(fitted_model.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

result = model.predict(input_X_test, batch_size=Size_Batch, verbose=Verbose)
new = []
for i in range(0, len(result)):
    if output_Y_test[i][0] == 1:
        new.append(result[i][0])
plt.boxplot(new)
plt.show()


def saddle_point(matrix):
    for lt in matrix:
        FlagMin = True
        j = 0
        mn = lt.index(min(lt))
        for i in lt:
            if lt[mn] == i:
                j = j + 1
            if j > 1:
                FlagMin = False
        for i in matrix:
            if lt[mn] < i[mn]:
                FlagMin = False
            if matrix.index(lt) != matrix.index(i) and lt[mn] == i[mn]:
                FlagMin = False
        if FlagMin:
            tr = (matrix.index(lt), mn)
            return tr
    return False

print(saddle_point([[8,3,0,1,2,3,4,8,1,2,3],[3,2,1,2,3,9,4,7,9,2,3],[7,6,0,1,3,5,2,3,4,1,1]]));


import pickle
d = dict(name='William', age=24, score=88)

# write_1
f = open('pickle.txt', 'wb')
pickle.dump(d, f)
f.close()

# write_2
print(pickle.dumps(d))
print()

# read_1
f = open('pickle.txt', 'rb')
r = pickle.load(f)
f.close()
print(r)
print()

# read_2
pickle_str = b'\x80\x03}q\x00(X\x03\x00\x00\x00ageq\x01K\x14X\x05\x00\x00\x00scoreq' \
             b'\x02KXX\x04\x00\x00\x00nameq\x03X\x03\x00\x00\x00Bobq\x04u.'
print(pickle.loads(pickle_str))


import json
import random
from ..exceptions import DuplicatedParameterException, UnknownDataLabelException, UnknownDataFormatException

class Dataset(object):
    """

    ``Dataset`` is an useful module that helps you to manage your data. It has a similar
    interface to list, supports "append", "extend", "slice" and "merge" methods.

    """
    def __init__(self, data_list=None, copy=True):
        """
        :param list data_list: This can be a list of :py:class:`.DataInstance`, a list of dict, a list of sentences or a list of tuples of type `(sentence, target)`. **Default:** None
        :param bool copy: Create a copy of :py:class:`.DataInstance` s in data_list. **Default:** True
        """
        self.__data = {}
        self.__next_idx = 0
        if data_list is not None:
            unsolved = []
            for data in data_list:
                if isinstance(data, dict) and ("idx" in data):
                    self.__data[ data["idx"] ] = DataInstance(**data)
                    self.__next_idx = max(self.__next_idx, data["idx"] + 1)
                elif isinstance(data, DataInstance) and data.id is not None:
                    self.__data[ data.id ] = data.copy() if copy else data
                    self.__next_idx = max(self.__next_idx, data.id + 1)
                else:
                    unsolved.append(data)

            for data in unsolved:
                if isinstance(data, tuple):
                    sent, target = data
                    self.__data[self.__next_idx] = DataInstance(
                        sent, target=target, idx=self.__next_idx
                    )
                elif type(data).__name__ == "str":
                    self.__data[self.__next_idx] = DataInstance(
                        data, idx=self.__next_idx
                    )
                elif isinstance(data, dict):
                    data["idx"] = self.__next_idx
                    self.__data[self.__next_idx] = DataInstance(
                        ** data
                    )
                elif isinstance(data, DataInstance):
                    data.id = self.__next_idx
                    self.__data[self.__next_idx] = data.copy() if copy else data
                else:
                    raise UnknownDataFormatException(data)
                self.__next_idx += 1
        else:
            pass

    def append(self, data_instance):
        """
        :param DataInstance data_instance: The :py:class:`.DataInstance` which you want to append.
        :return: Index of this new data instance.
        :rtype: int
        """
        ret_id = self.__next_idx
        data_instance = data_instance.copy()

        data_instance.index = ret_id
        self.__data[ret_id] = data_instance
        self.__next_idx += 1
        return ret_id

    def remove(self, index):
        """
        :param int index: The index of data instance that you want to remove.
        :return: Returns False if index is not found, otherwise True.
        :rtype: bool
        """
        if isinstance(index, DataInstance):
            data = index
            index, data.index = data.index, None
        if index in self.__data:
            del self.__data[index]
            return True
        else:
            return False

    def shuffle(self, inplace=False, copy=True):
        """
        :param bool inplace: If true, randomly shuffle dataset and returns itself.
        :param bool copy: If true, works on the copy of this dataset. This option would be ignored if ``inplace=True``.
        :rtype: Dataset

        This method would randomly shuffle dataset and reset indexes.
        """
        keys = list(self.__data.keys())
        random.shuffle(keys)
        if inplace:
            nw_dict = {}
            for idx, old_id in enumerate(keys):
                nw_dict[idx] = self.__data[old_id]
                nw_dict[idx].id = idx
            self.__data = nw_dict
            return self
        else:
            ret = [ self.__data[idx].copy() if copy else self.__data[idx] for idx in keys]
            return Dataset(ret, copy=False)

    def iter(self, shuffle=False, copy=True):
        """
        :param bool shuffle: Shuffle dataset. **Default:** False
        :param bool copy: Generates copy of data instances. **Default:** True
        :return: Returns iterator of all data instances.
        :rtype: generator
        """
        keys = list(self.__data.keys())
        if shuffle:
            random.shuffle(keys)
        else:
            keys = sorted(keys)
        def generator():
            for kw in keys:
                inst = self.__data[kw]
                yield inst.copy() if copy else inst
        return generator()

    def __iter__(self):
        return self.iter(shuffle=False, copy=False)

    def eval(self, clsf, batch_size=1, copy=True, ignore_known=True, ignore_meta=False):
        """
        :param Classifier clsf: classifier.
        :param int batch_size: Number of instances in a batch. **Default:** 1
        :param bool copy: Generate a copy for result. **Default:** True
        :param bool ignore_known: Ignore the data instances which already has a predicted label. **Default:** True
        :param bool ignore_meta: Ignore meta of data instances when evaluating. **Default:** False
        :return: Results with predictions.
        :rtype: Dataset

        This method evaluates your classifier on the dataset and generates a result.
        """
        ret = []
        def update(batch):
            batch_x = [ inst.x for inst in batch ]
            res = clsf.get_pred(batch_x)
            for i, inst in enumerate(batch):
                if copy:
                    inst = inst.copy()
                    ret.append(inst)
                inst.pred = res[i]
            return

        batch = []
        for kw, val in self.__data.items():
            if ignore_known and val.pred is not None:
                if copy:
                    ret.append(val.copy())
                continue

            if (not ignore_meta) and val.meta != {}:  # for instance with meta
                inst = val.copy() if copy else val
                inst.pred = clsf.get_pred([inst.x], inst.meta)[0]
                if copy:
                    ret.append(inst)
                continue

            batch.append(val)
            if len(batch) < batch_size:
                continue
            update(batch)
            batch = []
        if len(batch) > 0:
            update(batch)
            batch = []
        if copy:
            return Dataset(ret, copy=False)
        else:
            return self

    def __check(self, equal, ignore_unknown, keep_ids, copy):
        ret = []
        for kw, val in self.__data.items():
            if val.pred is None or val.y is None:
                if ignore_unknown:
                    continue
                raise UnknownDataLabelException(val)
            if (val.pred != val.y) == equal:
                continue
            inst = val.copy() if copy else val
            if copy and not keep_ids:
                inst.id = None
            ret.append(inst)
        return Dataset(ret, copy=False)

    def correct(self, ignore_unknown=True, keep_ids=True, copy=True):
        """
        :param bool ignore_unknown: Ignore the data instances which has no predicted label.
        :param bool keep_ids: Keep index of instances in the result. **Default:** True
        :param bool copy: If true, returns a copy of dataset. **Default:** True
        :rtype: Dataset

        This method returns a subset of the dataset which predicted label is equal to true label.
        """
        return self.__check(True, ignore_unknown, keep_ids, copy)

    def wrong(self, ignore_unknown=True, keep_ids=True, copy=True):
        """
        :param bool ignore_unknown: Ignore the data instances which has no predicted label.
        :param bool keep_ids: Keep index of instances in the result. **Default:** True
        :param bool copy: If true, returns a copy of dataset. **Default:** True
        :rtype: Dataset

        This method returns a subset of the dataset which predicted label is not equal to true label.
        """
        return self.__check(False, ignore_unknown, keep_ids, copy)

    def sample(self, num, keep_ids=True, copy=True):
        """
        :param int num: Number of sampling results.
        :param bool keep_ids: Keep index of instances in the result. **Default:** True
        :param bool copy: If true, returns a copy of dataset. **Default:** True
        :rtype: Dataset

        This method randomly samples ``num`` instances from the dataset.
        """
        keys = list(self.__data.keys())
        random.shuffle(keys)
        keys = keys[:num]
        keys = sorted(keys)

        ret = []
        for kw in keys:
            inst = self.__data[kw]
            if copy:
                inst = inst.copy()
                if not keep_ids:
                    inst.id = None
            ret.append(inst)
        return Dataset(ret, copy=False)

    def filter_label(self, label, keep_ids=True, copy=True):
        """
        :param int label: The ground truth label which you want to keep.
        :param bool keep_ids: Keep index of instances in the result. **Default:** True
        :param bool copy: If true, returns a copy of dataset. **Default:** True
        :return: Returns a subset of the dataset, consists of instances that ground truth label is equal to ``label``.
        :rtype: Dataset
        """
        ret = []
        for kw, val in self.__data.items():
            if val.y == label:
                inst = val.copy() if copy else val
                if copy and not keep_ids:
                    inst.id = None
                ret.append(inst)
        return Dataset(ret, copy=False)

    def filter_pred(self, label, keep_ids=True, copy=True):
        """
        :param int label: The predicted label which you want to keep.
        :param bool keep_ids: Keep index of instances in the result. **Default:** True
        :param bool copy: If true, returns a copy of dataset. **Default:** True
        :return: Returns a subset of the dataset, consists of instances that predicted label is equal to ``label``.
        :rtype: Dataset
        """
        ret = []
        for kw, val in self.__data.items():
            if val.pred == label:
                inst = val.copy() if copy else val
                if copy and not keep_ids:
                    inst.id = None
                ret.append(inst)
        return Dataset(ret, copy=False)

    def __setitem__(self, index, val):
        if not isinstance(index, int):
            raise TypeError("Key '%s' is not supported." % repr(index))
        if isinstance(val, DataInstance):
            val = val.copy()
            val.id = index
            self.__data[index] = val
        else:
            raise TypeError("Object '%s' is not allowd." % repr(val))

    def __getitem__(self, index):
        if isinstance(index, int) and not isinstance(index, bool):
            if index in self.__data:
                return self.__data[index]
            raise KeyError(index)
        elif isinstance(index, slice):
            st = index.start if index.start is not None else 0
            stop = index.stop if index.stop is not None else self.__next_idx
            step = index.step if index.step is not None else 1
            if st < 0 or stop < 0:
                return Dataset()
            if st >= stop or step <= 0:
                return Dataset()

            if step > 0:
                ret = []
                while st < stop:
                    if st in self.__data:
                        ret.append(self.__data[st])
                    st += step

            return Dataset(ret, copy=False)
        elif isinstance(index, bool):
            if index:
                return self.correct(ignore_unknown=True, keep_ids=True, copy=False)
            else:
                return self.wrong(ignore_unknown=True, keep_ids=True, copy=False)
        elif index is None:
            return self.filter_pred(None, keep_ids=True, copy=False)
        else:
            raise KeyError(index)

    def extend(self, dataset_b, copy=True, inplace=True):
        """
        :param Dataset dataset_b: Another dataset.
        :param bool copy: Use the copy of ``dataset_b``. **Default:** True
        :param bool inplace: If true, returns self, otherwise returns a new :py:class:`.Dataset` **Default:** True
        :return: Returns a dataset of result.
        :rtype: Dataset

        This method would reset indexes of ``dataset_b``.
        """
        assert isinstance(dataset_b, Dataset)
        ret = []
        for inst in dataset_b:
            if copy:
                inst = inst.copy()
            if inplace:
                inst.id = self.__next_idx
                self.__data[self.__next_idx] = inst
                self.__next_idx += 1
            else:
                ret.append(inst)
                inst.id = self.__next_idx + len(ret)
        if inplace:
            return self
        else:
            for kw, val in self.__data.items():
                inst = val.copy() if copy else val
                ret.append(inst)
            return Dataset(ret, copy=False)

    def merge(self, dataset_b, copy=True, inplace=True):
        """
        :param Dataset dataset_b: Another dataset.
        :param bool copy: Use the copy of ``dataset_b``. **Default:** True
        :param bool inplace: If true, returns self, otherwise returns a new :py:class:`.Dataset` **Default:** True
        :return: Returns a dataset of result.
        :rtype: Dataset

        This method merges two dataset and keeps indexes of all instances. If indexes are conflicted, the instance in ``dataset_b`` would be ignored.
        """
        assert isinstance(dataset_b, Dataset)
        ret = []
        for inst in dataset_b:
            if inst.id not in self.__data:
                if copy:
                    inst = inst.copy()
                if inplace:
                    self.__data[inst.id] = inst
                else:
                    ret.append(inst)
        if not inplace:
            for kw, val in self.__data.items():
                if copy:
                    inst = val.copy()
                else:
                    inst = val
                ret.append(inst)
            return Dataset(ret, copy=False)
        else:
            return self

    def __len__(self):
        return len(self.__data)

    def __contains__(self, key):
        return key in self.__data

    def __iadd__(self, val):
        return self.extend(val, copy=True, inplace=True)

    def __delitem__(self, index):
        if isinstance(index, int):
            if index in self.__data:
                del self.__data[index]
            raise KeyError(index)
        elif isinstance(index, slice):
            st = index.start if index.start is not None else 0
            stop = index.stop if index.stop is not None else self.__next_idx
            step = index.step if index.step is not None else 1
            if st < 0 or stop < 0:
                return
            if st >= stop or step <= 0:
                return
            while st < stop:
                if st in self.__data:
                    del self.__data[st]
                st += step
        elif isinstance(index, bool):
            for inst in self.__check(index, ignore_unknown=True, keep_ids=True, copy=False):
                del self.__data[inst.id]
        else:
            raise KeyError(index)

    def __add__(self, dataset_b):
        assert isinstance(dataset_b, Dataset)
        ret = []
        for kw, val in self.__data.items():
            inst = val.copy()
            inst.id = None
            ret.append(inst)
        for inst in dataset_b:
            inst = inst.copy()
            inst.id = None
            ret.append(inst)
        return Dataset(ret, copy=False)

    def data(self, copy=True):
        """
        :param bool copy: Use the copy of dataset. **Default:** True
        :return: A list contains all instances of this dataset.
        :rtype: list

        This method returns a dict which is JSON-serializable, and can be used to initialize :py:class:`.Dataset`. See :py:meth:`.Dataset.__init__`.
        """
        ret = []
        for kw, val in self.__data.items():
            ret.append(val.data(copy=copy))
        return ret

    def clear_pred(self, copy=False):
        """
        :param bool copy: Use the copy of dataset. If false, it would modify in place. **Default:** False
        :rtype: Dataset

        This method clears the predicted label for all instances (sets it to None), and returns the result dataset.
        """
        ret = []
        for kw, val in self.__data.items():
            if copy:
                inst = val.copy()
                ret.append(inst)
            else:
                inst = val
            inst.pred = None
        if copy:
            return Dataset(ret, copy=False)
        else:
            return self

    def clear_label(self, copy=False):
        """
        :param bool copy: Use the copy of dataset. If false, it would modify in place. **Default:** False
        :rtype: Dataset

        This method clears the ground truth label for all instances (sets it to None), and returns the result dataset.
        """
        ret = []
        for kw, val in self.__data.items():
            if copy:
                inst = val.copy()
                ret.append(inst)
            else:
                inst = val
            inst.y = None
        if copy:
            return Dataset(ret, copy=False)
        else:
            return self

    def copy(self):
        """
        :return: Returns a copy of this dataset.
        :rtype: Dataset
        """
        return Dataset( self.data() )

    def reset_index(self, inplace=False):
        """
        :param bool inplace: If true, resets in place, otherwise works on a copy of dataset. **Default:** False
        :return: A result dataset.
        :rtype: Dataset

        This method resets indexes of all instances to ``0 ... len(dataset) - 1``
        """
        ret = []
        for kw, val in self.__data.items():
            if inplace:
                inst = val
            else:
                inst = val.copy()
            ret.append(inst)
            inst.id = None

        if inplace:
            self.__data = {}
            self.__next_idx = 0
            for inst in ret:
                self.append(inst)
            return self
        else:
            return Dataset(ret)

class DataInstance(object):
    __KEY_MAP = {
        "id": ["id", "idx", "index"],
        "x": ["x", "x_orig", "sent", "sentence"],
        "y": ["y", "y_orig", "label"],
        "pred": ["pred", "y_pred"],
        "target": ["target"],
        "meta": ["meta"]
    }
    __KEY_ORDER = [
        "x", "y", "pred", "target", "meta", "id"
    ]
    def __find_key(self, kwargs, keys, default=None):
        ret = None
        okw = None
        for kw in keys:
            if kw in kwargs:
                if ret is None:
                    ret = kwargs[kw]
                    okw = kw
                else:
                    if kwargs[kw] != ret:
                        raise DuplicatedParameterException("%s = %d; %s = %d" % (okw, ret, kw, kwargs[kw]))
        if ret is None:
            ret = default
        return ret

    def __init__(self, *args, **kwargs):
        """
        :param str x: Original input sentence.
        :param int y: Ground truth label of input x. **Default:** None
        :param int pred: Predicted label. **Default:** None
        :param int target: Target label in attacker, leaves None for untargeted attack. **Default:** None
        :param int id: Index of data instance in dataset, leaves None for standalone data instance. **Default:** None
        :param dict meta: Meta data of this instance.
        """
        for i, val in enumerate(args):
            if i >= len(self.__KEY_ORDER):
                raise TypeError("__init__() takes %d positional argument but %d were given" % (len(self.__KEY_ORDER), len(args)))
            kwargs[self.__KEY_ORDER[i]] = val
        self.__x_orig = self.__find_key(kwargs, self.__KEY_MAP["x"], None)
        assert self.__x_orig is not None, "'X' shouldn't be None."
        self.__y_orig = self.__find_key(kwargs, self.__KEY_MAP["y"], None)
        self.__pred = self.__find_key(kwargs, self.__KEY_MAP["pred"], None)
        self.__meta = self.__find_key(kwargs, self.__KEY_MAP["meta"], {})
        self.__target = self.__find_key(kwargs, self.__KEY_MAP["target"], None)
        self.__id = self.__find_key(kwargs, self.__KEY_MAP["id"], None)

    def __getattr__(self, name):
        if name in self.__KEY_MAP["id"]:
            return self.__id
        elif name in self.__KEY_MAP["x"]:
            return self.__x_orig
        elif name in self.__KEY_MAP["y"]:
            return self.__y_orig
        elif name in self.__KEY_MAP["pred"]:
            return self.__pred
        elif name in self.__KEY_MAP["target"]:
            return self.__target
        elif name in self.__KEY_MAP["meta"]:
            return self.__meta
        elif name.startswith("_"):
            return super().__getattr__(name)
        elif name in self.__meta:
            return self.__meta[name]
        raise AttributeError(
            "'%s' object has no attribute '%s'\nmeta: %s"
            % (self.__class__.__name__, name, self.__meta.__repr__())
        )

    def __setattr__(self, name, value):
        if name in self.__KEY_MAP["id"]:
            self.__id = value
        elif name in self.__KEY_MAP["x"]:
            assert value is not None
            self.__x_orig = value
        elif name in self.__KEY_MAP["y"]:
            self.__y_orig = value
        elif name in self.__KEY_MAP["pred"]:
            self.__pred = value
        elif name in self.__KEY_MAP["target"]:
            self.__target = value
        elif name in self.__KEY_MAP["meta"]:
            assert isinstance(value, dict)
            self.__meta = value
        elif name.startswith("_"):
            super().__setattr__(name, value)
        else:
            self.__meta[name] = value

    def __delattr__(self, name):
        if name in self.__KEY_MAP["id"]:
            self.__id = None
        elif name in self.__KEY_MAP["x"]:
            raise AttributeError("'%s' doesn't support attribute deletion" % name)
        elif name in self.__KEY_MAP["y"]:
            self.__y_orig = None
        elif name in self.__KEY_MAP["pred"]:
            self.__pred = None
        elif name in self.__KEY_MAP["target"]:
            self.__target = None
        elif name in self.__KEY_MAP["meta"]:
            self.__meta = {}
        elif name.startswith("_"):
            super().__delattr__(name)
        elif name in self.__meta:
            del self.__meta[name]
        else:
            raise AttributeError(
                "'%s' object has no attribute '%s'\nmeta: %s"
                % (self.__class__.__name__, name, self.__meta.__repr__())
            )

    def __repr__(self):
        ret = "<%s" % self.__class__.__name__
        if self.__id is not None:
            ret += " index=%d" % self.__id
        if len(self.__x_orig) > 20:
            ret += " x=%s..." % self.__x_orig[:18]
        else:
            ret += " x=%s" % self.__x_orig

        if self.__y_orig is not None:
            ret += " y=%d" % self.__y_orig
        if self.__pred is not None:
            ret += " pred=%d" % self.__pred
        if self.__meta is not None:
            ret += " meta=%s" % self.__meta.__repr__()
        ret += ">"
        return ret

    def __str__(self):
        ret = "%s" % self.__class__.__name__
        if self.__id is not None:
            ret += " %d\n" % self.__id
        else:
            ret += "\n"

        ret += "x:\t%s\n" % self.__x_orig
        if self.__y_orig is not None:
            ret += "y:\t%d\n" % self.__y_orig
        else:
            ret += "y:\tunknown\n"
        if self.__pred is not None:
            ret += "pred:\t%d\n" % self.__pred
        else:
            ret += "pred:\tunknown\n"
        if self.__target is not None:
            ret += "target:\t%d\n" % self.__target
        ret += "meta:\t%s" % self.__meta.__repr__()
        return ret

    def __contains__(self, name):
        try:
            val = self.__getattr__(name)
        except AttributeError:
            return False
        return val is not None

    def copy(self):
        """
        :rtype: DataInstance

        This method returns a copy of itself.
        """
        return DataInstance(
            x_orig=self.__x_orig,
            y_orig=self.__y_orig,
            pred=self.__pred,
            target=self.__target,
            idx=self.__id,
            meta=self.__meta.copy(),
        )

    def data(self, copy=True):
        """
        :rtype: dict

        This method returns a dict contains all information of this instance.
        """
        ret = { "x_orig": self.__x_orig }
        if self.__y_orig is not None:
            ret["y_orig"] = self.__y_orig
        if self.__pred is not None:
            ret["prd"] = self.__pred
        if self.__target is not None:
            ret["target"] = self.__target
        if self.__id is not None:
            ret["idx"] = self.__id
        ret["meta"] = self.__meta.copy() if copy else self.__meta
        return ret


import json

from django.http import HttpResponse
from django.contrib.auth.models import User, AnonymousUser
from .models import Campaign

def create_campaign(request):
    if request.method != 'POST':
        return HttpResponse(status=405)
    if isinstance(request.user, AnonymousUser):
        return HttpResponse(status=403)
    if 'players' in request.POST:
        players = (User.objects.get(id=x) for x in request.POST['players'])
    else:
        players = tuple()
    if 'name' in request.POST:
        name = request.POST['name']
    else:
        name = ''
    new_campaign = Campaign(dm=request.user, name=name)
    new_campaign.save()  # Must have an id before many to many relationships can be saved
    for player in players:
        new_campaign.players.add(player)
    new_campaign.save()
    return HttpResponse(bytes(str(new_campaign.id), 'utf8'), status=201)

def _get_user(user_id):
    query = User.objects.filter(id=user_id)
    if len(query.all()) == 0:
        return None
    return query.all()[0]

def _campaign_patch(campaign, data):
    if 'dm' in data:
        new_dm = _get_user(data['dm'])
        if new_dm is None:
            return HttpResponse(status=400)
        campaign.dm = new_dm
    if 'remove_player' in data:
        player = _get_user(data['remove_player'])
        if player is None or player not in campaign.players.all():
            return HttpResponse(status=400)
        campaign.players.remove(player)
    if 'add_player' in data:
        player = _get_user(data['add_player'])
        if player is None:
            return HttpResponse(status=400)
        campaign.players.add(player)
    if 'private' in data:
        if not isinstance(data['private'], bool):
            return HttpResponse(status=400)
        campaign.private = data['private']
    if 'name' in data:
        if not isinstance(data['private'], str):
            return HttpResponse(status=400)
        campaign.private = data['name']
    campaign.save()
    return HttpResponse(status=204)

def _campaign_get(user, campaign):
    if campaign.private and user != campaign.dm and user not in campaign.players.all():
        return HttpResponse(status=403)
    return HttpResponse(json.dumps({
        'name': campaign.name,
        'dm': campaign.dm.username,
        'players': [p.username for p in campaign.players.all()]
    }), status=200)

def campaign(request, campaign_id):
    query = Campaign.objects.filter(id=campaign_id)
    if not query.exists():
        return HttpResponse(status=404)
    campaign, = query.all()
    if request.method == 'GET':
        return _campaign_get(request.user, campaign)
    elif request.method == 'DELETE':
        if request.user != campaign.dm:
            return HttpResponse(status=403)
        campaign.delete()
        return HttpResponse(status=204)
    elif request.method == 'PATCH':
        if request.user != campaign.dm:
            return HttpResponse(status=403)
        data = json.loads(request.body, encoding='utf8')
        return _campaign_patch(campaign, data)
    return HttpResponse(status=405)


from OpenGL.GL import glBegin, glColor3f, glEnd, glEndList, glLineWidth, glNewList, glNormal3f, glVertex3f, \
                      GL_COMPILE, GL_LINES, GL_QUADS
from OpenGL.GLU import gluDeleteQuadric, gluNewQuadric, gluSphere

G_OBJ_SPHERE = 2
def make_sphere():
	glNewList(G_OBJ_SPHERE,GL_COMPILE)
	quad = gluNewQuadric()
	gluSphere(quad,0.5,30,30)
	gluDeleteQuadric(quad)
	glEndList()

def init_primitives():
	make_sphere()

OpenGL/3D Modelling Tool via OpenGL/primitive.py


import logging
from typing import Any, Mapping, Optional, Sequence
from uuid import uuid4

from django.conf import settings
from django.utils import timezone
from rest_framework import serializers

from sentry.auth.access import SystemAccess
from sentry.incidents.logic import (
    ChannelLookupTimeoutError,
    InvalidTriggerActionError,
    get_slack_channel_ids,
)
from sentry.incidents.models import AlertRule, AlertRuleTriggerAction
from sentry.incidents.serializers import AlertRuleSerializer
from sentry.integrations.slack.client import SlackClient
from sentry.integrations.utils import get_identities_by_user
from sentry.mediators import project_rules
from sentry.models import (
    Identity,
    IdentityProvider,
    IdentityStatus,
    Integration,
    Organization,
    Project,
    Rule,
    RuleActivity,
    RuleActivityType,
    User,
    UserEmail,
)
from sentry.shared_integrations.exceptions import (
    ApiError,
    ApiRateLimitedError,
    DuplicateDisplayNameError,
)
from sentry.tasks.base import instrumented_task
from sentry.utils import json
from sentry.utils.json import JSONData
from sentry.utils.redis import redis_clusters

from .utils import (
    SLACK_RATE_LIMITED_MESSAGE,
    get_channel_id_with_timeout,
    get_slack_data_by_user,
    strip_channel_name,
)

logger = logging.getLogger("sentry.integrations.slack.tasks")

SLACK_FAILED_MESSAGE = (
    "The slack resource does not exist or has not been granted access in that workspace."
)

class RedisRuleStatus:
    def __init__(self, uuid: Optional[str] = None) -> None:
        self._uuid = uuid or self._generate_uuid()

        cluster_id = getattr(settings, "SENTRY_RULE_TASK_REDIS_CLUSTER", "default")
        self.client = redis_clusters.get(cluster_id)
        self._set_initial_value()

    @property
    def uuid(self) -> str:
        return self._uuid

    def set_value(
        self, status: str, rule_id: Optional[int] = None, error_message: Optional[str] = None
    ) -> None:
        value = self._format_value(status, rule_id, error_message)
        self.client.set(self._get_redis_key(), f"{value}", ex=60 * 60)

    def get_value(self) -> JSONData:
        key = self._get_redis_key()
        value = self.client.get(key)
        return json.loads(value)

    def _generate_uuid(self) -> str:
        return uuid4().hex

    def _set_initial_value(self) -> None:
        value = json.dumps({"status": "pending"})
        self.client.set(self._get_redis_key(), f"{value}", ex=60 * 60, nx=True)

    def _get_redis_key(self) -> str:
        return f"slack-channel-task:1:{self.uuid}"

    def _format_value(
        self, status: str, rule_id: Optional[int], error_message: Optional[str]
    ) -> str:
        value = {"status": status}
        if rule_id:
            value["rule_id"] = str(rule_id)
        if error_message:
            value["error"] = error_message
        elif status == "failed":
            value["error"] = SLACK_FAILED_MESSAGE

        # Explicitly typing to satisfy mypy.
        _value: str = json.dumps(value)
        return _value

@instrumented_task(name="sentry.integrations.slack.search_channel_id", queue="integrations")  # type: ignore
def find_channel_id_for_rule(
    project: Project,
    actions: Sequence[AlertRuleTriggerAction],
    uuid: str,
    rule_id: Optional[int] = None,
    user_id: Optional[int] = None,
    **kwargs: Any,
) -> None:
    redis_rule_status = RedisRuleStatus(uuid)

    try:
        project = Project.objects.get(id=project.id)
    except Project.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    user = None
    if user_id:
        try:
            user = User.objects.get(id=user_id)
        except User.DoesNotExist:
            pass

    organization = project.organization
    integration_id: Optional[int] = None
    channel_name: Optional[str] = None

    # TODO: make work for multiple Slack actions
    for action in actions:
        if action.get("workspace") and action.get("channel"):
            integration_id = action["workspace"]
            # we need to strip the prefix when searching on the channel name
            channel_name = strip_channel_name(action["channel"])
            break

    try:
        integration = Integration.objects.get(
            provider="slack", organizations=organization, id=integration_id
        )
    except Integration.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    # We do not know exactly how long it will take to paginate through all of the Slack
    # endpoints but need some time limit imposed. 3 minutes should be more than enough time,
    # we can always update later
    try:
        (prefix, item_id, _timed_out) = get_channel_id_with_timeout(
            integration, channel_name, 3 * 60
        )
    except DuplicateDisplayNameError:
        # if we find a duplicate display name and nothing else, we
        # want to set the status to failed. This just lets us skip
        # over the next block and hit the failed status at the end.
        item_id = None
        prefix = ""
    except ApiRateLimitedError:
        redis_rule_status.set_value("failed", None, SLACK_RATE_LIMITED_MESSAGE)
        return

    if item_id:
        for action in actions:
            # need to make sure we are adding back the right prefix and also the channel_id
            if action.get("channel") and strip_channel_name(action.get("channel")) == channel_name:
                action["channel"] = prefix + channel_name
                action["channel_id"] = item_id
                break

        kwargs["actions"] = actions
        kwargs["project"] = project

        if rule_id:
            rule = Rule.objects.get(id=rule_id)
            rule = project_rules.Updater.run(rule=rule, pending_save=False, **kwargs)
        else:
            rule = project_rules.Creator.run(pending_save=False, **kwargs)
            if user:
                RuleActivity.objects.create(
                    rule=rule, user=user, type=RuleActivityType.CREATED.value
                )

        redis_rule_status.set_value("success", rule.id)
        return
    # if we never find the channel name we failed :(
    redis_rule_status.set_value("failed")

@instrumented_task(  # type: ignore
    name="sentry.integrations.slack.search_channel_id_metric_alerts", queue="integrations"
)
def find_channel_id_for_alert_rule(
    organization_id: int,
    uuid: str,
    data: Any,
    alert_rule_id: Optional[int] = None,
    user_id: Optional[int] = None,
) -> None:
    redis_rule_status = RedisRuleStatus(uuid)
    try:
        organization = Organization.objects.get(id=organization_id)
    except Organization.DoesNotExist:
        redis_rule_status.set_value("failed")
        return

    user = None
    if user_id:
        try:
            user = User.objects.get(id=user_id)
        except User.DoesNotExist:
            pass

    alert_rule = None
    if alert_rule_id:
        try:
            alert_rule = AlertRule.objects.get(organization_id=organization_id, id=alert_rule_id)
        except AlertRule.DoesNotExist:
            redis_rule_status.set_value("failed")
            return

    try:
        mapped_ids = get_slack_channel_ids(organization, user, data)
    except (serializers.ValidationError, ChannelLookupTimeoutError, InvalidTriggerActionError) as e:
        # channel doesn't exist error or validation error
        logger.info(
            "get_slack_channel_ids.failed",
            extra={
                "exception": e,
            },
        )
        redis_rule_status.set_value("failed")
        return
    except ApiRateLimitedError as e:
        logger.info(
            "get_slack_channel_ids.rate_limited",
            extra={
                "exception": e,
            },
        )
        redis_rule_status.set_value("failed", None, SLACK_RATE_LIMITED_MESSAGE)
        return

    for trigger in data["triggers"]:
        for action in trigger["actions"]:
            if action["type"] == "slack":
                if action["targetIdentifier"] in mapped_ids:
                    action["input_channel_id"] = mapped_ids[action["targetIdentifier"]]
                else:
                    # We can early exit because we couldn't map this action's slack channel name to a slack id
                    # This is a fail safe, but I think we shouldn't really hit this.
                    redis_rule_status.set_value("failed")
                    return

    # we use SystemAccess here because we can't pass the access instance from the request into the task
    # this means at this point we won't raise any validation errors associated with permissions
    # however, we should only be calling this task after we tried saving the alert rule first
    # which will catch those kinds of validation errors
    serializer = AlertRuleSerializer(
        context={
            "organization": organization,
            "access": SystemAccess(),
            "user": user,
            "use_async_lookup": True,
            "validate_channel_id": False,
        },
        data=data,
        instance=alert_rule,
    )
    if serializer.is_valid():
        try:
            alert_rule = serializer.save()
            redis_rule_status.set_value("success", alert_rule.id)
            return
        # we can still get a validation error for the channel not existing
        except (serializers.ValidationError, ChannelLookupTimeoutError):
            # channel doesn't exist error or validation error
            redis_rule_status.set_value("failed")
            return
    # some other error
    redis_rule_status.set_value("failed")
    return

@instrumented_task(name="sentry.integrations.slack.link_users_identities", queue="integrations")  # type: ignore
def link_slack_user_identities(integration: Integration, organization: Organization) -> None:
    emails_by_user = UserEmail.objects.get_emails_by_user(organization)
    slack_data_by_user = get_slack_data_by_user(integration, organization, emails_by_user)

    idp = IdentityProvider.objects.get(
        type=integration.provider,
        external_id=integration.external_id,
    )
    date_verified = timezone.now()
    identities_by_user = get_identities_by_user(idp, slack_data_by_user.keys())

    for user, data in slack_data_by_user.items():
        # Identity already exists, the emails match, AND the external ID has changed
        if user in identities_by_user.keys():
            if data["slack_id"] != identities_by_user[user].external_id:
                # replace the Identity's external_id with the new one we just got from Slack
                identities_by_user[user].update(external_id=data["slack_id"])
            continue
        # the user doesn't already have an Identity and one of their Sentry emails matches their Slack email
        matched_identity, created = Identity.objects.get_or_create(
            idp=idp,
            external_id=data["slack_id"],
            defaults={"user": user, "status": IdentityStatus.VALID, "date_verified": date_verified},
        )
        # the Identity matching that idp/external_id combo is linked to a different user
        if not created:
            logger.info(
                "post_install.identity_linked_different_user",
                extra={
                    "idp_id": idp.id,
                    "external_id": data["slack_id"],
                    "object_id": matched_identity.id,
                    "user_id": user.id,
                    "type": idp.type,
                },
            )

# TODO: add retry logic
@instrumented_task(name="sentry.integrations.slack.post_message", queue="integrations", max_retries=0)  # type: ignore
def post_message(
    payload: Mapping[str, Any], log_error_message: str, log_params: Mapping[str, Any]
) -> None:
    client = SlackClient()
    try:
        client.post("/chat.postMessage", data=payload, timeout=5)
    except ApiError as e:
        extra = {"error": str(e), **log_params}
        logger.info(log_error_message, extra=extra)


# import itertools
import pdb
import time
import urllib
from collections import Counter
from typing import Union

import networkx as nx
import numpy as np
import obonet
import pandas as pd
from Bio import SeqIO
from kedro.extras.datasets.biosequence import BioSequenceDataSet
from kedro.extras.datasets.json import JSONDataSet
from kedro.extras.datasets.networkx import JSONDataSet as nxJSONDataSet
from kedro.io import PartitionedDataSet
from SPARQLWrapper import JSON, SPARQLWrapper

pd.set_option("display.max_rows", 500)
pd.set_option("display.max_columns", 500)

class ReturnDict(dict):
    def __missing__(self, key):
        return key

def _parse_annotation_id(x):
    """Removing identifier for each identification.
    ex: UniProtKB:P25762; parsing
    """
    return x.str.split(":").str[1]

def _parse_protein_length(prot_sequences, unique_gids):
    prot_sequences_dict = SeqIO.to_dict(prot_sequences)
    pdb.set_trace()
    return {gid: len(prot_sequences_dict[gid]) for gid in unique_gids}

def _merge_rows(df):
    df = df.groupby("gid").agg(set)
    res = [[",".join(map(str, x)) for x in df[col]] for col in df.columns]
    fdf = pd.DataFrame(res).T.set_index(df.index)
    fdf.columns = df.columns
    print("End")
    return fdf

def _merge_duplicate_info(df):
    """This function merges duplicate information in a dataframe and then merges based on
    a comma. It had some hardcoded column for index and groupby
    """
    unique_df = df[~df.gid.duplicated(keep=False)]
    unique_df = unique_df.set_index("gid")
    duplicated_df = df[df.gid.duplicated(keep=False)]
    tmp = duplicated_df.groupby("gid").agg(set)
    res = [[",".join(map(str, x)) for x in tmp[col]] for col in tmp.columns]
    joined_duplicates = pd.DataFrame(res).T.set_index(tmp.index)
    joined_duplicates.columns = tmp.columns
    final = pd.concat([unique_df, joined_duplicates], axis=0)
    return final

def _creating_edges_list_multilayer(annot, elist, edge, prokka_gff):
    """creating edges for multilayer networks using the list method
    [node1 layer1 node2 layer2 weight]

    What if a node comes from multiple bins. I need to check
    if both nodes have / then I am not accounting the types for both of them!
    """
    tmp = []
    edge_length = []
    edge_scaf = set()
    for node in edge:
        length, level, scaf_level = prokka_gff.loc[node][["length", "bin", "scaffold"]]
        # tmp.append(f"{node}|{level}")
        if "," in length:
            length = np.mean(np.array(length.split(",")).astype(np.int32))
        else:
            length = int(length)
        tmp.append(node)
        tmp.append(level)
        edge_length.append(length)
        edge_scaf.add(scaf_level)
    # if "low" in tmp[0] or "low" in tmp[1]:
    #    tmp.append(1)
    # else:
    #    tmp.append(-1)
    tmp.append(np.mean(edge_length))
    if len(edge_scaf) == 1:
        tmp.append(edge_scaf.pop())
    else:
        tmp.append("")
    tmp.append(annot)
    elist.append(tmp[:])
    return elist

def _merge_duplicate_seqrecords(seqrecords):
    """Removing protein sequences that are have the description and annotation of hypothetical protein
    If I merge two sequences that are the same, I need to merge the ids as well
    But if I merge those ids what will I do with the Nosema annotation for the gene?
    """
    unique_records = {}
    for rec in seqrecords:
        if "hypothetical protein" in rec.description:
            continue
        if rec.seq in unique_records:
            merged_rec = unique_records[rec.seq].id + "/" + rec.id
            unique_records[rec.seq].id = merged_rec
        else:
            unique_records[rec.seq] = rec
    return unique_records.values()

def _find_merged_ids(records):
    list_merged_id = [x.id for x in records if "/" in x.id]
    dict_merged_ids = {}
    for merged_id in list_merged_id:
        for original_id in merged_id.split("/"):
            dict_merged_ids[original_id] = merged_id
    return dict_merged_ids

def preprocess_prokka_sequences(
    partition_prokka_faa: PartitionedDataSet,
) -> Union[BioSequenceDataSet, JSONDataSet]:
    """Merging duplicate protein sequence from various partitions of prokka.

    partition_prokka_faa: A list of biopython sequence records from various partitions

    Returns:
    1. Biosequence dataset which loads fasta file
    2. JSON file of merged ids
    """
    start = time.time()
    combine_all = []
    for partition_load_func in partition_prokka_faa.values():
        partition_data = partition_load_func()
        combine_all.extend(partition_data)
    print(f"For loop {time.time() - start}")
    unique_records = _merge_duplicate_seqrecords(combine_all)
    merged_ids = _find_merged_ids(unique_records)
    return list(unique_records), merged_ids

def prokka_bins_gff(
    partition_prokka_gff: PartitionedDataSet,
) -> Union[pd.DataFrame, pd.DataFrame, JSONDataSet]:
    """Parsing prokka annotation information from the multiple runs
    I don't need to keep most of the information like gene or annotation info
    """
    list_gff_df = []
    for partition_load_func in partition_prokka_gff.values():
        list_gff_df.append(partition_load_func())
    concat_gff = pd.concat(list_gff_df, ignore_index=True)
    # concat_gff["scaf_level"] = concat_gff["scaffold"] + ":" + concat_gff["level"]
    prokka_bins = dict(concat_gff[["prokka_unique", "bin"]].values)
    prokka_gff = concat_gff.drop(["prokka_unique"], axis=1)

    uni_prokka_gff = prokka_gff[prokka_gff.annot.str.contains("UniProtKB")]
    rest_prokka_gff = prokka_gff[~prokka_gff.annot.str.contains("UniProtKB")]
    return uni_prokka_gff, rest_prokka_gff, prokka_bins

def preprocess_prokka_annotations(
    prot_sequences: BioSequenceDataSet,
    merged_gene_ids: JSONDataSet,
    annotations: pd.DataFrame,
) -> pd.DataFrame:
    """Preprocesses data for prokka annotations pulled from the gff files

    prot_sequences: A BioSequence dataset containing protein sequences
    merged_gene_ids: A JSON file containing a mapping of duplicated gene ids to a merged id
    annotations: A pandas dataframe containing information pulled from prokka's gff files

    Returns:
    A pandas dataframe containing processed prokka annotations, including gene id,
    """
    merged_gene_ids = ReturnDict(merged_gene_ids)
    annotations["gid"] = annotations.gid.map(merged_gene_ids)  # Duplicates
    # Why do a lot of the merged ids not show up in annnotations
    # pdb.set_trace()
    # all_bins = set(annotations.bin)
    # fixed_annotations = _merge_duplicate_info(annotations)
    annotations["length"] = annotations.gid.map(
        _parse_protein_length(prot_sequences, set(annotations.gid))
    )
    annotations = annotations[annotations.length > 100].copy()
    annotations["annot"] = _parse_annotation_id(annotations["annot"])
    # fixed_annotations["test"] = range(len(fixed_annotations))
    # fixed_annotations["test"] = fixed_annotations.test.astype(str)
    return annotations

def _sparql_uniprot_query(sparql, prot_list: list[str]):
    # Filter out the uniprot keywords
    prot_list = '("' + '") ("'.join(prot_list) + '")'
    sparql.setQuery(
        f"""
    PREFIX up: <http://purl.uniprot.org/core/>
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    SELECT DISTINCT
        ?ac
        ?goterm_id
    WHERE \u007b
        VALUES (?ac) \u007b{prot_list}\u007d
        BIND (IRI(CONCAT("http://purl.uniprot.org/uniprot/",?ac)) AS ?protein)
        ?protein a up:Protein .
        ?protein up:classifiedWith ?goTerm .
        BIND (REPLACE(STR(?goTerm), "^.*/([^/]*)$","$1") as ?goterm_id)
        OPTIONAL \u007b
        ?goTerm rdfs:subClassOf <http://purl.obolibrary.org/obo/GO_0003674> .
        ?goTerm rdfs:label ?moltype .
        \u007d
        FILTER(?moltype)

    \u007d ORDER BY DESC(?ac)
    """
    )
    res = []
    for i in range(4):
        try:
            res = sparql.query().convert()
            if res != []:
                break
        except urllib.error.HTTPError as exec:
            print(f"Sleeping for 30 seconds try {i+1}")
            print(f"HTTPError {exec}")
            time.sleep(30)
            continue
            # res = sparql.query().convert()
    return res

def _chunks(lst, n):
    for i in range(0, len(lst), n):
        yield lst[i : i + n]

def go_annotations(prokka_gff: pd.DataFrame) -> pd.DataFrame:
    """I need to add asyncio here so that I can send all the requests all at once instead of waiting for the
    computation to be done and then requesting again
    This will improve the speed of performance
    """
    sparql = SPARQLWrapper("https://sparql.uniprot.org/sparql/")
    sparql.setReturnFormat(JSON)
    uni_annots = set(prokka_gff.annot)
    xres = []
    for annot_bin in _chunks(list(uni_annots), 150):
        res = _sparql_uniprot_query(sparql, annot_bin)
        if res == []:
            print("Broken, skipping this bin")
            print(annot_bin)
            continue
        xx = res["head"]["vars"]
        rbinds = res["results"]["bindings"]
        xres.extend([[s["value"] for s in x.values()] for x in rbinds])
    df = pd.DataFrame(xres, columns=xx)
    tmp = df.groupby("ac").agg(set)
    res = [",".join(map(str, x)) for x in tmp["goterm_id"]]
    df2 = pd.DataFrame(res, index=tmp.index, columns=tmp.columns)
    dict_df = df2.goterm_id.to_dict()
    prokka_gff["go"] = prokka_gff.annot.map(dict_df)
    return prokka_gff

def prokka_edges(
    prokka_gff: pd.DataFrame, nosema_conc: pd.DataFrame, prot_sequences: BioSequenceDataSet
) -> Union[pd.DataFrame, pd.DataFrame]:
    """I need to add the nosema intensity here. I need to implement multiprocessing here. It is taking wayyyyy too long
    I need to also take into account what's being not included in prokka_gff
    If two proteins are in the same bin with the same annotation then they are the same!
    """
    prokka_gff = prokka_gff.fillna("")
    prokka_gff["length"] = prokka_gff.length.astype(str)
    # prokka_edges = []
    # prokka_gff["bin"] = ["b" + str(b) for b in prokka_gff.bin]
    # prokka_gff["gid"] = ["g" + str(g) for g in prokka_gff.gid]
    # annot_groups = prokka_gff.groupby("annot")["bin","gid"].agg(list)
    # Merging genes whose annotation and bin are the same, thus they are effectively the same

    prokka_gff2 = _merge_rows(prokka_gff)
    pdb.set_trace()
    pg = prokka_gff.reset_index()
    res = pg.groupby(["annot", "bin"]).gid.agg(list)
    g1 = res[res.map(len) > 1]
    print(g1.head())
    return prokka_gff, g1

def _flat_func(x):
    return [item for sublist in x for item in sublist]

def go_ontology(go_prokka: pd.DataFrame) -> nxJSONDataSet:
    # Read the taxrank ontology
    url = "http://purl.obolibrary.org/obo/go/go-basic.obo"
    graph = obonet.read_obo(url)
    mapping = dict(
        zip(graph.nodes.keys(), [x.replace(":", "_") for x in graph.nodes.keys()])
    )
    graph = nx.relabel_nodes(graph, mapping)
    id_to_name = {id_: data["name"] for id_, data in graph.nodes(data=True)}
    # nx.descendants(graph, "GO:0000036")

    # Analysis of go_prokka
    goh_prokka = go_prokka[~go_prokka.go.isnull()].copy()
    goh_prokka["go"] = goh_prokka.go.str.split(",")
    res = goh_prokka.groupby("annot").go.agg(list)
    # dictionary of uniprot mapped to go
    flat_res = {key: set(_flat_func(val)) for key, val in res.items()}
    df = (
        pd.DataFrame.from_dict(flat_res, orient="index")
        .rename_axis("uniprots")
        .reset_index()
    )
    go_df = df.melt(id_vars=["uniprots"], value_name="go_term").dropna()
    # go mapped to uniprot
    go_annots = go_df.groupby("go_term").uniprots.agg(set)
    total_annots = [x for x in go_annots.values]
    shared_go = Counter(_flat_func(total_annots))
    go_shared = {}
    for key, val in shared_go.items():
        go_shared[val] = go_shared.get(val, []) + [key]
    shared_go_counts = Counter(list(shared_go.values()))
    count_lengths = Counter([len(x) for x in flat_res.values()])
    go_annots = go_annots.reset_index()
    go_annots["name"] = go_annots.go_term.map(id_to_name)
    go_annots.uniprots = go_annots.uniprots.map(list)
    sources = [
        x for x in graph.nodes() if graph.out_degree(x) == 1 and graph.in_degree(x) == 0
    ]
    sour = set(go_annots.go_term) & set(sources)
    print(shared_go_counts)
    print(count_lengths)
    print(sour)
    xx = go_annots.go_term.to_list()
    sub = graph.subgraph(xx)
    return (
        dict(zip(go_df.uniprots, go_df.go_term)),
        dict(zip(go_annots.go_term, go_annots.uniprots)),
        sub,
    )


#!/usr/bin/python3

import serial
import rrdtool
import os.path

def createrrd():
    # 600 samples of 5 minutes  (2 days and 2 hours)
    # 700 samples of 30 minutes (2 days and 2 hours, plus 12.5 days)
    # 775 samples of 2 hours    (above + 50 days)
    # 797 samples of 1 day      (above + 732 days, rounded up to 797)
    if os.path.exists("rads.rrd"):
        print("Using existing RRD...")
    else:
        rrdtool.create("rads.rrd",
                "--start", "now",
                "--step", "300",
                "DS:cps:GAUGE:600:U:U",
                "DS:cpm:GAUGE:600:U:U",
                "RRA:AVERAGE:0.5:1:600",
                "RRA:AVERAGE:0.5:6:700",
                "RRA:AVERAGE:0.5:24:775",
                "RRA:AVERAGE:0.5:288:797",
                "RRA:MAX:0.5:1:600",
                "RRA:MAX:0.5:6:700",
                "RRA:MAX:0.5:24:775",
                "RRA:MAX:0.5:288:797")

ser = serial.Serial('/dev/serial/by-id/usb-FTDI_FT232R_USB_UART_AB0KPLG7-if00-port0',baudrate=9600, bytesize=8,parity='N',stopbits=1, timeout=3, xonxoff=0, rtscts=0)

print(ser.baudrate)
print(ser.name)

createrrd()

while (1):
    # b'CPS, 0, CPM, 13, uSv/hr, 0.07, SLOW\r\n'
    line = ser.readline()
    string = line.decode()
    chunks = string.split(',')
    if (len(chunks) == 7):
        cps = chunks[1]
        cpm = chunks[3]
        print("Logging N:"+cps+":"+cpm)
        rrdtool.update("rads.rrd","N:"+cps+":"+cpm)
    else:
        print("Bad read:", line)


import tkinter as tk

window = tk.Tk()
window.title("Address Entry Form")

frame = tk.Frame(master= window, relief = tk.SUNKEN, borderwidth =3)
frame.pack()

label1 = tk.Label(text = "First Name: ", master = frame )
entry1 = tk.Entry(master= frame, width=50)
label1.grid(row=0, column=0, sticky="e")
entry1.grid(row=0, column=1)

label2 = tk.Label(text = "Last Name: ", master = frame)
entry2 = tk.Entry(master= frame, width=50)
label2.grid(row=1, column=0, sticky="e")
entry2.grid(row=1, column=1)

label3 = tk.Label(text = "Address Line 1: ", master = frame )
entry3 = tk.Entry(master= frame, width=50)
label3.grid(row=2, column=0, sticky=tk.E)
entry3.grid(row=2, column=1)

label4 = tk.Label(text = "Address Line 2: ", master = frame )
entry4 = tk.Entry(master= frame, width=50)
label4.grid(row=3, column=0, sticky=tk.E)
entry4.grid(row=3, column=1)

label5 = tk.Label(text = "City: ", master = frame )
entry5 = tk.Entry(master= frame, width=50)
label5.grid(row=5, column=0, sticky=tk.E)
entry5.grid(row=5, column=1)

label6 = tk.Label(text = "State/Province: ", master = frame )
entry6 = tk.Entry(master= frame, width=50)
label6.grid(row=6, column=0, sticky=tk.E)
entry6.grid(row=6, column=1)

label7 = tk.Label(text = "Postal Code: ", master = frame )
entry7 = tk.Entry(master= frame, width=50)
label7.grid(row=7, column=0, sticky=tk.E)
entry7.grid(row=7, column=1)

label8 = tk.Label(text = "Country: ", master = frame )
entry8 = tk.Entry(master= frame, width=50)
label8.grid(row=8, column=0, sticky=tk.E)
entry8.grid(row=8, column=1)

buttons =tk.Frame()
buttons.pack(fill=tk.X,ipadx=5, ipady=5)

def submit_click():
    file = open("py.txt",'a')
    file.write("\n")
    file.write(entry1.get()+"\n")
    file.write(entry2.get()+"\n")
    file.write(entry3.get()+"\n")
    file.write(entry4.get()+"\n")
    file.write(entry5.get()+"\n")
    file.write(entry6.get()+"\n")
    file.write(entry7.get()+"\n")
    file.write(entry8.get()+"\n")
    file.close()

def drop():
    entry1.delete(first=0, last=tk.END)
    entry2.delete(first=0, last=tk.END)
    entry3.delete(first=0, last=tk.END)
    entry4.delete(first=0, last=tk.END)
    entry5.delete(first=0, last=tk.END)
    entry6.delete(first=0, last=tk.END)
    entry7.delete(first=0, last=tk.END)
    entry8.delete(first=0, last=tk.END)

button_submit = tk.Button(master= buttons,text= "Submit", command = submit_click)
button_submit.pack(side = tk.RIGHT, padx = 10, ipadx=10)

button_clear = tk.Button(master= buttons,text="Clear", command = drop)
button_clear.pack(side = tk.RIGHT, ipadx = 10)

window.mainloop()


# @File: MyHashTable
# @Author: Kevin Huo
# @LastUpdate: 12/10/2020 10:30 PM

class MyHashTable(object):
    """
    自己实现哈希表
    """
    def __init__(self, elements=None):
        """
        1. 哈希表的存储容量有不同的取值方式, 常见的方式是和需要存储的数据的长度相同
        注意: python中定义桶的数量的原则是:
            1.1 默认8
            1.2 当key的数量动态增长而且超过了当前桶容量时, 用一下规则增加桶容量:
                如果当前桶容量 < 50000, 那么每次对当前容量乘以4, 即 self.bucket_size = self.bucket_size * 4
                如果当前桶容量 >= 50000, 那么每次乘以2, 即self.bucket_size = self.bucket_size * 2

        2. 计算索引的方式也采用 散列值对容量取余 (还有一种做法, 是对小于等于容量的最近的一个质数取余, 但是2种办法都可以)
        3. 散列函数我不具体实现, 直接使用Python内置的 hash(), 但是我们需要知道，对一个哈希函数, 它有3个特点要满足:
            3.1 - 哈希值 (散列值) 的计算速度需要很快
            3.2 - 计算出的哈希值需要不可逆, 也就是说不能用哈希值倒推出原始键值.
            3.3 - 输入不同长度的键, 要保证输出的哈希值长度固定.
        """
        # 一共多少个桶 (python源代码中默认尺寸是8, 并且递增的方式是 8, 16, 32, 这种以2的幂增长)
        self.bucket_size = len(elements) if elements else 8
        # 这里初始化每个桶. 有很多种选择, 比如用简单的python内置列表, 或者用链表, 或者甚至用红黑树(仅当数据很大时使用)
        # 我们用python内置数组实现
        self.buckets = [[] for _ in range(self.bucket_size)]
        if elements:
            self._assign_buckets(elements)

    def __getitem__(self, key):
        """这个函数支持使用 d["key"] 的方式访问元素"""
        return self._get_value(key)

    def __setitem__(self, key, value):
        """该函数支持使用 d["key"] = new_v 的方式更新值"""
        self._write_val(key, value)

    def __str__(self):
        """在print(d) 时候被调用"""
        return str(self.buckets)

    def _assign_buckets(self, elements):
        """
        该函数仅当初始化 MyHashTable 时传入elements 时候被调用
        elements 有2种可能的传入方式:
        第一种 -- 数组 + 子数组形式 = [[key1, val1], [key2, val2], ...]
        第二种 -- 传入一个python内置字典 {"key1": "val1", "key2": "val2"}
        """
        check_list = elements.items() if isinstance(elements, dict) else elements

        for k, v in check_list:
            hashed_val = hash(k)
            bucket_idx = hashed_val % self.bucket_size
            self.buckets[bucket_idx].append([k, v])

    def _get_value(self, input_key):
        """哈希表的查询"""
        hashed_k = hash(input_key)
        idx = hashed_k % self.bucket_size
        for k, v in self.buckets[idx]:
            if k == input_key:
                return v

    def _write_val(self, input_key, new_val):
        """哈希表的写入
        如果self.buckets[idx] 中已有该key, 则是属于更新操作.
        如果self.buckets[idx] 为空, 则属于新建key-val操作
        """
        hashed_k = hash(input_key)
        idx = hashed_k % self.bucket_size
        # 空桶, 属于新建操作
        if not self.buckets[idx]:
            self.buckets[idx].append((input_key, new_val))
            return
        # 如果桶不为空, 那么属于更新操作
        for i in range(len(self.buckets[idx])):
            currk, currv = self.buckets[idx][i]
            if currk == input_key:
                # 更新v
                self.buckets[idx][i][1] = new_val
                return
        # 如果程序能执行到这里, 说明桶既不为空, 而且没有找到对应k, 那说明哈希冲突了, 我们实现的冲突解决方案是 "链表法", 也就是在当前桶中将所有
        # 具有相同哈希值的key-val对都存在里面.
        self.buckets[idx].append([input_key, new_val])

if __name__ == '__main__':
    raw = {"name": "huoke", "city": "lanzhou"}
    d = MyHashTable(raw)
    print(d["name"])
    # 新增
    d["age"] = 18
    print(d)
    # 更新
    d["city"] = "shanghai"
    print(d)


"""
지속적인 오류:  must be str, not int
문자열에 정수 더하는 경우 생김
list(s) 한다고 문자열의 요소가 chr로 고려되지 않음, 여전히 str (type 함수로 확인 )

+int형으로 변경할 때 (int)i 가 아니라 int(i)임

추가적인 오류: ValueError- invalid literal for int() with base 10 (str 오류)
"""
def upperMethod(s):
    res=""
    for i in list(s):
        if i<'a': #Capital case
            #upeer로 변환
            temp_i=ord(i)+32
            res+=chr(temp_i)
        else: res+=i
    return res

def rmFactor(arr, factor):
    res=[]
    for i in arr:
        if i!=factor:
            res.append(i)
    return res

def solution(cacheSize, cities):

    if cacheSize==0:
        return len(cities)*5
    cache=[]
    time=0
    for i in cities:
        i=upperMethod(i)

        #cache hit(1)
        if i in cache: #O(n)
            cache=rmFactor(cache,i)
            cache.append(i)
            time+=1

        #cache miss(5)
        else:
            if len(cache)==cacheSize: #cache full
                cache.pop(0)
                cache.append(i)
                time+=5

            else:
                cache.append(i)
                time+=5
    return time

print(solution(2,["Jeju", "Pangyo", "NewYork", "newyork"]))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr  2 14:37:24 2017

@author: gjacopo
"""

from __future__ import print_function

import warnings
from collections import OrderedDict

import requests

try:
    import numpy as np#analysis:ignore
except:
    raise IOError("Package numpy not imported - Requested")

try:
    import pandas as pd
    PDVERS = int(pd.__version__.split('.')[1])
except:
    PDVERS = 0 # unknown
    raise IOError("Package pandas not imported - Requested")

try:
    import json
except:
    try:
        import simplejson as json#analysis:ignore
    except:
        warnings.warn("Package simplejson/json not imported")

try:
    #import pyjstat
    from pyjstat import pyjstat
except:
    warnings.warn("Package pyjstat not imported - Used for JSONSTAT format")

__GEO_LABELS  = [('EU28', 'European Union (28 countries)'),
               # ('EU27':, 'European Union (27 countries)'),
               # ('EA19', 'Euro area (19 countries)'),
               # ('EA18', 'Euro area (18 countries)'),
               # ('EU', 'European Union (EU6-1972, EU9-1980, EU10-1985, EU12-1994, EU15-2004, EU25-2006, EU27-2013, EU28)'),
               # ('EA', 'Euro area (EA11-2000, EA12-2006, EA13-2007, EA15-2008, EA16-2010, EA17-2013, EA18-2014, EA19)'),
               ('AT', 'Austria'),
               ('BE', 'Belgium'),
               ('BG', 'Bulgaria'),
               ('CY', 'Cyprus'),
               ('CZ', 'Czech Republic'),
               ('DE', 'Germany (until 1990 former territory of the FRG)'),
               ('DK', 'Denmark'),
               ('EE', 'Estonia'),
               ('EL', 'Greece'),
               ('ES', 'Spain'),
               ('FI', 'Finland'),
               ('FR', 'France'),
               ('HR', 'Croatia'),
               ('HU', 'Hungary'),
               ('IE', 'Ireland'),
               ('IT', 'Italy'),
               ('LT', 'Lithuania'),
               ('LU', 'Luxembourg'),
               ('LV', 'Latvia'),
               ('MT', 'Malta'),
               ('NL', 'Netherlands'),
               ('PL', 'Poland'),
               ('PT', 'Portugal'),
               ('RO', 'Romania'),
               ('SE', 'Sweden'),
               ('SI', 'Slovenia'),
               ('SK', 'Slovakia'),
               ('UK', 'United Kingdom'),
               ('CH', 'Switzerland'),
               ('NO', 'Norway'),
               ('IS', 'Iceland'),
               ('MK', 'the Former Yugoslav Republic of Macedonia'),
               ('RS', 'Serbia'),
               ('TR', 'Turkey')]
GEO_LABELS  = OrderedDict(__GEO_LABELS)

__AGE_LABELS = [('TOTAL', 'Total population'),
                ('Y_LT6', 'Less than 6 years'),
                ('Y6-10', 'From 6 to 10 years'),
                ('Y11-15', 'From 11 to 15 years'),
                ('Y_LT16', 'Less than 16 years'),
                ('Y16-19', 'From 16 to 19 years'),
                ('Y16-24', 'From 16 to 24 years'),
                ('Y_GE18', '18 years or over'),
                ('Y18-24', 'From 18 to 24 years'),
                ('Y20-24', 'From 20 to 24 years'),
                ('Y25-29', 'From 25 to 29 years'),
                ('Y25-34', 'From 25 to 34 years'),
                ('Y25-49', 'From 25 to 49 years'),
                ('Y35-44', 'From 35 to 44'),
                ('Y45-54', 'From 45 to 54'),
                ('Y50-64', 'From 50 to 64 years'),
                ('Y55-64', 'From 55 to 64 years'),
                ('Y65-74', 'From 65 to 74 years'),
                ('Y_GE75', '75 years or over')
                ]

AGE_LABELS  = OrderedDict(__AGE_LABELS)

SEX_LABELS  = {'M': 'Males',
               'F': 'Females',
               'T': 'Total'}

UNIT_LABELS = {'THS_PER': 'Thousand persons',
               'PC_POP': 'Percentage of total population'
               }

STATUS_LABELS = {'POP': 'Population',
                'EMP': 'Employed persons',
                'SAL': 'Employees',
                'NSAL': 'Employed persons except employees',
                'NEMP': 'Not employed persons',
                'UNE': 'Unemployed persons',
                'RET': 'Retired persons',
                'INAC_OTH': 'Other inactive persons'}

class EstatDataFrame(object):

    PROTOCOL        = "http"
    API_LANG        = "en"
    API_FMT         = "json"
    API_DOMAIN      = 'ec.europa.eu/eurostat/wdds'
    API_VERS        = 2.1
    API_URL         = "{}://{}/rest/data/v{}/{}/{}".format(
                  PROTOCOL, API_DOMAIN, API_VERS, API_FMT, API_LANG
                  )

    def __init__(self, **kwargs):
        self.__indicator = []
        self.__url = ""
        self.__response = None
        if kwargs == {}:
            return
        if 'indicator' in kwargs:
            self.__indicator = kwargs.pop('indicator')

    @property
    def indicator(self):
        return self.__indicator

    @property
    def url(self):
        return self.__url

    @property
    def response(self):
        return self.__response

    def build_url(self, **kwargs):
        url = "{}/{}?".format(EstatDataFrame.API_URL, self.indicator[0])
        #if 'geo' in kwargs:
        #    url = "{}geo={}&".format(url, kwargs.pop('geo', None))
        _izip_replicate = lambda d : [(k,i) if isinstance(d[k], (tuple,list))        \
                else (k, d[k]) for k in d for i in d[k]]
        #_no_replicate = lambda d : d.items()
        filters = '&'.join(['{k}={v}'.format(k=k, v=v) for (k, v) in _izip_replicate(kwargs)])
        try:
            self.__url = "{}{}".format(url, filters)
        except:
            pass
        return self.url

    def get_response(self):
        # request the URL
        session = requests.session()
        try:
            response = session.head(self.url)
            response.raise_for_status()
        except:
            raise IOError("ERROR: wrong request formulated")
        else:
            print ("OK: status={}".format(response.status_code))
        # load the data
        try:
            response = session.get(self.url)
        except:
            raise IOError('error retrieveing response from URL')
        try:
            if EstatDataFrame.API_FMT == 'json':
                self.__response = response.json()
            elif EstatDataFrame.API_FMT == 'unicode':
                self.__response = response.text
        except:
            pass
        return self.response

    def load_json(self, pivot=None, index=None, use_label=True):
        try:
            # pyjstat produces dataframe in stacked shape
            dataset = pyjstat.Dataset.read(self.url)
        except:
            raise IOError("Dataframe not created")
        df = dataset.write('dataframe')
        if use_label is True:
            for name in dataset['dimension']:
                dname = dataset['dimension'][name]['category']['label']
                df[name] = df[name].map(dict((v,k) for k,v in dname.items()))
        if pivot is None:
            return df
        if index is None:
            index = set(df.columns).difference({pivot}.union({'value'}))
            # or: index = [c for c in df.columns if c not in {pivot, 'ivalue'}]
        if PDVERS < 18:
            df = df.set_index(index + [pivot]).unstack(pivot)
        else:
            df = df.pivot_table(index=index, columns=pivot, values='value')
        df.reset_index(drop=False, inplace=True)
        try:
            #df.index.name = None
            #df = df.reindex(df.index.rename(None))
            df.columns.name = None
        except:
            pass
        return df

    def __call__(self, **kwargs):
        # input data loading and formatting
        url = self.build_url(**kwargs)#analysis:ignore
        resp = self.get_response()#analysis:ignore
        df = self.load_json(pivot='time')
        return df


import sys
sys.path.append('..')
import numpy as np
from collections import defaultdict
import numpy as np
import random
from copy import deepcopy
import algorithms.TD.TD_lambda_backwards_eval_SARSA as TDeval

class TD_on_policy(object):
    """
    This is an implementation of a TD policy iteration algorithm, using an on-policy strategy.
    The evaluation loop is going to be backwards TD lambda evaluation of one episode at a time, updating Q(v,s) in an online fashion.
    The improvement step will follow an epsilon greedy or maybe epsilon soft strategy.

    Args:
        env: OpenAI env
        improvement_rounds: Number of rounds of policy evaluation-->improvement complete cycles before termination
        discount_factor: Gamma discount factor for future rewards
        learning_rate: During evaluation step, aka alpha, controls size of our q function update
        lambda_value: During evaluarion step, plays a role in eligibility decay function
        eval_episodes: Number of episods for the evaluation loop to use to update the Q(s,a) function

    Returns:
        List of tabular policy dictionaries, each of the form {(s1, s2, s3): action, (s4, s5, s6): action}
        along the way to the final policy found
    """

    def __init__(self, env, discount_factor, learning_rate, lambda_value):
        # Create your policy evaluator you will use in prediction steps
        self.policy_evaluator = TDeval.TD_lambda_backwards_eval_SARSA(env=env,
                                                    learning_rate=learning_rate,
                                                    lambda_value=lambda_value,
                                                    discount_factor=discount_factor)
        self.nA = env.action_space.n
        print('''---Creating Iterator Object---
discount factor: {}
learning rate (alpha): {}
lambda: {}'''.format(discount_factor, learning_rate, lambda_value))

    def iterate(self, iteration_loops, print_every_n, eval_episodes, epsilon_soft):
        """
        Main loop of this policy iteration.
        Performs policy evaluation to update an action-value function.
        Then epsilon-greedily or perhaps epsilon softly picks a new policy based on this updated value function.
        Repeat a number of times.
        "And so on to viscosity" - Lupe Fiasco
        """
        current_policy_function = lambda state_input: random.randint(0,self.nA-1) # Initialize with random policy
        current_Q = None
        current_E = None
        iterations_completed = 0
        list_of_Q_dicts = [] # Maintain Qs throughout for comparison later
        while iterations_completed < iteration_loops:
            current_E, current_Q = self.policy_evaluator.evaluate(policy_function=current_policy_function, existing_Q=current_Q, num_episodes=eval_episodes, existing_E=current_E) # Use policy evaluator to update action-value function
            current_policy_function = self.epsilon_greedily_update_policy(current_Q=current_Q,
                                                             iterations_completed=iterations_completed,
                                                             epsilon_soft=epsilon_soft) # Use new Q function to epsilon greedily get new policy
            if iterations_completed % print_every_n == 0:
                print("Completed {} full policy iterations and saved most recent policy".format(iterations_completed))
                print(current_Q)
                list_of_Q_dicts.append(deepcopy(current_Q))
            iterations_completed += 1
        return list_of_Q_dicts

    def epsilon_greedily_update_policy(self, current_Q, iterations_completed, epsilon_soft):
        """
        Given a q function and number of loops completed,
        create an epsilon greedy policy from it.
        Returns a policy function which chooses an action from state.
        Epsilon = 1/iteration_number
        """
        iteration = iterations_completed+1
        # epsilon = min(1/np.log(iterations_completed+.0001),1)
        # epsilon = 1/iteration
        epsilon = epsilon_soft # Epsilon soft policy, always this epsilon random chance
        def new_policy(state):
            heads = True if random.random() < epsilon else False # Flip our epsilon greedy coin
            if heads: # If heads comes up, choose random action
                return random.randint(0, self.nA-1)
            else: # If tails comes up, choose greedy option
                return np.argmax(current_Q[state])
        return new_policy


"""
This script extracts node position(x,y,z) in MIDAS software through .mgt file
"""
import re
import numpy as np
with open('practical.mgt','r') as f:
	content=f.read()
pattern=re.compile(r'(\*NODE)(.*)(\*ELEMENT)',re.S)
extra=pattern.search(content).group(2)
with open('test.txt','w') as p:
	p.write(extra)
data=np.loadtxt('test.txt',delimiter=',',comments=';',skiprows=1)
datac=data[:,1:]
np.savetxt('testc.txt',datac,delimiter=',')


import matplotlib.image as mpimg
import cv2
import imageio
from pathlib import Path
import os
import numpy as np
from tqdm import tqdm

import dlib
detector = dlib.get_frontal_face_detector() #切割器
def clip_image(image, boundary):
    top = np.clip(boundary.top(), 0, np.Inf).astype(np.int16)
    bottom = np.clip(boundary.bottom(), 0, np.Inf).astype(np.int16)
    left = np.clip(boundary.left(), 0, np.Inf).astype(np.int16)
    right = np.clip(boundary.right(), 0, np.Inf).astype(np.int16)
    image = cv2.resize(image[top:bottom, left:right],(128,128))
    return image

path = "E:/VGGFACE2/msra/"
path = Path(path)
file_dirs = [x for x in path.iterdir() if x.is_dir()]
print("总文件夹数目为：",len(file_dirs))    #86876
start = 22000
end = start + 2000
print("现在计算的是",start,"-",end)
counter = start
for file_dir in tqdm(file_dirs[start:end]):
    new_path = "C:/facepairs/msra_" + str(counter)
    if os.path.exists(new_path):
        counter += 1
    else:
        os.makedirs(new_path)
        counter += 1

        image_path_list = list(file_dir.glob('*.jpg'))

        image_counter = 0
        for image_path in image_path_list:
            image = np.array(mpimg.imread(image_path))
            boundarys = detector(image, 2)

            if len(boundarys) == 1:
                image_new = clip_image(image, boundarys[0])
                image_path_new = new_path + "/msra_" + str(counter) + "_" + str(image_counter) + ".jpg"  # 这里可以对保存的地点调整路径
                imageio.imsave(image_path_new, image_new)
                image_counter += 1

# with tf.device("/CPU:0"):
#     image_path = "E:/face_DATA/lfw_160_none/Aaron_Eckhart/Aaron_Eckhart_0001.jpg"
#     img = mpimg.imread(image_path)
#     #img_1 = cv2.resize(img_1, (128, 128))
#     image_path_new = "C:/face_DATA/lfw_160_none/Aaron_Eckhart/Aaron_Eckhart_0001.jpg"
#     imageio.imsave(image_path_new, img)

    #image = tf.image.random_flip_left_right(image)
    #image = tf.image.random_jpeg_quality(image,80,100)


#!/usr/bin/env python3
"""Utils unit tests"""

from parameterized import parameterized
import unittest
from unittest.mock import patch
import utils

class TestAccessNestedMap(unittest.TestCase):
    """Class testing access_nested_map"""

    @parameterized.expand([
        ({"a": 1}, ("a",), 1),
        ({"a": {"b": 2}}, ("a",), {"b": 2}),
        ({"a": {"b": 2}}, ("a", "b"), 2)
    ])
    def test_access_nested_map(self, nested_map, path, expected):
        """Test access_nested_map"""
        self.assertEqual(utils.access_nested_map(nested_map, path), expected)

    @parameterized.expand([
        ({}, ("a",)),
        ({"a": 1}, ("a", "b"))
    ])
    def test_access_nested_map_exception(self, nested_map, path):
        """Test access_nested_map's raise when nested_map is not a Mapping"""
        self.assertRaises(KeyError, utils.access_nested_map, nested_map, path)

class TestGetJson(unittest.TestCase):
    """Class testing get_json"""

    @parameterized.expand([
        ("http://example.com", {"payload": True}),
        ("http://holberton.io", {"payload": False})
    ])
    def test_get_json(self, test_url, test_payload):
        """Test get_json"""
        with patch('requests.get') as mocked:
            mocked.return_value.json.return_value = test_payload
            self.assertEqual(utils.get_json(test_url), test_payload)
            mocked.assert_called_once()

class TestMemoize(unittest.TestCase):
    """ Class testing memoize """

    def test_memoize(self):
        """Test memoize"""
        class TestClass:
            def a_method(self):
                return 42

            @utils.memoize
            def a_property(self):
                return self.a_method()

        with patch.object(TestClass, 'a_method', return_value=42) as mocked:
            test_class = TestClass()
            self.assertEqual(test_class.a_property, mocked.return_value)
            self.assertEqual(test_class.a_property, mocked.return_value)
            mocked.assert_called_once()


# !/usr/bin python3
# encoding: utf-8 -*-
# @author: 沙陌 微信：Matongxue_2
# @Time: 2022-01-16 14:42
# @Copyright：北京码同学

# 几个高阶函数，了解下，主要是做面试题的时候可能会用到

# 计算给定序列中各个数字的2次方,生成一个新的列表
from functools import reduce

list1 = [2,4,6,7,2,6] # 新的是[4,16,36,49,4,36]

def ercifang(i):
    return i**2
# map(ercifang, list1) 含义是，将list1中的每个数据传递给函数ercifang进行计算，得到新结果
print(list(map(ercifang, list1)))
# 使用匿名函数
print(list(map(lambda i:i**2, list1)))
list2 = ['20cm','10mm','30mm','50cm','70cm']
# 提出list2这个列表中以cm结尾数据，形成新的列表
def jduge_cm(i:str): #i:str  注意这是声明入参的类型是字符串
    if i.endswith('cm'):
        return i
print(list(map(jduge_cm, list2)))

# 计算一个列表的所有项的乘几
list3 = [2,4,6,7,2,6]
# 需要导入from functools import reduce
print(reduce(lambda x, y: x * y, list3))

# 得到一个列表中所有的偶数
list4 = [1,2,3,4,5,6,7,8,9,10]
print(list(filter(lambda x: x % 2 == 0, list4)))

list5 = ['20cm','10mm','30mm','50cm','70cm']
# 提出list2这个列表中以cm结尾数据，形成新的列表
def jduge_cm(i:str): #i:str  注意这是声明入参的类型是字符串
    return i.endswith('cm')
print(list(filter(lambda i: i.endswith('cm'), list5)))
print(list(filter(jduge_cm, list5)))
#filter 的第一个参数是函数，这个函数的返回值是布尔值，表示是否满足条件


#!/usr/bin/python3
from mikatools import *
from sys import argv
from os import path

orig = argv[1]
target = argv[2]
print(orig, target)
if not path.exists(orig) and not path.exists(target):
	d = {"info":"take a look at UralicNLP model catalogue...\n\nFor people developing this resource, include a metadata.json to your repository to change this message."}
else:
	d = json_load(orig)
json_dump(d, target)