import os
from PIL import Image
import numpy as np
import shutil

def mkdirs(paths):
    if isinstance(paths, list) and not isinstance(paths, str):
        for path in paths:
            mkdir(path)
    else:
        mkdir(paths)

# if directory does not already exist, create path
def mkdir(path):
    if not os.path.exists(path):
        os.makedirs(path)

# return image saved in path
def load_image(path):
    with Image.open(path).convert('L') as img:      # convert image to b&w
        img = np.array(img)
        img = np.expand_dims(img, axis=2)       # expand shape of array, new axis is placed in position 2
        return img

#
def load_label(txt):        # txt = dataroot, cap_scheme, label, label + '_train/test.txt'
    with open(txt, 'r') as fp_txt:
        labels = fp_txt.read()
        labels = labels.split('#')
        return labels

# return dictionary created from character set used in captcha scheme
def char2dict(char_set):
    dict = {}
    for i, char in enumerate(char_set):
        dict[i] = char
    return dict

# return character dictionary used in given captcha scheme
def create_dict(cap_scheme):
    char_set = []
    scheme = CaptchaSchemes()
    if cap_scheme == 'min24':
        char_set = scheme.min24
    return char2dict(char_set)

# vector -> text
# vector: indicate where character position in dictionary
def vec2text(vect, dict):
    text = ''
    for i, key in enumerate(vect):
        value = dict[key]
        text += value
    return text

# print result
def print_predict(opt):
    scheme = CaptchaSchemes()
    preds = opt.preds
    preds_right_num = 0     # initialize
    char_set = []       # initialize
    total_num = len(preds)
    print('total_num : ', total_num)
    if opt.cap_scheme == 'min24':       # if the scheme is 민원24
        char_set = scheme.min24     # return character set of min24
    dict = char2dict(char_set)      # and create dictionary based on set
    if opt.isTune:
        label = 'real'
    else:
        label = 'synthetic'    # create and sort labels
    if opt.phase == 'train':
        real_label_name = os.path.join(opt.dataroot, opt.cap_scheme, label, label + '_train.txt')
    else:       # if phase is validation or testing
        real_label_name = os.path.join(opt.dataroot, opt.cap_scheme, label, label + '_test.txt')
    real_labels = load_label(real_label_name)
    for i, pred in enumerate(preds):        # for all images
        pred = vec2text(pred, dict)     # recognized text value of iamge
        if pred.lower() == real_labels[i].lower():      # convert to lowercase and compare
            preds_right_num += 1
            print('Correct Prediction')
        print('No.{} \t Predict: {} \t Real: {}' .format(i, pred, real_labels[i]))
    print('Recognition Accuracy : ', preds_right_num/total_num)

# define character set of each captcha scheme
class CaptchaSchemes:
    def __init__(self):
        self.min24 = [
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
        ]

#
if __name__ == '__main__':
    dict = create_dict('min24')     # create dictionary for min24 captcha scheme
    c = 'Z'

    def char2pos(c):
        k = -1
        for (key, value) in dict.items():
            if value == c:
                k = key
        return k

    key = char2pos(c)
    print(key)

def create_labels():
    img_path = '../data/min24/synthetic/min240_4246/'
    img_save_path = '../data/min24/synthetic/min244000-4246/'

    j = 0
    for i in range(4000, 4247, 1):
        shutil.copyfile(img_path + str(i) + '.jpg',
                        img_save_path + str(j) + '.jpg')
        j += 1


line = input()

nums = line.split()

n = int(nums[0])
m = int(nums[1])

board = []

for row in range(n):
    board.append([0] * m)

num = 1
for row in range(n):
    if row % 2 == 0: # when the line is an even number
        for col in range(m): # 0~4
            board[row][col] = num # In the place of (row, col), inset num
            num += 1 # increase num by 1
    if row % 2 == 1:
        for col in range(m-1, -1, -1):
            board[row][col] = num
            num += 1

for row in board:
    for col in row:
        print(col, end = ' ')
    print()


from django import forms

from onequiz.tests.BaseTest import BaseTest
from quiz.forms import EssayQuestionCreateForm

class EssayQuestionCreateFormTest(BaseTest):

    def setUp(self, path=None) -> None:
        self.basePath = path
        super(EssayQuestionCreateFormTest, self).setUp('')

    def testFieldsAndType(self):
        form = EssayQuestionCreateForm()
        self.assertEqual(len(form.base_fields), 5)

        self.assertTrue(isinstance(form.base_fields.get('figure'), forms.ImageField))
        self.assertEqual(form.base_fields.get('figure').label, 'Figure (Optional)')
        self.assertTrue(isinstance(form.base_fields.get('figure').widget, forms.ClearableFileInput))

        self.assertTrue(isinstance(form.base_fields.get('content'), forms.CharField))
        self.assertEqual(form.base_fields.get('content').label, 'Content (Optional)')
        self.assertTrue(isinstance(form.base_fields.get('content').widget, forms.Textarea))

        self.assertTrue(isinstance(form.base_fields.get('explanation'), forms.CharField))
        self.assertEqual(form.base_fields.get('explanation').label, 'Explanation (Optional)')
        self.assertTrue(isinstance(form.base_fields.get('explanation').widget, forms.Textarea))

        self.assertTrue(isinstance(form.base_fields.get('mark'), forms.IntegerField))
        self.assertEqual(form.base_fields.get('mark').label, 'Mark')
        self.assertTrue(isinstance(form.base_fields.get('mark').widget, forms.NumberInput))

        self.assertTrue(isinstance(form.base_fields.get('answer'), forms.CharField))
        self.assertEqual(form.base_fields.get('answer').label, 'Answer')
        self.assertTrue(isinstance(form.base_fields.get('answer').widget, forms.Textarea))

    def testFigureAndContentIsEmpty(self):
        testParams = self.TestParams(
            mark=50,
            answer='test answer',
        )
        form = EssayQuestionCreateForm(data=testParams.getData())
        self.assertFalse(form.is_valid())
        self.assertEqual(2, len(form.errors))
        self.assertTrue(form.has_error('figure'))
        self.assertTrue(form.has_error('content'))
        self.assertEqual('Cannot leave both figure and content empty.', form.errors.get('figure')[0])
        self.assertEqual('Cannot leave both figure and content empty.', form.errors.get('content')[0])

    def testMarkIsLessThanZero(self):
        testParams = self.TestParams(
            content='test content',
            mark=-1,
            answer='test answer',
        )
        form = EssayQuestionCreateForm(data=testParams.getData())
        self.assertFalse(form.is_valid())
        self.assertEqual(1, len(form.errors))
        self.assertTrue(form.has_error('mark'))
        self.assertEqual('Mark cannot be a negative number.', form.errors.get('mark')[0])

    def testAnswerFieldIsEmpty(self):
        testParams = self.TestParams(
            content='test content',
            mark=80,
        )
        form = EssayQuestionCreateForm(data=testParams.getData())
        self.assertFalse(form.is_valid())
        self.assertEqual(1, len(form.errors))
        self.assertTrue(form.has_error('answer'))
        self.assertEqual('Answer cannot be left empty.', form.errors.get('answer')[0])

    def testEssayQuestionObjectCreated(self):
        testParams = self.TestParams(
            content='test content',
            explanation='test explanation',
            mark=80,
            answer='test answer',
        )

        form = EssayQuestionCreateForm(data=testParams.getData())
        self.assertTrue(form.is_valid())

        newEssayQuestion = form.save()
        # self.assertIsNone(newEssayQuestion.question.figure)
        self.assertEqual(testParams.content, newEssayQuestion.question.content)
        self.assertEqual(testParams.explanation, newEssayQuestion.question.explanation)
        self.assertEqual(testParams.mark, newEssayQuestion.question.mark)
        self.assertEqual(testParams.answer, newEssayQuestion.answer)

    class TestParams:

        def __init__(self, figure=None, content=None, explanation=None, mark=None, answer=None):
            self.figure = figure
            self.content = content
            self.explanation = explanation
            self.mark = mark
            self.answer = answer

            pass

        def getData(self):
            data = {
                'figure': self.figure,
                'content': self.content,
                'explanation': self.explanation,
                'mark': self.mark,
                'answer': self.answer,
            }
            return data


import numpy as np
import matplotlib.pyplot as pt
import pandas as pd
from sklearn.tree import DecisionTreeClassifier

data = pd.read_csv("dataset/train.csv")
print(data.columns)
data = data.to_numpy()

classifier = DecisionTreeClassifier()

x_train = data[0:21000, 1:]
y_train = data[0:21000, 0]

classifier.fit(x_train, y_train)

# Now testing the dataset
x_test = data[21000:,1:]
y_test = data[21000:, 0]

d = x_test[8]
d.shape = (28, 28)
pt.imshow(255-d, cmap="gray")

pt.show()

# testing the accuracy
p = classifier.predict(x_test)
count = 0
for i in range(0, 21000):
    count+=1 if p[i] == y_test[i] else 0

print("Accuracy = ", (count/21000)*100)


import csv

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from model.datasets import SampleDataset

def collect_data(data: dict, csv_path, is_tp):
    # recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max
    reader = csv.DictReader(open(csv_path, "r"))
    fieldnames = reader.fieldnames

    for row in reader:
        species_id = row['species_id''']
        songtype_id = row['songtype_id']
        t_min = float(row['t_min'])
        t_max = float(row['t_max'])
        f_min = float(row['f_min'])
        f_max = float(row['f_max'])
        duration = t_max - t_min

        key = f'{species_id}|{songtype_id}|{is_tp}'
        if key not in data:
            data[key] = {'count': 0, 'duration': [], 'f_min': [], 'f_max': []}

        data[key]['count'] = data[key]['count'] + 1
        data[key]['duration'].append(duration)
        data[key]['f_min'].append(f_min)
        data[key]['f_max'].append(f_max)

def collect_mean_values(data):
    def _get_stat(data_item):
        return np.min(data_item), np.max(data_item), np.mean(data_item)

    for key, value in data.items():
        value['duration'] = _get_stat(value['duration'])
        value['f_min'] = _get_stat(value['f_min'])
        value['f_max'] = _get_stat(value['f_max'])

def analyze_data(path, is_tp):
    data = pd.read_csv(path)
    data[SampleDataset.k_key] = data['species_id'].astype(str) + '|' + data['songtype_id'].astype(str)
    data[SampleDataset.k_duration] = data['t_max'] - data['t_min']
    # data = data.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id], ascending=[True, True])

    gr_min = data.groupby(SampleDataset.k_key).min(numeric_only=True)
    gr_mean = data.groupby(SampleDataset.k_key).mean(numeric_only=True)
    gr_max = data.groupby(SampleDataset.k_key).max(numeric_only=True)

    gr_min = gr_min.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame
    gr_mean = gr_mean.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame
    gr_max = gr_max.sort_values([SampleDataset.k_species_id, SampleDataset.k_songtype_id])  # type:pd.DataFrame

    gr_orig = gr_min[[SampleDataset.k_species_id, SampleDataset.k_songtype_id]]
    gr_orig['tp'] = is_tp

    columns = [SampleDataset.k_f_min, SampleDataset.k_f_max, SampleDataset.k_duration]
    gr_min = gr_min[columns].add_suffix('_min')
    gr_mean = gr_mean[columns].add_suffix('_mean')
    gr_max = gr_max[columns].add_suffix('_max')

    new_data = pd.concat([gr_orig, gr_min, gr_mean, gr_max], axis=1)
    columns = [
        SampleDataset.k_species_id, SampleDataset.k_songtype_id, 'tp',
        SampleDataset.k_f_min + '_min', SampleDataset.k_f_min + '_mean', SampleDataset.k_f_min + '_max',
        SampleDataset.k_f_max + '_min', SampleDataset.k_f_max + '_mean', SampleDataset.k_f_max + '_max',
        SampleDataset.k_duration + '_min', SampleDataset.k_duration + '_mean', SampleDataset.k_duration + '_max',
    ]
    new_data = new_data[columns]

    return new_data

def main():
    statistics_data = {}
    statistics_data_tp = {}
    statistics_data_fp = {}

    # tips = sns.load_dataset("tips")
    # sns.displot(tips, x="size")
    # plt.show()

    # collect_data(statistics_data, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data_tp, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_tp.csv', 1)
    # collect_data(statistics_data_fp, r'd:\Projects\Kaggle\rfcx-species-audio-detection_data\train_fp.csv', 0)

    data_tp = analyze_data(r'../data/train_tp.csv', 1)
    data_fp = analyze_data(r'../data/train_fp.csv', 0)
    data_all = pd.concat([data_tp, data_fp])

    data_all.to_csv(r'../data/stat.csv', index=False)
    exit(0)

    # recording_id,species_id,songtype_id,t_min,f_min,t_max,f_max
    csv_tp = pd.read_csv(r'../data/train_tp.csv')
    csv_tp['key'] = csv_tp['species_id'].astype(str) + '|' + csv_tp['songtype_id'].astype(str)
    csv_tp['duration'] = csv_tp['t_max'] - csv_tp['t_min']
    csv_tp.sort_values(['f_min', 'f_max'], ascending=[True, True])

    csv_fp = pd.read_csv(r'../data/train_fp.csv')
    csv_fp['key'] = csv_fp['species_id'].astype(str) + '|' + csv_fp['songtype_id'].astype(str)
    csv_fp['duration'] = csv_fp['t_max'] - csv_fp['t_min']
    csv_fp.sort_values(['f_min', 'f_max'], ascending=[True, True])

    species_id_tp = sorted(set(csv_tp['species_id']))
    species_id_fp = sorted(set(csv_fp['species_id']))
    keys_tp = sorted(set(csv_tp['key']))
    keys_fp = sorted(set(csv_fp['key']))

    min_tp = csv_tp['f_min'].min()
    mean_tp = csv_tp['f_min'].mean()
    max_tp = csv_tp['f_max'].max()
    min_fp = csv_fp['f_min'].min()
    mean_fp = csv_fp['f_min'].mean()
    max_fp = csv_fp['f_max'].max()

    d_min_tp = csv_tp['duration'].min()
    d_mean_tp = csv_tp['duration'].mean()
    d_max_tp = csv_tp['duration'].max()
    d_min_fp = csv_fp['duration'].min()
    d_mean_fp = csv_fp['duration'].mean()
    d_max_fp = csv_fp['duration'].max()

    df_1 = csv_tp[['key', 'f_min']]
    df_1['f_type'] = 'min'
    df_1.rename(columns={'f_min': 'f'}, inplace=True)
    df_2 = csv_tp[['key', 'f_max']]
    df_2['f_type'] = 'max'
    df_2.rename(columns={'f_max': 'f'}, inplace=True)

    df = pd.concat([df_1, df_2])
    df = df.sort_values(['f'], ascending=[False])
    # grouped = csv_tp.groupby("key")
    # users_sorted_average = pd.DataFrame({col: vals['key'] for col, vals in grouped}).mean().sort_values(
    #     ascending=True)

    # sns.displot(csv_tp, x='key', y='duration', height=5, aspect=3)
    sns.displot(csv_tp, x='key', y='duration', height=5, aspect=3)
    plt.show()

    # sns.catplot(x="key", y="f_min",data=csv_tp)
    sns.catplot(x="key", y="f", hue='f_type', data=df)
    plt.show()
    exit(1)

    collect_mean_values(statistics_data_tp)

    keys = sorted(statistics_data_fp.keys())
    counts = [statistics_data_fp[k]['count'] for k in keys]

    sns.displot(x=keys, y=counts, height=5, aspect=3)
    plt.rcParams["figure.figsize"] = (10, 5)
    plt.show()

if __name__ == '__main__':
    main()


import numpy as np

from tello.lib.libh264decoder import H264Decoder
from tello.data_providers.data_provider import DataProvider
from tello.utils import TELLO_CAMERA_PORT, TELLO_IP, TELLO_COMMAND_PORT

class CameraDataProvider(DataProvider):
    """
    Provider of the latest RGB camera image from the Tello drone as :obj:`np.array` with shape
    (height x width x 3).

    .. note::

        Internally the camera provider works like this: It collects all messages (data) belonging to
        a single image frame. The data is encoded with H264 so an H264 decoder is used to decode the
        content of the collected data. Finally, the decoded data is converted into a numpy array.

    """

    def __init__(self):
        super(CameraDataProvider, self).__init__(TELLO_CAMERA_PORT)

        self.decoder = H264Decoder()

    # ------------------------------------- PUBLIC INTERFACE ------------------------------------- #

    def start(self):
        super(CameraDataProvider, self).start()
        # Trigger the streaming service of the drone
        self._socket.sendto("streamon".encode(), (TELLO_IP, TELLO_COMMAND_PORT))

    # ----------------------------------- PROTECTED INTERFACE ------------------------------------ #

    def _receiving_thread_fn(self):
        """
        Collect and decode camera images frames from the Tello drone and update the latest data with
        the newest camera image.
        """
        while True:
            encoded_frame_data = self._collect_frame_data()

            decoded_frames = self.decoder.decode(encoded_frame_data)

            if decoded_frames:
                # Unpack the latest frame from the output of the H264 decoder (there should be only
                # one, but it returns an array anyway)
                (decoded_frame, width, height, _) = decoded_frames[-1]
                rgb_image = self._decoded_frame_to_numpy(decoded_frame, width, height)

                self._set_latest_data(rgb_image)
            else:
                print("WARNING: No frames decoded!")

    def _collect_frame_data(self):
        """
        Keep receiving data from the bound socket until the end of one image frame. Then return it
        as one chunk of data for decoding.

        .. note::

            Go to https://github.com/dji-sdk/Tello-Python/blob/master/Tello_Video/tello.py for the
            original implementation.

        """
        frame_data = "".encode()  # Received data are binary strings

        while True:
            # Blocking receive because we want to update the latest data only if we get new data
            message, ip = self._socket.recvfrom(2048)
            # There can be leftover responses from initialization
            if message == "ok".encode():
                continue

            frame_data += message

            if len(message) != 1460:
                break

        return frame_data

    def _decoded_frame_to_numpy(self, decoded_frame, width, height):
        """
        Convert the decoded string into a (height x width x 3) RGB image.

        Args:
            decoded_frame(str): Decoded string from the H264 decoder
            width(int): Width of the image frame
            height(int): Height of the image frame

        Return:
            :obj:`np.array` RGB camera image (height x width x 3)
        """
        frame = np.fromstring(decoded_frame, dtype=np.ubyte, count=len(decoded_frame), sep="")
        rgb_image = frame.reshape((height, width, 3))
        return rgb_image


import pandas as pd
import glob
import seaborn
import matplotlib.pyplot as plt

pd.set_option('display.width', 500)
pd.set_option('display.max_columns', 36)
files = glob.glob('C:/*.csv')
df = pd.concat((pd.read_csv(file) for file in files), ignore_index=True)
df = df.apply(lambda x: x.str.replace('"', '')).apply(lambda x: x.str.replace('=', ''))

def to_datetime(columnname):
    for items in list(columnname):
        df[items] = pd.to_datetime(df[items])

to_datetime(['Entered', 'Received', 'Completed'])

# create new column
df['Entered - Received'] = df['Entered'] - df['Received']

df = df.stack().reset_index()  # stack is pivot

new_df = pd.DataFrame(df)
new_df = new_df.xs(['level_1', 0], axis=1, drop_level=True)  # keep the level you want
new_df = new_df.sort_values(0)  # sort value
new_df = new_df.drop_duplicates(subset=0)  # drop duplicate, subset is the column that you want to drop duplicates on
new_df['allocation'] = new_df[0].diff()  # subtract from previous row for the timestamp column
new_df = new_df.rename(columns={'level_1': 'function', 0: 'timestamp'})  # rename column

new_df['minutes'] = pd.to_timedelta(new_df['allocation']).astype('timedelta64[m]')  # convert time to minutes
new_df2 = pd.DataFrame(data=new_df[['function', 'minutes']])
new_df2 = new_df2.groupby('function', as_index=False).sum()  # groupby function
new_df2 = pd.DataFrame(new_df2)

graph = seaborn.barplot(x=new_df2['function'], y=new_df2['minutes'])

for p in graph.patches:
    graph.annotate(str(p.get_height()), (p.get_x() * 1.005, p.get_height() * 1.005))

plt.legend(['Entered', 'Completed'], )
graph.get_legend().legendHandles[0].set_color('blue')
graph.get_legend().legendHandles[1].set_color('yellow')
plt.xlabel('Function')
plt.ylabel('Minutes')
plt.style.use('seaborn')
plt.title('Minutes per function')
plt.show()


import os
import sys
from random import Random, choices
import numpy as np
import configparser
from rtinspector import RealtimeInspector, RTIResult
from mario_constant import *

############################################################################
# Dependency Matrix Converter
############################################################################
class DMatConverter():
    """
    numpy array to string converter for key storing in hashtable
    """
    def Decode(sequence, bsize):
        res = np.array([ord(c) for c in sequence], dtype=int)
        return res.reshape(bsize)

    def Encode(mat):
        res = ''.join([chr(ele) for ele in mat.flatten().tolist()])
        return res

class DMat_11:
    """ Dependency Matrix [1 1] """
    def PrevState(matrix, unit):
        return DMatConverter.Encode(matrix[0:unit,0:unit])

    def NextState(matrix, unit):
        return DMatConverter.Encode(matrix[0:unit,unit:unit*2])

    @classmethod
    def Parse(cls, matrix, unit):
        return (cls.PrevState(matrix, unit), cls.NextState(matrix, unit))

class DMat_1101:
    """ Dependency Matrix [1 1; 0 1] """
    def PrevState(matrix, unit):
        return DMatConverter.Encode( np.concatenate((matrix[0:unit, 0:unit], matrix[unit:unit*2, unit:unit*2]),axis=1) )

    def NextState(matrix, unit):
        return DMatConverter.Encode(matrix[0:unit,unit:unit*2])

    @classmethod
    def Parse(cls, matrix, unit):
        return (cls.PrevState(matrix, unit), cls.NextState(matrix, unit))

class DMat_1111:
    """ Dependency Matrix [1 1; 1 1] """
    def PrevState(matrix, unit):
        return DMatConverter.Encode( np.concatenate((matrix[0:unit, 0:unit], matrix[unit:unit*2, 0:unit], matrix[unit:unit*2, unit:unit*2]),axis=1))

    def NextState(matrix, unit):
        return DMatConverter.Encode(matrix[0:unit,unit:unit*2])

    @classmethod
    def Parse(cls, matrix, unit):
        return (cls.PrevState(matrix, unit), cls.NextState(matrix, unit))

############################################################################
# Mario Level Object
############################################################################
class MarioLevel:
    """
    Load level from file and convert into numpy array.
    """
    def __init__(self, path):
        self.data = None
        self.load_level(path)
        pass

    def load_level(self, path):
        f = open(path, 'r')
        # load the each lines of level
        lines = []
        for line in f.readlines():
            lines.append(line.strip())

        h = len(lines)
        w = len(lines[0])
        self.data = np.zeros((h, w), dtype=int)
        self.data.fill(MarioSprite.EMPTY.value)

        i = 0
        for line in lines:
            # convert into np.array and remove the unnecessary sprites
            self.data[i,:] = np.array([MarioSprite.EMPTY.value if ord(c) in ALL_SPRITES_TO_BE_REMOVED else ord(c) for c in line.strip()][0:w])
            i += 1

        f.close()

    def get_data(self):
        return self.data

    def EncodeLevel(data):
        lines = data.tolist()
        out = []
        for line in lines:
            out.append(''.join([chr(c) for c in line]))
        return '\n'.join(out)

############################################################################
# Dependency Table
############################################################################
class DependencyTable:
    """
    Stores the markov states(t), states(t+1) and frequency
    """
    def __init__(self):
        self._table= {}
        pass

    def train(self, states):
        # train
        prev_state = states[0]
        next_state = states[1]
        if not self._table.get(prev_state):
            self._table[prev_state] = {}
        count = self._table[prev_state].get(next_state)
        if count == None: self._table[prev_state][next_state] = 0
        self._table[prev_state][next_state] +=1
        pass

    def generate_state(self, state):
        # generate
        data = None
        table = self._table.get(state)
        if not table: return data
        data = choices([k for k in table.keys()], [v for v in table.values()])
        if type(data) == list: data = data[0]
        return data

############################################################################
# Mario Core Markov Chains training and generator
############################################################################
# * use decode, encode only in this class
class MarioCoreMC:
    """
    Mario Core Markov Chain object
    """
    def __init__(self, unitsize):
        self._unit = BLOCK_UNIT if not unitsize else unitsize

        # dependency tables
        self._dt_wall = DependencyTable()
        self._dt_bottom = DependencyTable() # same with dt_11

        self._dt_11 = DependencyTable()
        self._dt_1101 = DependencyTable()
        self._dt_1111 = DependencyTable()
        pass

    def train(self, level):
        """ train the level """
        ldata = level.get_data()
        nrows = ldata.shape[0]/self._unit

        ######
        # 1. save the wallet information
        self._dt_wall.train((DEFAULT_STATE, DMatConverter.Encode( ldata[:, 0:self._unit])))

        ######
        # 2. train the bottom lines
        n = int((nrows-1)*self._unit)
        bottoms = ldata[n:n+self._unit, :]

        length = bottoms.shape[1]
        if length % self._unit is not 0: length -= (length % self._unit)

        for xoff in range(self._unit,length,self._unit):
            nstate = bottoms[:,xoff:xoff+self._unit]
            mat = bottoms[:,xoff-self._unit:xoff+self._unit]
            self._dt_bottom.train(DMat_11.Parse(mat, self._unit))

        ######
        # 3. train from bottom to top and left to right : to 3 dependency table
        for xoff in range(self._unit, length, self._unit):
            for yoff in range(ldata.shape[0] - 2*self._unit, -1, -self._unit):
                # train to D1101 D1111
                mat = ldata[yoff:yoff+self._unit*2, xoff-self._unit:xoff+self._unit]
                self._dt_1101.train(DMat_1101.Parse(mat, self._unit))
                self._dt_1111.train(DMat_1111.Parse(mat, self._unit))

                # train to D11
                self._dt_11.train(DMat_11.Parse(ldata[yoff:yoff+self._unit, xoff-self._unit:xoff+self._unit], self._unit))
        pass

    def generate(self, output, rti):
        """ generate new level """
        (h, w) = output.shape
        rti.reset(output, self._unit)

        ######
        # 1. generate the left wall
        wall = self._dt_wall.generate_state(DEFAULT_STATE)
        wall = DMatConverter.Decode(wall, (h, self._unit))
        output[:, 0:self._unit] = wall[:,:]

        ######
        # 2. generate the bottom
        #output[h-self._unit:h, 0:self._unit] = self._start_block[:,:]
        retry = True
        if w%self._unit != 0:
            w -= w%self._unit
        while retry:
            cur = output[h-self._unit:h, 0:self._unit]
            for xoff in range(self._unit, w, self._unit):
                generated = self._dt_bottom.generate_state(DMatConverter.Encode(cur))
                if not generated: break
                cur = DMatConverter.Decode(generated, (self._unit, self._unit))
                output[h-self._unit:h, xoff:xoff+self._unit] = cur[:,:]

            retry = not rti.check_bottom()

        #####
        # 3. generate whole tiles
        for xoff in range(self._unit, w, self._unit):
            for yoff in range(h - 2*self._unit, -1, -self._unit):
                res = rti.inspect((xoff, yoff)) ## inspect whenever changed new column

                # check current position
                generated = None

                # dependency matrix methods
                while res is not RTIResult.TRUE:
                    #
                    if res is RTIResult.DMAT_1111:
                        cur = output[yoff:yoff+self._unit*2, xoff-self._unit:xoff+self._unit]
                        generated = self._dt_1111.generate_state(DMat_1111.PrevState(cur, self._unit))
                    #
                    elif res is RTIResult.DMAT_1101:
                        cur = output[yoff:yoff+self._unit*2, xoff-self._unit:xoff+self._unit]
                        generated = self._dt_1101.generate_state(DMat_1101.PrevState(cur, self._unit))
                    #
                    elif res is RTIResult.DMAT_11:
                        cur = output[yoff:yoff+self._unit, xoff-self._unit:xoff+self._unit]
                        generated = self._dt_11.generate_state(DMat_11.PrevState(cur, self._unit))
                    #
                    elif res is RTIResult.ERROR:
                        print("retry.... this happens when blocksize is too big or training data is not enouch.")
                        break
                        # sys.exit(1)

                    res = rti.check_block(generated)
                    if res is RTIResult.TRUE:
                        output[yoff:yoff+self._unit, xoff:xoff+self._unit] = DMatConverter.Decode(generated, (self._unit, self._unit))[:,:]
                ######################## end while

        # placing MARIO_START MARIO_EXIT
        startpos = rti.mario_start_pos()
        exitpos = rti.mario_exit_pos()

        if startpos:
            output[startpos[1], startpos[0]] = MarioSprite.MARIO_START.value
        output[exitpos[1], exitpos[0]] = MarioSprite.MARIO_EXIT.value

        # TODO self._rt_inspector.place enemy / coin / items
        pass #generate()

############################################################################
# Group Generator
############################################################################
class GroupQLevelGenerator:
    """
    Mario Level Generator of GroupQ
    """

    def __init__(self, config_path):
        self._config = None
        self.load_config(config_path)

        # load config
        self._unitsize = self._config[CONFIG_PARAMS].get('block_unit_size')
        if not self._unitsize: self._unitsize = DEFAULT_BLOCK_UNIT
        else: self._unitsize = int(self._unitsize)

        self._rt_inspector = RealtimeInspector(self._config)
        self._markovchain = MarioCoreMC(self._unitsize)
        # generating output
        self._output = np.zeros(DEST_SIZE, dtype=int)

    def load_config(self, config_path):
        # load config
        arg_load = config_path
        self._config = configparser.ConfigParser()
        self._config.read(arg_load)

    def train_levels(self):
        """ training the  mario levels from input paths in config file """
        te = TimeEstimate().start()

        # looping input paths
        for key in self._config[CONFIG_INPUTPATH]:
            base_path = self._config[CONFIG_INPUTPATH][key]
            # load levels
            flist = os.listdir(base_path)
            files = []
            [files.append(f) if f.endswith('txt') else None for f in flist]
            for filename in files:
                level = MarioLevel(os.path.join(base_path, filename))
                # train level
                self._markovchain.train( level )
        return te.stop()

    def generate_level(self):
        """generate the mario level"""
        te = TimeEstimate().start()

        # reset output and call genrate method of markov chain object
        self._output.fill(MarioSprite.EMPTY.value)
        self._markovchain.generate(self._output, self._rt_inspector)

        return te.stop()

    def save_file(self, idx):
        """ save the generated file to lvl_{idx}.txt """
        # ready for save
        filename = os.path.join(self._config[CONFIG_OUTPUTPATH]["path"], "lvl-{}.txt".format(idx))
        f = open(filename, 'w')
        f.write(MarioLevel.EncodeLevel(self._output))
        f.close()
        return filename

    def generate_levels(self):
        """ generate the levels and save to path"""
        nlevels = self._config[CONFIG_PARAMS].get('num_levels')
        nlevels = int(nlevels) if nlevels else DEFAULT_NUM_OF_LEVELS

        for i in range(nlevels):
            t = self.generate_level()
            fn = self.save_file(i+1)
            print("level generating time : {:.3f} secs, saved to : '{}'".format(t, fn))


# 2회차 코드
def solution(s):
    s = s[2:-2]
    s = s.split("},{")
    tmp = []
    for x in s:
        x = x.split(",")
        tmp.append(set(x))
    tmp.sort(key=lambda x:len(x))

    res = [int(*tmp[0])]
    for i in range(1, len(tmp)):
        nextNum = tmp[i] - tmp[i-1]
        res.append(int(*nextNum))
    return res

# 1회차 코드
def solution(s):
    s = s[2:-2]
    l = s.split("},{")
    res = []
    for x in l:
        tmp = x.split(",")
        res.append(tmp)
    res.sort(key=lambda x:len(x))

    l1 = res[0]
    for i in range(1, len(res)):
        l2 = res[i]
        num = list(set(l2) - set(l1))
        l1.extend(num)
    return [int(n) for n in l1]

''' 첫번째 풀이 pass -> split 제대로 못해서 반복문 사용하여 문자열 구분시도
def solution(s):
    s = s[1:-1]
    tmpL, l = [], []
    num, check = "", 0
    for i in range(len(s)):
        if s[i] == "{":
            check = 1
        elif s[i] == "," and check == 1:
            tmpL.append(num)
            num = ""
        elif s[i] == "}":
            tmpL.append(num)
            l.append(tmpL)
            check = 0
            num = ""
            tmpL = []
        elif s[i].isdecimal():
            num += s[i]
    l.sort(key=lambda x:len(x))

    l1 = l[0]
    for i in range(1, len(l)):
        l2 = l[i]
        num = list(set(l2) - set(l1))
        l1.extend(num)
    return [int(n) for n in l1]
'''


class SuperMarket(object):

    def __init__(self, name, location, product="sarukulu"):
        self.name = name
        self.location = location

        self.inSlip = []
        self.inBasket = []

    # slip preparation

    def add2Slip(self, item):

            if item in self.inSlip:
                print("Oops: '{0}' already available. Check in list".format(item))

            else:
                self.inSlip.append(item)
                print("'{0}' add item to list".format(item))

    def delFromSlip(self, item):

            if item in self.inSlip:
                print("'{0}' removed from slip".format(item))
                self.inSlip.remove(item)

    def add2Basket(self, item):

            if item in self.inBasket:
                print("Oops: '{0}' already available. Check in list".format(item))

            elif item not in self.inSlip:
                print("Oops: '{0}' is not in slip.".format(item))

            else:
                self.inBasket.append(item)
                print("'{0}' add item to list".format(item))

    def delFromBasket(self, item):

            if item in self.inBasket:
                print("'{0}' removed from slip".format(item))
                self.inBasket.remove(item)

    def audit(self):

        pending = set(self.inSlip) - set(self.inBasket)
        if len(pending) > 0:
            print("Yet to buy - {0}".format(pending))
        else:
            print("Done with shoping")

    def show(self):
        print("inSlip   :", self.inSlip)
        print("inBasket :", self.inBasket)


from water_in_jugs_qlearning import WaterInJugsQLearning, Jug
import itertools

def main():

    jugs = [Jug('Jug1', 8, 8), Jug('Jug2', 5, 0), Jug('Jug3', 3, 0)]
    actions = itertools.permutations(range(len(jugs)), 2)
    wj_arena = WaterInJugsQLearning(
        start_state=jugs,
        actions=actions,
        goal_qty=4,
        gamma=0.8,
        max_episodes=1000,
        epsilon_greedy=True)
    solution_steps, scores, eps_list = wj_arena.train()

    print('*** SOLUTION ***')
    for step in solution_steps:
        print(step)

    print('*** SCORES ***')
    for score, e in zip(scores, eps_list):
        print("{score:.2f};{epsilon:.2f}".format(score=score, epsilon=e))

if __name__ == "__main__":
    main()


import pandas as pd
import argparse
#cd Documents/education/galvanize/code/grader

parser = argparse.ArgumentParser(descrption='Find top students')
parser.add_argument('filename', metavar='F',type=str)
parser.add_argument('percentage', metavar='F',type=int)
args = parser.parse_args()
print('args:', args)
filename = args.filename
percentage = args.percentage

#df = pd.read_csv('data/grades.csv')

df = pd.read_csv(filename)
#df.head(2)
df1 = df.iloc[:,[0,1,3,5]]
#df1.head(2)
#df1.apply(lambda z: print(type(z)))
#df1.apply(lambda z: print(len(z)))
df1.apply(lambda row: (row[1]), axis=1)
#df1.head(2)
df1['final'] = df1.apply(lambda row: (row[1]+row[2]+(2*row[3]))/400, axis=1)
#df1.head(2)
#df1.head(2)
#num_students = int(len(df) * 0.01)
num_students = int(len(df1) * percentage/100)
#num_students
indicies = df1.final.argsort()
#indicies.head()
#df1.final[1424]
filtered_students = df.iloc[indicies[-num_students:]][::-1]
#filtered_students
filtered_students.student_id.values
ids = filtered_students.student_id.values
print(ids)


import json
import argparse
import urllib2
from urlparse import urljoin

class ServoClient(object):
    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.entryPoint = "http://%s:%s" % (host, port)
        motorsUrl = self.query("/")["motors"]["url"]
        motors = self.query(motorsUrl)
        self.motorurl = motors[0]["url"]

    def query(self, url, verb="GET", json_data = None):
        data = None
        data_content_type = None
        # Encode json data object
        if json_data is not None:
            data = json.dumps(json_data)
            data_content_type = 'text/json'

        headers = {}
        if data_content_type:
            headers['Content-Type'] = data_content_type

        # create the url
        url = urljoin(self.entryPoint, url)

        # prepare the request
        req = urllib2.Request(url, data, headers=headers)
        req.get_method = lambda: verb

        # issue the request and return the response
        resp = urllib2.urlopen(req)
        body = resp.read()

        if body:
            return json.loads(body)

    def isOn(self):
        return self.query(self.motorurl)["status"] == "on"

    def set_state(self, st):
        self.query(self.motorurl, "PUT", {"status":st})

    def pulse(self, dt):
        self.query(self.motorurl, "PUT", {"status":"on", "duration": dt})

    def start(self):
        self.set_state("on")

    def stop(self):
        self.set_state("off")

if __name__=='__main__':
    parser = argparse.ArgumentParser(description='HTTP Server')
    parser.add_argument('host', type=str, help='The host to connect too')
    parser.add_argument('port', type=int, help='Listening port for HTTP Server', default=8080, nargs='?')
    parser.add_argument('state', help='The new state of the servo',  type=str)
    args = parser.parse_args()

    s = ServoClient(args.host, args.port)
    s.set_state(args.state)


INTOCODE_NOT_RUNNING = 0
INTCODE_HALTED = 99
INTCODE_ERROR = 100
INTCODE_WAITING_FOR_INPUT = 3

class IntCode:
  def __init__(self, memory):
    self.__memory = memory
    self.__position = 0
    self.__state = INTOCODE_NOT_RUNNING
    self.__verbose = False
    self.__inputs = []
    self.__outputs = []

  def state(self):
    return self.__state

  def verbose(self, verbose):
    self.__verbose = verbose

  def inputs(self, inputs):
    self.__inputs = inputs

  def outputs(self):
    return self.__outputs

  def __getParameter(self, offset, mode):
    if mode == 0:
      pos_of_value = self.__memory[self.__position + offset]
      value = self.__memory[pos_of_value]
      return (value, '@{}:{}'.format(pos_of_value, value))
    value = self.__memory[self.__position + offset]
    return (value, str(value))

  def run(self):
    while True:
      instruction = self.__memory[self.__position]
      opcode = instruction % 100
      param1_mode = int(instruction // 100 % 10)
      param2_mode = int(instruction // 1000 % 10)
      param3_mode = int(instruction // 10000 % 10)

      if opcode == 1:  # ADD
        a, label_of_a = self.__getParameter(1, param1_mode)
        b, label_of_b = self.__getParameter(2, param2_mode)
        assert param3_mode == 0
        pos_of_c = self.__memory[self.__position + 3]
        c = a + b
        if self.__verbose:
          print('ADD', label_of_a, '+', label_of_b,
                '=', '@' + str(pos_of_c), c)
        self.__memory[pos_of_c] = c
        self.__position += 4

      elif opcode == 2:  # MUL
        a, label_of_a = self.__getParameter(1, param1_mode)
        b, label_of_b = self.__getParameter(2, param2_mode)
        assert param3_mode == 0
        pos_of_c = self.__memory[self.__position + 3]
        c = a * b
        if self.__verbose:
          print('MUL', label_of_a, '*', label_of_b,
                '=', '@' + str(pos_of_c), c)
        self.__memory[pos_of_c] = c
        self.__position += 4

      elif opcode == 3:  # INPUT
        assert param1_mode == 0
        pos_of_input = self.__memory[self.__position + 1]
        if self.__verbose:
          print('INPUT', '@' + str(pos_of_input))
        if len(self.__inputs) == 0:
          self.__state = INTCODE_WAITING_FOR_INPUT
          break
        str_value = self.__inputs[0]
        del self.__inputs[0]
        self.__memory[pos_of_input] = int(str_value)
        self.__position += 2

      elif opcode == 4:  # OUTPUT
        # if param1_mode != 0:
        #   print('Suprise :', instruction)
        # assert param1_mode == 0
        pos_of_output = self.__memory[self.__position + 1]
        if self.__verbose:
          print('output', '@' + str(pos_of_output))
        value = self.__memory[pos_of_output]
        self.__outputs.append(value)
        self.__position += 2

      elif opcode == 5:  # JUMP-IF-TRUE
        condition, label_of_condition = self.__getParameter(1, param1_mode)
        offset, label_of_offset = self.__getParameter(2, param2_mode)
        if self.__verbose:
          print('JUMP-IF-TRUE', label_of_condition, '? ->', label_of_offset)
        if condition != 0:
          self.__position = offset
        else:
          self.__position += 3

      elif opcode == 6:  # JUMP-IF-FALSE
        condition, label_of_condition = self.__getParameter(1, param1_mode)
        offset, label_of_offset = self.__getParameter(2, param2_mode)
        if self.__verbose:
          print('JUMP-IF-FALSE', label_of_condition, '? ->', label_of_offset)
        if condition == 0:
          self.__position = offset
        else:
          self.__position += 3

      elif opcode == 7:  # LESS-THAN
        a, label_of_a = self.__getParameter(1, param1_mode)
        b, label_of_b = self.__getParameter(2, param2_mode)
        assert param3_mode == 0
        pos_of_c = self.__memory[self.__position + 3]
        if self.__verbose:
          print('LESS-THAN', label_of_a, '<', label_of_b,
                '? ->', '@' + str(pos_of_c))
        if a < b:
          self.__memory[pos_of_c] = 1
        else:
          self.__memory[pos_of_c] = 0
        self.__position += 4

      elif opcode == 8:  # EQUAL
        a, label_of_a = self.__getParameter(1, param1_mode)
        b, label_of_b = self.__getParameter(2, param2_mode)
        assert param3_mode == 0
        pos_of_c = self.__memory[self.__position + 3]
        if self.__verbose:
          print('EQUAL', label_of_a, '==', label_of_b,
                '? ->', '@' + str(pos_of_c))
        if a == b:
          self.__memory[pos_of_c] = 1
        else:
          self.__memory[pos_of_c] = 0
        self.__position += 4

      elif opcode == 99:  # HALT
        self.__state = INTCODE_HALTED
        break

      else:
        self.__state = INTCODE_ERROR
        print('unknown',
              '@{}:{}'.format(self.__position, self.__memory[self.__position]))
        break

def exec(memory, verbose, inputs=[]):
  intcode = IntCode(memory)
  intcode.verbose(verbose)
  intcode.inputs(inputs)
  intcode.run()
  return intcode.outputs()


#!/usr/bin/env python3
import rospy
from geometry_msgs.msg import Twist
from sensor_msgs.msg import Joy
from std_msgs.msg import Empty
from std_msgs.msg import String

# Author: Andrew Dai
# This ROS Node converts Joystick inputs from the joy node
# into commands for turtlesim

# Receives joystick messages (subscribed to Joy topic)
# then converts the joysick inputs into Twist commands
# axis 1 aka left stick vertical controls linear speed
# axis 0 aka left stick horizonal controls angular speed
def callback(data):

    if data.buttons[0] == 1:
        rospy.loginfo("iniciou %s")
        pub.publish()
        # takeoff.publish({})

# Intializes everything
def start():
    # publishing to "turtle1/cmd_vel" to control turtle1
    global pub

    pub = rospy.Publisher('/ardrone/takeoff',Empty,queue_size=10)

    # subscribed to joystick inputs on topic "joy"
    rospy.Subscriber("joy", Joy, callback)
    # starts the node
    rospy.init_node('Joy2Dronetakeoff')
    rospy.spin()

if __name__ == '__main__':
    start()


import pandas as pd
from random import randint
from Functions import population_size_calculator, max_nr_substations_in_chromosome
from Object_Defs import centroid, collection_of_centroids, \
        substation, collection_of_substations, chromosome, population
from Processes import generator, process, Operator
from Recorders import CollectionOfProgramRuns, CollectionOfPopulations, MinimalistRecorder
import time
from Statistics import Stats

if __name__ == "__main__":
    start_time = time.localtime()
    ## -------------- Adding loads from csv to centroid class -----------------

    loads = pd.read_csv('loads normalized capacity.csv')
    load_transition = loads[['0', '1', '2']]
    all_centroids = collection_of_centroids()
    for x,y,peak in load_transition.values:
        one_centroid = centroid(x,y,peak)
        all_centroids.add_centroid(one_centroid)

    ## ----------- Important parameters ---------------------------------------

    nr_program_runs = 50
    pop_size = 500
    generations = 250

    elitist_percentage = 5
    operator_dict = {Operator.crossover:0.7, Operator.mutate_substation_location:0.25, Operator.mutate_substation_size:0.05}
    operator_adaptive = 0  # 0 or 1
    old_pop_preservation_rate = 0.3  # 0 < 1

    local_search = 1 # 0 or 1
    local_search_generations = 1000
    local_search_pop_size = 10
    local_search_operator_dict = {Operator.mutate_substation_location:1}

    recorder = MinimalistRecorder()
    stats = Stats()
#    col_of_prog_runs = CollectionOfProgramRuns(all_centroids)

    for run_nr in range(nr_program_runs):
        print(f'Program run nr {str(run_nr).zfill(2)}')

        ## ------------- Instantiate population -------------------------------

#        number_of_substation_options = len(substation.get_max_cap_to_cost_index(substation).keys())
#        pop_size = population_size_calculator(all_centroids.get_number_of_centroids(), number_of_substation_options)
        pop = population(pop_size)
#        print('Population size = ' + str(pop.get_max_population_size()))

        ## ------------- Fill initial population with chromosomes -------------

        for i in range(pop.get_max_population_size()):
            new_chromosome_len = randint(1, max_nr_substations_in_chromosome(all_centroids.get_number_of_centroids()))
            coll_of_substations = collection_of_substations()
            for i in range(new_chromosome_len): #adding substations to coll_of_substations
                new_substation = generator.substation(all_centroids.get_x_bounds(), all_centroids.get_y_bounds())
                coll_of_substations.add_substation(new_substation)
            new_chromosome = chromosome(generator.gene(all_centroids, coll_of_substations))
            process.update_chrom_cost(new_chromosome)
            pop.add_chromosome(new_chromosome)

        ## ------------------ Other Initialization ----------------------------

        operator = Operator(operator_dict, adaptive=operator_adaptive)  # Instantiates an operator class instance for every new prgram run
#        col_of_pops = CollectionOfPopulations(all_centroids)

        ## ------------------ Evolution process -------------------------------
        generation_nr = 0
        while not(recorder.termination_condition(2)) and generation_nr < generations:
            elite_pop = population(pop_size)
            collection_of_chromosomes = pop.get_elitist(elitist_percentage)
            for chrom in collection_of_chromosomes:
                elite_pop.add_chromosome(chrom)
            new_pop = process.evolve(pop, elite_pop, all_centroids, operator, generation_nr, old_pop_preservation_rate)
            success_percentage_list = operator.update()  # Update weights every program run
            ## --------------- Recording --------------------------------------
#            col_of_pops.add_population(new_pop)  # recorder that records every chromosome instance
            if run_nr == nr_program_runs-1:  # Is this the last program run?
                recorder.add_operator_success_percentage(success_percentage_list, operator)
            recorder.add_generation_best_cost(new_pop.get_elitist(100)[0].get_cost(), generation_nr)
            ## --------------- Ancillary --------------------------------------
#            stats.generation(new_pop, operator, generation_nr)
            ## ----------------------------------------------------------------
            pop = new_pop
            generation_nr += 1

        ## ------------------- Local Search -----------------------------------
        if local_search == 1:
            local_search_operator_list = [Operator.mutate_substation_location]
            local_search_operator = Operator(local_search_operator_dict, adaptive=0)

            generation_nr = 0
            while generation_nr < local_search_generations:
                elite_pop = population(local_search_pop_size)
                elite_pop.add_chromosome(pop.get_best_chromosome())
                best_chrom_in_pop = population(local_search_pop_size)
                best_chrom_in_pop.add_chromosome(pop.get_best_chromosome())
                # Only use the best chromosome from the last population to create mutations.
                new_pop = process.evolve(best_chrom_in_pop, elite_pop, all_centroids, local_search_operator, generation_nr, 0)
                if generation_nr % 100 == 0:
                    recorder.add_generation_best_cost(new_pop.get_elitist(100)[0].get_cost(), generation_nr)
#                stats.generation(new_pop, operator, generation_nr)
                pop = new_pop
                generation_nr += 1
        ## --------------------------------------------------------------------
#        col_of_prog_runs.add_collection_of_pops(col_of_pops)
        recorder.add_best_program_run_chromosome(new_pop.get_elitist(100)[0])
        if run_nr == nr_program_runs-1:
            recorder.add_final_pop_chrom_info(new_pop)

    ## -------------- More Recording ------------------------------------------
    recorder.save_best_program_run_costs()
    recorder.save_final_population()
    recorder.save_operator_success_count()
    recorder.pickle_out(recorder)

#    stats.chromosome(pop.get_best_chromosome())

    ## -------------- Plotting ------------------------------------------------
#    recorder.plot_all_run_costs()

    print(f'Start Time: {start_time}')
    print(f'Finish Time: {time.localtime()}')


# from system
import logging
from beaker.session import Session, SessionObject
from beaker.middleware import SessionMiddleware
from pylons import tmpl_context as c

# Put after imports so log is not overwridden by an imported module.
log = logging.getLogger(__name__)

class KwmoSession(Session):
    def __init__(self, request, environ=None, id=None, invalidate_corrupt=False,
             use_cookies=True, type=None, data_dir=None,
             key='beaker.session.id', timeout=None, cookie_expires=True,
             cookie_domain=None, secret=None, secure=False,
             namespace_class=None, **namespace_args):

        self.environ = environ

        # Call parent constructor.
        Session.__init__(self, request, id, invalidate_corrupt,
             use_cookies, type, data_dir,
             key, timeout, cookie_expires,
             cookie_domain, secret, secure,
             namespace_class, **namespace_args)

    def _create_id(self):
        # Call parent _create_id method.
        Session._create_id(self)
        if self.environ and self.environ.has_key('wsgiorg.routing_args') and (len(self.environ['wsgiorg.routing_args']) > 1):
            if 'workspace_id' in self.environ['wsgiorg.routing_args'][1]:
                if self.environ['wsgiorg.routing_args'][1]['controller'] in ['admin_teambox', 'admin_file_download']:
                    self.id = 'wsadminid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                    log.debug('kwmo admin session id created ' + self.id)
                else:
                    self.id = 'wsid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                    log.debug('kwmo session id created ' + self.id)
            else:
                log.info('_failed to create kwmo session id, creating session with cookie id ' + self.id)
        else:
            log.warn('__failed to create kwmo session id, creating session with cookie id ' + self.id)

    def load(self):
        if not self.is_new:
            if self.environ and self.environ.has_key('wsgiorg.routing_args') and (len(self.environ['wsgiorg.routing_args']) > 1):
                if 'workspace_id' in self.environ['wsgiorg.routing_args'][1]:
                    if self.environ['wsgiorg.routing_args'][1]['controller'] in ['admin_teambox', 'admin_file_download']:
                        self.id = 'wsadminid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                        log.debug('kwmo admin session loaded ' + self.id)
                    else:
                        self.id = 'wsid' + self.environ['wsgiorg.routing_args'][1]['workspace_id'] + '__' + self.id
                        #log.debug('kwmo session loaded ' + self.id)

                else:
                    log.info('_failed to load kwmo session, loading session with cookie id ' + self.id)
            else:
                log.warn('__failed to load kwmo session, loading session with cookie id ' + self.id)

        #call parent load
        Session.load(self)

    def save(self):
        # Save permissions in session if needed.
        if c.perms.dirty:
            # Save permissions.
            log.debug("Permissions have changed: saving them into session.")
            self['perms'] = c.perms.to_dict()

        # Call parent save method.
        log.debug("Saving session.")
        Session.save(self)

class KwmoSessionObject(SessionObject):
    def _session(self):
        """Lazy initial creation of session object"""
        if self.__dict__['_sess'] is None:
            params = self.__dict__['_params']
            environ = self.__dict__['_environ']
            self.__dict__['_headers'] = req = {'cookie_out':None}
            req['cookie'] = environ.get('HTTP_COOKIE')
            if params.get('type') == 'cookie':
                self.__dict__['_sess'] = CookieSession(req, **params)
            else:
                self.__dict__['_sess'] = KwmoSession(req, environ, use_cookies=True,
                                                 **params)
        return self.__dict__['_sess']

class KwmoSessionMiddleware(SessionMiddleware):
    def __call__(self, environ, start_response):
        session = KwmoSessionObject(environ, **self.options)
        if environ.get('paste.registry'):
            if environ['paste.registry'].reglist:
                environ['paste.registry'].register(self.session, session)
        environ[self.environ_key] = session
        environ['beaker.get_session'] = self._get_session

        def session_start_response(status, headers, exc_info = None):
            if session.dirty() or (session.accessed() and self.options.get('auto')):
                session.persist()
                if session.__dict__['_headers']['set_cookie']:
                    cookie = session.__dict__['_headers']['cookie_out']
                    if cookie:
                        headers.append(('Set-cookie', cookie))
            return start_response(status, headers, exc_info)
        return self.wrap_app(environ, session_start_response)


import pytest
import random
import requests
import json
from Tools import Request
from Tools import RandomParam

def result_judge(data):
    try:
        message = json.loads(data.text)
        assert message['code'] == 0 or  message['code'] == 200, json.loads(data.text)
    except:
        assert False, data.text

@pytest.mark.repeat(5)
def test_selectJiHuiChangeLog():
    url = 'http://localhost:8089/crm-manager-master/sys/jiHuiZongKu/selectJiHuiChangeLog'
    data = RandomParam.get_param(['userGroupId|int|1|'])
    result = Request.get_request(url, data)
    result_judge(result)

@pytest.mark.repeat(5)
def test_selectGouTongJiLuInDifferentPeriod():
    url = 'http://localhost:8089/crm-manager-master/sys/jiHuiZongKu/selectGouTongJiLuInDifferentPeriod'
    data = RandomParam.get_param(['tableId|int|0|','userGroupId|int|1|','page|int|1|1','step|int|1|10'])
    result = Request.post_request(url, data)
    result_judge(result)


import datetime
import os
import pprint

from pymongo import MongoClient
from bson.objectid import ObjectId

client = MongoClient(MONGODB_URI)

db = client.bank

accounts_collection = db.accounts

select_by_balance = {"$match": {"balance":{"$lt":1000}}}

select_by_avg = {
    "$group": {
        "_id":"$account_type",
        "avg_balance":{"$avg":"balance"}
    }
}

pipeline  =[select_by_balance,select_by_avg]

result = accounts_collection.aggregate(pipeline)

for item in result:
    pprint.pprint(item)

client.close()


from enum import Enum

class warriorRank(Enum):
    Pushover= 1
    Novice= 2
    Fighter= 3
    Warrior= 4
    Veteran= 5
    Sage= 6
    Elite= 7
    Conqueror= 8
    Champion= 9
    Master= 10
    Greatest= 11

class Warrior():

    def __init__(self) -> None:
        self.level: int = 1
        self.rank_counter = 1
        self.rank: str = warriorRank(self.rank_counter).name
        self.experience = 100
        self.achievements = []

        self.MAX_EXPERIENCE = 10000
        self.MAX_LEVEL = self.MAX_EXPERIENCE // 100

    def manage_experience_and_rank(self, experience: int) -> None:

        """Manages the whole experience and rank calculation"""

        if self.experience + experience >= self.MAX_EXPERIENCE:
            self.experience = self.MAX_EXPERIENCE
            self.level = self.MAX_LEVEL
            self.set_rank_counter()
            self.set_rank()
        else:
            self.experience += experience
            if self.validate_level(self.level):
                self.set_level()
            self.set_rank_counter()
            self.set_rank()

    def set_achievement(self, achievement: str) -> None:

        """Tracks the training achievements"""

        self.achievements.append(achievement)

    def set_level(self) -> None:

        """Calculates and sets the current level of the warrior"""

        self.level = self.experience // 100

    def calculate_rank(self, level: int) -> int:

        """Calculates the current rank of the warrior based on his level"""

        if self.validate_level(level):
            return level // 10 + 1
        return 11

    def set_rank_counter(self) -> None:

        """Sets the rank counter based on the current level of the warrior"""

        self.rank_counter = self.calculate_rank(self.level)

    def set_rank(self) -> None:

        """Sets the current rank of the warrior based on rank counter and rank enum"""

        self.rank = warriorRank(self.rank_counter).name

    def calculate_battle_experience(self, opponent_level: int) -> int:

        """Calculates the whole experience gained from battles based on the level difference between both warriors"""

        if self.level == opponent_level:
            self.manage_experience_and_rank(10)

        elif self.level - 1 == opponent_level:
            self.manage_experience_and_rank(5)

        elif self.level - 2 >= opponent_level:
            self.manage_experience_and_rank(0)

        else:
            level_diff = opponent_level - self.level

            self.manage_experience_and_rank(20 * level_diff * level_diff)

    def check_defeat(self, opponent_level: int) -> bool:

        """Function to evaluate if fighter can compete with opponent or
        if opponent is to strong"""

        return (self.level <= opponent_level - 5) and (warriorRank(self.rank_counter).value < self.calculate_rank(opponent_level))

    def comment_fight(self, opponent_level: int) -> str:

        """Function to manage fight commenting after succesful fight"""

        if self.level >= opponent_level + 2:
            return "Easy fight"
        elif self.level >= opponent_level:
            return "A good fight"
        else:
            return "An intense fight"

    def validate_level(self, level: int) -> bool:

        """Validates the current level"""

        return 0 < level <= self.MAX_LEVEL

    def training(self, training_data: list[str, int, int]) -> str:

        """Function to manage the training abilities of a fighter"""

        description, experience, minumum_level = training_data

        if self.level >= minumum_level:
            self.set_achievement(description)
            self.manage_experience_and_rank(experience)
            return description

        else:
            return "Not strong enough"

    def battle(self, opponent_level: int) -> str:

        """Manages the battle of two warriors inclusive experience management etc."""

        if not self.validate_level(opponent_level):
            return "Invalid level"

        elif self.check_defeat(opponent_level):
            return "You've been defeated"

        else:
            fight_comment = self.comment_fight(opponent_level)
            self.calculate_battle_experience(opponent_level)
            return fight_comment


import requests
from typing import Literal, List, Union
import time
import functools

WIKIPEDIA_API_URL = 'https://en.wikipedia.org/w/api.php'

def params_info(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    inprop: Literal['protection', 'talkid', 'watched', 'watchers', 'visitingwatchers', 'notificationtimestamp', 'subjectid', 'url', 'readable', 'preload', 'displaytitle'] = None,
):
    """
    Example URL: https://en.wikipedia.org/w/api.php?action=query&format=json&prop=info&titles=Algebra
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'info',
        'titles': titles,
        'pageids': pageids,
        'inprop': inprop
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_categorymembers(
    cmtitle: str = None,
    cmpageid: int = None,
    cmtype: Literal['page', 'subcat', 'file'] = None,
    cmlimit: int = 'max',
):
    """
    API Reference: https://www.mediawiki.org/wiki/API:Categorymembers

    Example URL: https://en.wikipedia.org/w/api.php?action=query&format=json&list=categorymembers&cmtype=subcat&cmtitle=Category:Algebra&cmlimit=max
    """

    assert (cmtitle is None) ^ (cmpageid is None), 'Either cmtitle or cmpageid must be specified.'
    assert cmtype in ['page', 'subcat', 'file'], 'cmtype must be one of "page", "subcat", "file".'

    params = {
        'action': 'query',
        'format': 'json',
        'list': 'categorymembers',
        'cmtitle': cmtitle,
        'cmpageid': cmpageid,
        'cmtype': cmtype,
        'cmlimit': cmlimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_categories(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    cllimit: int = 'max',
):
    """
    List all categories the pages belong to.

    Example https://en.wikipedia.org/w/api.php?action=query&prop=categories&titles=Albert%20Einstein&format=json
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'categories',
        'titles': titles,
        'pageids': pageids,
        'cllimit': cllimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_iwlinks(
        titles: Union[str, List[str]] = None,
        pageids: Union[int, List[int]] = None,
        iwlimit: int = 'max',
    ):
    """
    Returns all interwiki links from the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&prop=iwlinks&titles=Albert%20Einstein
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'iwlinks',
        'titles': titles,
        'pageids': pageids,
        'iwlimit': iwlimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_links(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    plnamespace: int = None,
    pllimit: int = 'max',
):
    """
    Returns all links from the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&prop=links&titles=Algebra&
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'links',
        'titles': titles,
        'pageids': pageids,
        'plnamespace': plnamespace,
        'pllimit': pllimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_extlinks(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    elprotocol: Literal['http', 'https', 'ftp', 'ftps'] = None,
    ellimit: int = 'max',
):
    """
    Returns all external links from the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&prop=extlinks&titles=Algebra
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'extlinks',
        'titles': titles,
        'pageids': pageids,
        'elprotocol': elprotocol,
        'ellimit': ellimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_linkshere(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    lhnamespace: int = None,
    lhshow: Literal['redirect', 'nonredirect'] = None,
    lhlimit: int = 'max',
):
    """
    Returns all pages that link to the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&list=backlinks&bltitle=Algebra&bllimit=max
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'list': 'backlinks',
        'titles': titles,
        'pageids': pageids,
        'lhnamespace': lhnamespace,
        'lhshow': lhshow,
        'lhlimit': lhlimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_images(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    imlimit: int = 'max',
):
    """
    Returns all images from the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&prop=images&titles=Astronomy&format=json
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'images',
        'titles': titles,
        'pageids': pageids,
        'imlimit': imlimit
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_pageviews(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    pvipdays: int = 30,
):
    """
    Returns the number of pageviews for the given pages.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&titles=Mathematics&prop=pageviews&pvipdays=30
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'pageviews',
        'titles': titles,
        'pageids': pageids,
        'pvipdays': pvipdays
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_siteviews():
    """
    Returns the number of views of wiki site.

    Example https://en.wikipedia.org/w/api.php?action=query&format=json&meta=siteviews
    """
    params = {
        'action': 'query',
        'format': 'json',
        'meta': 'siteviews',
    }
    return params

def params_mostviewed():
    """
    Returns the most viewed pages. (based on last day's pageview count)
    Example https://en.wikipedia.org/w/api.php?action=query&format=json&list=mostviewed
    """
    params = {
        'action': 'query',
        'format': 'json',
        'list': 'mostviewed',
    }
    return params

def params_content_parse(
    page: str = None,
    prop: Literal['text', 'wikitext'] = None
):
    """
    Returns content of given pages, either as HTML or wikitext.

    Example https://en.wikipedia.org/w/api.php?action=parse&format=json&page=Algebra&prop=text&formatversion=2
    """
    params = {
        'action': 'query',
        'format': 'json',
        'prop': prop,
        'page': page,
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params

def params_content_extracts(
    titles: Union[str, List[str]] = None,
    pageids: Union[int, List[int]] = None,
    exchars: int = None,
    exsentences: int = 'max',
    exlimit: int = 'max',
):
    """
    Returns content of given pages, as extracts.

    Example https://en.wikipedia.org/w/api.php?action=query&prop=extracts&exsentences=10&exlimit=1&titles=Pet_door&explaintext=1&formatversion=2
    """
    assert (titles is None) ^ (pageids is None), 'Either titles or pageids must be specified.'
    assert (exchars is None) ^ (exsentences is None), 'Either exchars or exsentences must be specified.'

    params = {
        'action': 'query',
        'format': 'json',
        'prop': 'extracts',
        'titles': titles,
        'pageids': pageids,
        'exchars': exchars,
        'exsentences': exsentences,
        'exintro': None if isinstance(titles, str) or isinstance(pageids, int) else "",
        'exlimit': exlimit,
        'explaintext': 1,
        'formatversion': 2,
    }
    params = {k: v for k, v in params.items() if v is not None}
    return params


def debugExceptions():
    '''
    From http://code.activestate.com/recipes/65287/
    After this is run, attach a hook to exception events to launch debugger in post-mortem mode upon error
    '''
    import sys
    def info(type, value, tb):
       if hasattr(sys, 'ps1') or not sys.stderr.isatty():
          # we are in interactive mode or we don't have a tty-like
          # device, so we call the default hook
          sys.__excepthook__(type, value, tb)
       else:
          import traceback, pdb
          # we are NOT in interactive mode, print the exception...
          traceback.print_exception(type, value, tb)
          print
          # ...then start the debugger in post-mortem mode.
          pdb.pm()
    sys.excepthook = info


from heapq import heappush, heappop

def solution(land, height):
  answer = 0
  n = len(land)
  # ➊ 주변 노드 탐색을 위한 di, dj
  di = [-1, 0, 1, 0]
  dj = [0, 1, 0, -1]
  visited = [[False] * n for _ in range(n)]
  # ➋ 시작 노드 추가
  q = []

  heappush(q, [0, 0, 0])  # [비용, i, j]

  # ➌ BFS + 우선 순위 큐로 다음 노드 관리
  while q:
    cost, i, j = heappop(q)
    # ➍ 아직 방문하지 않은 경로만 탐색
    if not visited[i][j]:
      visited[i][j] = True
      # ➎ 현재까지 비용을 합산
      answer += cost
      for d in range(4):
        ni, nj = i + di[d], j + dj[d]
        # ➏ 유효한 인덱스일 경우
        if 0 <= ni < n and 0 <= nj < n:
          temp_cost = abs(land[i][j] - land[ni][nj])
          # ➐ 입력으로 주어진 height보다 높이차가 큰 경우
          if temp_cost > height:
            new_cost = temp_cost
          else:
            new_cost = 0
          # ➑ 다음 경로를 푸시
          heappush(q, [new_cost, ni, nj])

  return answer


from Lab1 import methods
from Lab1.container import Container

if __name__ == '__main__':
    input_str = input('Write keyword: ')
    input_file = input('Write input file: ')
    output_file = input('Write output file: ')
    app = Container(input_str)
    app.encrypt(input_file, output_file)
    key = app.decrypt(output_file)
    print(key)
    print(methods.string_to_bitstr(key))


import sql
import asyncio

HEADER = 8
HOST = "0.0.0.0"
PORT = 12345
FORMAT = "utf-8"
ADDR = (HOST, PORT)
DISCONNECT_MESSAGE = "!DISCONNECT"

async def handle(reader:asyncio.StreamReader, writer:asyncio.StreamWriter):
    while True:
        try:
            print(writer.get_extra_info('peername'))
            data = await reader.read(100)
            message = data.decode()
            addr = writer.get_extra_info('peername')
            print(f"Received {message!r} from {addr!r}")
            print(f"Send: {message!r}")
            writer.write(data)
            await writer.drain()
            if message == DISCONNECT_MESSAGE:
                print("Close the connection")
                writer.close()
                await writer.wait_closed()
                break
        except:
            print("Close the connection")
            writer.close()
            await writer.wait_closed()
            break
async def main():
    server = await asyncio.start_server(handle, '127.0.0.1', 8888)
    addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
    print(f'Serving on {addrs}')
    async with server:
        await server.serve_forever()
asyncio.run(main())


#!/usr/bin/python3
"""Handles communications with the evaluation server"""
import sys
import socket
import base64
import time
import argparse
import threading
import errno
import queue
import traceback
import os
from Crypto.Cipher import AES
from Crypto import Random

class PowerState:
    def __init__(self):
        self.lock = threading.Lock()
        self.voltage = 0.0  # voltage (in V)
        self.current = 0.0  # current (in A)
        self.cumpower = 0

class EvalClient:
    def __init__(self, address, secret_key):
        if len(secret_key) not in (16, 24, 32):
            raise TypeError('AES key must be either 16, 24, or 32 bytes long')
        rndfile = Random.new()
        self.iv = rndfile.read(16)
        self.address = address
        self.secret_key = secret_key
        self.sock = None

    @property
    def is_connected(self):
        return self.sock is not None

    def connect(self):
        """Attempts to connect to the server.

        Returns True if the connection was successful, False otherwise.
        """
        if self.sock:
            return True
        try:
            self.sock = socket.create_connection(self.address)
        except OSError as e:
            print(e, file=sys.stderr)
            return False
        return True

    def send(self, action, voltage, current, power, cumpower):
        assert self.is_connected
        cipher = AES.new(self.secret_key, AES.MODE_CBC, self.iv)
        msg = '#{action}|{voltage}|{current}|{power}|{cumpower}'.format(
                action=action, voltage=voltage, current=current,
                power=power, cumpower=cumpower)
        msg = msg.encode()
        padding = b' ' * (AES.block_size - len(msg) % AES.block_size)
        msg = padding + msg
        msg = self.iv + cipher.encrypt(msg)
        msg = base64.b64encode(msg)
        try:
            self.sock.sendall(msg)
        except OSError:
            self.sock = None
            raise

class EvalServerThread(threading.Thread):
    """Eval server communication thread"""

    def __init__(self, client, pow_state, action_queue, interval=10):
        super().__init__(name='EvalServerThread', daemon=True)
        self.client = client
        self.pow_state = pow_state
        self.action_queue = action_queue
        self.interval = interval  # Connection interval (in s)

    def run(self):
        try:
            self._run()
        except:
            traceback.print_exc()
            os._exit(1)

    def _run(self):
        while True:
            while not self.client.is_connected:
                print('connecting to', self.client.address, file=sys.stderr)
                if not self.client.connect():
                    time.sleep(self.interval)
            # Receive item
            action = self.action_queue.get()
            # Read voltage/current
            with self.pow_state.lock:
                voltage = self.pow_state.voltage
                current = self.pow_state.current
            # Try to send it to the evaluation server
            try:
                self.client.send(action=action, voltage=voltage,
                                 current=current, power=voltage*current,
                                 cumpower=self.pow_state.cumpower)
            except OSError as e:
                print(e, file=sys.stderr)
            finally:
                self.action_queue.task_done()

class MlThread(threading.Thread):
    """Thread that reads from the machine-learning stream"""

    def __init__(self, filepath, action_queue):
        super().__init__(name='MlThread', daemon=True)
        self.filepath = filepath
        self.action_queue = action_queue

    def run(self):
        try:
            while True:
                self.run_one()
        except:
            traceback.print_exc()
            os._exit(1)

    def run_one(self):
        print('opening', self.filepath, '...', file=sys.stderr)
        with open(self.filepath, 'r') as f:
            print('opened', self.filepath)
            for line in f:
                try:
                    self.action_queue.put_nowait(line.strip())
                except queue.Full:
                    pass

class PowThread(threading.Thread):
    """Thread that reads from the power-measurement stream"""

    def __init__(self, filepath, pow_state):
        super().__init__(name='PowThread', daemon=True)
        self.filepath = filepath
        self.pow_state = pow_state

    def to_current(self, sensor_value):
        "Converts the raw sensor value to current (in A)"

        VOLTAGE_REF = 5.1
        RS = 0.1  # Shunt resistor value (in ohms)

        # Remap the ADC value into a voltage number
        sensor_value = (sensor_value * VOLTAGE_REF) / 1023

        # Follow the equation given by the INA169 datasheet to determine
        # the current flowing through RS. Assume RL = 10k
        # Is = (Vout * 1k) / (RS * RL)
        return sensor_value / (10 * RS)

    def to_voltage(self, sensor_value):
        "Convert the raw potential sensor value to voltage (in V)"

        VOLTAGE_REF = 5.1
        return min(2 * (sensor_value * VOLTAGE_REF) / 1023, VOLTAGE_REF)

    def run(self):
        try:
            while True:
                self.run_one()
        except:
            traceback.print_exc()
            os._exit(1)

    def run_one(self):
        print('opening', self.filepath, '...', file=sys.stderr)
        with open(self.filepath, 'r') as f:
            print('opened', self.filepath, file=sys.stderr)
            for line in f:
                raw_voltage, raw_current = [x.strip() for x in line.split(',')]
                raw_voltage = int(raw_voltage)
                raw_current = int(raw_current)
                with self.pow_state.lock:
                    self.pow_state.voltage = self.to_voltage(raw_voltage)
                    self.pow_state.current = self.to_current(raw_current)

class CumpowerThread(threading.Thread):
    """Thread that updates cumpower"""

    def __init__(self, pow_state):
        super().__init__(name='CumpowerThread', daemon=True)
        self.pow_state = pow_state

    def run(self):
        try:
            self._run()
        except:
            traceback.print_exc()
            os._exit(1)

    def _run(self):
        while True:
            with self.pow_state.lock:
                self.pow_state.cumpower += (self.pow_state.voltage *
                        self.pow_state.current)
            time.sleep(1)

def main(args, prog=None):
    p = argparse.ArgumentParser(prog=prog)
    p.add_argument('--secret', default='hihihihihihihihi')
    p.add_argument('mlstream')
    p.add_argument('powstream')
    p.add_argument('server')
    p.add_argument('port', type=int)
    args = p.parse_args()
    wifi_client = EvalClient((args.server, args.port), args.secret)
    pow_state = PowerState()
    action_queue = queue.Queue(1)
    wifi_thread = EvalServerThread(wifi_client, pow_state, action_queue)
    ml_thread = MlThread(args.mlstream, action_queue)
    pow_thread = PowThread(args.powstream, pow_state)
    cumpower_thread = CumpowerThread(pow_state)
    wifi_thread.start()
    ml_thread.start()
    pow_thread.start()
    cumpower_thread.start()
    wifi_thread.join()
    ml_thread.join()
    pow_thread.join()
    cumpower_thread.join()

if __name__ == '__main__':
    main(sys.argv[1:], sys.argv[0])


import datetime
import functools
import json
import logging
import pathlib
import time
from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, Union

import numpy as np
from matplotlib import pyplot as plt
from msi_zarr_analysis import VERSION
from msi_zarr_analysis.ml.dataset import GroupCollection
from msi_zarr_analysis.ml.dataset.cytomine_ms_overlay import (
    collect_spectra_zarr,
    get_overlay_annotations,
    translate_annotation_mapping_overlay_to_template,
)
from msi_zarr_analysis.utils.autocrop import autocrop
from msi_zarr_analysis.utils.check import open_group_ro
from msi_zarr_analysis.utils.cytomine_utils import (
    get_lipid_dataframe,
    get_page_bin_indices,
)
from sklearn.base import BaseEstimator
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier
from sklearn.model_selection import GridSearchCV, StratifiedGroupKFold, StratifiedKFold
from sklearn.tree import DecisionTreeClassifier

@functools.lru_cache(maxsize=1)
def datetime_str() -> str:
    "return a datetime representation, constant through the program"
    return datetime.datetime.now().strftime("%W-%w_%H-%M-%S")

def build_segmentation_mask(
    zarr_path: str,
    estimator: BaseEstimator,
) -> np.ndarray:

    z_group = open_group_ro(zarr_path)
    z_ints = z_group["/0"]
    z_lens = z_group["/labels/lengths/0"]

    # assume small dataset
    n_ints = z_ints[:, 0, ...]
    n_lens = z_lens[0, 0, ...]
    selection_mask = n_lens > 0

    # select valid spectra only
    dataset = n_ints[:, selection_mask].T
    prediction = estimator.predict(dataset)

    # build prediction 2D mask
    prediction_mask = np.zeros(shape=n_lens.shape, dtype=np.uint8)
    prediction_mask[selection_mask] = prediction

    # build colored mask
    #   no prediction: transparent
    #   red: class 1
    #   green: class 2
    mask = np.stack(
        [
            np.where(prediction_mask, 200, 50),  # R
            np.where(prediction_mask, 50, 200),  # G
            np.where(prediction_mask, 50, 30),  # B
            np.where(n_lens, 255, 0),  # A
        ],
        axis=-1,
    )

    return mask

class MLConfig(NamedTuple):
    choices: Dict[Type[BaseEstimator], Dict[str, Any]]
    cv_fold_outer: Optional[int] = None
    cv_fold_inner: Optional[int] = None

class DSConfig(NamedTuple):
    image_id_overlay: int  # Cytomine ID for the overlay image
    local_overlay_path: str  # local path of the (downloaded) overlay
    lipid_tm: str  # name of the lipid to base the template matching on

    project_id: int  # project id
    annotated_image_id: int  # image with the annotations

    zarr_path: str  # path to the non-binned zarr image

    classes: Dict[str, List[int]]

    save_image: Union[bool, str] = False

    transform_rot90: int = 0
    transform_flip_ud: bool = False
    transform_flip_lr: bool = False

    annotation_users_id: Tuple[int] = ()  # select these users only

    zarr_template_path: str = None  # use another group for the template matching

def run(
    base: str,
    config_path: str,
    ml_config: MLConfig,
    bin_csv_path: str,
    *ds_config: DSConfig,
    grouped_cv: bool,
):
    if not ds_config:
        raise ValueError("a list of dataset configuration is required")

    from cytomine import Cytomine

    with open(config_path) as config_file:
        config_data = json.loads(config_file.read())
        host_url = config_data["HOST_URL"]
        pub_key = config_data["PUB_KEY"]
        priv_key = config_data["PRIV_KEY"]

    lipid_df = get_lipid_dataframe(bin_csv_path)

    # build ds
    with Cytomine(host_url, pub_key, priv_key):

        dataset_to_be_merged: List[GroupCollection] = []

        for ds_config_itm in ds_config:

            # build all datasets and merge them if there are more than one
            page_idx, bin_idx, *_ = get_page_bin_indices(
                ds_config_itm.image_id_overlay, ds_config_itm.lipid_tm, bin_csv_path
            )

            annotation_dict = get_overlay_annotations(
                ds_config_itm.project_id,
                ds_config_itm.image_id_overlay,
                ds_config_itm.classes,
                select_users=ds_config_itm.annotation_users_id,
            )

            annotation_dict = translate_annotation_mapping_overlay_to_template(
                annotation_dict,
                ds_config_itm.zarr_template_path,
                bin_idx,
                ds_config_itm.local_overlay_path,
                page_idx,
                ds_config_itm.transform_rot90,
                ds_config_itm.transform_flip_ud,
                ds_config_itm.transform_flip_lr,
            )

            dataset_to_be_merged.append(
                collect_spectra_zarr(
                    annotation_dict,
                    lipid_df.Name,
                    ds_config_itm.zarr_path,
                )
            )

        if len(dataset_to_be_merged) == 1:
            ds = dataset_to_be_merged[0]
        else:
            ds = GroupCollection.merge_collections(*dataset_to_be_merged)

    estimator = model_selection_assessment(
        ds,
        ml_config,
        grouped_cv=grouped_cv,
    )

    dir = pathlib.Path("results") / datetime_str() / "class_masks"
    dir.mkdir(parents=True, exist_ok=True)
    path = str(dir / base)

    with Cytomine(host_url, pub_key, priv_key):

        for ds_config_itm in ds_config:
            mask = build_segmentation_mask(ds_config_itm.zarr_path, estimator)
            stem = pathlib.Path(ds_config_itm.zarr_path).stem

            # build all datasets and merge them if there are more than one
            page_idx, bin_idx, *_ = get_page_bin_indices(
                ds_config_itm.image_id_overlay, ds_config_itm.lipid_tm, bin_csv_path
            )

            template_group = open_group_ro(ds_config_itm.zarr_template_path)
            template_lipid = template_group["/0"][bin_idx, 0, ...]

            crop_idx = autocrop(template_lipid)
            mask = mask[crop_idx]
            template_lipid = template_lipid[crop_idx]

            fig, ax = plt.subplots(dpi=250)

            ax.imshow(template_lipid, interpolation="nearest")
            ax.imshow(mask, interpolation="nearest", alpha=0.4)

            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

            fig.tight_layout()
            fig.savefig(path + "_" + stem + "_annotated.png")
            plt.close(fig)

            fig, ax = plt.subplots(dpi=250)

            ax.imshow(template_lipid, interpolation="nearest")

            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

            fig.tight_layout()
            fig.savefig(path + "_" + stem + ".png")
            plt.close(fig)

def model_selection_assessment(
    collection: GroupCollection,
    ml_config: MLConfig,
    *,
    grouped_cv: bool,
):
    logger = logging.getLogger()

    collection.dataset.check_dataset(print_=True)
    class_names = collection.dataset.class_names()
    logger.info(f"terms: {class_names}")

    # StratifiedKFold ignores groups
    CVType = StratifiedGroupKFold if grouped_cv else StratifiedKFold
    # one for each loop
    outer = CVType(n_splits=ml_config.cv_fold_outer, shuffle=True, random_state=31)
    inner = CVType(n_splits=ml_config.cv_fold_inner, shuffle=True, random_state=32)

    def select_model_(ds_x: np.ndarray, ds_y: np.ndarray, groups: np.ndarray):
        "select one model that best fits a dataset"

        best_res = None
        best_model = None
        best_score = -1

        for model, arg_dict in ml_config.choices.items():

            folds = list(inner.split(ds_x, ds_y, groups))
            search = GridSearchCV(model(), arg_dict, cv=folds, n_jobs=-1)
            search.fit(ds_x, ds_y)

            logger.debug(
                "in-loop: %s, %s, %s", model, search.best_params_, search.best_score_
            )

            if search.best_score_ > best_score:
                best_res = search
                best_model = model
                best_score = search.best_score_

        if best_model is None or best_res is None:
            raise ValueError("empty choice dict!")

        return best_model, best_res

    def assessment_(ds_x: np.ndarray, ds_y: np.ndarray, groups: np.ndarray):
        "assess the quality of the selection procedure"

        scores = []

        for train_idx, test_idx in outer.split(ds_x, ds_y, groups):
            train_set = (ds_x[train_idx], ds_y[train_idx])
            test_set = (ds_x[test_idx], ds_y[test_idx])

            model, res = select_model_(*train_set, groups[train_idx])

            try:
                estimator = model(**res.best_params_, n_jobs=-1)
            except TypeError:
                estimator = model(**res.best_params_)
            estimator.fit(*train_set)

            scores.append(estimator.score(*test_set))

        return np.array(scores)

    logger.info("starting assessment procedure")
    start = time.time()
    scores = assessment_(collection.data, collection.target, collection.groups)
    logger.info("duration: %s", time.time() - start)
    logger.info("scores: %s (%s pm %s)", scores, scores.mean(), scores.std(ddof=1))

    logger.info("starting selection procedure")
    start = time.time()
    model, res = select_model_(collection.data, collection.target, collection.groups)
    logger.info("duration: %s", time.time() - start)
    logger.info("model: %s, parameters: %s", model, res.best_params_)

    logging.info("training model")
    try:
        estimator = model(**res.best_params_, n_jobs=-1)
    except TypeError:
        estimator = model(**res.best_params_)
    estimator.fit(collection.data, collection.target)

    avg_gini = estimator.feature_importances_
    if hasattr(estimator, "estimators_"):
        std_gini = np.std(
            [tree.feature_importances_ for tree in estimator.estimators_], axis=0
        )
    else:
        std_gini = np.full_like(avg_gini, np.nan)

    # 1: feature importance
    logging.info("feature importances:")
    for idx, (feature, importance, stddev) in enumerate(
        zip(collection.dataset.attribute_names(), avg_gini, std_gini)
    ):
        if len(feature) > 20:
            feature = feature[:17] + "..."
        logging.info("  %02d (%20s) : %.4f +/- %.4f", idx, feature, importance, stddev)

    return estimator

def main():
    def filter(record: logging.LogRecord) -> bool:
        if record.name == "root":
            return True

        if record.name == "cytomine.client":
            record.name = "cyt-client"
            return record.levelno != logging.DEBUG

        return False

    # setup logger
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger.addFilter(filter)

    formatter = logging.Formatter(
        "%(asctime)s.%(msecs)03d [%(name)s] [%(levelname)s] : %(message)s",
        datefmt="%j %H:%M:%S",
    )

    # main handler is stdout
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.NOTSET)
    stream_handler.setFormatter(formatter)
    stream_handler.addFilter(filter)
    logger.addHandler(stream_handler)

    logger.info("starting %s (VERSION: %s)", __file__, str(VERSION))

    data_sources = [
        {
            "name": "region_13",
            "args": {
                "image_id_overlay": 545025763,
                "local_overlay_path": "datasets/Adjusted_Cytomine_MSI_3103_Region013-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 545025783,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis13_binned.zarr",
            },
            "base": "datasets/comulis13",
        },
        {
            "name": "region_14",
            "args": {
                "image_id_overlay": 548365416,
                "local_overlay_path": "datasets/Region014-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 548365416,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis14_binned.zarr",
            },
            "base": "datasets/comulis14",
        },
        {
            "name": "region_15",
            "args": {
                "image_id_overlay": 548365463,
                "local_overlay_path": "datasets/Region015-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 548365463,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis15_binned.zarr",
            },
            "base": "datasets/comulis15",
        },
    ]

    normalizations = [
        "",
        "_norm_2305",
        "_norm_max",
        "_norm_tic",
        "_norm_vect",
    ]

    problem_classes = {
        "LS SC": {  # both merged
            "LS": [544926097, 544926081],
            "SC": [544926052, 544924846],
        },
        "LS_n LS_p": {  # irradiation on LS
            "LS_n": [544926097],
            "LS_p": [544926081],
        },
        "SC_n SC_p": {  # irradiation on SC
            "SC_n": [544926052],
            "SC_p": [544924846],
        },
        "LS_n SC_n": {  # LS vs SC when irradiated
            "LS_n": [544926097],
            "SC_n": [544926052],
        },
        "LS_p SC_p": {  # LS vs SC before irradiation
            "LS_p": [544926081],
            "SC_p": [544924846],
        },
    }

    ml_config = MLConfig(
        choices={
            ExtraTreesClassifier: {
                "n_estimators": [200, 500, 1000],
                "max_features": ["sqrt", None],
                "max_depth": [20, None],
            },
            RandomForestClassifier: {
                "n_estimators": [200, 500, 1000],
                "max_features": ["sqrt", None],
                "max_depth": [20, None],
            },
            DecisionTreeClassifier: {
                "max_depth": [1, 20, None],
            },
        },
        cv_fold_inner=5,
        cv_fold_outer=5,
    )

    for normalization in normalizations:

        for name, classes in problem_classes.items():

            ds_lst = []
            for source in data_sources:
                zarr_path = source["base"] + normalization + "_binned.zarr"
                ds_lst.append(
                    DSConfig(
                        **source["args"],
                        classes=classes,
                        save_image=False,
                        zarr_path=zarr_path,
                    )
                )

            for grouped_cv in [True, False]:

                base = (
                    name
                    + (normalization or "_no_norm")
                    + ("_gcv" if grouped_cv else "_cv")
                )
                file_handler = logging.FileHandler(
                    f"logs/classification_task/{base}.log", mode="a"
                )
                file_handler.setLevel(logging.INFO)
                file_handler.setFormatter(formatter)
                file_handler.addFilter(filter)
                logger.addHandler(file_handler)

                logger.info("norm: '%s', problem: '%s'", normalization, name)

                run(
                    base,
                    "config_cytomine.json",
                    ml_config,
                    "mz value + lipid name.csv",
                    *ds_lst,
                    grouped_cv=grouped_cv,
                )

                logger.info("done")

                logger.removeHandler(file_handler)

if __name__ == "__main__":
    main()