import subprocess as sp
import os
import numpy as np
from pathlib import Path
import xml.etree.ElementTree as ET
import sys
import tempfile
import shutil
import time
import glob
from tempfile import TemporaryDirectory
from distutils.dir_util import copy_tree
import pytest
import re

driver_models = [
    "lj",
    "sg",
    "harm",
    "harm3d",
    "morse",
    "zundel",
    "qtip4pf",
    "pswater",
    "eckart",
    "ch4hcbe",
    "ljpolymer",
    "doublewell",
    "doublewell_1D",
    "MB",
]

def get_driver_info(
    example_folder,
    driver_info_file="driver.txt",
    driver="gas",
    socket_mode="unix",
    port_number=33333,
    address_name="localhost",
    flags=[],
):
    """This function looks for the existence of .driver_info file
    to run the example with a meaningfull driver. If the file doesn't
    exist, the driver gas is assigned."""
    try:
        with open(Path(example_folder) / driver_info_file) as f:
            flags = list()
            ncount = 0
            while True:
                line = f.readline()
                if not line:
                    break
                elif "driver" in line:
                    driver = line.split()[1]
                elif "socket_mode" in line:
                    socket_mode = line.split()[1]
                elif "port" in line:
                    port_number = line.split()[1]
                elif "address" in line:
                    address_name = line.split()[1]
                elif "flags" in line:
                    flags.append({line.split()[1]: line.split()[2:]})
                ncount += 1
            if ncount == 0:
                raise ValueError("driver.txt is empty")
    except FileNotFoundError:
        raise FileNotFoundError(
            "({}) An input.xml file was found but a driver.txt not".format(
                example_folder
            )
        )
    except:
        pass

    if driver not in driver_models:
        driver = "gas"

    driver_info = {
        "model": driver,
        "socket_mode": socket_mode,
        "port_number": port_number,
        "address_name": address_name,
        "flag": flags,
    }

    return driver_info

def get_info_test(parent):
    """This function recursively searches for examples
    and checks for the presence of the required additional info
    required (i.e. driver.txt)
    """
    folders = [x[0] for x in os.walk(parent)]
    reg_tests = list()

    for ff in folders:
        if os.path.isfile(Path(ff) / "input.xml"):
            try:
                driver_info = get_driver_info(ff)
                reg_tests.append([ff, driver_info])
            except FileNotFoundError:
                raise FileNotFoundError(
                    "({}) An input.xml file was found but a driver.txt not".format(ff)
                )
            except ValueError:
                raise ValueError(
                    'Please specify "model address      port mode" inside driver.txt'
                )

    return reg_tests

class Runner(object):
    """This class handles the creation of tmp directories,
    the i-pi call, the driver call, and finally
    it checks that the generated outputs are the expected ones.
    """

    def __init__(
        self,
        parent,
        call_ipi="i-pi input.xml",
        call_driver="i-pi-driver",
        check_errors=True,
        check_numpy_output=True,
        check_xyz_output=True,
    ):
        """Store parent directory and commands to call i-pi and driver
        call_ipi: command to call i-pi
        call_driver: list of commands to call drivers
        """

        self.parent = parent
        self.call_ipi = call_ipi
        self.call_driver = call_driver
        self.check_error = check_errors
        self.check_numpy_output = check_numpy_output
        self.check_xyz_output = check_xyz_output

    def _run(self, info, nid):
        """This function tries to run the example in a tmp folder and
        checks if ipi has ended without error.
        After that the output is checked against a reference
        arguments:
            cwd: folder where all the original regression tests are stored
        """

        cwd = info[0]
        self.files = []
        self.forms = []

        try:
            # Create temp file and copy files
            self.tmp_dir = Path(tempfile.mkdtemp())
            print("temp folder: {}".format(self.tmp_dir))

            files = os.listdir(self.parent / cwd)
            for f in files:
                shutil.copy(self.parent / cwd / f, self.tmp_dir)
            driver_info = get_driver_info(self.tmp_dir)

            # Creating the clients list
            input_name = self.tmp_dir / "input.xml"
            output_name = self.tmp_dir / "input.xml"
            try:
                tree = ET.parse(input_name)
            except:
                print("The error is in the format or the tags of the xml!")
            root = tree.getroot()

            clients = list()

            for s, ffsocket in enumerate(root.findall("ffsocket")):
                name = ffsocket.attrib["name"]
                mode = driver_info["socket_mode"]
                ffsocket.attrib["mode"] = mode

                for element in ffsocket:
                    port = driver_info["port_number"]
                    if element.tag == "port":
                        element.text = str(port)
                    elif element.tag == "address":
                        dd = driver_info["address_name"] + "_" + str(nid) + "_" + str(s)
                        element.text = dd
                        address = dd

                model = driver_info["model"]
                clients.append([model, address, port, mode])

                for flag in driver_info["flag"]:
                    for k, v in flag.items():
                        clients[s].append(k)
                        clients[s].extend(v)

            tree.write(open(output_name, "wb"))

            # Run i-pi
            ipi = sp.Popen(
                self.call_ipi,
                cwd=(self.tmp_dir),
                shell=True,
                stdout=sp.PIPE,
                stderr=sp.PIPE,
            )
            time.sleep(2)

            with open(self.tmp_dir / "files_to_check.txt") as f:
                lines = f.readlines()
            for nl, line in enumerate(lines):
                if nl > 1:
                    self.files.append(line.split()[0])
                    self.forms.append(line.split()[1])

            # Run drivers by defining cmd2 which will be called, eventually
            flag_indeces = list()

            driver = list()
            cmd2 = list()
            for client in clients:
                if client[3] == "unix":
                    clientcall = self.call_driver + " -m {} -h {} -u ".format(
                        client[0], client[1]
                    )
                    cmd2.append(clientcall)
                elif client[3] == "inet":
                    cmd2.append(
                        self.call_driver
                        + " -m {} -h {} -p {}".format(client[0], client[1], client[2])
                    )
                else:
                    raise ValueError("Driver mode has to be either unix or inet")

                cmd = cmd2[0]
                if any("-" in str(s) for s in client):
                    flag_indeces = [
                        i for i, elem in enumerate(client) if "-" in str(elem)
                    ]
                    for i, ll in enumerate(flag_indeces):
                        if i < len(flag_indeces) - 1:
                            cmd2 += "{} {}".format(
                                client[ll],
                                ",".join(client[ll + 1 : flag_indeces[i + 1]][:]),
                            )
                        else:
                            cmd2.append(
                                " {} {}".format(
                                    client[ll], ",".join(client[ll + 1 :][:])
                                )
                            )
                    cmd += cmd2[1]
                driver.append(
                    sp.call(cmd, cwd=(cwd), shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
                )

        except sp.TimeoutExpired:
            raise RuntimeError(
                "Time is out. Aborted during {} test. \
              Error {}".format(
                    str(cwd), ipi.communicate(timeout=2)[0]
                )
            )

        except FileNotFoundError:
            raise ("{}".format(str(cwd)))

        except ValueError:
            raise ("{}".format(str(cwd)))
        try:
            for filename in glob.glob("/tmp/ipi_*"):
                os.remove(filename)
        except:
            pass

        if self.check_error:
            self._check_error(ipi)
        if self.check_numpy_output:
            self._check_numpy_output(cwd)
        if self.check_xyz_output:
            self._check_xyz_output(cwd)

    def _check_error(self, ipi):
        """ This function checks if ipi has exited with errors"""

        ipi_error = ipi.communicate(timeout=120)[1].decode("ascii")
        if ipi_error != "":
            print("IPI ERROR OCCURED: {}".format(ipi_error))
        assert "" == ipi_error

    def _check_numpy_output(self, cwd):
        """This function checks if the numpy-accessible datafiles are 'all_close' to the
        reference file provided

        The checked 'numpy-accessible' files are collected in the files_to_check.txt
        where there filetype is specified, too.
        """

        for ii, refname in enumerate(self.files):
            if self.forms[ii] == "numpy":
                try:
                    ref_output = np.loadtxt(Path(cwd) / refname)
                except IOError:
                    raise IOError(
                        'Please provide a reference properties output named "{}"'.format(
                            refname
                        )
                    )
                except ValueError:
                    raise ValueError(
                        "Please check ref_simulation.out in {}".format(str(self.parent))
                    )

                fname = refname[4:]
                test_output = np.loadtxt(self.tmp_dir / fname)

                try:
                    np.testing.assert_allclose(
                        test_output, ref_output, rtol=1.0e-7, atol=1.0e-15
                    )
                    print("No anomaly during the regtest for {}".format(refname))
                except AssertionError:
                    raise AssertionError(
                        "ANOMALY: Disagreement between reference and {} in {}".format(
                            fname, str(self.parent)
                        )
                    )

    def _check_xyz_output(self, cwd):
        """This function checks if the ref_simulation.XXXXX.xyz files are 'all_close'
        to the reference file provided.

        The checked .xyz files are collected in the files_to_check.txt where there filetype is
        also specified.

        Common options for XXXXX and the tags they can be generated with:
            pos_c = <trajectory filename='pos_c' stride='1' > x_centroid </trajectory>
            frc_c = <trajectory filename='frc_c' stride='1' > f_centroid </trajectory>
            vel_c = <trajectory filename='vel_c' stride='1' > v_centroid </trajectory>
            mom_c = <trajectory filename='mom_c' stride='1' > p_centroid </trajectory>

        In case of a multi-bead simulations, please check of the positions and
        forces on the first bead, too.
            pos_0 = <trajectory filename='pos' stride='1' bead='0'> positions </trajectory>
            frc_0 = <trajectory filename='frc' stride='1' bead='0'> forces </trajectory>
            vel_0 = <trajectory filename='vel' stride='1' bead='0'> velocities </trajectory>
            mom_0 = <trajectory filename='mom' stride='1' bead='0'> momenta </trajectory>

        """

        for ii, refname in enumerate(self.files):
            if self.forms[ii] == "xyz":
                ref_structs = []
                try:
                    with open(Path(cwd) / refname) as ref:
                        ref_natoms = int(ref.readline())
                        for s, ll in enumerate(ref.readlines()):
                            if (s + 1) % (ref_natoms + 2) != 0 and (s + 1) % (
                                ref_natoms + 2
                            ) != 1:
                                ref_structs.append(ll.split()[1:])
                    reff = [[float(v) for v in r] for r in ref_structs]
                    ref_xyz = np.array(reff)
                except IOError:
                    raise IOError(
                        "Please provide a reference file named {} in {}".format(
                            refname, str(self.parent / cwd)
                        )
                    )

                except ValueError:
                    raise ValueError(
                        "Please check the values for the file named {} in {}".format(
                            refname, str(self.parent / cwd)
                        )
                    )

                fname = refname[4:]
                structs = []
                with open(self.tmp_dir / fname) as f:
                    natoms = int(f.readline())
                    for s, ll in enumerate(f.readlines()):
                        if (s + 1) % (natoms + 2) != 0 and (s + 1) % (natoms + 2) != 1:
                            structs.append(ll.split()[1:])
                    testt = [[float(v) for v in r] for r in structs]
                    test_xyz = np.array(testt)

                try:
                    np.testing.assert_allclose(
                        test_xyz, ref_xyz, rtol=1.0e-7, atol=1.0e-8
                    )
                    print("No anomaly during the regtest for {}".format(refname))
                except AssertionError:
                    raise AssertionError(
                        "ANOMALY: Disagreement between reference and {} in {}".format(
                            fname, str(self.parent / cwd)
                        )
                    )


"""
This is just a raw importer for the sprint.
"""

import os
import psycopg2
import psycopg2.extras
import simplejson

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk_index

es = Elasticsearch()

class NominatimExporter(object):

    ITER_BY = 100

    def __init__(self, credentials, itersize=1000, limit=None, **kwargs):
        self.credentials = credentials
        print('***************** Export init ***************')
        self.conn = self.create_connexion()
        print('Connected')
        self.cur = self.conn.cursor("nominatim", cursor_factory=psycopg2.extras.DictCursor)
        print('Cursor created')
        self.cur.itersize = itersize
        self.limit = limit
        self.kwargs = kwargs

    def create_connexion(self):
        return psycopg2.connect(**self.credentials)

    def __enter__(self):
        sql = """SELECT osm_type,osm_id,class as osm_key,type as osm_value,admin_level,rank_search,rank_address,
            place_id,parent_place_id,calculated_country_code as country_code, postcode, housenumber,
            (extratags->'ref') as ref, street,
            ST_X(ST_Centroid(geometry)) as lon,
            ST_Y(ST_Centroid(geometry)) as lat,
            name->'name' as name,
            name->'name:de' as name_de,
            name->'name:fr' as name_fr,
            name->'name:en' as name_en,
            name->'short_name' as short_name,
            name->'official_name' as official_name, name->'alt_name' as alt_name,
            (extratags->'place') as extra_place
            FROM placex
            ORDER BY geometry
            """
        self.cur.execute(sql)
        print('Query executed with itersize', self.cur.itersize)
        return self

    def get_name_clause(self):
        return "name->'name' as name"

    def add_parents(self, row):
        if not "context_name" in row:
            row['context_name'] = []
        self.add_parent(row, row)

    def add_parent(self, child, row):
        if child['parent_place_id']:
            sql = """SELECT parent_place_id, type as osm_value, class as osm_key, {name}, admin_level FROM placex WHERE place_id={parent_place_id}"""
            sql = sql.format(**{
                'parent_place_id': child['parent_place_id'],
                'name': self.get_name_clause()
            })
            cur = self.conn.cursor(str(child['parent_place_id']), cursor_factory=psycopg2.extras.DictCursor)
            cur.execute(sql)
            parent = cur.fetchone()
            cur.close()
            self.add_parent_data(parent, row)
            self.add_parent(parent, row)

    def add_parent_data(self, parent, row):
        if not parent:
            return
        parent = dict(parent)
        name = parent['name']
        if name and not name in row['context_name']:
            row["context_name"].append(name)
        if (parent.get('osm_key') == "boundary"
           and parent.get('osm_value') == "administrative"
           and parent.get('admin_level') == 8):
            row['city'] = {
                'default': name
            }

    def __exit__(self, *args):
        self.cur.close()
        self.conn.close()

    def __iter__(self):
        return self.cur.__iter__()

    def to_json(self, raw):
        row = dict(raw)
        self.add_parents(row)
        row['coordinate'] = {
            'lon': raw['lon'],
            'lat': raw['lat']
        }
        row['name'] = {
            'default': raw['name'],
            'de': raw['name_de'],
            'fr': raw['name_fr'],
            'en': raw['name_en'],
        }
        row['id'] = '_'.join([raw['osm_type'], str(raw['osm_id'])])
        row['importance'] = 0.75 - raw['rank_search'] / 40
        row['context'] = {
            'default': row['context_name']
        }
        return row

class BaseConsumer(object):

    def __iter__(self):
        credentials = {
            'dbname': os.environ.get('DBNAME', 'nominatim')
        }
        with NominatimExporter(credentials) as exporter:
            for raw in exporter:
                yield exporter.to_json(raw)

class ESImporter(BaseConsumer):

    INDEX_CHUNK_SIZE = 10000
    ES_INDEX = "photon"

    def __call__(self):
        count = 0
        data = []
        print('Starting with ES index', self.ES_INDEX)
        for row in self:
            data.append(row)
            count += 1
            if count >= self.INDEX_CHUNK_SIZE:
                self.index(data)
                count = 0
                data = []
        if data:
            self.index(data)
        print('Done!')

    def format(self, row):
        return row

    def suggest_payload(self, row):
        return {
            "latlon": row['latlon'],
            'type': row['type'],
            'source': row['source'],
            'class': row['class'],
        }

    def exclude_row(self, row):
        return False

    def index(self, data):
        print('Start indexing batch of', len(data))
        bulk_index(es, data, index=self.ES_INDEX, doc_type='place', refresh=True)
        print('End indexing of current batch')

    def join(self, l, sep=", "):
        return sep.join([str(s) for s in l if str(s).strip()])

    def set_id(self, row, *args):
        row['_id'] = self.join(args, "_")

class JSONBatchDump(BaseConsumer):
    """Format for ES batch import."""

    MAX_ROWS = 100000

    def __call__(self):
        data = []
        count = 0
        index = 0
        print('Starting export')

        def do_write(data, index):
            with open('dump{}.eson'.format(index), mode='w', encoding='utf-8') as f:
                f.write('\n'.join(data))

        for row in self:
            data.append('{"index": {}}')
            data.append(simplejson.dumps(row))
            count += 1
            if count % 10000 == 0:
                print('Processed:', count, index)
            if count >= self.MAX_ROWS:
                do_write(data, index)
                data = []
                count = 0
                index += 1
        do_write(data)
        print('Done!')

if __name__ == "__main__":
    importer = JSONBatchDump()
    importer()


# Необходимо собрать аналитику о товарах. Реализовать словарь, в котором каждый ключ — характеристика товара,
# например название, а значение — список значений-характеристик, например список названий товаров.

list = []
userChoice = input("Хотите добавить товар (д/н)? ")

if userChoice.lower() == "д":
    index = 0

    while userChoice == "д":
        name = input("Наименование: ")
        price = input("Цена: ")
        amount = input("Количество: ")
        units = input("Единицы: ")

        index += 1
        list.append((index, {"наименование": name, "цена": float(price), "количество": int(amount), "eд": units}))
        userChoice = input("Добавить ещё товар (д/н)? ").lower()

    resultDict = dict()

    for item in list:
        for key in item[1]:
            if resultDict.get(key) is None:
                resultDict[key] = [item[1][key]]
            elif item[1][key] not in resultDict[key]:
                resultDict[key].append(item[1][key])

    print("Итог: ")
    print(resultDict)
print("Работа завершена")

Урок1.2/Задача6.py


import tkinter as tk
def action1():
    a=radokvodu.get()
    s=a.split(",")
    for i in range(0,len(s)):
        s[i]=float(s[i])
    print(s)
    d=sum(s)/len(s)
    d=str(round(d,2))
    clova.config(text="середнє значення "+d)
    button.config(text='Введіть ще раз')
    radokvodu.delete(0, tk.END)
window = tk.Tk()
fr = tk.Frame(master=window,relief=tk.RAISED, borderwidth=10)
clova = tk.Label(master=fr,text="введіть цифри через кому", foreground='black',background='yellow',width=50,height=5)
radokvodu = tk.Entry(foreground="white", background="blue", width=30)
button = tk.Button(text="Пуск",width=50,height=2,background="brown",foreground="yellow", command=action1 )

fr.pack()
clova.pack()
radokvodu.pack()
button.pack()
window.mainloop()


class Settings:

    def __init__(self):
        self.width = 30  # Cell
        self.height = 20    # Cell
        self.obstaclePercent = 10   # Percent

        self.mutation = 50  # Percent
        self.elitism = 3  # Percent
        self.mutationProp = .7

        self.populationSize = max(150, int(self.width * self.height / 3))

        self.drawType = 1  # Values = 0 and higher integers


'''
CS5001
Final Project
Spring 2023
MyName: Zhixiao Wang

This is a driver file of a program named Artwork Viewer.

If you want to test this program, I suggest you choose the USA, Germany because the data is moderate and easy to observe.
'''

import pandas as pd
import tkinter as tk

from get_and_clean_art_data import *
from get_and_clean_artist_data import *
from user_interaction import *

ART_URL = "https://opendata.vancouver.ca/api/explore/v2.1/catalog/datasets/public-art/exports/csv?lang=en&timezone=America%2FLos_Angeles&use_labels=true&delimiter=%3B"
ARTIST_URL = "https://opendata.vancouver.ca/api/explore/v2.1/catalog/datasets/public-art-artists/exports/csv?lang=en&timezone=America%2FLos_Angeles&use_labels=true&delimiter=%3B"

def main():

    try:

        # 1. Extract and clean the data we need from the website
        # finally we get two lists of lists, art_data and artist_data
        content_of_arts = get_art_csv_file(ART_URL)
        work_title = get_work_title_from_art_table(content_of_arts)
        type, status, material, neighbourhood = get_other_info_from_art_table(content_of_arts)
        artist_id, year = get_artist_id_and_year_from_art_table(content_of_arts)
        art_data = get_public_art_data(work_title, type, status, material, neighbourhood, artist_id, year)

        content_of_artists = get_artist_csv_file(ARTIST_URL)
        id, first_name, last_name = get_basic_info_from_artist_table(content_of_artists)
        country = get_country_from_artist_table(content_of_artists)
        artist_data = get_public_arist_data(id, first_name, last_name, country)

        # 2.Create two data frames according to these two lists of lists
        # data frames are used in user interaction to provide a sequence of options
        df_artist = pd.DataFrame(artist_data, columns=["artist id", "first name", "last name", "country"])
        df_art = pd.DataFrame(art_data, columns = ["work title", "artist id", "type", "status", "material", "neighbourhood", "year"])

        # 3. Interacting with the user
        # user will choose the country of artists and the category of their artworks
        # create the top-level window
        root = tk.Tk()
        frames = set_root_and_create_child_frames(root)

        # df_art and df_artist are only used to create a option list, not for data analysis
        create_start_page(df_art, df_artist, frames)

        root.mainloop()

    except NameError:
        print("NameError occurred")
    except TypeError as te:
        print(type(te), te)
    except ValueError as ve:
        print(type(ve), ve)
    except AttributeError as ae:
        print(type(ae), ae)
    except Exception as e:
        print("Other errors occurred", type(e), e)

if __name__ == "__main__":
    main()


import os
import math
import torch
import itertools
import torchvision
from torch.autograd import Variable
from TripletNet import TripletNet

def shuffle_data(data, seed = 0):
	image_ids, labels = data
	shuffled_image_ids = []
	shuffled_labels = []
	num_images = len(image_ids)
	torch.manual_seed(seed)
	perm = list(torch.randperm(num_images))
	for i in range(num_images):
		shuffled_image_ids.append(image_ids[perm[i]])
		shuffled_labels.append(labels[perm[i]])
	return shuffled_image_ids, shuffled_labels

def make_minibatches(data, minibatch_size = 16,  seed = 0, shuffle = 'random'):
	X, Y = data
	m = len(X)
	minibatches = []
	if shuffle == 'sequential':
		shuffled_X, shuffled_Y = X, Y

	elif shuffle == 'random':
		shuffled_X, shuffled_Y = shuffle_data(data, seed = seed)

	num_complete_minibatches = math.floor(m/minibatch_size)
	for k in range(0, num_complete_minibatches):
		minibatch_X = shuffled_X[k * minibatch_size : k * minibatch_size + minibatch_size]
		minibatch_Y = shuffled_Y[k * minibatch_size : k * minibatch_size + minibatch_size]
		minibatches.append((minibatch_X, minibatch_Y))

	rem_size = m - num_complete_minibatches * minibatch_size
	if m % minibatch_size != 0:
		minibatch_X = shuffled_X[num_complete_minibatches * minibatch_size : m]
		minibatch_Y = shuffled_Y[num_complete_minibatches * minibatch_size : m]
		minibatches.append((minibatch_X, minibatch_Y))

	return minibatches

def batch2images_tensor(minibatch_X, dataloader, gpu_device):
	minibatch_size = len(minibatch_X)
	images_tensor = torch.zeros(minibatch_size, 3, 224, 224)
	for i in range(minibatch_size):
		x = minibatch_X[i]
		x_image = dataloader.get_image(x)
		images_tensor[i, :, :, :] = x_image
	images_tensor = Variable(images_tensor)
	if torch.cuda.is_available():
		with torch.cuda.device(gpu_device):
			images_tensor = images_tensor.cuda()
	return images_tensor

def gen_triplets(minibatch, id2embeds, embedding_dim, gpu_device, mode = 'all'):
	X, Y = minibatch
	Y_prod = itertools.product(Y, repeat=3)
	X_prod = itertools.product(X, repeat=3)
	triplet = []
	for x, y  in zip(X_prod, Y_prod):
		xa, xp, xn = x
		ya, yp, yn = y
		if (ya == yp) and (ya!=yn) and (xa!=xp):
			triplet.append((xa, xp, xn))

	num_triplets = len(triplet)
	anchor = torch.zeros(num_triplets, embedding_dim)
	positive = torch.zeros(num_triplets, embedding_dim)
	negative = torch.zeros(num_triplets, embedding_dim)
	if torch.cuda.is_available():
		with torch.cuda.device(gpu_device):
			anchor = anchor.cuda()
			positive = positive.cuda()
			negative = negative.cuda()

	for i in range(num_triplets):
		xa, xp, xn = triplet[i]
		anchor[i, :] = id2embeds[xa]
		positive[i, :] = id2embeds[xp]
		negative[i, :] = id2embeds[xn]

	return anchor, positive, negative

def label2embeds_list2dict(labels_list, embeds):
	label2embeds = {}
	num_labels = len(labels_list)
	for i in range(num_labels):
		label = labels_list[i]
		label2embeds[label] = embeds[i, :]
	return label2embeds

def final_label2embeds(triplet_net, train_dataloader, gpu_device):
	labels_list = []
	image_ids = []
	for label, images_list in train_dataloader.images_dict.items():
		image_ids.append(images_list[0])
		labels_list.append(label)

	images_tensor = batch2images_tensor(image_ids, train_dataloader, gpu_device)
	with torch.no_grad():
		embeds = triplet_net.embedding(images_tensor)
	label2embeds = label2embeds_list2dict(labels_list, embeds)
	return label2embeds

def who_is_it(label2embeds, embed):
	labels = []
	num_labels = len(label2embeds)
	embedding_dim = embed.shape[0]
	embeds = torch.zeros(num_labels, embedding_dim)
	i = 0
	for label, cur_embed in label2embeds.items():
		labels.append(label)
		embeds[i, :] = cur_embed
		i += 1
	dist = torch.pow(embeds - embed, 2).sum(dim = 1)
	index = torch.argmin(dist).tolist()
	return labels[index]

def accuracy(data, dataloader, label2embeds, triplet_net, gpu_device):
	image_ids, Y = data
	num_data = len(Y)
	embedding_dim = triplet_net.embedding.embedding_dim
	embeds = torch.zeros(num_data, embedding_dim)
	minibatch_size = 32
	minibatches = make_minibatches(data, minibatch_size = minibatch_size,  seed = 0, shuffle = 'sequential')
	start = 0
	end = 0
	for cur_minibatch in minibatches:
		minibatch_X, _ = cur_minibatch
		cur_minibatch_size = len(minibatch_X)
		end += cur_minibatch_size
		images_tensor = batch2images_tensor(minibatch_X, dataloader, gpu_device)
		with torch.no_grad():
			embeds[start:end, :] = triplet_net(images_tensor)
		start = end

	acc = 0
	# pred = []

	for i in range(num_data):
		embed = embeds[i]
		target_label = Y[i]
		predicted_label = who_is_it(label2embeds, embed)
		if predicted_label == target_label:
			acc += 1
			# pred.append(predicted_label)
	return acc, num_data


from pwn import *
import os

context.arch = 'amd64'
pty = process.PTY
is_local = False
is_remote = False

elf_local = ELF("/home/kali/CTF/PCTF/print_shop/printshop")
if len(sys.argv) == 1:
    is_local = True
    p = process(elf_local.path, stdin=pty, stdout=pty)

elif len(sys.argv) > 1:
    #  nc chal.pctf.competitivecyber.club 4444
    is_remote = True
    if len(sys.argv) == 3:
        host = sys.argv[1]
        port = sys.argv[2]
    else:
        host, port = sys.argv[1].split(':')
    p = remote(host, port)

def debug(cmd=''):
    if is_local: gdb.attach(p,cmd)

#debug(cmd='''
#        b *adminBook+114
#      ''')


#!/usr/bin/python3

from statistics import median

values = [] #array to store names
counter=0 #counter to check number of entries
while counter<13:
    value = int(input("Enter values: ")) #taking input from the user
    counter=counter+1 #incrementing the counter variable by 1
    values.append(value) #appending the entered value to the values array

maximum = max(values)
minimum = min(values)
average = sum(values)/len(values)
med = median(values)
sum = sum(values)
print('Max: ', maximum)
print("Min: ", minimum)
print("Avg: ", average)
print("Med: ", med)
print("Sum: ", sum)


import json
from random import randint

import vk_api
from vk_api.longpoll import VkLongPoll, VkEventType

import dialogflow_v2 as dialogflow

import os
from dotenv import load_dotenv

load_dotenv()

TOKEN_VK = os.environ.get('TOKEN_VK')
GOOGLE_APPLICATION_CREDENTIALS = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')

def handle_message(event, vk_api):
    client = dialogflow.SessionsClient()

    with open(GOOGLE_APPLICATION_CREDENTIALS, 'r') as file:
        config = json.load(file)

    session = client.session_path(config['project_id'], event.user_id)
    text_input = dialogflow.types.TextInput(
        text=event.text,
        language_code='ru'
    )
    query_input = dialogflow.types.QueryInput(text=text_input)

    response = client.detect_intent(
        session=session,
        query_input=query_input
    )

    if not response.query_result.intent.is_fallback:
        vk_api.messages.send(
            user_id=event.user_id,
            message=response.query_result.fulfillment_text,
            random_id=randint(1, 1000)
        )

if __name__ == "__main__":
    vk_session = vk_api.VkApi(token=TOKEN_VK)
    vk_api = vk_session.get_api()
    longpoll = VkLongPoll(vk_session)
    for event in longpoll.listen():
        if event.type == VkEventType.MESSAGE_NEW and event.to_me:
            handle_message(event, vk_api)


"""
Unit Test Code for Master
"""

import pytest
from experiment_scheduler.master.master import Master
from experiment_scheduler.master.grpc_master.master_pb2 import (
    TaskStatus,
    TaskLogFile,
    ExperimentsStatus,
    AllExperimentsStatus,
    AllTasksStatus,
    MasterResponse,
)
from experiment_scheduler.task_manager.grpc_task_manager.task_manager_pb2 import (
    TaskStatus as TMTaskStatus,
)

class MockRequest:
    """
    Mock Simple Request (used commonly)
    """

    def __init__(self, task_id):
        self.task_id = task_id

class MockEditRequest:
    """
    Mock class for edit_task
    """

    def __init__(self, task_id, cmd, task_env):
        self.task_id = task_id
        self.cmd = cmd
        self.task_env = task_env

class MockExperimentRequest:
    """
    Mock class for ExperimentRequest (protobuf format)
    """

    def __init__(self, experiment_id):
        self.experiment_id = experiment_id

class MockExperimentTasks:
    """
    Mock class for ORM ExperimentTasks
    """

    def __init__(self):
        self.id = "TestID"  # pylint:disable=invalid-name
        self.tasks = [
            MockTask("1", TaskStatus.Status.RUNNING),
            MockTask("2", TaskStatus.Status.RUNNING),
        ]

class MockRequestExperiments:
    """
    Mock class for request_experiments
    """

    def __init__(self, name, tasks):
        self.name = name
        self.tasks = tasks

class MockExperiment:
    """
    Mock class for ORM Experiment
    """

    def get(self):
        """
        ORM get Method
        :return:
        """
        return MockExperimentTasks()

    def list(self):
        """
        ORM list method
        :return:
        """
        return [MockExperimentTasks(), MockExperimentTasks()]

class MockTask:
    """
    Mock class for Task object
    """

    def __init__(  # pylint:disable=too-many-arguments
        self,
        task_id="task_id",
        status=None,
        name="test",
        task_manager_id="task_manager",
        task_env=None,
        command="python test.py",
    ):
        if task_env is None:
            task_env = {"test_key": "test_val"}
        self.id = task_id  # pylint:disable=invalid-name
        self.name = name
        self.status = status
        self.task_manager_id = task_manager_id
        self.task_env = task_env
        self.command = command

    def list_success(self):
        """
        list mock for success case
        :return:
        """
        return [
            MockTask("test_task_id", TaskStatus.Status.RUNNING),
            MockTask("2", TaskStatus.Status.RUNNING),
            MockTask("3", TaskStatus.Status.RUNNING),
        ]

    def list_fail(self):
        """
        list mock for failed case
        :return:
        """
        return [
            MockTask("1", TaskStatus.Status.RUNNING),
            MockTask("2", TaskStatus.Status.RUNNING),
            MockTask("3", TaskStatus.Status.RUNNING),
        ]

    def commit(self):
        """
        commit should not do anything while test.
        :return:
        """
        return None

class MockTaskManager:
    """
    Mock class for ORM TaskManager
    """

    def __init__(self, address, path=""):
        self.address = address
        self.log_file_path = path
        self.id = "testID"  # pylint:disable=invalid-name

class MockProcessMonitor:
    """
    Mock class for Process Monitor Class
    """

    def kill_task(self):
        """
        Mock mehtod for kill_task
        :return:
        """
        return TaskStatus(task_id="test_task_id", status=TaskStatus.Status.KILLED)

    def get_task_log(self):
        """
        Mock method for get_task_log
        :return:
        """
        return "task completed"

    def run_task_success(self):
        """
        Mock method for run_task in success case
        :return:
        """
        return TMTaskStatus(task_id="test1", status=TaskStatus.Status.RUNNING)

    def run_task_failed(self):
        """
        Mock method for run_task in failure case
        :return:
        """
        return TMTaskStatus(task_id="test1", status=TaskStatus.Status.NOTSTART)

class MockThread:
    """
    Make Thread not work during test
    """

    @classmethod
    def start(cls):
        """
        Thread's Start should do nothing
        :return:
        """
        return

class TestMaster:
    """
    Master Test class
    """

    @pytest.fixture
    def master(self, mocker):
        """
        Fixture for Master class
        :param mocker:
        :return:
        """
        # mocker.patch("threading.Thread", return_value=MockThread)
        mocker.patch(
            "experiment_scheduler.master.process_monitor.ProcessMonitor._health_check",
            return_value=None,
        )
        mocker.patch(
            "experiment_scheduler.master.master.Master._execute_command",
            return_value=None,
        )
        master = Master()
        return master

    def test_get_task_managers(self, master, mocker):
        """
        Test for get_task_managers
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch("ast.literal_eval", return_value=["test_address"])
        task_managers = master.get_task_managers()
        assert task_managers == ["test_address"]
        mocker.patch("os.getenv", return_value="test_address2")
        task_managers = master.get_task_managers()
        assert task_managers == ["test_address2"]

    def test_select_task_manager(self, master, mocker):
        """
        Test for select_task_manager
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch("os.getenv", return_value="test_address1 test_address2")
        master = Master()
        selected_task_manager = master.select_task_manager()
        assert selected_task_manager == "test_address1"

        selected_task_manager = master.select_task_manager(1)
        assert selected_task_manager == "test_address2"

    def test_request_experiments(self, master):
        """
        Test for request_experiments
        :param master:
        :return:
        """
        master = Master()
        request = MockRequestExperiments(
            name="test_experiment",
            tasks=[MockTask(name="test_task", command="test_command")],
        )
        context = ""
        response = master.request_experiments(request, context)
        assert response.experiment_id.startswith("test_experiment-")
        assert response.response == MasterResponse.ResponseStatus.SUCCESS

    def test_get_task_log(self, master, mocker):
        """
        Test for get_task_log
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.RUNNING
            ),
        )
        mocker.patch(
            "experiment_scheduler.db_util.task_manager.TaskManager.get",
            return_value=MockTaskManager(address="test_address"),
        )
        master = Master()
        request = MockRequest(task_id="test_task_id")
        context = ""
        response = master.get_task_log(request, context)
        assert next(response) == TaskLogFile(
            error_message=bytes("Check task id", "utf-8")
        )

        mocker.patch(
            "experiment_scheduler.db_util.task_manager.TaskManager.get",
            return_value=MockTaskManager(address="test_address", path="not empty"),
        )
        mocker.patch(
            "experiment_scheduler.master.process_monitor.ProcessMonitor.get_task_log",
            return_value=MockProcessMonitor().get_task_log(),
        )
        response = master.get_task_log(request, context)
        assert "".join(response) == "task completed"

    def test_get_task_status(self, master, mocker):
        """
        test for get_task_status
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch("experiment_scheduler.db_util.task.Task.get", return_value=None)
        request = MockRequest(task_id="test_task_id")
        context = ""
        response = master.get_task_status(request, context)
        assert response.task_id == "test_task_id"
        assert response.status == TaskStatus.Status.NOTFOUND

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.RUNNING
            ),
        )
        response = master.get_task_status(request, context)
        assert response.task_id == "test_task_id"
        assert response.status == TaskStatus.Status.RUNNING

    def test_kill_task(self, master, mocker):
        """
        test for kill_task
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch("experiment_scheduler.db_util.task.Task.get", return_value=None)
        request = MockRequest(task_id="test_task_id")
        context = ""
        response = master.kill_task(request, context)
        assert response == TaskStatus(
            task_id="test_task_id", status=TaskStatus.Status.NOTFOUND
        )

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.NOTSTART
            ),
        )
        response = master.kill_task(request, context)
        assert response == TaskStatus(
            task_id="test_task_id", status=TaskStatus.Status.KILLED
        )

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id",
                status=TaskStatus.Status.RUNNING,
                task_manager_id=None,
            ),
        )
        response = master.kill_task(request, context)
        assert response == TaskStatus(
            task_id="test_task_id", status=TaskStatus.Status.KILLED
        )

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.RUNNING
            ),
        )
        mocker.patch(
            "experiment_scheduler.db_util.task_manager.TaskManager.get",
            return_value=MockTaskManager(address="test_address"),
        )
        mocker.patch(
            "experiment_scheduler.master.process_monitor.ProcessMonitor.kill_task",
            return_value=MockProcessMonitor().kill_task(),
        )
        response = master.kill_task(request, context)
        assert response == TaskStatus(
            task_id="test_task_id", status=TaskStatus.Status.KILLED
        )

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.DONE
            ),
        )
        response = master.kill_task(request, context)
        assert response == TaskStatus(
            task_id="test_task_id", status=TaskStatus.Status.DONE
        )

    def test_get_all_tasks(self, master, mocker):
        """
        Test for get_all_tasks
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch(
            "experiment_scheduler.db_util.experiment.Experiment.get",
            return_value=MockExperiment().get(),
        )
        master = Master()
        request = MockExperimentRequest(experiment_id="test_experiment_id")
        context = ""
        response = master.get_all_tasks(request, context)
        assert response == AllExperimentsStatus(
            experiment_status_array=[
                ExperimentsStatus(
                    experiment_id="test_experiment_id",
                    task_status_array=AllTasksStatus(
                        task_status_array=[
                            TaskStatus(task_id="1", status=TaskStatus.Status.RUNNING),
                            TaskStatus(task_id="2", status=TaskStatus.Status.RUNNING),
                        ]
                    ),
                )
            ]
        )

        mocker.patch(
            "experiment_scheduler.db_util.experiment.Experiment.list",
            return_value=MockExperiment().list(),
        )
        request = MockExperimentRequest(experiment_id=None)
        response = master.get_all_tasks(request, context)

        assert response == AllExperimentsStatus(
            experiment_status_array=[
                ExperimentsStatus(
                    experiment_id="TestID",
                    task_status_array=AllTasksStatus(
                        task_status_array=[
                            TaskStatus(task_id="1", status=TaskStatus.Status.RUNNING),
                            TaskStatus(task_id="2", status=TaskStatus.Status.RUNNING),
                        ]
                    ),
                ),
                ExperimentsStatus(
                    experiment_id="TestID",
                    task_status_array=AllTasksStatus(
                        task_status_array=[
                            TaskStatus(task_id="1", status=TaskStatus.Status.RUNNING),
                            TaskStatus(task_id="2", status=TaskStatus.Status.RUNNING),
                        ]
                    ),
                ),
            ]
        )

    def test_execute_task(self, master, mocker):
        """
        test for execute_task
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask(
                task_id="test_task_id", status=TaskStatus.Status.RUNNING
            ),
        )
        mocker.patch(
            "experiment_scheduler.db_util.task_manager.TaskManager.get",
            return_value=MockTaskManager(address="test_task_manager_address"),
        )
        mocker.patch(
            "experiment_scheduler.master.process_monitor.ProcessMonitor.run_task",
            return_value=MockProcessMonitor().run_task_success(),
        )

        master = Master()
        task_manager_address = "test_task_manager_address"
        task_id = "test_task_id"
        response = master.execute_task(task_manager_address, task_id)
        assert response.status == TaskStatus.Status.RUNNING

        mocker.patch(
            "experiment_scheduler.master.process_monitor.ProcessMonitor.run_task",
            return_value=MockProcessMonitor().run_task_failed(),
        )
        response = master.execute_task(task_manager_address, task_id)
        assert response.status == TaskStatus.Status.NOTSTART

    def test_edit_task(self, master, mocker):
        """
        Test for edit_task
        :param master:
        :param mocker:
        :return:
        """
        mocker.patch(
            "experiment_scheduler.db_util.task.Task.list",
            return_value=MockTask("1", TaskStatus.Status.RUNNING).list_success(),
        )
        mocker.patch(
            "experiment_scheduler.db_util.task.Task.get",
            return_value=MockTask("1", TaskStatus.Status.RUNNING),
        )
        master = Master()
        request = MockEditRequest(
            task_id="test_task_id", cmd="test_cmd", task_env={"key": "val"}
        )
        context = ""
        response = master.edit_task(request, context)
        assert response.experiment_id == "0"
        assert response.response == MasterResponse.ResponseStatus.SUCCESS

        mocker.patch(
            "experiment_scheduler.db_util.task.Task.list",
            return_value=MockTask("1", TaskStatus.Status.RUNNING).list_fail(),
        )
        response = master.edit_task(request, context)
        assert response.experiment_id == "0"
        assert response.response == MasterResponse.ResponseStatus.FAIL


import gym
from gym import Env
from gym.error import DependencyNotInstalled

from mazeGame import MazeGame
from gym import spaces
import numpy as np

metadata = {
    "render_modes": ["human", "rgb_array"],
}

class MazeEnv(Env):

    # observation: [distance of sensor] * 6 + [end direction] * 1
    # action: {0: turn left, 1: turn right, 2: walk forward, 3: walk backward}
    def __init__(self, maze_idx, render_mode=None):
        self.metadata = metadata

        self.mazeGame = MazeGame(maze_idx)
        self.action_space = spaces.Discrete(4)
        self.observation_space = spaces.Box(low=0, high=200, shape=(7,))

        self.end_distance = self.mazeGame.end.distance(self.mazeGame.robot_pos)
        self.TIME_PENALTY = -1

        self.render_mode = render_mode
        self.screen = None
        self.ZOOM = 4
        self.SCREEN_SIZE = 200 * self.ZOOM

    def step(self, action):
        self.mazeGame.action(action)
        terminated, truncated = self.mazeGame.isEnd()
        observation = self.mazeGame.observation()
        observation = self._go2eo(observation)
        reward = self._reward()

        info = {'robot_pos': self.mazeGame.robot_pos, 'robot_face': self.mazeGame.robot_face,
                'start': self.mazeGame.start, 'end': self.mazeGame.end, 'time': self.mazeGame.time,
                'time_limit': self.mazeGame.time_limit, 'TIME_PENALTY': self.TIME_PENALTY}
        self.end_distance = self.mazeGame.end.distance(self.mazeGame.robot_pos)
        return observation, reward, terminated, truncated, info

    def reset(self, *args, **kwargs):
        self.end_distance = self.mazeGame.end.distance(self.mazeGame.start)
        self.mazeGame.reset()
        observation = self.mazeGame.observation()
        observation = self._go2eo(observation)
        info = {'robot_pos': self.mazeGame.robot_pos, 'robot_face': self.mazeGame.robot_face,
                'start': self.mazeGame.start, 'end': self.mazeGame.end, 'time': self.mazeGame.time,
                'time_limit': self.mazeGame.time_limit, 'TIME_PENALTY': self.TIME_PENALTY}
        return observation, info

    def render(self, mode='human'):
        if self.render_mode is None:
            gym.logger.warn(
                "You are calling render method without specifying any render mode. "
                "You can specify the render_mode at initialization, "
                f'e.g. gym("{self.spec.id}", render_mode="rgb_array")'
            )
            return

        try:
            import pygame
            from pygame import gfxdraw
        except ImportError:
            raise DependencyNotInstalled(
                "pygame is not installed, run `pip install gym[classic_control]`"
            )
        if self.screen is None:
            pygame.init()
            if self.render_mode == "human":
                pygame.display.init()
                self.screen = pygame.display.set_mode(
                    (self.SCREEN_SIZE, self.SCREEN_SIZE)
                )
            else:  # mode == "rgb_array"
                self.screen = pygame.Surface((self.SCREEN_SIZE, self.SCREEN_SIZE))

        self.screen.fill((255, 255, 255))
        for line in self.mazeGame.maze_lines:
            pygame.draw.line(
                self.screen,
                (0, 0, 0),
                self.mc2pc(*line[:2]),
                self.mc2pc(*line[2:]),
                1
            )
        pygame.draw.circle(
            self.screen,
            (0, 0, 255),
            self.mc2pc(self.mazeGame.start.x, self.mazeGame.start.y),
            5 * self.ZOOM,
        )
        pygame.draw.circle(
            self.screen,
            (255, 0, 0),
            self.mc2pc(self.mazeGame.end.x, self.mazeGame.end.y),
            5 * self.ZOOM,
        )
        pygame.draw.circle(
            self.screen,
            (0, 255, 0),
            self.mc2pc(self.mazeGame.robot_pos.x, self.mazeGame.robot_pos.y),
            5 * self.ZOOM,
        )

        pygame.draw.line(
            self.screen,
            (255, 255, 255),
            self.mc2pc(self.mazeGame.robot_pos.x, self.mazeGame.robot_pos.y),
            self.mc2pc(self.mazeGame.robot_pos.x + self.mazeGame.robot_face[0] * 5, self.mazeGame.robot_pos.y + self.mazeGame.robot_face[1] * 5),
            1
        )

        if self.render_mode == "human":
            pygame.event.pump()
            pygame.display.flip()

        elif self.render_mode == "rgb_array":
            return np.transpose(
                np.array(pygame.surfarray.pixels3d(self.screen)), axes=(1, 0, 2)
            )

    # game observation to environment observation
    def _go2eo(self, observation):
        return np.array(observation["sensor"] + [observation["end_direction"]], dtype=np.float32)

    #
    def _reward(self):
        new_distance = self.mazeGame.end.distance(self.mazeGame.robot_pos)
        move_reward = self.end_distance - new_distance
        return move_reward + self.TIME_PENALTY

    # math coordinate to pygame coordinate
    def mc2pc(self, x, y):
        return x * self.ZOOM, self.SCREEN_SIZE - y * self.ZOOM


import select
import signal
import serial
from time import sleep
#from binascii import *

# export filtering
__all__ = ['GPS_reader']

def LOG(msg=''):
    print('[GPS_reader]%s' % msg)

class GPS_reader(object):
    # debug level
    DEBUG = 1
    # for interrupt handler and looping control
    _THREADED = False
    _STOP_EVENT = None
    # serial port
    PORT = '/dev/ttyUSB0'
    BAUDRATE = 9600
    # number of last NMEA info to store
    NMEA_NUM = 3
    # type of NMEA info to collect
    NMEA_TYPE = ("GPBOD", "GPBWC", "GPGGA", "GPGLL", "GPGSA", "GPGSV", "GPHDT",
                 "GPR00", "GPRMA", "GPRMB", "GPRMC", "GPRTE", "GPTRF", "GPSTN",
                 "GPVBW", "GPVTG", "GPWPL", "GPXTE", "GPZDA")

    def __init__(self):
        try:
            self._ser = serial.Serial(port=self.PORT, baudrate=self.BAUDRATE)
        except:
            LOG('[ERR] cannot open %s' % self.PORT)
            self._ser = None
        #
        self.infos = {}
        for nmea_t in self.NMEA_TYPE:
            self.infos[nmea_t] = []
        #
        self._listening = False
        self._reading = False
        #
        if self._ser:
            LOG(' reading position information over %s' % self.PORT)
        #
        if not self._THREADED:
            def handle_int(signum, frame):
                self.stop()
                LOG(' SIGINT: quitting')
            signal.signal(signal.SIGINT, handle_int)

    def stop(self):
        self._listening = False
        if self._ser:
            while self._reading:
                sleep(0.001)
            self._ser.close()

    def looping(self):
        if not self._listening:
            return False
        else:
            if not self._THREADED:
                return True
            elif hasattr(self._STOP_EVENT, 'is_set') \
            and not self._STOP_EVENT.is_set():
                return True
            return False

    def listen(self):
        if not self._ser:
            return
        self._listening = True
        while self.looping():
            try:
                self._reading = True
                l = self._ser.readline()
                self._reading = False
            # catch various errors that happen when the serial port is not
            # available in some way
            except (OSError, ValueError, serial.SerialException):
                break
            else:
                self.process(l)

    def process(self, buf='\n'):
        if self.DEBUG > 1:
            LOG(' process buf: %s' % buf)
        # grep NMEA_TYPE we want to get
        nmea_t = buf[1:6]
        if nmea_t in self.NMEA_TYPE:
            self.infos[nmea_t].append(buf[7:-2])
            if len(self.infos[nmea_t]) > self.NMEA_NUM:
                self.infos[nmea_t].remove( self.infos[nmea_t][0] )
            if self.DEBUG:
                LOG(' got %s: %s' % (nmea_t, buf[7:-2]))

    def get_last_info(self, nmea_t='GPRMC'):
        if not self._ser:
            return ''
        if nmea_t in self.NMEA_TYPE:
            if len(self.infos[nmea_t]):
                return self.infos[nmea_t][-1]


"""OpenAPI Tool."""

from typing import List, Optional

import requests

from llama_index.readers.schema.base import Document
from llama_index.tools.tool_spec.base import BaseToolSpec

class OpenAPIToolSpec(BaseToolSpec):
    """OpenAPI Tool

    This tool can be used to parse an OpenAPI spec for endpoints and operations
    Use the RequestsToolSpec to automate requests to the openapi server
    """

    spec_functions = ["load_openapi_spec"]

    def __init__(self, spec: Optional[dict] = None, url: Optional[str] = None):
        import yaml

        if spec and url:
            raise ValueError("Only provide one of OpenAPI dict or url")
        elif spec:
            pass
        elif url:
            response = requests.get(url).text
            spec = yaml.load(response, Loader=yaml.Loader)
        else:
            raise ValueError("You must provide a url or OpenAPI spec as a dict")

        parsed_spec = self.process_api_spec(spec)
        self.spec = Document(text=str(parsed_spec))

    def load_openapi_spec(self) -> List[Document]:
        """
        You are an AI agent specifically designed to retrieve information by making web requests to an API based on an OpenAPI specification.

        Here's a step-by-step guide to assist you in answering questions:

        1. Determine the base URL required for making the request

        2. Identify the relevant paths necessary to address the question

        3. Find the required parameters for making the request

        4. Perform the necessary requests to obtain the answer

        Returns:
            Document: A List of Document objects.
        """
        return [self.spec]

    def process_api_spec(self, spec: dict) -> dict:
        """Perform simplification and reduction on an OpenAPI specification.

        The goal is to create a more concise and efficient representation
        for retrieval purposes.
        """

        def reduce_details(details: dict) -> dict:
            reduced = {}
            if details.get("description"):
                reduced["description"] = details.get("description")
            if details.get("parameters"):
                reduced["parameters"] = [
                    param
                    for param in details.get("parameters", [])
                    if param.get("required")
                ]
            if "200" in details["responses"]:
                reduced["responses"] = details["responses"]["200"]
            return reduced

        def dereference_openapi(openapi_doc):
            """Dereferences a Swagger/OpenAPI document by resolving all $ref pointers."""
            try:
                import jsonschema
            except ImportError:
                raise ImportError(
                    "The jsonschema library is required to parse OpenAPI documents. "
                    "Please install it with `pip install jsonschema`."
                )

            resolver = jsonschema.RefResolver.from_schema(openapi_doc)

            def _dereference(obj):
                if isinstance(obj, dict):
                    if "$ref" in obj:
                        with resolver.resolving(obj["$ref"]) as resolved:
                            return _dereference(resolved)
                    return {k: _dereference(v) for k, v in obj.items()}
                elif isinstance(obj, list):
                    return [_dereference(item) for item in obj]
                else:
                    return obj

            dereferenced_doc = _dereference(openapi_doc)
            return dereferenced_doc

        spec = dereference_openapi(spec)
        endpoints = []
        for route, operations in spec["paths"].items():
            for operation, details in operations.items():
                if operation in ["get", "post", "patch"]:
                    endpoint_name = f"{operation.upper()} {route}"
                    description = details.get("description")
                    endpoints.append(
                        (endpoint_name, description, reduce_details(details))
                    )

        return {
            "servers": spec["servers"],
            "description": spec["info"].get("description"),
            "endpoints": endpoints,
        }


import tensorflow as tf
import config as cfg
import tensorflow.contrib.slim as slim

epsilon = 1e-9

class CapsuleNet():
    def __init__(self,is_training = True):
        self.image_size = cfg.IMAGE_SIZE
        self.category_num = cfg.CATEGORY_NUM
        self.input_img = tf.placeholder(tf.float32,[None,cfg.IMAGE_SIZE,cfg.IMAGE_SIZE,3],"input_img")
        self.input_label = tf.placeholder(tf.uint8,[None],"input_label")
        self.buildNet(self.input_img,is_training = is_training)
        if is_training:
            self.getloss(self.input_label)
            self.summary()

    def getloss(self,input_label):
        #The margin loss
        positive = tf.square(tf.maximum(0., cfg.MAX_POSITIVE_THRES - self.output_len))
        negative = tf.square(tf.maximum(0., self.output_len - cfg.MIN_NEGATIVE_THRES))
        positive_label = tf.cast(tf.one_hot(input_label,cfg.CATEGORY_NUM),tf.float32)
        negative_label = 1. - positive_label
        margin_loss = tf.reduce_sum(positive_label * positive + cfg.NEGATIVE_LOSS_WEIGHT + negative_label * negative,axis=1)
        self.margin_loss = tf.reduce_mean(margin_loss)

        #The reconstruction loss
        raw_img = tf.reshape(self.input_img,[cfg.BATCH_SIZE,-1])
        reconstruct_loss = tf.reduce_sum(tf.square(self.decoded - raw_img),axis=1)
        self.reconstruct_loss = tf.reduce_mean(reconstruct_loss)

        self.total_loss = self.margin_loss + cfg.REGULARIZATION_WEIGHT * self.reconstruct_loss

    #Build the network, input the image data, output 10 categories possibility.
    def buildNet(self,input_img,is_training = True):
        with tf.variable_scope("variable_scope"):
            net = slim.conv2d(input_img, 256, 13, 1,padding='VALID',
                              weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),scope='conv0')
            net = slim.conv2d(net,cfg.CAPSULE1_VEC_NUM * cfg.CAPSULE1_VEC_LEN, 9, 2,
                              padding='VALID',weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),scope='capsule1')

            featureMap_size = net.shape[1].value

            capsule1 = tf.reshape(net, [-1, featureMap_size * featureMap_size * cfg.CAPSULE1_VEC_NUM,
                                        cfg.CAPSULE1_VEC_LEN])
            capsule1 = self.squash(capsule1)
            '''
            # 2 layer route takes too much time training, so just takes one layer routing.
            capsule2 = self.route(capsule1, cfg.CAPSULE2_VEC_NUM, cfg.CAPSULE2_VEC_LEN,
                                  scope="capsule2")
            capsule3 = self.route(capsule2, cfg.CAPSULE3_VEC_NUM, cfg.CAPSULE3_VEC_LEN,
                                  scope="capsule3")
            '''
            capsule3 = self.route(capsule1, cfg.CAPSULE3_VEC_NUM, cfg.CAPSULE3_VEC_LEN,
                                  scope="capsule3")
            self.output_capsule = capsule3
            self.output_len = tf.sqrt(tf.reduce_sum(tf.square(capsule3),axis=2) + epsilon)
            #The length of each vector indicate the possiblity of the category for each capsule
            if is_training:
                mask = tf.one_hot(self.input_label, depth=cfg.CATEGORY_NUM)
            else:
                mask = tf.argmax(self.output_len, axis=1)
                mask = tf.one_hot(mask, depth=cfg.CATEGORY_NUM)
            mask = tf.reshape(mask,[-1,cfg.CAPSULE3_VEC_NUM,1])
            mask = tf.cast(mask,tf.float32)
            masked_capsule = tf.matmul(capsule3,mask,transpose_a=True)
            masked_capsule = tf.squeeze(masked_capsule)

            with tf.variable_scope('Decoder'):
                fc1 = slim.fully_connected(masked_capsule, 512, scope='fc1')
                fc2 = slim.fully_connected(masked_capsule, 1024, scope='fc2')
                self.decoded = slim.fully_connected(masked_capsule, cfg.IMAGE_SIZE*cfg.IMAGE_SIZE*3, scope='decode',activation_fn=tf.sigmoid)

    def route(self,input,output_vector_num,output_vector_len,scope):
        input_vec_num = input.shape[1].value
        input_vec_len = input.shape[2].value
        with tf.variable_scope(scope):
            W = tf.get_variable('Weight', shape=(1, input_vec_num, output_vector_num, input_vec_len, output_vector_len),
                                dtype=tf.float32,
                            initializer=tf.truncated_normal_initializer(0.0, 0.01))
            W = tf.tile(W, [cfg.BATCH_SIZE, 1, 1, 1, 1])
            input = tf.reshape(input,[cfg.BATCH_SIZE,input_vec_num,1,input_vec_len,1])
            input = tf.tile(input,[1,1,output_vector_num,1,1])
            # from input vector to output vector,
            # each input vector will generate output_vector_num output vector,
            # total input_vec_num input vectors generate input_vec_num * output_vector_num output vector,
            # the output vector lenght is output_vector_len
            # W_mul_input shape should be [batch, input_vec_num, output_vector_num, output_vector_len,1]
            # the last dimension keeps there as we will need dot product later
            output_vectors = tf.matmul(W,input,transpose_a=True)# from input vec to output vec

            #routing_mat, the last 2 dimension is just for calculation convenience
            routing_mat = tf.zeros([cfg.BATCH_SIZE, input_vec_num, output_vector_num,1,1])
            u_stopped = tf.stop_gradient(output_vectors, name='stop_gradient')

            for iter in range(cfg.ROUTE_ITER_COUNT):
                with tf.variable_scope('iter_' + str(iter)):
                    #weight_routing_mat is the weight(possiblity) from each input vector routing to output_vector_num output vectors
                    weight_routing_mat = tf.nn.softmax(routing_mat, dim=2)
                    if iter < cfg.ROUTE_ITER_COUNT - 1:
                        output = u_stopped * weight_routing_mat

                        output = tf.reduce_sum(output, axis=1, keep_dims=True)
                        output = self.squash(output)
                        #in order to make vector dot multiplication, expand the last dimension
                        output = tf.reshape(output,[cfg.BATCH_SIZE,1,output_vector_num,output_vector_len,1])

                        v_tile = tf.tile(output, [1, input_vec_num, 1, 1, 1])
                        #u_dot_v dimension is [batch,input_vector_num,output_vector_num,1,1]
                        #Add it to routing_mat
                        u_dot_v = tf.matmul(u_stopped, v_tile, transpose_a=True)
                        routing_mat = routing_mat + u_dot_v
                    #The last iteration
                    elif iter == cfg.ROUTE_ITER_COUNT - 1:
                        output_vec = output_vectors * weight_routing_mat
                        output_vec = tf.reduce_sum(output_vec, axis=1)
                        output_vec = tf.squeeze(output_vec)
                        output_vec = self.squash(output_vec)
            #output_vec shape [batch,output_vector_num,output_vector_len]
            return output_vec

    def squash(self, vec):
        vec_L2 = tf.reduce_sum(tf.square(vec), -1, keep_dims=True)
        scalar_factor = vec_L2 / (1 + vec_L2)
        unit_vec = vec  / tf.sqrt(vec_L2 + epsilon)
        vec_squashed = scalar_factor * unit_vec
        return vec_squashed

    def summary(self):
        tf.summary.scalar('total_loss', self.total_loss)
        tf.summary.scalar('reconstruct_loss', self.reconstruct_loss)
        tf.summary.scalar('margin_loss', self.margin_loss)

        correct_prediction = tf.equal(tf.cast(self.input_label,tf.int64), tf.argmax(self.output_len, axis=1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))


#!/usr/bin/env python3

import argparse, os, pickle
import numpy as np
import pandas as pd
from more_itertools import first
from tqdm import tqdm

from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.utils.miscellaneous import bbox_overlaps, intersect_2d
from maskrcnn_benchmark.data.datasets.evaluation.vg.sgg_eval import _triplet
from utils import load, save, \
        analyze_method_dir, load_dataset, load_eval_results

def match_boxes(boxes_1, boxes_2, iou_thres):
    assert boxes_1.ndim == boxes_2.ndim == 2
    assert boxes_1.shape[1] == boxes_2.shape[1]
    assert boxes_1.shape[1] % 4 == 0
    n = boxes_1.shape[1] // 4
        # number of objects (1/2 for object/relationships counts)
    boxes_1 = np.split(boxes_1, n, 1)
    boxes_2 = np.split(boxes_2, n, 1)
    ious = [bbox_overlaps(bs_1, bs_2) for bs_1, bs_2 in zip(boxes_1, boxes_2)]
    return (np.stack(ious, axis=-1) >= iou_thres).all(-1) # over objects

def compute_statistics(gt_classes, pred_classes, gt_boxes, pred_boxes,
        iou_thres, unknown_class, object_class_indices):
        # count gts for each number of unknowns and for each classification
        # against preds;
        # assume that the predictions (pred_classes and pred_boxes) are sorted
        # by scores
    assert gt_classes.ndim == pred_classes.ndim
    class_match = intersect_2d(gt_classes, pred_classes)     # (#gts, #preds)
    box_match = match_boxes(gt_boxes, pred_boxes, iou_thres) # (#gts, #preds)
    assert class_match.shape == box_match.shape
    num_gt_unknowns = (
            gt_classes[:, object_class_indices] == unknown_class).sum(1)

    result_dict = {20: {}, 50: {}, 100: {}}
    for k in result_dict.keys():
        mask = box_match[:, :k] # consider top-k preds only

        results = {n: {'correct': 0, 'wrong': 0, 'background': 0}
            for n in range(len(object_class_indices) + 1)}
            # for each number of unknown objects (class labels corresponding to
            # objects along the 2nd axis of gt_classes are indicated by
            # object_class_indices)
        for gt_ind, (pred_mask, n) in enumerate(zip(mask, num_gt_unknowns)):
            pred_inds = pred_mask.nonzero()[0]
            c =      'background' if len(pred_inds) == 0                  \
                else 'correct'    if class_match[gt_ind, pred_inds].any() \
                else 'wrong'
            results[n.item()][c] += 1
        result_dict[k] = pd.DataFrame(results)
    return result_dict

def concat(dfs):
    return pd.concat(dfs, keys=range(len(dfs))).unstack().reset_index(drop=True)

def aggregate_df_dicts(df_ds):
    assert all(df.keys() == df_ds[0].keys() for df in df_ds)
    return {k: concat(v) for k, v in
            zip(df_ds[0].keys(), zip(*(dfs.values() for dfs in df_ds)))}

def main():
    # Parse command-line arguments.
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config-file',
            default='configs/e2e_relation_X_101_32_8_FPN_1x.yaml')
    parser.add_argument('-C', '--no-cache', dest='cache', action='store_false')
    parser.add_argument('-p', '--precision', type=int, default=2)
    parser.add_argument('-N', '--num', type=int)
    parser.add_argument('path')
    args = parser.parse_args()
    _, dataset_name, result_dir = analyze_method_dir(args.path)

    if args.config_file is not None:
        cfg.merge_from_file(args.config_file)

    # Load dataset info.
    dataset = load_dataset(dataset_name)
    num_obj_classes = len(dataset.ind_to_classes   )
    num_rel_classes = len(dataset.ind_to_predicates)
    unknown_class = dataset.classes_to_ind['__unknown__']
    iou_thres = cfg.TEST.RELATION.IOU_THRESHOLD

    # Load data.
    eval_results = load_eval_results(result_dir)
    groundtruths = eval_results['groundtruths']
    predictions  = eval_results['predictions' ]

    # Compute statistics for each image.
    obj_filename = os.path.join(result_dir, 'obj_stats.pkl')
    rel_filename = os.path.join(result_dir, 'rel_stats.pkl')
    if args.cache and os.path.exists(obj_filename):
        obj_stats = load(obj_filename)
        rel_stats = load(rel_filename)
    else:
        obj_stats = []
        rel_stats = []
        image_ids = list(range(len(groundtruths)))
        if args.num is not None:
            image_ids = image_ids[:args.num]
        for image_id in tqdm(image_ids):
            groundtruth = groundtruths[image_id]
            prediction  = predictions [image_id]
            gt_boxes         = groundtruth.bbox                       .numpy()
            gt_classes       = groundtruth.get_field('labels'        ).numpy()
            gt_rels          = groundtruth.get_field('relation_tuple').numpy()
            pred_boxes       = prediction.bbox                        .numpy()
            pred_classes     = prediction.get_field('pred_labels'    ).numpy()
            obj_scores       = prediction.get_field('pred_scores'    ).numpy()
            pred_rel_inds    = prediction.get_field('rel_pair_idxs'  ).numpy()
            pred_rel_classes = prediction.get_field('pred_rel_labels').numpy()

            # Compute object statistics.
            assert gt_classes.ndim == pred_classes.ndim == 1
            inds = np.argsort(obj_scores, axis=0)[::-1] # descending
            sorted_pred_boxes   = pred_boxes  [inds]
            sorted_pred_classes = pred_classes[inds]
                # sort predicted objects here (not to be used for relationships
                # because they assume the original order)
            obj_stats.append(compute_statistics(
                    gt_classes         [:, np.newaxis],
                    sorted_pred_classes[:, np.newaxis],
                    gt_boxes, sorted_pred_boxes, iou_thres, unknown_class, [0]))
                # class arrays must be 2D (as in relationship counts)

            # Compute relationship statistics.
            pred_rels = np.column_stack((pred_rel_inds, pred_rel_classes))
            gt_triplets, gt_triplet_boxes, _ = _triplet(
                    gt_rels, gt_classes, gt_boxes)
            pred_triplets, pred_triplet_boxes, _ = _triplet(
                    pred_rels, pred_classes, pred_boxes)
                # predicted relationships are already sorted in
                # relation_head.PostProcessor
            rel_stats.append(compute_statistics(gt_triplets, pred_triplets,
                    gt_triplet_boxes, pred_triplet_boxes,
                    iou_thres, unknown_class, [0, 2]))
                # consider subject and object classes only in unknown counting
                # for each GT

        obj_stats = aggregate_df_dicts(obj_stats)
        rel_stats = aggregate_df_dicts(rel_stats)
        save(obj_filename, obj_stats)
        save(rel_filename, rel_stats)

    pd.set_option('display.float_format', f'{{:.{args.precision}f}}'.format)
    dfs = {}
    for k in obj_stats.keys():
        df = pd.concat({name: df.mean(axis=0).unstack().T
            for name, df in [('obj', obj_stats[k]), ('rel', rel_stats[k])]}
        ).stack().unstack([0, 2]) # two DataFrames side by side
        df = df.rename_axis([f'@{k}', '#unknowns'], axis=1)
        print(df)
        dfs[k] = df

    output = os.path.join(args.path, 'eval')
    os.makedirs(output, exist_ok=True)
    for k, df in dfs.items():
        df.to_pickle(os.path.join(output, f'{k}.pkl'))

if __name__ == '__main__':
    main()


import glob
import os
import pandas as pd

def concatenate(path):
    files = glob.glob(
        os.path.join(path, "*.txt"))  # advisable to use os.path.join as this makes concatenation OS independent

    df_from_each_file = (pd.read_csv(f, sep='\t', header=None, names=["id", "labels", "text_a", "text_b"]) for f in
                         files)
    df = pd.concat(df_from_each_file, ignore_index=True)

    return df


import time
import logging
from lxml import etree

from pyjt.Proxy import Proxy
from pyjt.Inspector import Inspector

log = logging.getLogger(__name__)

class Locator:
    """ The Locator determines if a component matches specific search
        criteria.

        Args:
            **kwargs:   Search parameters as keyword arguments.

        Search criteria:
            The locator constructor receives a list of <name>:<value>
            keyword arguments and stores them.

            For each argument, the locator will call the component's
            get<Name>() function and compares it's return value to <value>.
            Only if the values are equal, the component matches this
            search criteria.
    """
    def __init__(self, **kwargs):
        self._filters = kwargs
        self._contains = []
        self._has = []

    def update(self, **kwargs):
        """ Add more search criteria's to this locator.

            Args:
                **kwargs:   The search criteria's to add.
                            Any existing search criteria will be overwritten.

            Returns:
                A reference to this locator object (self)
        """
        self._filters.update(**kwargs)
        return self

    def contains(self, **kwargs):
        """ Ensure the control has a sub-component with the given attributes
            somewhere in the tree.

            Args:
                **kwargs:   The search criteria for the sub-component
                            that has to be a part of the component candiate.

            Returns:
                A reference to this locator object (self)
        """
        self._contains.append(Locator(**kwargs))
        return self

    def has(self, **kwargs):
        """ Ensure the control has a direct sub-component with the given
            attributes.
        """
        self._has.append(Locator(**kwargs))
        return self

    def matches(self, component):
        """ The matches function determines wether the given component matches
            matches the search criteria.

            You can implement your own matcher by deriving a class from this
            Locator and implement this function.

            Args:
                component:  The component that should match this search
                            criteria.
            Returns:
                True, if the component matches this search criteria, otherwise False.
        """

        if not _matches(component, **self._filters):
            return False

        log.debug(f"found {component}, checking childs...")
        for locator in self._contains:
            log.debug(f"contains {locator}")
            if ComponentFinder._locate(window=component, locator=locator) is None:
                return False

        for locator in self._has:
            log.debug(f"has {locator}")
            if not _child_matches(component, locator):
                return False
        return True

    def __repr__(self):
        return f"Locator({self._filters})"

def _child_matches(component, locator):
    log.debug(f"_child_matches({component}, {locator}")
    for subcontrol in component.getComponents():
        if locator.matches(subcontrol):
            log.debug(f"_child_matches({component}, {locator}: found {subcontrol}")
            return True
    log.debug(f"_child_matches({component}, {locator}: FAILED")
    return False

class Timer:
    def __init__(self, timeout=None, cycle=None):
        self._timeout = timeout
        self._cycle = cycle
        self._first = True

    def start(self):
        self._started = time.time()
        return self

    def elapsed(self, seconds=None):
        if not self._first:
            time.sleep(self._cycle)
        self._first = False
        now = time.time()
        timeout = seconds if seconds else self._timeout
        return self._started + timeout <= now

class ComponentFinder:
    DEFAULT_TIMEOUT = 1
    DEFAULT_DELAY = 1

    @staticmethod
    def find(window, locator=None, timeout=None, **kwargs):
        timeout = timeout if timeout else ComponentFinder.DEFAULT_TIMEOUT
        timer = Timer(timeout, ComponentFinder.DEFAULT_DELAY).start()
        locator = locator if locator else Locator(**kwargs)

        while not timer.elapsed():
            log.debug(f"retry({locator})")
            hit = ComponentFinder._locate(window, locator)
            if hit is not None:
                return hit
        return None

    @staticmethod
    def find_by_xpath(control, xpath, timeout=None):
        timeout = timeout if timeout else ComponentFinder.DEFAULT_TIMEOUT
        timer = Timer(timeout, ComponentFinder.DEFAULT_DELAY).start()
        xp = etree.XPath(xpath)

        while not timer.elapsed():
            log.debug(f"retry({xpath})")
            (tree, mapping) = Inspector.etreemap(control)
            el = xp(tree)
            if not el:
                continue
            _id = el[0].get("_id")
            log.warning(_id)
            return mapping[int(_id)]

    @staticmethod
    def findIn(func, locator=None, timeout=None):
        timeout = timeout if timeout else ComponentFinder.DEFAULT_TIMEOUT
        timer = Timer(timeout, ComponentFinder.DEFAULT_DELAY).start()

        while not timer.elapsed():
            log.debug(f"retry({locator})")
            windows = func()
            for window in windows:
                hit = ComponentFinder._locate(window, locator)
                if hit is not None:
                    return hit
        return None

    @staticmethod
    def _locate(window, locator):
        log.debug(f"locate({locator})")
        if locator.matches(window):
            return window
        components = window.getComponents()
        for cmpt in components:
            log.debug(f"found component: {cmpt.getName()} {type(cmpt)}")
            hit = ComponentFinder._locate(cmpt, locator)
            if hit:
                return hit
        return None

@staticmethod
def _matches(cmpt, **kwargs):
    # log.debug(f"_matches({cmpt}, name={name}, text={text})")
    for name, value in kwargs.items():
        if name == 'role':
            cc = cmpt.object if isinstance(cmpt, Proxy) else cmpt
            log.debug(f"\tcheck01 {name} {type(cc)}={value}")
            check = isinstance(cc, value)
            log.debug(f"\t\tcheck01: {check}")
            if not check:
                return False
            continue
        fname = f"get{name[0].capitalize()}{name[1:]}"
        log.debug(f"check({name}, {value}) -> {fname}: {cmpt}")
        if not hasattr(cmpt, fname):
            log.debug(f"\tcheck({name}) component has no attr {fname}")
            return False
        fnc = getattr(cmpt, fname)
        _value = fnc()
        log.debug(f"\tcheck {value} == {_value}")
        if value is None and _value is None:
            continue
        if not _value:
            return False
        if value not in _value:
            return False
    if not cmpt.isShowing():
        log.debug("\tcheck9")
        return False
    log.debug("\tcheck10")
    return True


import random
import json
from collections import defaultdict

import numpy as np
from rdkit.Chem.Scaffolds import MurckoScaffold
from rdkit.Chem import MolFromSmiles

from ._base_loader import _BaseDataLoader

def _myShuffle(x, *s):
    x[slice(*s)] = random.sample(x[slice(*s)], len(x[slice(*s)]))

class JsonLoader(_BaseDataLoader):
    def __init__(self, path, *_, rand_seed=None, **kwargs):
        """
        path: path to the csv file
        kwargs: keyword arguments for the pandas.read_csv function
        """
        super().__init__(path)
        self.kwargs = kwargs
        if rand_seed is not None:
            random.seed(rand_seed)

    @property
    def data_list(self):
        """Load the json file

        Returns:
            The data in json file. dict or list in most cases.
        """
        try:
            return self._data_list
        except AttributeError:
            with open(self.path, "r") as f:
                self._data_list = json.load(f)
            return self._data_list

    def _split(self, ratio, shuffle, nl_symbol):
        indices = list()
        for idx, data in enumerate(self.data_list):
            if nl_symbol in data[2]:
                continue
            indices.append(idx)
        if shuffle:
            random.shuffle(indices)
        splitter = int(len(indices) * ratio)
        training_indices = indices[:splitter]
        testing_indices = indices[splitter:]
        return training_indices, testing_indices

    def _scaffold_split(self, ratio, shuffle, nl_symbol):
        # create dict of the form {scaffold_i: [idx1, idx....]}
        all_scaffolds = defaultdict(list)
        n_samples = 0
        for i, data in enumerate(self.data_list):
            if nl_symbol in data[2]:  # partially labeled data
                continue
            smiles = data[0]
            if MolFromSmiles(smiles) is None:
                continue
            scaffold = MurckoScaffold.MurckoScaffoldSmiles(
                smiles=smiles, includeChirality=True
            )
            all_scaffolds[scaffold].append(i)
            n_samples += 1

        # sort from largest to smallest sets
        all_scaffolds = {key: sorted(value) for key, value in all_scaffolds.items()}
        all_scaffold_sets = [
            scaffold_set
            for (scaffold, scaffold_set) in sorted(
                all_scaffolds.items(), key=lambda x: (len(x[1]), x[1][0]), reverse=True
            )
        ]

        # shuffle the order of the sets that have less than 5 members
        # make label distribution more even
        if shuffle:
            for i, scaffold_set in enumerate(all_scaffold_sets):
                if len(scaffold_set) <= 5:
                    start_point = i
                    break
            _myShuffle(all_scaffold_sets, start_point, None)

        # get train, valid, and test indices
        train_cutoff = ratio * n_samples
        train_idx, test_idx = [], []
        for scaffold_set in all_scaffold_sets:
            if len(train_idx) + len(scaffold_set) > train_cutoff:
                test_idx.extend(scaffold_set)
            else:
                train_idx.extend(scaffold_set)

        assert len(set(train_idx).intersection(set(test_idx))) == 0

        return train_idx, test_idx

    def load_data(
        self, ratio=0.9, shuffle=True, nl_symbol="_", scaffold_splitting=False
    ):
        """Load the training and testing sets.
        Args:
            ratio (float): the ratio between training and testing set.
            shuffle (bool): if shuffle the indices.
            nl_symbol: the symbol denoting data with no label in the dataset.
            scaffold_splitting (bool): Use scaffold splitting. (ICLR2020, ContextPred)

        Returns:
            data (list): [x_train, y_train, x_test, y_test]
        """
        if scaffold_splitting:
            training_indices, testing_indices = self._scaffold_split(
                ratio, shuffle, nl_symbol
            )
        else:
            training_indices, testing_indices = self._split(ratio, shuffle, nl_symbol)
        x_train, y_train, x_test, y_test = list(), list(), list(), list()
        for idx in training_indices:
            x_train.append(self.data_list[idx][1])
            y_train.append(self.data_list[idx][2])
        for idx in testing_indices:
            x_test.append(self.data_list[idx][1])
            y_test.append(self.data_list[idx][2])
        x_train = np.array(x_train, dtype=np.float)
        y_train = np.array(y_train)
        x_test = np.array(x_test, dtype=np.float)
        y_test = np.array(y_test)
        return x_train, y_train, x_test, y_test

    def load_unlabeled(self, nl_symbol="_"):
        """Load the unlabeled data.
        params:
        nl_symbol: the symbol denoting data with no label dataset.
        ================================================================
        return:
        data (numpy array): unlabeled dataset with respect to the specified
                            label column
        """
        unlabeled = list()
        labels = list()
        for data in self.data_list:
            if nl_symbol not in data[2]:
                continue
            unlabeled.append(np.array(data[1]))
            labels.append(data[2])
        return np.stack(unlabeled), labels


import numpy as np
import matplotlib.pyplot as plt
from quadprog import solve_qp
from numpy.linalg import matrix_rank as rank,inv
from numpy import hstack,vstack
from plot_polygon import plot_convex_shape

def Acc(T,alpha=1e-8):
    return np.array([[400.0/7.0*pow(T,7),40*pow(T,6),120.0/5.0*pow(T,5),10*pow(T,4),0,0],
                     [40*pow(T,6),144/5*pow(T,5),18*pow(T,4),8*pow(T,3),0,0],
                     [120.0/5.0*pow(T,5),18*pow(T,4),12*pow(T,3),6*pow(T,2),0,0],
                     [10*pow(T,4),8*pow(T,3),6*pow(T,2),4*T,0,0],
                     [0,0,0,0,alpha,0],
                     [0,0,0,0,0,alpha]])

def nt(t):
    return np.array([[pow(t,5),pow(t,4),pow(t,3),pow(t,2),t,1]])
def ddnt(t):
    return np.array([[20*pow(t,3),12*pow(t,2),6*t,2,0,0]])
def dnt(t):
    return np.array([[5*pow(t,4),4*pow(t,3),3*pow(t,2),2*t,1,0]])

def diagm(matlist: list,rcol = 0):
    mat  = None
    for it in matlist :
        if mat is None:
            mat = it
        else:
            mat = np.block([[mat,np.zeros((mat.shape[0],it.shape[1]-rcol))],
                            [np.zeros((it.shape[0],mat.shape[1]-rcol)),it]])
    return mat
#n
def diagv(veclist: list,rcol = 0):
    vec  = None
    for it in veclist :
        if vec is None:
            vec = it
        else:
            vec = np.block([vec,np.zeros(rcol),it])
    return vec

"""
for some problem provide the solutions
min 1/2 x^T G x  -a^Tx
s.t. C.T x>= b
"""
def reduce_convex(polygon_set,s=0.1,w=0.05):
    """reduce the shape of the support polygons"""
    reduce_polygon = []
    for (vertex,duration) in polygon_set:
        num_vertices = len(vertex)
        new_vertex = vertex.copy()
        if num_vertices>2:
            print(">2")
            c = np.zeros(num_vertices)
            vec_n = np.zeros((num_vertices,2))
            for i in range(num_vertices):
                direct = vertex[(i + 1) % num_vertices] - vertex[i]
                normal = np.array([direct[1], -direct[0]])#inside polygons direct
                normal /= np.linalg.norm(normal)
                new_vertex[i] +=s*normal
                vec_n[i] = normal
                c[i] = -normal@new_vertex[i]
                # new_vertex[(i + 1) % num_vertices] +=s*normal
            for i in range(num_vertices):
                new_vertex[i] = -np.linalg.inv(vec_n[[i,(i+1)% num_vertices]])@c[[i,(i+1)% num_vertices]]
            reduce_polygon.append((new_vertex,duration))
        else:
            direct = vertex[1] - vertex[0]
            direct /=np.linalg.norm(direct)
            normal = np.array([direct[1], -direct[0]])#inside polygons direct
            normal /= np.linalg.norm(normal)
            v1 = vertex[0]+w*normal+s*direct
            v2 = vertex[1]+w*normal-s*direct
            v3 = vertex[1]-w*normal-s*direct
            v4 = vertex[0]-w*normal+s*direct
            reduce_polygon.append(([v1,v2,v3,v4],duration))
            print(2)
    return reduce_polygon

dx = 0.2
dy = 0.2
delta = 0.01
#leg sequence was changed, and the  3rd column is the RR leg, and the 4th column is the RF leg
polygons = [(np.array([[0.5,   0.25],[0.5   ,-0.25] ,[-0.5   ,-0.25],[-0.5   ,0.25]]),0.05),#0.05-0.0 all down until LF lift
            (np.array([              [0.5   ,-0.25] ,[-0.5   ,-0.25],[-0.5   ,0.25]]),0.15),#0.2-0.05,LF lift until RR lift
            (np.array([              [0.5   ,-0.25]                 ,[-0.5   ,0.25]]),0.10),#0.3-0.2 RR lift until LF touch
            (np.array([[0.5+dx,0.25],[0.5   ,-0.25]                 ,[-0.5   ,0.25]]),0.15),#0.45-0.3LF touch until RR touch
            (np.array([[0.5+dx,0.25],[0.5   ,-0.25] ,[-0.5+dx,-0.25],[-0.5   ,0.25]]),0.10),#0.55-0.45 all touch until LR lift
            (np.array([[0.5+dx,0.25],                [-0.5+dx,-0.25],[-0.5   ,0.25]]),0.15),#0.70-0.55 FR lift until RL lift
            (np.array([[0.5+dx,0.25],                [-0.5+dx,-0.25]               ]),0.10),#0.80-0.70 RL lift until FR touch
            (np.array([[0.5+dx,0.25],[0.5+dx,-0.25] ,[-0.5+dx,-0.25]               ]),0.15),#0.95-0.8FR touch until RL touch
            (np.array([[0.5+dx,0.25],[0.5+dx,-0.25] ,[-0.5+dx,-0.25],[-0.5+dx,0.25]]),0.05)]#1.0 -0.95 RL touch until next sequnece

n = 2 #number of segments
d = 2 #number of coordinates

duration=[polygons[j][1] for j in range(len(polygons))]
c_point = [sum(polygons[j][0])/4 for j in range(len(polygons))]
stp=c_point[0]
dstp=[0.,0.,0.]
ddstp=[0.,0.,0.]
fp=c_point[1]
#
Q_all = np.zeros((0,0))
a_all = np.zeros(0)
Ceq_all = np.zeros((0,0))
beq_all = np.zeros(0)
Ciq_all = np.zeros((0,0))
biq_all = np.zeros(0)

for i in range(n):
    #for the sampling of the line, we have some others formulations
    Q_ = None
    a_ = None
    Ceq_ = None
    beq_ = None
    #for the qp slover
    Ciq_ = None
    biq_ = None
    if i == 0 :#first segment
        Accmat = diagm([Acc(duration[i]) for j in range(d)])
        #
        pCeq = diagm([nt(0) for j in range(d)])
        pbeq = hstack([stp[j] for j in range(d)])
        #
        pdCeq = diagm([dnt(0) for j in range(d)])
        pdbeq = hstack([dstp[j] for j in range(d)])
        #
        pddCeq = diagm([ddnt(0) for j in range(d)])
        pddbeq = hstack([ddstp[j] for j in range(d)])
        #
        if n>1:
            Ceq = diagm([hstack([nt(duration[i]),np.zeros((1,6*(d-1))),-nt(0)]) for j in range(d)],6*d)
            beq = hstack([0. for j in range(d)])
            #
            dCeq = diagm([hstack([dnt(duration[i]),np.zeros((1,6*(d-1))),-dnt(0)]) for j in range(d)],6*d)
            dbeq = hstack([0. for j in range(d)])
            #
            ddCeq = diagm([hstack([ddnt(duration[i]),np.zeros((1,6*(d-1))),-ddnt(0)])for j in range(d)],6*d)
            ddbeq = hstack([0. for j in range(d)])
        Q_= Accmat
        # a=hstack([np.zeros(6*d)])
        Ceq_=diagm([vstack([pCeq,pdCeq,pddCeq]),vstack([Ceq,dCeq,ddCeq])],6*d)
        beq_=hstack([pbeq,pdbeq,pddbeq,beq,dbeq,ddbeq])
        # Ciq
        # biq

    if i > 1 and i < n-1:#middle segment
        Accmat = diagm([Acc(duration[i]) for j in range(d)])
        #
        Ceq = diagm([hstack([nt(duration[i]),np.zeros((1,6*(d-1))),-nt(0)]) for j in range(d)],6*d)
        beq = hstack([0. for j in range(d)])
        #
        dCeq = diagm([hstack([dnt(duration[i]),np.zeros((1,6*(d-1))),-dnt(0)]) for j in range(d)],6*d)
        dbeq = hstack([0. for j in range(d)])
        #
        ddCeq = diagm([hstack([ddnt(duration[i]),np.zeros((1,6*(d-1))),-ddnt(0)])for j in range(d)],6*d)
        ddbeq = hstack([0. for j in range(d)])
        #
        Q_=Accmat
        # a
        Ceq_=vstack([Ceq,dCeq,ddCeq])
        beq_=hstack([beq,dbeq,ddbeq])
        # Ciq
        # biq

    if i == n-1:#final segment
        Accmat = diagm([Acc(duration[i])for j in range(d)])
        Tm = diagm([nt(duration[i]).T@nt(duration[i]) for j in range(d)])
        am = hstack([nt(duration[i])*fp[j] for j in range(d)])
        am = am.reshape(-1)
        #
        Ciq = diagm([vstack([nt(duration[i]),-nt(duration[i])]) for j in range(d)])
        biq = hstack([hstack([fp[j]-delta,-(fp[j]+delta)]) for j in range(d)])
        #
        # dCeq = diagm([dnt(t)for j in range(d)])
        # dbeq = hstack([0. for j in range(d)])
        # #
        # ddCeq = diagm([ddnt(t)for j in range(d)])
        # ddbeq = hstack([0. for j in range(d)])
        Q_=Accmat+Tm
        a_=am
        # Ceq=vstack([Ceq,dCeq,ddCeq])
        # beq=hstack([beq,dbeq,ddbeq])
        Ciq_ = Ciq
        biq_ = biq
    #calculate Q,a,c,b, eq constraints
    if not Q_ is None:
        Q_all= diagm([Q_all,Q_],Q_all.shape[1]-i*6*d)
    if not a_ is None:
        a_all=diagv([a_all,a_],i*6*d-a_all.shape[0])
    if not Ceq_ is None:
        Ceq_all= diagm([Ceq_all,Ceq_],Ceq_all.shape[1]-i*6*d)
        beq_all=diagv([beq_all,beq_])
    else:
        Ceq_all= hstack([Ceq_all,np.zeros((Ceq_all.shape[0],(i+1)*6*d-Ceq_all.shape[1]))])
    if not Ciq_ is None:
        Ciq_all= diagm([Ciq_all,Ciq_],Ciq_all.shape[1]-i*6*d)
        biq_all=diagv([biq_all,biq_])
    else:
        Ciq_all= hstack([Ciq_all,np.zeros((Ciq_all.shape[0],(i+1)*6*d-Ciq_all.shape[1]))])

eqns = Ceq_all.shape[0]
C = vstack([Ceq_all,Ciq_all])
b = hstack([beq_all,biq_all])
xf, f, xu, iters, lagr, iact = solve_qp(Q_all,a_all,C.T,b,eqns)

N =10
traj_p = np.zeros((d,n*N))
traj_dp = np.zeros((d,n*N))
traj_ddp = np.zeros((d,n*N))
tot_time = np.zeros(n*N)

for i in range(n):
    time = np.linspace(0,duration[i],N)
    for j in range(N):
        tot_time[j+i*N] = tot_time[i*N-1]+time[j]
        for k in range(d):
            traj_p[k,j+i*N] = nt(time[j])@xf[i*6+k*6:i*6+(k+1)*6]
            traj_dp[k,j+i*N] = dnt(time[j])@xf[i*6+k*6:i*6+(k+1)*6]
            traj_ddp[k,j+i*N] = ddnt(time[j])@xf[i*6+k*6:i*6+(k+1)*6]

LABEL={
    0:'x',1:'y',2:'z'
}

plt.figure()
plt.plot(tot_time,traj_p[1,:],label='pos')
plt.legend()
plt.grid()

plt.figure()
plt.plot(tot_time,traj_dp[1,:],label='vel')
plt.legend()
plt.grid()

plt.figure()
plt.plot(tot_time,traj_ddp[1,:],label="acc")
plt.legend()
plt.grid()
# plt.figure()
# plt.show(tra)