#
#   sprite sheet to c64 sprite converter
#   (c)2020 by awsm of Mayday!
#   http://www.awsm.de
#
#   converts a bitmap with a spritesheet
#   into a sprite data binary
#
#
#   usage:
#   spritesheet_to_spd(input_file,output_file, sprite_width, sprite_height)
#   input_file: name of the PNG image. Image must be (24*X)*(21*X) pixels
#   output_file: name of the sprite file (usually ends with .spd)
#   sprite_width: usually 24, which is the width of a hires C64 sprite
#   sprite_height: usually 21, which is the height of a hires C64 sprite
#
#   example:
#   spritesheet_to_spd("spritesheet.png","sprites.spd", 24, 21)
#   loads a PNG file called "spritesheet.png" and exports it
#   as "sprites.spd" in C64 SpritePad format
#

from PIL import Image
import math

def get_sprites(image_path,x,y):
    i = Image.open(image_path, 'r')
    i.pixels = list(i.getdata())
    i.x, i.y = i.size
    i.col = i.x / x
    i.row = i.y / y
    i.total = int(i.row * i.col)
    i.sprite_width = x
    i.sprite_height = y
    return i

def convert_sprite(sprites,number):
    sprite_length = sprites.sprite_width * sprites.sprite_height
    row = int(math.ceil(number / sprites.col)) -1
    row_offset = row * (sprite_length * sprites.col)
    col = number - row*sprites.col -1
    col_offset = col * sprites.sprite_width
    start = int(row_offset + col_offset)
    end = int(start + sprite_length)

    line = 0
    pixel_count = 0
    bit_array = []

    for i in range(start,end):
        pixel_count = pixel_count + 1

        pixel_pos = i + line
        pixel = sprites.pixels[pixel_pos]

        if pixel == 0:
            print(" ", end='')
            bit_array.append(0)
        else:
            print("#", end='')
            bit_array.append(1)

        if pixel_count == sprites.sprite_width:
            print()
            pixel_count = 0
            line = line + sprites.x -sprites.sprite_width
    return bit_array

def bits_to_bytes(data):
    byte_counter = 0
    byte = ""
    bytes_array = []

    for pixel in data:
        byte = byte + str(pixel)

        if byte_counter == 7:
            byte_counter = 0
            hex_number = int(byte,2)
            bytes_array.append(hex_number)
            byte = ""
        else:
            byte_counter = byte_counter + 1
    bytes_array.append(1)
    return bytes_array

def write_file(filename, data):
    if not filename:
        filename = "test"

    newFile = open(filename, "wb")
    file_array = [11,8,6]
    file_array = file_array + data
    newFileByteArray = bytearray(file_array)
    newFile.write(newFileByteArray)

def png_to_spd(input_filename, output_filename, sprite_width, sprite_height ):
    image = get_sprites(input_filename, sprite_width,sprite_height)

    all_sprites = []

    for i in range(1,image.total+1):
        sprite_data = convert_sprite(image,i)
        sprite_bytes = bits_to_bytes(sprite_data)
        all_sprites = all_sprites + sprite_bytes

    write_file(output_filename,all_sprites)

#-----------------------------------------------------------------

png_to_spd("test.png","test2.spd", 24, 21)


import json
import os
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import GridSearchCV, train_test_split
import mlflow
from util import (
    DataPrepUtil, initialize_column_transformer, initialize_pipeline, plot_roc_curve,
    plot_learning_curve, config, record_hyperparameters
)

if __name__ == "__main__":
    # Data cleaning step
    data_prep_util = DataPrepUtil()
    X_train, y_train = data_prep_util.load_train_data(config.get('paths').get('train_data'))
    X_train, X_test, y_train, y_test = train_test_split(
        X_train, y_train, test_size=config.get('test_size'), random_state=config.get('random_state'),
        stratify=y_train
    )

    # Hyperparameter tuning, metric and parameter logging
    mlflow.set_experiment(config.get('mlflow').get('exp_name'))
    RUN_NAME = os.path.basename(__file__).split('.')[0]
    with mlflow.start_run(run_name=RUN_NAME):
        print('Starting run {}...'.format(RUN_NAME))
        mlflow.log_param('test_size', config.get('test_size'))
        pipeline = initialize_pipeline(
            initialize_column_transformer(scale_values=True),
            KNeighborsClassifier()
        )
        param_grid = {
            'model__n_neighbors': config.get('hyperparams').get('knn').get('n_neighbors'),
            'model__weights': config.get('hyperparams').get('knn').get('weights')
        }
        gs_clf = GridSearchCV(
            pipeline, param_grid=param_grid,
            scoring=config.get('scoring'), cv=config.get('k'), verbose=1
        )
        gs_clf.fit(X_train, y_train)
        mlflow.log_params(gs_clf.best_params_)
        mlflow.log_metrics({
                '_'.join([config.get('scoring'), 'train']): gs_clf.score(X_train, y_train),
                '_'.join([config.get('scoring'), 'test']): gs_clf.score(X_test, y_test)
        })

        print('Plotting ROC curve...')
        img_fn = 'roc_{}.png'.format(RUN_NAME)
        plot_roc_curve(gs_clf, X_test, y_test, img_fn)

        # print('Plotting learning curve...')
        # img_fn = 'learning_curve_{}.png'.format(RUN_NAME)
        # plot_learning_curve(gs_clf, X_train, y_train, img_fn)

        print('Recording hyperparameters used...')
        txt_fn = 'hyperparams_{}.txt'.format(RUN_NAME)
        record_hyperparameters(txt_fn, config.get('hyperparams').get('knn'))


from collections.abc import Iterable
from itertools import chain

takes_synonyms = ['takes', 'take', 'consume', 'consumes']
gives_synonyms = ['gives', 'give', 'produce', 'produces']
constraints_synonyms = ['constraints', "dependency", "dependencies"]
reciprocal_synonyms = ['reciprocal']
name_synonyms = ['name']
type_synonyms = ['type']
module_synonyms = ['module']
link_synonyms = ['link', 'links']
index_synonyms = ['index']

def first_intersection(a, b):
    if isinstance(b, Iterable):
        for e in a:
            if e in b:
                return e
    return None

class Element:
    @classmethod
    def get_values_from_parameter(cls, parameter):
        """
        Parses the parameter values
        There are several types of parameter values:
            Single value:       1           gpio
            List of values:     1 2 3
            Diapason of values: -1..3
            List of diapasons:  1..3 5..7
            Mixed type:         1 3..5 7
        There are several types of parameter constraints:
            Single constraint: =1   ~1
            List of single constraints: =1 =-3
            Diapason constraint: =1..3  ~5..7
        Values/constraints in lists have a logical relationship of "and"
        :param: parameter: string or list to parse
        :return: dictionary with a values and constraints
        """
        values = []
        constraints = []
        parameters = []
        if isinstance(parameter, str):
            if ' ' in parameter:
                # Complex parameter
                parameters = parameter.split(' ')
            else:
                # Simple parameter
                parameters.append(parameter)
        for param in parameters:
            if '=' == param[0]:
                # Value must be equal to param, or must be in diapason
                if '..' in param:
                    boundaries = param.split('..')
                    constraints.append((boundaries[0][1:], boundaries[-1]))
                else:
                    constraints.append((param[1:]))
            elif '~' == param[0]:
                # Value must be not equal to param, or must be out of diapason
                if '..' in param:
                    boundaries = param.split('..')
                    constraints.append([boundaries[0][1:], boundaries[-1]])
                else:
                    constraints.append([param[1:]])
            else:
                values.append(param)

        return {'values': values, 'constraints': constraints}

class Pin(Element):
    @classmethod
    def parse(cls, name, parent_module, descriptor):
        parameters = {}
        reciprocal = {}
        links = []
        scope = ''
        if descriptor:
            if type(descriptor) == str:
                links.append(descriptor)
            else:
                for pin_attribute_name, pin_attribute_value in descriptor.items():
                    if pin_attribute_name in name_synonyms:
                        name = pin_attribute_value
                    elif pin_attribute_name in reciprocal_synonyms:
                        # todo: parse
                        reciprocal = pin_attribute_value
                    elif pin_attribute_name in link_synonyms:
                        links.append(pin_attribute_value)
                    elif pin_attribute_name == 'scope':
                        assert isinstance(pin_attribute_value, str)
                        scope = {'values': pin_attribute_value}
                    else:
                        parameters[pin_attribute_name] = cls.get_values_from_parameter(pin_attribute_value)

        # If type is unknown, get type and index from name
        type_synonym = first_intersection(type_synonyms, parameters)
        if not type_synonym:
            type_synonym = type_synonyms[0]
            listed_name = name.split('_')
            if listed_name[-1].isdigit():
                parameters[type_synonym] = {'values': ['_'.join(listed_name[:-1])]}
                index_synonym = first_intersection(index_synonyms, parameters)
                if not index_synonym:
                    index_synonym = index_synonyms[0]
                    parameters[index_synonym] = {'values': [int(listed_name[-1])]}
            else:
                parameters[type_synonym] = {'values': [name]}

        # Add type constraints (types must matches for connecting)
        for type_synonym in type_synonyms:
            if type_synonym in parameters:
                for pin_type_name in parameters[type_synonym]['values']:
                    if 'constraints' not in parameters[type_synonym]:
                        parameters[type_synonym]['constraints'] = []
                    if not parameters[type_synonym]['constraints']:
                        parameters[type_synonym]['constraints'].append(pin_type_name)

        # Add module constraints to the pin
        for type_synonym in type_synonyms:
            if type_synonym in parameters:
                for pin_type_name in parameters[type_synonym]['values']:
                    if pin_type_name in parent_module.constraints:
                        for parameter_name in parent_module.constraints[pin_type_name]:
                            if parameter_name not in parameters[type_synonym]:
                                parameters[parameter_name] = {'values': [], 'constraints': []}
                            for pin_type_value in parent_module.constraints[pin_type_name][parameter_name]['values']:
                                parameters[parameter_name]['values'].append(pin_type_value)
                            for pin_type_constraint in parent_module.constraints[pin_type_name][parameter_name]['constraints']:
                                parameters[parameter_name]['constraints'].append(pin_type_constraint)

        pin = cls(parent_module, parameters)
        pin.name = name
        pin.reciprocal = reciprocal
        pin.links = links
        return pin, scope

    def __init__(self, module, parameters={}):
        self.name = None
        self.module = module
        self.connected_to = []
        self.parameters = parameters
        self.reciprocal = {}
        self.links = []

    def __str__(self):
        repr = 'Pin {}: parameters: {}'.format(self.name, self.parameters)
        for connected_pin in self.connected_to:
            repr += '\n        Connected to: {}:{}'.format(connected_pin.module.name, connected_pin.name)
        return repr

    def __repr__(self):
        return self.__str__()

    def connect(self, pin):
        if self.is_pins_connectable(pin):
            self.connected_to.append(pin)
            pin.connected_to.append(self)

    def is_pins_connectable(self, pin):
        """
        Checks parameters for matching.
        :param: pin2: pin to check a connection possibility
        :return: True/False - connection is possible / connection is not possible
        """
        match = False
        pin1 = self
        pin2 = pin
        for _ in range(2):
            for parameter in pin1.parameters:
                if 'constraints' in pin1.parameters[parameter] and pin1.parameters[parameter]['constraints']:
                    pin1_constraints = pin1.parameters[parameter]['constraints']
                    if parameter in pin2.parameters:
                        pin2_values = pin2.parameters[parameter]['values']
                        # Compare values and constraints
                        values_is_satisfied = [False] * len(pin2_values)
                        for cnstr in pin1_constraints:
                            constraint_is_satisfied = False
                            constraint_index = 0
                            for vl in pin2_values:
                                if isinstance(cnstr, tuple):
                                    # Value must be in diapason
                                    # todo
                                    pass
                                elif isinstance(cnstr, list):
                                    # Value must NOT be in diapason
                                    # Diapason
                                    # todo
                                    pass
                                else:
                                    # Other type
                                    # The value must equal to the constraint
                                    if cnstr in vl:
                                        values_is_satisfied[constraint_index] = True if values_is_satisfied[constraint_index] is False else False
                                        constraint_is_satisfied = True if constraint_is_satisfied is False else False
                                constraint_index += 1
                            if not constraint_is_satisfied:
                                # Constraint is not satisfied
                                return False
                        if False in values_is_satisfied:
                            # At list one value out of constraints
                            return False
                    else:
                        # There are not values to compare with constraints
                        # Pins is not connectable
                        return False
                    # All constraints and values is satisfied
                    match = True
            pin2 = self
            pin1 = pin
        return match

class Module(Element):
    parse_recurse_deep_counter = 0

    @classmethod
    def parse(cls, parent, path, descriptor):
        cls.parse_recurse_deep_counter += 1

        links = {}
        items = descriptor.items()
        module = cls(parent, None, path)
        for attribute_name, attribute_value in items:
            type_synonym = first_intersection(type_synonyms, attribute_value)
            if attribute_name in name_synonyms:
                module.name = attribute_value
            elif type_synonym and attribute_value[type_synonym] in module_synonyms:
                # Module in root
                module.children.append(cls.parse(module, path + ':' + attribute_name, attribute_value))
            elif attribute_name in takes_synonyms:
                for take_name, take_value in attribute_value.items():
                    type_synonym = first_intersection(type_synonyms, take_value)
                    if type_synonym and take_value[type_synonym] in module_synonyms:
                        # Element is module, parse it recursively
                        module.children.append(cls.parse(module, path + ':' + take_name, take_value))
                    else:
                        # Element is pin, parse it
                        pin, scope = Pin.parse(take_name, module, take_value)
                        module.add_pin_to_module_by_scope(pin, scope, False)
            elif attribute_name in gives_synonyms:
                for give_name, give_value in attribute_value.items():
                    type_synonym = first_intersection(type_synonyms, give_value)
                    if type_synonym and give_value[type_synonym] in module_synonyms:
                        # Element is module, parse it recursively
                        module.children.append(cls.parse(module, path + ':' + give_name, give_value))
                    else:
                        # Element is pin, parse it
                        pin, scope = Pin.parse(give_name, module, give_value)
                        module.add_pin_to_module_by_scope(pin, scope, True)
            elif attribute_name in constraints_synonyms:
                for pin_type in attribute_value:
                    module.constraints[pin_type] = {}
                    if attribute_value[pin_type]:
                        for parameter in attribute_value[pin_type]:
                            module.constraints[pin_type][parameter] =\
                                cls.get_values_from_parameter(attribute_value[pin_type][parameter])
            elif attribute_name in link_synonyms:
                links.update(attribute_value)
            else:
                type_synonym = first_intersection(type_synonyms, attribute_value)
                if type_synonym in module_synonyms:
                    # Element is module, parse it recursively
                    module.children.append(cls.parse(module, path + ':' + attribute_name, attribute_value))
                else:
                    module.attributes[attribute_name] = attribute_value

        # Resolve links
        for give_pin_name in module.give_pins:
            for link_pin_name in module.give_pins[give_pin_name].links:
                if link_pin_name in module.take_pins:
                    module.links[module.give_pins[give_pin_name]] = module.take_pins[link_pin_name]
                    module.links[module.take_pins[link_pin_name]] = module.give_pins[give_pin_name]
                else:
                    # Link error, no such pin
                    print('\nLink error, no such pin:', link_pin_name, '\n')
        for take_pin_name in module.take_pins:
            for link_pin_name in module.take_pins[take_pin_name].links:
                if link_pin_name in module.give_pins:
                    module.links[module.take_pins[take_pin_name]] = module.give_pins[link_pin_name]
                    module.links[module.give_pins[link_pin_name]] = module.take_pins[take_pin_name]
                else:
                    # Link error, no such pin
                    print('\nLink error, no such pin:', link_pin_name, '\n')
        for take_pin_name in links:
            if take_pin_name in module.take_pins:
                if type(links[take_pin_name]) and links[take_pin_name] in module.give_pins:
                    module.links[module.take_pins[take_pin_name]] = module.give_pins[links[take_pin_name]]
                    module.links[module.give_pins[links[take_pin_name]]] = module.take_pins[take_pin_name]
                else:
                    # Link error, no such pin
                    print('\nLink error, no such pin:', module.links[take_pin_name], '\n')
            else:
                # Link error, no such pin
                print('\nLink error, no such pin:', take_pin_name, '\n')

        cls.parse_recurse_deep_counter -= 1

        # Root module post parse actions
        if cls.parse_recurse_deep_counter == 0:
            # Resolve all connections with children
            for module_a in chain(module.children, [module]):
                for module_b in chain(module.children, [module]):
                    if module_a != module_b:
                        module_a.connect(module_b)
        return module

    def __init__(self, parent, name, path):
        self.parent = parent
        self.name = name
        self.path = path
        self.take_pins = {}
        self.give_pins = {}
        self.attributes = {}
        self.children = []
        self.constraints = {}
        self.links = {}

    def __str__(self):
        repr = '\nModule {}: name: {}, attributes: {}'.format(self.path, self.name, self.attributes)
        for pin in self.give_pins:
            repr += '\n    ->' + str(self.give_pins[pin])
        for pin in self.take_pins:
            repr += '\n    <-' + str(self.take_pins[pin])
        for link in self.links:
            repr += '\n        Link: ' + link.name + ' - ' + self.links[link].name
        for child in self.children:
            repr += '\n    ' + str(child)
        return repr

    def __repr__(self):
        return self.__str__()

    def connect(self, module):
        print('Trying to connect module {} to {}'.format(self.name, module.name))
        for pin in self.take_pins:
            for pin_candidate in module.give_pins:
                # Check all constraints
                for pin_parameter in self.take_pins[pin].parameters:
                    for pin_candidat_parameter in module.give_pins[pin_candidate].parameters:
                        if pin_parameter == pin_candidat_parameter:
                            # print('    ', pin_parameter, self.take_pins[pin].parameters[pin_parameter])
                            self.take_pins[pin].connect(module.give_pins[pin_candidate])

    def add_pin_to_module_by_scope(self, pin, scope, is_pin_give=False):
        # Add pin to modules according scope
        module = self
        if is_pin_give:
            module.give_pins[pin.name] = pin
        else:
            module.take_pins[pin.name] = pin
        if scope:
            while module:
                if module.parent:
                    if module.name in scope['values']:
                        if is_pin_give:
                            module.give_pins[pin.name] = pin
                        else:
                            module.take_pins[pin.name] = pin
                        break
                else:
                    # It's a root module
                    if 'root' in scope['values']:
                        if is_pin_give:
                            module.give_pins[pin.name] = pin
                        else:
                            module.take_pins[pin.name] = pin
                module = module.parent


import os, glob, sys
import numpy as np
from PyQt5 import QtGui, QtCore
import pyqtgraph as pg
from . import guiparts
from facemap import roi, utils
from natsort import natsorted
import pickle
from PyQt5.QtWidgets import (QFileDialog, QMessageBox)

def open_file(parent, file_name=None):
    if file_name is None:
        file_name = QFileDialog.getOpenFileName(parent,
                            "Open movie file", "", "Movie files (*.h5 *.mj2 *.mp4 *.mkv *.avi *.mpeg *.mpg *.asf *m4v)")
    # load ops in same folder
    if file_name:
        parent.filelist = [[file_name[0]]]
        load_movies(parent)

def open_folder(parent, folder_name=None):
    if folder_name is None:
        folder_name = QFileDialog.getExistingDirectory(parent,
                            "Choose folder with movies")
    # load ops in same folder
    if folder_name:
        extensions = ['*.mj2','*.mp4','*.mkv','*.avi','*.mpeg','*.mpg','*.asf']
        file_name = []
        for extension in extensions:
            files = glob.glob(folder_name+"/"+extension)
            files = [folder_name+"/"+os.path.split(f)[-1] for f in files]
            file_name.extend(files)
        for folder in glob.glob(folder_name+'/*/'):
            for extension in extensions:
                files = glob.glob(os.path.join(folder_name,folder,extension))
                files = [folder_name+"/"+folder+"/"+os.path.split(f)[-1] for f in files]
                file_name.extend(files)
        if len(file_name) > 1:
            choose_files(parent, file_name)
            load_movies(parent)

def choose_files(parent, file_name):
    parent.filelist = file_name
    LC=guiparts.ListChooser('Choose movies', parent)
    result = LC.exec_()
    if len(parent.filelist)==0:
        parent.filelist=file_name
    parent.filelist = natsorted(parent.filelist)
    if len(parent.filelist)>1:
        dm = QMessageBox.question(
            parent,
            "multiple videos found",
            "are you processing multiple videos taken simultaneously?",
            QMessageBox.Yes | QMessageBox.No,
        )
        if dm == QMessageBox.Yes:
            print('multi camera view')
            # expects first 4 letters to be different e.g. cam0, cam1, ...
            files = []
            iview = [os.path.basename(parent.filelist[0])[:4]]
            for f in parent.filelist[1:]:
                fbeg = os.path.basename(f)[:4]
                inview = np.array([iv==fbeg for iv in iview])
                if inview.sum()==0:
                    iview.append(fbeg)
            print(iview)
            for k in range(len(iview)):
                ij = 0
                for f in parent.filelist:
                    if iview[k] == os.path.basename(f)[:4]:
                        if k==0:
                            files.append([])
                        files[ij].append(f)
                        ij +=1
            parent.filelist = files
        else:
            print('single camera')
            files = parent.filelist.copy()
            parent.filelist = []
            for f in files:
                parent.filelist.append([f])

    else:
        parent.filelist = [parent.filelist]
    parent.filelist = natsorted(parent.filelist)

def open_proc(parent, file_name=None):
    if file_name is None:
        file_name = QFileDialog.getOpenFileName(parent,
                        "Open processed file", filter="*.npy")
        file_name = file_name[0]
    try:
        proc = np.load(file_name, allow_pickle=True).item()
        parent.filenames = proc['filenames']
        good=True
    except:
        good=False
        print("ERROR: not a processed movie file")
    if good:
        v = []
        good = load_movies(parent, filelist=parent.filenames)
        if good:
            if 'fullSVD' in proc:
                parent.fullSVD = proc['fullSVD']
            else:
                parent.fullSVD = True
            k=0 # number of processed things
            parent.proctype = [0,0,0,0,0,0,0,0]
            parent.wroi = [0,0,0,0,0,0,0,0]

            if 'motSVD' in proc:
                parent.processed = True
            else:
                parent.processed = False

            iROI=0
            parent.typestr = ['pupil', 'motSVD', 'blink', 'run']
            if parent.processed:
                parent.col = []
                if parent.fullSVD:
                    parent.cbs1[k].setText('fullSVD')
                    parent.cbs2[k].setText('fullSVD')
                    parent.lbls[k].setText('fullSVD')
                    parent.lbls[k].setStyleSheet("color: white;")
                    parent.proctype[0] = 0
                    parent.col.append((255,255,255))
                    k+=1
                parent.motSVDs = proc['motSVD']
                parent.running = proc['running']
                parent.pupil = proc['pupil']
                parent.blink = proc['blink']
            else:
                k=0

            kt = [0,0,0,0]
            # whether or not you can move the ROIs
            moveable = not parent.processed
            if proc['rois'] is not None:
                for r in proc['rois']:
                    dy = r['yrange'][-1] - r['yrange'][0]
                    dx = r['xrange'][-1] - r['xrange'][0]
                    pos = [r['yrange'][0]+parent.sy[r['ivid']], r['xrange'][0]+parent.sx[r['ivid']], dy, dx]
                    parent.saturation.append(r['saturation'])
                    parent.rROI.append([])
                    parent.reflectors.append([])
                    if 'pupil_sigma' in r:
                        psig = r['pupil_sigma']
                        parent.pupil_sigma = psig
                        parent.sigmaBox.setText(str(r['pupil_sigma']))
                    else:
                        psig = None
                    parent.ROIs.append(roi.sROI(rind=r['rind'], rtype=r['rtype'], iROI=r['iROI'], color=r['color'],
                                        moveable=moveable, parent=parent, saturation=r['saturation'], pupil_sigma=psig,
                                        yrange=r['yrange'], xrange=r['xrange'], pos=pos, ivid=r['ivid']))
                    if 'reflector' in r:
                        for i,rr in enumerate(r['reflector']):
                            pos = [rr['yrange'][0], rr['xrange'][0], rr['yrange'][-1]-rr['yrange'][0], rr['xrange'][-1]-rr['xrange'][0]]
                            parent.rROI[-1].append(roi.reflectROI(iROI=r['iROI'], wROI=i, pos=pos, parent=parent,
                                                yrange=rr['yrange'], xrange=rr['xrange'], ellipse=rr['ellipse']))
                    if parent.fullSVD:
                        parent.iROI = k-1
                    else:
                        parent.iROI = k
                    parent.ROIs[-1].ellipse = r['ellipse']
                    parent.sl[1].setValue(int(parent.saturation[parent.iROI] * 100 / 255))
                    parent.ROIs[parent.iROI].plot(parent)
                    if parent.processed:
                        if k < 5:
                            parent.cbs1[k].setText('%s%d'%(parent.typestr[r['rind']], kt[r['rind']]))
                            parent.cbs2[k].setText('%s%d'%(parent.typestr[r['rind']], kt[r['rind']]))
                            parent.lbls[k].setText('%s%d'%(parent.typestr[r['rind']], kt[r['rind']]))
                            parent.lbls[k].setStyleSheet("color: rgb(%s,%s,%s);"%
                                                        (str(int(r['color'][0])), str(int(r['color'][1])), str(int(r['color'][2]))))
                            parent.wroi[k] = kt[r['rind']]
                            kt[r['rind']]+=1
                            parent.proctype[k] = r['rind'] + 1
                            parent.col.append(r['color'])
                    k+=1
            parent.kroi = k

            # initialize plot
            parent.cframe = 1
            if parent.processed:
                for k in range(parent.kroi):
                    parent.cbs1[k].setEnabled(True)
                    parent.cbs2[k].setEnabled(True)
                if parent.fullSVD:
                    parent.cbs1[0].setEnabled(True)
                    parent.cbs1[0].setChecked(True)
                    parent.cbs2[0].setEnabled(True)
                parent.plot_processed()
            parent.next_frame()

def load_movies(parent, filelist=None):
    if filelist is not None:
        parent.filelist = filelist
    try:
        cumframes, Ly, Lx, containers = utils.get_frame_details(parent.filelist)
        nframes = cumframes[-1]
        good = True
    except Exception as e:
        print("ERROR: not a supported movie file")
        print(e)
        good = False
    if good:
        parent.reset()
        parent.video = containers
        parent.filenames = parent.filelist
        parent.nframes = nframes
        parent.cumframes = np.array(cumframes).astype(int)
        parent.Ly = Ly
        parent.Lx = Lx
        parent.p1.clear()
        parent.p2.clear()
        if len(parent.Ly)<2:
            parent.LY = parent.Ly[0]
            parent.LX = parent.Lx[0]
            parent.sx = np.array([int(0)])
            parent.sy = np.array([int(0)])
            parent.vmap = np.zeros((parent.LY,parent.LX), np.int32)
        else:
            # make placement of movies
            Ly = np.array(parent.Ly.copy())
            Lx = np.array(parent.Lx.copy())

            LY, LX, sy, sx = utils.video_placement(Ly, Lx)
            parent.vmap = -1 * np.ones((LY,LX), np.int32)
            for i in range(Ly.size):
                parent.vmap[sy[i]:sy[i]+Ly[i], sx[i]:sx[i]+Lx[i]] = i
            parent.sy = sy
            parent.sx = sx
            parent.LY = LY
            parent.LX = LX

        parent.fullimg = np.zeros((parent.LY, parent.LX, 3))
        parent.imgs = []
        parent.img = []
        for i in range(len(parent.Ly)):
            parent.imgs.append(np.zeros((parent.Ly[i], parent.Lx[i], 3, 3)))
            parent.img.append(np.zeros((parent.Ly[i], parent.Lx[i], 3)))
        #parent.movieLabel.setText(os.path.dirname(parent.filenames[0][0]))
        parent.save_path = os.path.dirname(parent.filenames[0][0])
        parent.frameDelta = int(np.maximum(5,parent.nframes/200))
        parent.frameSlider.setSingleStep(parent.frameDelta)
        if parent.nframes > 0:
            parent.update_frame_slider()
            parent.update_buttons()
        parent.cframe = 1
        parent.loaded = True
        parent.processed = False
        parent.jump_to_frame()
    return good

def save_folder(parent):
    folderName = QFileDialog.getExistingDirectory(parent,
                        "Choose save folder")
    # load ops in same folder
    if folderName:
        parent.save_path = folderName
        if len(folderName) > 30:
            parent.savelabel.setText("..."+folderName[-30:])
        else:
            parent.savelabel.setText(folderName)

def get_pose_file(parent):
    # Open a folder and allow selection of multiple files with extension *.h5 only
    # Returns a list of files
    filelist = []
    filelist = QFileDialog.getOpenFileNames(parent, 'Open Pose File', parent.save_path, '*.h5')
    if filelist[0] == '':
        return
    else:
        parent.poseFilepath = natsorted(filelist[0])
        parent.poseFileLoaded = True
        parent.load_labels()
        parent.Labels_checkBox.setChecked(True)
        parent.update_status_bar("Pose file(s) loaded")

def load_cluster_labels(parent):
    try:
        file_name = QFileDialog.getOpenFileName(parent,
                        "Select cluster labels file", "", "Cluster label files (*.npy *.pkl)")[0]
        extension = file_name.split(".")[-1]
        if extension == "npy":
            parent.loaded_cluster_labels = np.load(file_name, allow_pickle=True)
            parent.is_cluster_labels_loaded = True
        elif extension == "pkl":
            with open(file_name, 'rb') as f:
                parent.loaded_cluster_labels = pickle.load(f)
                parent.is_cluster_labels_loaded = True
        else:
            return
    except Exception as e:
        msg = QMessageBox(parent)
        msg.setIcon(QMessageBox.Warning)
        msg.setText("Error: not a supported filetype")
        msg.setStandardButtons(QMessageBox.Ok)
        msg.exec_()
        print(e)

def load_umap(parent):
    try:
        file_name = QFileDialog.getOpenFileName(parent,
                        "Select UMAP data file", "", "UMAP label files (*.npy *.pkl)")[0]
        extension = file_name.split(".")[-1]
        if extension == "npy":
            embedded_data = np.load(file_name, allow_pickle=True)
        elif extension == "pkl":
            with open(file_name, 'rb') as f:
                embedded_data = pickle.load(f)
        else:
            return
        return embedded_data
    except Exception as e:
        msg = QMessageBox(parent)
        msg.setIcon(QMessageBox.Warning)
        msg.setText("Error: not a supported filetype")
        msg.setStandardButtons(QMessageBox.Ok)
        msg.exec_()
        print(e)

def load_trace_data(parent):
    try:
        file_name = QFileDialog.getOpenFileName(parent,
                        "Select data file", "", "(*.npy *.pkl)")[0]
        extension = file_name.split(".")[-1]
        if extension == "npy":
            dat = np.load(file_name, allow_pickle=True)
        elif extension == "pkl":
            with open(file_name, 'rb') as f:
                dat = pickle.load(f)
        else:
            return
        return dat
    except Exception as e:
        msg = QMessageBox(parent)
        msg.setIcon(QMessageBox.Warning)
        msg.setText("Error: not a supported filetype")
        msg.setStandardButtons(QMessageBox.Ok)
        msg.exec_()
        print(e)

def save_clustering_output(output, parent):
    filename, ext = parent.filenames[0][0].split(".")
    filename = filename.split("/")[-1]    # Use video filename
    savename = os.path.join(parent.save_path, ("%s_facemap_clusters.npy"%filename))
    np.save(savename, output)
    parent.update_status_bar("Clustering output saved: "+savename)


import xml.etree.ElementTree as etree

arquivo = etree.parse('./Exemplo.xml')

root = arquivo.getroot()
# print(root.attrib)
# print(root[0])

# for i in root[0]:
#     print(i.tag, i.attrib, i.text)

for i in root.findall('food'):
    print(i.find('item').text)
    print(i.find('descricao').text)


from hashlib import md5
from typing import List

def get_file_hash(absolute_path:str) -> str:
    with open(absolute_path, 'rb') as f:
        hash_obj = md5()
        hash_obj.update(f.read())
        return hash_obj.hexdigest()

def get_list_hash(data:List[str], encoding:str = 'latin1') -> str:
    hash_obj = md5()
    for chunk in sorted(data):
        hash_obj.update(chunk.encode(encoding))
    return hash_obj.hexdigest()


from datetime import datetime

class AERONETConstants:
    def __init__(self):

        self.DATEREF = datetime(1970, 1, 1, 0, 0, 0)
        self.ThuillierMethodDefault = 'interp'

        #CSV Original File

        #NCFile
        self.dim_time = 'TimeIndex'
        self.dim_wl = 'Central_wavelenghts'


'''
Problem
-------------------
Given an array nums of n integers and an integer target, find three integers in nums such that the sum is closest to target. Return the sum of the three integers. You may assume that each input would have exactly one solution.

Constraints
-------------------
3 <= nums.length <= 10^3\
-10^3 <= nums[i] <= 10^3\
-10^4 <= target <= 10^4\

Thinking
-------------------

'''
from typing import List

def threeSumClosest(nums: List[int], target: int) -> int:
    nums.sort()
    closest_sum = 2**31-1
    for i in range(len(nums)):
        j, k = i+1, len(nums)-1
        while j < k:
            curr_sum = nums[i] + nums[j] + nums[k]
            if curr_sum == target:
                return curr_sum
            if abs(curr_sum-target) < abs(closest_sum-target):
                closest_sum = curr_sum
            if curr_sum < target:
                j = j+1
            else:
                k = k-1
    return closest_sum

if __name__ == "__main__":
    print(threeSumClosest([-1, 2, 1, -4], 1))


from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

from issues.views import *

urlpatterns = patterns('',
    # Examples:
    url(r'^$', home, name='home'),
    url(r'^issues/$', issues_list, name='issue_list'),
    url(r'^issues/(\d+)/', issue_articles_list),
    url(r'^weblog/', include('weblog.urls.entries')),
    url(r'^weblog/categories/', include('weblog.urls.categories')),

    url(r'^admin/', include(admin.site.urls)),
)


#
# @lc app=leetcode.cn id=718 lang=python3
#
# [718] 最长重复子数组
#
from typing import List
# @lc code=start

class Solution:
    def findLength(self, A: List[int], B: List[int]) -> int:
        m = len(A)
        n = len(B)
        dp = [0] * (n+1)
        last_dp = [0] * (n+1)
        ans = 0
        for i in range(m):
            for j in range(n):
                if A[i] == B[j]:
                    dp[j+1] = last_dp[j] + 1
                else:
                    dp[j+1] = 0
                ans = max(ans, dp[j+1])
            # print(f'{i=}, {dp=}')
            last_dp = dp[:]

        return ans

# @lc code=end
if __name__ == "__main__":
    f = Solution().findLength
    assert f([1, 2, 3, 2, 1], [3, 2, 1, 4, 7]) == 3
    assert f([], []) == 0
    assert f([1, 2], [1]) == 1
    assert f([3, 2, 1, 2, 3], [2, 1, 2]) == 3
    assert f([0, 1, 1, 1, 1], [1, 0, 1, 0, 1]) == 2

718.最长重复子数组.py


# !/usr/bin/env python3
# coding: utf-8

from src.minority_game import  MinorityGameWithRandomChoice
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
if __name__ == "__main__":
    # set agent num
    agent_num = 201
    # set round number from 10000 to 100000
    rounds = range(10000,110000,10000)

    mean_list = []
    stdd_list = []
    round_num = 100000

    # Run the game for different round number
    game = MinorityGameWithRandomChoice(agent_num, round_num)
    mean_list,stdd_list = game.run_game()

    # print result
    print(mean_list)
    print(stdd_list)
    # plot mean
    fig = plt.figure(figsize=(8, 4))
    plt.plot(rounds,mean_list)
    plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.2)
    fig.suptitle('Mean Value')
    plt.xlabel('Iteration number')
    plt.ylabel('Average numbers of winners')
    fig.savefig('ex3-1-mean.jpg')

    # plot SD
    fig2 = plt.figure(figsize=(8, 4))
    plt.plot(rounds,stdd_list)
    plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.2)
    fig2.suptitle('Standard Deviation')
    plt.xlabel('Iteration number')
    plt.ylabel('Average numbers of winners')
    fig2.savefig('ex3-1-std.jpg')


"""
This file contains the function to fine-tune the embedding
of the extractions.
AutoEncoder is modified from
https://github.com/stangelid/oposum/tree/master/mate
"""
from itertools import cycle
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn.init import xavier_uniform_ as xavier_uniform
from torch.nn.utils import clip_grad_norm
from evaluation import Plot

from tqdm import tqdm
import time

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Define Triplet Margin Cosine Loss.
def triplet_margin_cosine_loss(anchor, positive, negative, margin=1.0, eps=1e-8, sum_loss=False):
	assert anchor.dim() == 2, "Anchor and positive must be 2D matrices."
	assert negative.dim() <= 3, "Negative must be 2D (1 negative sample) or 3D matrix (multiple negatives)."
	assert margin > 0.0, 'Margin should be positive value.'

	d_p, d_n = None, None
	if positive.dim() == 2:
		d_p = F.cosine_similarity(anchor, positive, eps=eps).unsqueeze(1)
	else:
		d_p = F.cosine_similarity(anchor.unsqueeze(1), positive, dim=2, eps=eps)

	if negative.dim() == 2:
		d_n = F.cosine_similarity(anchor, negative, eps=eps).unsqueeze(1)
	else:
		d_n = d_n = F.cosine_similarity(anchor.unsqueeze(1), negative, dim=2, eps=eps)

	dist_hinge = torch.clamp(margin - d_p.unsqueeze(1) + d_n, min=0.0).mean(dim=1)

	if not sum_loss:
		loss = torch.mean(dist_hinge)
	else:
		loss = torch.sum(dist_hinge)

	return loss

class TripletMarginCosineLoss(nn.Module):
	def __init__(self, margin=1.0, eps=1e-8, sum_loss=False):
		super(TripletMarginCosineLoss, self).__init__()
		self.margin = margin
		self.eps = eps
		self.sum_loss = sum_loss

	def forward(self, anchor, positive, negative):
		return triplet_margin_cosine_loss(anchor, positive, negative, self.margin, self.eps, self.sum_loss)

class AttentionEncoder(nn.Module):
	"""Segment encoder that produces segment vectors as the weighted
	average of word embeddings.
	"""
	def __init__(self, vocab_size, emb_size, bias=True, M=None, b=None):
		"""Initializes the encoder using a [vocab_size x emb_size] embedding
		matrix. The encoder learns a matrix M, which may be initialized
		explicitely or randomly.

		Parameters:
			vocab_size (int): the vocabulary size
			emb_size (int): dimensionality of embeddings
			bias (bool): whether or not to use a bias vector
			M (matrix): the attention matrix (None for random)
			b (vector): the attention bias vector (None for random)
		"""
		super(AttentionEncoder, self).__init__()
		self.lookup = nn.Embedding(vocab_size, emb_size)
		self.M = nn.Parameter(torch.Tensor(emb_size, emb_size))
		if M is None:
			xavier_uniform(self.M.data)
		else:
			self.M.data.copy_(M)
		if bias:
			self.b = nn.Parameter(torch.Tensor(1))
			if b is None:
				self.b.data.zero_()
			else:
				self.b.data.copy_(b)
		else:
			self.b = None

	def forward(self, inputs):
		"""Forwards an input batch through the encoder"""
		x_wrd = self.lookup(inputs)
		x_avg = x_wrd.mean(dim=1)

		x = x_wrd.matmul(self.M)
		x = x.matmul(x_avg.unsqueeze(1).transpose(1,2))
		if self.b is not None:
			x += self.b

		x = torch.tanh(x)
		a = F.softmax(x, dim=1)

		z = a.transpose(1,2).matmul(x_wrd)
		z = z.squeeze()
		if z.dim() == 1:
			return z.unsqueeze(0)
		return z

	def set_word_embeddings(self, embeddings, fix_w_emb=True):
		"""Initialized word embeddings dictionary and defines if it is trainable"""
		self.lookup.weight.data.copy_(embeddings)
		self.lookup.weight.requires_grad = not fix_w_emb

class AutoEncoder(nn.Module):
	"""Modified from AspectAutoencoder, however, the goal is to encode
	target spans and fine-tune the embedding of target span by cluster
	assignments and additional classification signals.
	"""
	def __init__(self, target_cols, aux_c_sizes, vocab_size, emb_size, n_cluster,
			n_sample=10, w_emb=None, c_emb=None,
			attention=True, fix_w_emb=True, fix_c_emb=False):
		"""Initialization
		Parameters:
			target_cols (String[]): column names that need to be embedded
			aux_c_sizes (list of ints): number of classes for each
				auxiliary classification task.
			vocab_size (int): vocabular size
			emb_size (int): embedding dimension
			n_cluster (int): number of clusters
			num_sample (int): number of positive/negative examples
			w_emb (vocab_size * emb_size tensor): word embedding weights
			c_emb (n_cluster * emb_size tensor): cluster center embeeding
				weights
			attention (True or False): whether use attention encoder to encode
				target fields
			fix_w_emb (True or False): fix the Embedding layer or not
			fix_c_emb (True or False): fix the center matrix or not
		"""
		super(AutoEncoder, self).__init__()
		self.n_cols = len(target_cols)
		self.aux_c_sizes = aux_c_sizes
		self.vocab_size = vocab_size
		self.emb_size = emb_size
		self.n_cluster = n_cluster
		self.n_sample = n_sample
		self.attention = attention
		self.fix_w_emb = fix_w_emb
		self.fix_c_emb = fix_c_emb

		# Define Embedding Layer
		self.col_encoders = []
		for i in range(self.n_cols):
			col_encoder = None
			if not self.attention:
				col_encoder = nn.EmbeddingBag(vocab_size, emb_size)
				self.add_module("EmbeddingBag_%d"%i, col_encoder)
			else:
				col_encoder = AttentionEncoder(vocab_size, emb_size)
				self.add_module("AttentionEncoder_%d"%i, col_encoder)
			self.col_encoders.append(col_encoder)

		assert emb_size % self.n_cols == 0, "Cannot recover emb size."
		self.emb_linear = nn.Linear(emb_size, int(emb_size/self.n_cols))

		# Initialize wording embedding weights
		if w_emb is None:
			for col_encoder in self.col_encoders:
				xavier_uniform(col_encoder.weight.data)
		else:
			self._assert_size(w_emb, (vocab_size, emb_size), "Word embedding")
			for col_encoder in self.col_encoders:
				if not attention:
					col_encoder.weight.data.copy_(w_emb)
					col_encoder.weight.requires_grad = not fix_w_emb
				else:
					col_encoder.set_word_embeddings(w_emb, fix_w_emb)

		# Initalize representative weights.
		if c_emb is None:
			self.c_emb = nn.Parameter(torch.tensor(self.n_cluster, emb_size))
			xavier_uniform(self.c_emb.data)
		else:
			self._assert_size(c_emb, (self.n_cluster, emb_size), "Center")
			self.c_emb = nn.Parameter(torch.Tensor(c_emb.size()))
			self.c_emb.data.copy_(c_emb)
			self.c_emb.requires_grad = not fix_c_emb

		# Define cluster classification layer
		self.lin = nn.Linear(emb_size, self.n_cluster)

		# Define other classification layers
		self.aux_lins = []
		for c_size in self.aux_c_sizes:
			lin = nn.Linear(emb_size, c_size)
			self.add_module("Classifier_%d" % c_size, lin)
			self.aux_lins.append(lin)

		self.softmax = nn.Softmax(dim=1)

	def forward_one(self, input_arry):
		# Encode every fields in the targets
		encs = []
		for i in range(len(input_arry)):
			if not self.attention:
				offsets = Variable(torch.arange(0, input_arry[i].numel(), input_arry[i].size(1), out=input_arry[i].data.new().long()))
				encs.append(self.emb_linear(self.col_encoders[i](input_arry[i].view(-1), offsets)))
			else:
				encs.append(self.emb_linear(self.col_encoders[i](input_arry[i])))

		# Combine all fields
		#enc = self.enc_combiner(torch.cat(encs, dim=1))
		enc = torch.cat(encs, dim=1)

		# Find cluster assignment distribution
		x = self.lin(enc)
		c_probs = self.softmax(x)
		return enc, x, c_probs

	def forward(self, input_arry, input_other=None):
		enc, x, c_probs = self.forward_one(input_arry)

		if input_other is not None:
			enc_other, x_other, c_probs_other = self.forward_one(input_other)
			return c_probs, c_probs_other

		reconn_enc = torch.mm(x, self.c_emb)
		# Run classifers
		a_probss = [self.softmax(lin(enc)) for lin in self.aux_lins]

		# Select positive and negative examples
		self.set_target(enc, c_probs)
		#self.set_target_by_centers(c_probs)
		return c_probs, a_probss, enc, reconn_enc

	def set_target(self, enc, c_probs):
		batch_size, emb_size = enc.size()
		neg_mask = self._create_mask(batch_size, c_probs, self.n_sample, emb_size, mtype='neg')
		self.negative = Variable(enc.expand(batch_size, batch_size, emb_size).gather(1, neg_mask))
		pos_mask = self._create_mask(batch_size, c_probs, self.n_sample, emb_size, mtype='pos')
		self.positive = Variable(enc.expand(batch_size, batch_size, emb_size).gather(1, pos_mask))

	def get_targets(self):
		assert self.positive is not None, 'Positive targets not set; needs a forward pass first'
		assert self.negative is not None, 'Negative targets not set; needs a forward pass first'
		return self.positive, self.negative

	def set_target_by_centers(self, c_probs):
		batch_size, n_cluster = c_probs.size()
		neg_mask = torch.multinomial(c_probs, self.n_sample).unsqueeze(2).expand(batch_size, self.n_sample, self.emb_size)
		pos_mask = torch.multinomial(1/(c_probs+1e-10), self.n_sample).unsqueeze(2).expand(batch_size, self.n_sample, self.emb_size)
		#print(neg_mask.size(), self.c_emb.size(), self.c_emb.expand(batch_size, n_cluster, self.emb_size).size())
		self.positive = Variable(self.c_emb.expand(batch_size, n_cluster, self.emb_size).gather(1, neg_mask))
		self.negative = Variable(self.c_emb.expand(batch_size, n_cluster, self.emb_size).gather(1, pos_mask))

	def _create_mask(self, batch_size, probs, n_sample, emb_size, mtype='neg',
					 eps=1e-20):
		n_sample = min(batch_size - 1, n_sample)
		probs_e = probs.unsqueeze(0).expand(batch_size, batch_size, -1)
		prob_1 = probs_e.flatten(0,1)
		prob_2 = probs_e.transpose(1,0).flatten(0,1)
		sim = F.cosine_similarity(prob_1, prob_2, dim=1)
		weight = 1-sim if mtype=='neg' else sim
		weight = weight.view(batch_size, batch_size)
		diagnal_mask = torch.eye(batch_size, batch_size, device=device).bool()
		weight.masked_fill_(diagnal_mask, torch.tensor(0., device=device))
		weight = torch.max(weight, torch.tensor(eps, device=device))
		mask = torch.multinomial(weight, n_sample)
		mask = mask.unsqueeze(2).expand(batch_size, n_sample, emb_size)
		return mask

	def _assert_size(self, tensor, size, name):
		assert tensor.size() == size, "%s: %s != %s" % (name,
			str(tensor.size()), str(size))

class EmbEncoder:
	def __init__(self, target_cols, auxiliary_cols, edges, w_emb, n_sample,
		n_epochs, batch_size, lr=0.5, n_inititer=5, max_attemp=1):
		""" Initialization
		Parameters:
			target_cols (String[]): column names that need to be embedded
			auxiliary_cols (list of dicts): columns that contain additional
				labels, together with the label info.
				e.g., {"col": "attribute", "n_class": 17, "loss_ratio":0.5}

			w_emb (vocab_size * emb_size tensor): initial tensor for Embedding
				layer.
			n_sample (int): number of positive / negative examples used
				for reconstruction loss
			n_epochs (int): number of epochs for training the embedding.
			lr (float): learning rate
			n_inititer (int): number of iterations to fine-tune cluster
				classification layers (lin)
		"""
		self.target_cols = target_cols
		self.auxiliary_cols = auxiliary_cols
		self.edge_loss_ratio = edges["loss_ratio"]
		self.edge_loss_lr = edges["lr"]
		self.w_emb = torch.tensor(w_emb)
		self.n_sample = n_sample
		self.n_epochs = n_epochs
		self.lr = lr
		self.batch_size = batch_size
		self.n_inititer = n_inititer
		self.max_attemp = max_attemp

	def run(self, aux_c_sizes, row_iter, label_iter, c_emb, spans, gold,
			output_dir, pair_iter=None, aux_weights=None):
		best_enc = None
		best_loss = None
		best_labels = None
		for i in range(self.max_attemp):
			print("\t", "attemp %d" % i)
			if pair_iter is None:
				fine_tuned_enc, loss, fine_tuned_label = self.refine(aux_c_sizes, row_iter,
					label_iter, c_emb, spans=spans, gold=gold,
					output_dir=output_dir, aux_weights=aux_weights)
			else:
				fine_tuned_enc, loss, fine_tuned_label = self.refine_with_pair(aux_c_sizes,
					row_iter, pair_iter, label_iter, c_emb, spans=spans, gold=gold,
					output_dir=output_dir, aux_weights=aux_weights)
			if best_loss is None or loss < best_loss:
				best_enc = fine_tuned_enc
				best_loss = loss
				best_labels = fine_tuned_label
		return best_enc, best_loss, fine_tuned_label

	def refine_with_pair(self, aux_c_sizes, row_iter, pair_iter, label_iter, c_emb,
			   spans=None, gold=None, output_dir=None, output_interval=5, aux_weights=None):
		if gold is None or gold[0] is None:
			output_dir = None
		n_cluster, emb_size = c_emb.size()
		vocab_size, _ = self.w_emb.size()

		# Initialize the Encoder model
		model = AutoEncoder(self.target_cols, aux_c_sizes,
			vocab_size, emb_size, n_cluster, n_sample=self.n_sample,
			w_emb=self.w_emb, c_emb=c_emb, attention=True,
			fix_w_emb=True, fix_c_emb=True)
		model.to(device)
		model.train()

		# Define optimizer
		params = filter(lambda p: p.requires_grad, model.parameters())
		optimizer = torch.optim.Adam(model.parameters(), lr=self.lr)
		kl_div_optimizer = torch.optim.Adam(model.parameters(), lr=self.edge_loss_lr, amsgrad=True)

		# Define loss functions
		rec_loss = TripletMarginCosineLoss()
		cls_loss = nn.CrossEntropyLoss()
		kl_loss = nn.KLDivLoss(reduction='batchmean')

		fine_tuned_enc = None
		total_loss = None
		fine_tuned_label = None
		pbar = tqdm(total = self.n_epochs)
		times = [0. for i in range(4)]

		aux_losses = []
		if aux_weights is not None:
			for aux_weight in aux_weights:
				aux_losses.append(nn.CrossEntropyLoss(weight=torch.FloatTensor(aux_weight).to(device)))

		for epoch in range(self.n_epochs+1):
			encs = []
			probs = []
			curr_loss = 0
			for idx, (batch, pair_batch, c_labels) in enumerate(zip(row_iter, cycle(pair_iter), label_iter)):
				start_time = time.time()
				# Process data
				target_idss, a_labelss = batch
				target_idss = target_idss.transpose(1, 0)
				a_labelss = a_labelss.transpose(1, 0)
				target_idss = [ids.to(device) for ids in target_idss]
				a_labelss = [labels.to(device) for labels in a_labelss]

				# Process pairs
				left_ids, right_ids = pair_batch
				left_ids, right_ids = left_ids.transpose(1, 0), right_ids.transpose(1, 0)
				left_ids = [ids.to(device) for ids in left_ids]
				right_ids = [ids.to(device) for ids in right_ids]

				# Process clustered labels
				c_labels = c_labels.to(device)
				c_probs, a_probss, enc, reconn_enc = model(target_idss)
				times[0] += time.time()-start_time
				start_time = time.time()

				# Update fine tune rep_probs loss.
				positives, negatives = model.get_targets()

				if idx < self.n_inititer:
					loss = cls_loss(c_probs, c_labels)
				else:
					loss = rec_loss(reconn_enc, enc, negatives)

				# Update classification loss, e.g., for type or sentiment.
				for i, (a_probs, a_labels) in enumerate(zip(a_probss, a_labelss)):
					#print(a_labels, torch.var(a_labels.double()))
					if aux_weights is None:
						aux_loss = self.auxiliary_cols[i]["loss_ratio"]*cls_loss(a_probs, a_labels)
					else:
						aux_loss = self.auxiliary_cols[i]["loss_ratio"]*aux_losses[i](a_probs, a_labels)
					loss += aux_loss

				# Back propogation
				optimizer.zero_grad()
				loss.backward(retain_graph=True)
				optimizer.step()
				curr_loss += loss.detach().item()

				# Update KL loss for pairs
				if self.edge_loss_ratio > 0:
					prob_left, prob_right = model(left_ids, right_ids)
					edge_loss = self.edge_loss_ratio*(-kl_loss(prob_left.log(), prob_right))

					# Back propogation
					kl_div_optimizer.zero_grad()
					edge_loss.backward()
					torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
					kl_div_optimizer.step()
					curr_loss += edge_loss.detach().item()
					#print("\t", edge_loss.detach().item())

					#for name, param in model.named_parameters():
					#	if param.requires_grad:
					#		print(epoch, idx, name, param.grad)
					#		break

				times[1] += time.time() - start_time
				start_time = time.time()

				times[2] += time.time()-start_time
				start_time = time.time()

				# Update variables
				encs.append(enc.detach().cpu())
				probs.append(c_probs.detach().cpu().argmax(1))
				times[3] += time.time()-start_time

			#print(curr_loss)

			if total_loss is None or curr_loss < total_loss:
				fine_tuned_enc = torch.cat(encs, dim=0)
				total_loss = curr_loss
				fine_tuned_label = torch.cat(probs, dim=0)

			pbar.update(1)

			# Print intermediate embeddings
			if output_dir is None or epoch % output_interval != 0:
				continue
			#Plot(output_dir, fine_tuned_enc, spans, gold, str(epoch))

		#print(times)
		return fine_tuned_enc, total_loss, fine_tuned_label

	def refine(self, aux_c_sizes, row_iter, label_iter, c_emb,
			   spans=None, gold=None, output_dir=None, output_interval=5, aux_weights=None):
		""" Fine-turning embeddings
		Parameters:
			aux_c_sizes (list of ints): number of classes for each
				auxiliary classification task.
			row_iter: batches of data points
			label_iter: batches of cluster labels
			c_emb: representative embedding tensor from cluster algorithm
		"""
		if gold is None or gold[0] is None:
			output_dir = None
		n_cluster, emb_size = c_emb.size()
		vocab_size, _ = self.w_emb.size()

		# Initialize the Encoder model
		model = AutoEncoder(self.target_cols, aux_c_sizes,
			vocab_size, emb_size, n_cluster, n_sample=self.n_sample,
			w_emb=self.w_emb, c_emb=c_emb, attention=True,
			fix_w_emb=True, fix_c_emb=True)
		model.to(device)
		model.train()

		# Define optimizer
		params = filter(lambda p: p.requires_grad, model.parameters())
		optimizer = torch.optim.Adam(model.parameters(), lr=self.lr)

		# Define loss functions
		rec_loss = TripletMarginCosineLoss()
		cls_loss = nn.CrossEntropyLoss()

		aux_losses = []
		if aux_weights is not None:
			for aux_weight in aux_weights:
				aux_losses.append(nn.CrossEntropyLoss(weight=torch.FloatTensor(aux_weight).to(device)))

		fine_tuned_enc = None
		total_loss = None
		fine_tuned_label = None
		pbar = tqdm(total = self.n_epochs)
		times = [0. for i in range(4)]

		for epoch in range(self.n_epochs+1):
			encs = []
			probs = []
			curr_loss = 0
			for idx, (batch, c_labels) in enumerate(zip(row_iter, label_iter)):
				start_time = time.time()
				target_idss, a_labelss = batch
				target_idss = target_idss.transpose(1, 0)
				a_labelss = a_labelss.transpose(1, 0)
				target_idss = [ids.to(device) for ids in target_idss]
				a_labelss = [labels.to(device) for labels in a_labelss]
				c_labels = c_labels.to(device)
				c_probs, a_probss, enc, reconn_enc = model(target_idss)
				times[0] += time.time()-start_time
				start_time = time.time()

				# Update fine tune rep_probs loss.
				positives, negatives = model.get_targets()

				if idx < self.n_inititer:
					loss = cls_loss(c_probs, c_labels)
				else:
					loss = rec_loss(enc, positives, negatives)
				#print(loss.item())

				# Update classification loss, e.g., for type or sentiment.
				for i, (a_probs, a_labels) in enumerate(zip(a_probss, a_labelss)):
					#print(a_labels, torch.var(a_labels.double()))
					if aux_weights is None:
						aux_loss = self.auxiliary_cols[i]["loss_ratio"]*cls_loss(a_probs, a_labels)
					else:
						aux_loss = self.auxiliary_cols[i]["loss_ratio"]*aux_losses[i](a_probs, a_labels)
					loss += aux_loss
					#print(aux_loss.item())

				times[1] += time.time() - start_time
				start_time = time.time()

				# Back propogation
				optimizer.zero_grad()
				loss.backward(retain_graph=True)
				optimizer.step()

				times[2] += time.time()-start_time
				start_time = time.time()

				# Update variables
				curr_loss += loss.detach().item()
				encs.append(enc.detach().cpu())
				probs.append(c_probs.detach().cpu().argmax(1))
				times[3] += time.time()-start_time

			if total_loss is None or curr_loss < total_loss:
				fine_tuned_enc = torch.cat(encs, dim=0)
				total_loss = curr_loss
				fine_tuned_label = torch.cat(probs, dim=0)

			pbar.update(1)
			# Print intermediate embeddings
			if output_dir is None or epoch % output_interval != 0:
				continue
			#Plot(output_dir, fine_tuned_enc, spans, gold, str(epoch))

		return fine_tuned_enc, total_loss, fine_tuned_label


# -*- coding: utf-8 -*-

from toolium.pageobjects.page_object import PageObject
from selenium.common.exceptions import NoSuchElementException

from toolium.pageelements import *
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.select import Select
import selenium.webdriver.support.expected_conditions as WAITCON
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from toolium.pageelements import InputText, Button
import random
import string
import time
from selenium.webdriver.common.by import By

class TabsPageObject(PageObject):
    """
    :Class represents web page elements for Login page.
    """

    def init_page_elements(self):

        """ Open page elements

        :returns: this page object instance
        """
        self.genre = Button(By.XPATH, '//*[@text="Songs by genre"]', wait=True)
        self.rock = Button(By.XPATH, '//*[@text="Rock songs"]', wait=True)
        self.song = Button(By.XPATH, '//*[@text="Silent Partner"]', wait=True)

    def SelectGenre(self):
        """ Select genre group

        :returns: this page object instance
        """
        try:
            self.logger.info("\n=====================================Genre======================================")
            self.logger.debug("Opening genres")
            self.logger.debug("Checking if the button is present : %s", self.genre)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.genre.wait_until_visible()
            self.genre.click()
            time.sleep(2)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.genre))
            return None

    def Rock(self):
        """ Select Rock music genre

        :returns: this page object instance
        """
        try:
            self.logger.info("\n===============================Selecting Rock music from genre=================================")
            self.logger.debug("Choose a genre")
            self.logger.debug("Checking if the pageElement is present : %s", self.rock)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.rock.wait_until_visible()
            self.rock.click()
            time.sleep(2)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.rock))
            return None

    def SelectedSong(self):
        """ Select play selected song

        :returns: this page object instance
        """
        try:
            self.logger.info("\n===============================Playing awakening=================================")
            self.logger.debug("Playing song.")
            self.logger.debug("Checking if the pageElement is present : %s", self.song)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.song.wait_until_visible()
            self.song.click()
            time.sleep(10)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.song))
            return None


from matplotlib import pyplot as plt
# 可视化

plt.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文

fig = plt.figure()
plt.text(0.66, 0.41, "戴")

plt.show()

import re

data = "如果放到POST中将出错"

import jieba
jieba.suggest_freq(("中", "将"), True)
seg_list = jieba.cut(data,  HMM=False)

seg_list1 = jieba.cut_for_search(data)
print("Full mode:", "/ ".join(seg_list))
print("Full mode:", "/ ".join(seg_list1))
result = jieba.tokenize(data)
print(result)
for i in result:
    print(i)