import os
import sys
import zipfile

#获取脚本路径, 如果是在IDE下运行，只能用此方式获取
def get_cur_dir():
    path = sys.path[0]
    #判断为脚本文件还是py2exe编译后的文件，如果是脚本文件，则返回的是脚本的目录，如果是py2exe编译后的文件，则返回的是编译后的文件路径
    if os.path.isdir(path):
        return path

    elif os.path.isfile(path):
        return os.path.dirname(path)

def recurse_files(input_path, result):
    files = os.listdir(input_path)

    for file in files:
        if os.path.isdir(os.path.join(input_path,file)):
            if file != 'users': #过滤文件夹
                recurse_files(os.path.join(input_path,file), result)
        else:
            ext = os.path.splitext(file)[1]
            if input_path == cur_dir:#根目录只添加特定类型文件
                if ext == '.dll' or ext == '.exe' or ext == '.qm':
                    result.append(os.path.join(input_path,file))
            else:
                result.append(os.path.join(input_path,file))

if __name__ == '__main__':
    cur_dir = get_cur_dir()

    zip_file_name = os.path.join(cur_dir, 'debug.zip');
    if os.path.isfile(zip_file_name): os.remove(zip_file_name)
    zip = zipfile.ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED)

    filelists = []
    recurse_files(cur_dir, filelists)

    for file in filelists:
        arcName = file #在压缩包内名字
        if os.path.dirname(file) == cur_dir:
            arcName = os.path.basename(file) #文件在要压缩目录下，则只为文件名
        else:
            dir = os.path.dirname(file)
            dir = dir.replace(cur_dir + '\\', '') #只从要压缩的目录下当前目录名开始
            arcName = dir + '\\' + os.path.basename(file)

        zip.write(file, arcName)
        #zip.write(file) #会出现全路径

    # 调用了close方法才会保证完成压缩
    zip.close()


import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np

def convert_tag2dict(tag, dictkeys):
    dd = {}
    for key in dictkeys:
        if key in tag:
            lowestindex = tag.find(key)
            firstquote = tag.find('\"', lowestindex)
            secondquote = tag.find('\"', firstquote+1)
            dd[key]=tag[firstquote+1:secondquote]
        else:
            dd[key]=None
        pass
    return dd

outputfile = open("./data/outputfile.txt", 'w')
#  <Record type="HKCategoryTypeIdentifierSleepAnalysis"
# sourceName="윤창희의 Apple Watch" sourceVersion="9.3.1" creationDate="2023-04-25 08:09:26 +0900"
# startDate="2023-04-25 02:00:21 +0900" endDate="2023-04-25 02:08:21 +0900" value="HKCategoryValueSleepAnalysisAsleepCore">
data_format=["creationDate", "startDate", "endDate", "value"]
datas = []
with open("./data/rawdata.txt", 'r') as ych_health_file:

    lines = ych_health_file.readlines()
    for aline in lines:
        if ("HKCategoryTypeIdentifierSleepAnalysis" in aline) and ("Apple Watch" in aline):
            # aline = aline.strip('<>')
            # outputfile.write(aline)
            d=convert_tag2dict(aline, data_format)
            datas.append(d)
            outputfile.write(str(d)+'\n')

outputfile.close()


import os

import web3
from etherscan.contracts import Contract as EtherscanContract

w3_infura = web3.Web3(
    web3.Web3.HTTPProvider(
        f"https://mainnet.infura.io/v3/{os.environ['WEB3_INFURA_PROJECT_ID']}",
    ),
)

def init_contract(address: str):

    address = web3.Web3.toChecksumAddress(address)
    contract_abi = EtherscanContract(
        address=address,
        api_key=os.environ["ETHERSCAN_API_KEY"],
    ).get_abi()
    contract = w3_infura.eth.contract(address=address, abi=contract_abi)
    return contract


"""Music library manager for Mist program. Uses SQL Alchemy for persistant storage."""

__author__ = "Jonathan Allen"
__all__ = ["SqlaSession", "DatabaseError", "NonexistantEntryError",
           "ConflictingEntriesError", "DirectoryError", "Repository", "Song",
           "SongFile", "Tag", "Playlist", "signalEmitter"]

import os
import re
import shutil
import types
import thread
import Queue

from PySide import QtCore

from metaedit import *
from jonathan import dirtools
from jonathan.dictools import *

#===============================================================================
# Constants
#===============================================================================
REQ_SONG_DATA = ["TITLE", "ALBUM", "ARTIST"]
SUPPORTED_FILE_TYPES = [".mp3", ".m4a", ".ogg", ".oga", ".flac"]
SUPPORTED_MIMETYPES = ["audio/mpeg", "audio/flac", "audio/mp4", "audio/ogg"]

#===============================================================================
# SQL Alchemy Pre-setup
#===============================================================================
import sqlalchemy
from sqlalchemy.orm import scoped_session, sessionmaker, relationship, session
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.types import TypeDecorator, Unicode

_sqlaEngine = None
SqlaSession = scoped_session(sessionmaker())
_guiSession = None
_SqlaBase = declarative_base()
_workerRunning = True
_workerSession = None
_workerThread = None
workerQueue = Queue.Queue()

#===============================================================================
# Exceptions
#===============================================================================
class DatabaseError(Exception):
    pass

class NonexistantEntryError(DatabaseError):
    pass

class ConflictingEntriesError(DatabaseError):
    pass

class DirectoryError(Exception):
    pass

#===============================================================================
# Thread related function decorators
#===============================================================================
def attachSession(function):
    def wrapper(self, *args, **kwargs):
        self.sqlaSession = SqlaSession()
        try:
            if self not in self.sqlaSession: self.sqlaSession.add(self)
            return function(self, *args, **kwargs)
        except InvalidRequestError: pass
    return wrapper

def initialize(function):
    def wrapper(self, *args, **kwargs):
        if not getattr(self, 'initialized', None): self.setup()
        return function(self, *args, **kwargs)
    return wrapper

#===============================================================================
# SQL Alchemy Types
#===============================================================================
class UnicodeString(TypeDecorator):
    """Safely coerce Python bytestrings to Unicode
    before passing off to the database."""

    impl = Unicode

    def process_bind_param(self, value, dialect):
        if isinstance(value, str):
            value = value.decode('utf-8')
        return value

#===============================================================================
# SQL Alchemy Tables
#===============================================================================
"""
Secondary table to associate tags with songs. Relationship is bidirectional and many-to-many.
"""
songTagTable = sqlalchemy.Table('songTag', _SqlaBase.metadata,
    sqlalchemy.Column('songId', sqlalchemy.Integer, sqlalchemy.ForeignKey('song.id')),
    sqlalchemy.Column('tagId', sqlalchemy.Integer, sqlalchemy.ForeignKey('tag.id')),
)

"""
Secondary table to associate songs with playlists. Relationship is unidirectional and many-to-many.
"""
playlistSongTable = sqlalchemy.Table('playlistSong', _SqlaBase.metadata,
    sqlalchemy.Column('playlistId', sqlalchemy.Integer, sqlalchemy.ForeignKey('playlist.id')),
    sqlalchemy.Column('songId', sqlalchemy.Integer, sqlalchemy.ForeignKey('song.id'))
)

class Repository(_SqlaBase):
    """
    Table to store information about repositories. Stores repository name, repository
    directory, repository settings, and pending repository transfers.
    """
    __tablename__ = 'repository'
    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key = True)
    name = sqlalchemy.Column(UnicodeString)
    location = sqlalchemy.Column(UnicodeString)
    monitored = sqlalchemy.Column(sqlalchemy.Boolean)
    managed = sqlalchemy.Column(sqlalchemy.Boolean)
    directoryStructure = sqlalchemy.Column(UnicodeString)
    pendingTransfers = sqlalchemy.Column(sqlalchemy.PickleType(mutable = True))
    songFiles = relationship("SongFile", backref = 'repository')

    DEFAULT_DIRECTORY_STRUCTURE = "%ARTIST%/%ALBUM%/%TITLE%"

    def __init__(self, name, location, \
                 directoryStructure = DEFAULT_DIRECTORY_STRUCTURE, \
                 monitored = True, managed = True):
        self.name = name
        self.location = location
        self.monitored = monitored
        self.managed = managed
        self.directoryStructure = directoryStructure
        self.pendingTransfers = {}

    @attachSession
    @initialize
    def __contains__(self, item):
        if isinstance(item, types.IntType):
            return any([item == song.id for song in self.getSongs()])
        if isinstance(item, Song):
            return item in self.getSongs()
        raise TypeError("{} is not of type Song or Song.id".format(item))

    @attachSession
    def __repr__(self):
        songsList = [songItem.songId for songItem in self.songFiles]
        return "<Repository({}, {}, {}, {}, {}, {})>". \
                format(self.name, self.location, self.directoryStructure,
                       self.managed, self.monitored, songsList)

    @attachSession
    def __str__(self):
        songsList = [songItem.songId for songItem in self.songFiles]
        output = "Repository:"
        output += "\n\tName: {}".format(self.name)
        output += "\n\tLocation: {}".format(self.location)
        output += "\n\tDirectoryStructure: {}".format(self.directoryStructure)
        output += "\n\tManaged: {}".format(self.managed)
        output += "\n\tMonitored: {}".format(self.monitored)
        output += "\n\tSongs: {}".format(songsList)
        return output

    @attachSession
    def delete(self):
        repositoryName = self.name
        [self.sqlaSession.delete(songFile) for songFile in self.songFiles]
        self.sqlaSession.delete(self)
        self.sqlaSession.commit()
        signalEmitter.repositoryRemoved.emit(repositoryName)

    @attachSession
    def setup(self):
        self.initialized = True
        try:
            self.directory = ManageDirectory(self)
        except DirectoryError:
            signalEmitter.repositoryDisabled.emit(self.id)
            self.directory = None
        if self.directory:
            for songId, (action, arguments) in self.pendingTransfers.items():
                song = self.sqlaSession.query(Song).get(songId)
                if action == "add":
                    self.addSongs({arguments[0], song}, arguments[1])
                elif action == "remove":
                    self.removeSongs([song])
                del self.pendingTransfers[songId]

    @attachSession
    @initialize
    def update(self):
        if not self.directory or not self.monitored: return
        [self.sqlaSession.delete(songFile) for songFile in self.songFiles
                    if not os.path.exists(songFile.location)]
        self.sqlaSession.commit()

        directoryFiles = self.directory.songs
        missingSongs = songs = self.getSongs()
        missingSongs.difference_update(directoryFiles)
        [self.removeSongs([song], False) for song in missingSongs]

        newSongs = directoryFiles - songs

        signalEmitter.longTask.emit("Updating Repository: " + self.name, len(newSongs))
        self.addSongs(newSongs, keepFiles = not self.managed)
        signalEmitter.longTaskDone.emit()

        self.directory.fixAllPaths()

    @attachSession
    @initialize
    def setName(self, value):
        if self.name == value: return
        if self.sqlaSession.query(Repository).filter(Repository.name == value).all():
            raise DatabaseError("Repository", {"name":value}, DatabaseError.Conflicting)
        signalEmitter.repositoryRemoved.emit(self.name)
        self.name = value
        self.sqlaSession.commit()
        signalEmitter.repositoryAdded.emit(self.id)

    @attachSession
    @initialize
    def setLocation(self, value):
        if self.location == value: return
        if self.sqlaSession.query(Repository).filter(Repository.location == value).all():
            raise DatabaseError("Repository", {"location":value}, DatabaseError.Conflicting)
        self.location = value
        self.sqlaSession.commit()
        self.setup()
        self.update()

    @attachSession
    @initialize
    def setManaged(self, value):
        if self.managed == value: return
        self.managed = value
        self.sqlaSession.commit()
        self.update()

    @attachSession
    @initialize
    def setMonitored(self, value):
        if self.monitored == value: return
        self.monitored = value
        self.sqlaSession.commit()
        self.update()

    @attachSession
    @initialize
    def getSongs(self):
        return set([songFile.song for songFile in self.songFiles])

    @attachSession
    @initialize
    def addSongs(self, songs, keepFiles = False):
        songs = songs - self.getSongs()

        for (song, location) in songs.items():
            if not self.directory:
                self.pendingTransfers[song.id] = ("add", [location, keepFiles])
            else:
                if self.managed:
                    location = self.directory.fixFilePath(location, song, keepFiles)
                self.songFiles.append(SongFile(song, location))
        self.sqlaSession.commit()
        signalEmitter.repositoryChanged.emit(self.id)

    @attachSession
    @initialize
    def removeSongs(self, songs, keepFiles = False):
        self.pendingTransfers = self.pendingTransfers - songs

        songs = set(songs).intersection(self.getSongs())

        if not self.directory:
            for song in songs:
                if song in self:
                    self.pendingTransfers[song.id] = ("remove", [])

        else:
            songFiles = [songFile for songFile in self.songFiles if songFile.song in songs]
            for songFile in songFiles:
                if not keepFiles or self.monitored:
                    if os.path.exists(songFile.location):
                        os.remove(songFile.location)
                self.sqlaSession.delete(songFile)
        self.sqlaSession.commit()
        signalEmitter.repositoryChanged.emit(self.id)

class Song(_SqlaBase):
    __tablename__ = 'song'
    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key = True)
    title = sqlalchemy.Column(UnicodeString)
    tags = relationship("Tag", secondary = songTagTable,
                                    backref = "songs", lazy = "dynamic")

    DEFAULT_METADATA = {
                        "ARTIST": "Unknown",
                        "ALBUM": "Unknown"
    }

    def __init__(self, title):
        self.title = title

    @attachSession
    def __contains__(self, item):
        if isinstance(item, types.IntType):
            return any([item == tag.id for tag in self.tags])
        if isinstance(item, Tag):
            return item in self.tags.all()
        raise TypeError("{} is not of type Tag or Tag.id".format(item))

    @attachSession
    def __repr__(self):
        locations = [songFile.location for songFile in self.songFiles]
        return "<Song({}, {}, {})>".\
                    format(self.id, self.title, locations)

    @attachSession
    def __str__(self):
        metadata = self.getMetadata()
        metadata.keysUpper()
        locations = [songFile.location for songFile in self.songFiles]
        output = "Song:"
        output += "\n\tId: {}".format(self.id)
        output += "\n\tName: {}".format(self.title)
        output += "\n\tArtist: {}".format(metadata["ARTIST"])
        output += "\n\tAlbum: {}".format(metadata["ALBUM"])
        output += "\n\tLocations: {}".format(locations)
        del metadata["ARTIST"]
        del metadata["ALBUM"]
        del metadata["TITLE"]
        output += "\n\tTags: {}".format(metadata)
        return output

    @attachSession
    def partialUpdate(self, location):
        """Augments current data with metadata from location"""

        songMetadataHandler = Metadata(location)
        songMetadata = songMetadataHandler.read() + Song.DEFAULT_METADATA
        self.addTags([getTag(tagName, tagValue) for (tagName, tagValue)
                      in songMetadata.items()], write = True)

    @attachSession
    def update(self, location):
        """Resets all data to metadata from location"""

        songMetadataHandler = Metadata(location)
        songMetadata = songMetadataHandler.read()
        if "TITLE" not in songMetadata:
            self.title = getUnknownTitle()
        else:
            self.title = songMetadata["TITLE"]
            del songMetadata["TITLE"]
        self.sqlaSession.commit()
        self.addTags([getTag(tagName, tagValue) for (tagName, tagValue)
                      in songMetadata.items()], write = True)
        self.checkForDuplicates()

    @attachSession
    def delete(self):
        self.sqlaSession.query(SongFile).filter(SongFile.song == self).delete()

        playlists = self.sqlaSession.query(Playlist) \
                                .join(Playlist.songs).filter(Song.id == self.id)
        [playlist.removeSong(self) for playlist in playlists]
        self.sqlaSession.commit()
        self.removeTags(self.tags, write = False)

        songId = self.id
        self.sqlaSession.delete(self)
        self.sqlaSession.commit()
        signalEmitter.songsRemoved.emit([songId])

    @attachSession
    def getSongFiles(self):
        return set(self.sqlaSession.query(SongFile).filter(SongFile.song == self).all())

    @attachSession
    def setTitle(self, value):
        self.title = value
        self.sqlaSession.commit()
        signalEmitter.songsChanged.emit([self.id])

    @attachSession
    def getMetadata(self):
        return StringDictionary(Song.DEFAULT_METADATA.items()
                    + [(tag.name, tag.value) for tag in self.tags]
                    + [("TITLE", self.title)])

    @attachSession
    def addTags(self, tags, write = False):
        tags = [tag for tag in tags if tag not in self]
        for newTag in tags:
            self.removeTags([tag for tag in self.tags
                             if tag.name == newTag.name], write = False)
            self.tags.append(newTag)
        if write:
            data = ExtendedDictionary([(newTag.name, newTag.value) for newTag in tags])
            locations = self.getLocations()
            for location in locations.values():
                try:
                    Metadata(location).write(data)
                except:
                    pass
        self.sqlaSession.commit()
        self.checkForDuplicates()
        signalEmitter.songsChanged.emit([self.id])

    @attachSession
    def removeTags(self, tags, write = False):
        tags = [tag for tag in tags if tag in self]
        [self.tags.remove(tag) for tag in tags]
        self.sqlaSession.commit()
        if write:
            for location in self.getLocations().values():
                fileMetadata = Metadata(location)
                [fileMetadata.removeTag(tag.name) for tag in tags]

        signalEmitter.songsChanged.emit([self.id])

    @attachSession
    def modifyLocation(self, new):
        [setattr(songFile, "location", new) for songFile in self.getSongFiles()]
        self.sqlaSession.commit()
        signalEmitter.songsChanged.emit([self.id])

    @attachSession
    def getLocation(self, repository = None):
        locations = self.getLocations()
        if repository:
            if repository in locations:
                return locations[repository]
        else:
            activeRepositories = [repository for repository
                                  in locations.keys() if repository.directory]
            if activeRepositories:
                return locations[activeRepositories[0]]
        return None

    @attachSession
    def getLocations(self):
        return ExtendedDictionary((self.sqlaSession.query(Repository) \
                     .get(songFile.repositoryId), songFile.location) \
                    for songFile in self.songFiles)

    @attachSession
    def writeMetadata(self):
        for location in self.getLocations().values():
            try:
                songMetadataHandler = Metadata(location)
                songMetadataHandler.write(self.getMetadata())
            except NonexistantFileError:
                self.songFiles.filter(SongFile.location == location).delete()
            except UnsupportedFiletypeError:
                pass

    def merge(self, others):
        songFiles = [songFiles for songFiles
                     in reduce(lambda x, y: x + y.getLocations.items(), others, [])]
        [repository.addSong(location, self) for (repository, location) in songFiles]
        playlists = set([])
        for song in others:
            [playlists.add(playlist) for playlist
                        in self.sqlaSession.query(Playlist)
                                    .filter(Playlist.songs.contains(song))]
        [playlist.addSong(song) for playlist in playlists]
        [other.delete() for other in others]

    def checkForDuplicates(self):
        metadata = self.getMetadata()
        duplicates = self.sqlaSession.query(Song) \
                    .filter(Song.title == self.title).filter(Song.id != self.id) \
                    .filter(Song.tags.contains(getTag("ARTIST", metadata["ARTIST"]))) \
                    .filter(Song.tags.contains(getTag("ALBUM", metadata["ALBUM"]))).all()
        self.merge(duplicates)

class SongFile(_SqlaBase):
    __tablename__ = 'songfile'
    repositoryId = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('repository.id'),
                primary_key = True)
    songId = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('song.id'),
                primary_key = True)
    location = sqlalchemy.Column(UnicodeString)
    song = relationship(Song, backref = "songFiles")

    def __init__(self, song, location):
        self.song = song
        self.songId = song.id
        self.location = location

    @attachSession
    def __repr__(self):
        repository = self.sqlaSession.query(Repository).get(self.repositoryId)
        return "<SongFile({}, {})>".format(repository.name, self.location)

    @attachSession
    def __str__(self):
        repository = self.sqlaSession.query(Repository).get(self.repositoryId)
        output = "SongFile:"
        output += "\n\tRepository: {}".format(repository.name)
        output += "\n\tLocation: {}".format(self.location)
        output += "\n\tSong Id: {}".format(self.songId)
        return output

class Tag(_SqlaBase):
    __tablename__ = 'tag'
    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key = True)
    name = sqlalchemy.Column(UnicodeString)
    value = sqlalchemy.Column(UnicodeString)

    def __init__(self, name, value):
        name = name.upper()
        if name == "TITLE": raise ValueError("Cannot make TITLE tag.")
        self.name = name
        self.value = value

    @attachSession
    def __repr__(self):
        return "<Tag({}: {})>".format(self.name, self.value)

    @attachSession
    def __str__(self):
        songsList = [song.id for song in self.songs]
        output = "Tag:"
        output += "\n\tName: {}".format(self.name.capitalize())
        output += "\n\tValue: {}".format(self.value)
        output += "\n\tSongs: {}".format(songsList)
        return output
    @attachSession
    def delete(self):
        tagId = self.id
        [song.removeTags([self]) for song in self.songs]

        self.sqlaSession.delete(self)
        self.sqlaSession.commit()
        signalEmitter.tagsRemoved.emit([tagId])

    @attachSession
    def setValue(self, value):
        if self.value == value: return
        existingTag = self.sqlaSession.query(Tag).filter(Tag.name == self.name) \
            .filter(Tag.value == value).first()
        if existingTag:
            existingTag.merge(self)
        else:
            self.value = value
            signalEmitter.tagsChanged.emit([self.id])

    def merge(self, other):
        for song in other.songs:
            song.addTags([self], write = True)
        other.delete()

class Playlist(_SqlaBase):
    __tablename__ = 'playlist'
    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key = True)
    name = sqlalchemy.Column(UnicodeString)
    songs = relationship("Song",
                    secondary = playlistSongTable,
                    backref = "playlists")

    def __init__(self, name):
        self.name = name

    @attachSession
    def __contains__(self, item):
        if isinstance(item, types.IntType):
            return any([item == song.id for song in self.songs])
        if isinstance(item, Song):
            return item in self.songs
        raise TypeError("{} is not of type Song or Song.id".format(item))

    @attachSession
    def __repr__(self):
        songsList = [song.id for song in self.songs]
        return "<Playlist({}, {})>".format(self.name, songsList)

    @attachSession
    def __str__(self):
        songsList = [song.id for song in self.songs]
        output = "Playlist:"
        output += "\n\tName: {}".format(self.name)
        output += "\n\tSongs: {}".format(songsList)
        return output

    @attachSession
    def delete(self):
        name = self.name
        self.sqlaSession.delete(self)
        self.sqlaSession.commit()
        signalEmitter.playlistRemoved.emit(name)

    @attachSession
    def setName(self, value):
        if self.name == value: return
        if self.sqlaSession.query(Playlist).filter(Playlist.name == value).all():
            raise DatabaseError("Playlist", {"name":value}, DatabaseError.Conflicting)
        signalEmitter.playlistRemoved.emit(self.name)
        self.name = value
        self.sqlaSession.commit()
        signalEmitter.playlistAdded.emit(self.id)

    @attachSession
    def addSongs(self, songs):
        songs = set(songs)
        songs.difference_update(self.songs)
        [self.songs.append(song) for song in songs]
        self.sqlaSession.commit()
        signalEmitter.playlistChanged.emit(self.id)

    @attachSession
    def removeSongs(self, songs):
        songs = set(songs).intersection(self.songs)
        [self.songs.remove(song) for song in songs]
        self.sqlaSession.commit()
        signalEmitter.playlistChanged.emit(self.id)

#===============================================================================
# Setup engine and initialize session
#===============================================================================
def setupSqlAlchemy(dbDirectory):
    if not _sqlaEngine: _setupSqlEngine(dbDirectory)
    _SqlaBase.metadata.create_all()

def resetSql():
    _SqlaBase.metadata.drop_all()
    setupSqlAlchemy("")

def _setupSqlEngine(dbFile):
    global _sqlaEngine
    global _guiSession
    global _SqlaBase
    global _workerThread

    _sqlaEngine = sqlalchemy.create_engine("sqlite:///{}".format(dbFile))
    SqlaSession.configure(bind = _sqlaEngine)
    session.Session.close_all()
    _sqlaMasterSession = SqlaSession()
    _SqlaBase.metadata.bind = _sqlaEngine

    _workerThread = thread.start_new_thread(_runBackgroundThread, ())

#===============================================================================
# PySide Signals
#===============================================================================
class SignalEmitter(QtCore.QObject):
    repositoryAdded = QtCore.Signal(int)
    repositoryChanged = QtCore.Signal(int)
    repositoryRemoved = QtCore.Signal(str)
    repositoryDisabled = QtCore.Signal(int)

    playlistAdded = QtCore.Signal(int)
    playlistChanged = QtCore.Signal(int)
    playlistRemoved = QtCore.Signal(str)

    songsAdded = QtCore.Signal(list)
    songsRemoved = QtCore.Signal(list)
    songsChanged = QtCore.Signal(list)

    tagsAdded = QtCore.Signal(list)
    tagsRemoved = QtCore.Signal(list)
    tagsChanged = QtCore.Signal(list)

    longTask = QtCore.Signal(str, int)
    longTaskIncrement = QtCore.Signal(int)
    longTaskDone = QtCore.Signal()

    def __init__(self, parent = None):
        QtCore.QObject.__init__(self, parent)

signalEmitter = SignalEmitter()

#===============================================================================
# Debugging Tools
#===============================================================================
def dump(fileName):
    sqlaSession = SqlaSession()
    output = "Repositories:\n"
    for item in sqlaSession.query(Repository).all():
        output += "\n" + item.__str__()
    output += "\n\n" + "="*80 + "\n\nPlaylists:\n"
    for item in sqlaSession.query(Playlist).all():
        output += "\n" + item.__str__()
    output += "\n\n" + "="*80 + "\n\nSongs:\n"
    for item in sqlaSession.query(Song).all():
        output += "\n" + item.__str__()
    output += "\n\n" + "="*80 + "\n\nTags:\n"
    for item in sqlaSession.query(Tag).all():
        output += "\n" + item.__str__()
    output += "\n\n" + "="*80 + "\n\nSongFiles:\n"
    for item in sqlaSession.query(SongFile).all():
        output += "\n" + item.__str__()
    open(fileName, 'w').write(output)

def _runBackgroundThread():
    global _backgroundSession
    _backgroundSession = SqlaSession()
    while _workerRunning:
        task, args = workerQueue.get()
        task(*args)
        workerQueue.task_done()
    thread.exit()

#===============================================================================
# Repository Functions
#===============================================================================
def createRepository(name, location, \
                     directoryStructure = Repository.DEFAULT_DIRECTORY_STRUCTURE, \
                     monitored = True, managed = True):
    sqlaSession = SqlaSession()
    if sqlaSession.query(Repository).filter(Repository.name == name).all():
        raise DatabaseError("Repository", {"name": name}, DatabaseError.Conflicting)
    if sqlaSession.query(Repository).filter(Repository.location == location).all():
            raise DatabaseError("Repository", {"location":location}, DatabaseError.Conflicting)
    repository = Repository(name, location, directoryStructure, monitored, \
                               managed)
    sqlaSession.add(repository)
    sqlaSession.commit()
    workerQueue.put([loadRepository, (repository.id,)])
    return repository

def loadAllRepositories():
    sqlaSession = SqlaSession()
    [workerQueue.put([loadRepository, (repository.id,)]) for repository in sqlaSession.query(Repository.id)]

def loadRepository(id):
    sqlaSession = SqlaSession()
    repository = sqlaSession.query(Repository).get(id)
    repository.update()
    signalEmitter.repositoryAdded.emit(id)

#===============================================================================
# Playlist Functions
#===============================================================================
def createPlaylist(name):
    sqlaSession = SqlaSession()
    if sqlaSession.query(Playlist).filter(Playlist.name == name).all():
        raise DatabaseError("Playlist", {"name": name}, DatabaseError.Conflicting)
    playlist = Playlist(name)
    sqlaSession.add(playlist)
    sqlaSession.commit()
    signalEmitter.playlistAdded.emit(playlist.id)
    return playlist

def loadAllPlaylists():
    sqlaSession = SqlaSession()
    for playlist in sqlaSession.query(Playlist.id):
        signalEmitter.playlistAdded.emit(playlist.id)

#===============================================================================
# Song Functions
#===============================================================================
def loadAllSongs():
    sqlaSession = SqlaSession()
    songs = sqlaSession.query(Song).all()
    signalEmitter.songsAdded.emit([song.id for song in songs])

def createSong(location):
    sqlaSession = SqlaSession()
    if not os.path.exists(location): raise NonexistantFileError("{} does not exist.".format(location))
    songMetadataHandler = Metadata(location)
    metadata = songMetadataHandler.read(REQ_SONG_DATA)
    if "TITLE" not in metadata:
        metadata["TITLE"] = getUnknownTitle()
    song = getSongByMetadata(metadata["TITLE"], metadata.subset(["ARTIST", "ALBUM"]))
    if song: return song
    song = Song(metadata["TITLE"])
    sqlaSession.add(song)
    sqlaSession.commit()
    song.update(location)
    signalEmitter.songsAdded.emit([song.id])
    return song

def getUnknownTitle():
    sqlaSession = SqlaSession()
    unknownSongs = sqlaSession.query(Song).filter(Song.title.like("unknown%")).all()
    counter = 0
    for song in unknownSongs:
        try:
            counter = max(counter, int(song.title.replace("unknown", "")))
        except ValueError: pass
    return "unknown{}".format(counter)

def getSong(id):
    sqlaSession = SqlaSession()
    return sqlaSession.query(Song).get(id)

def getSongByLocation(location):
    sqlaSession = SqlaSession()
    songFile = sqlaSession.query(SongFile).filter(SongFile.location == location) \
                .first()
    if songFile: return songFile.song
    try:
        fileMetadataHandler = Metadata(location)
    except FileError: return None
    metadata = fileMetadataHandler.read(REQ_SONG_DATA)

    song = getSongByMetadata(metadata["TITLE"], metadata.subset(["ARTIST", "ALBUM"]))
    if song: return song
    return createSong(location)

def getSongByMetadata(title, metadata):
    sqlaSession = SqlaSession()
    possibleMatches = sqlaSession.query(Song).filter(Song.title == title)
    for (key, value) in metadata.items():
        possibleMatches = possibleMatches.filter(Song.tags.contains(getTag(key, value)))
    return possibleMatches.first()

#===============================================================================
# Tag Functions
#===============================================================================

def loadAllTags():
    sqlaSession = SqlaSession()
    tags = sqlaSession.query(Tag).all()
    [tag.delete() for tag in tags if not tag.songs]
    signalEmitter.tagsAdded.emit([tag.id for tag in sqlaSession.query(Tag).all()])

def getTag(name, value):
    sqlaSession = SqlaSession()
    name = name.upper()
    tag = sqlaSession.query(Tag).filter(Tag.name == name) \
                .filter(Tag.value == value).first()
    if tag: return tag
    tag = Tag(name, value)
    sqlaSession.add(tag)
    sqlaSession.commit()
    signalEmitter.tagsAdded.emit([tag.id])
    return tag

class ManageDirectory(object):
    """
    Song directory manager
    """
    instances = {}
    def __init__(self, repository):
        self.repository = repository
        self.songs = ExtendedDictionary()
        self.refreshDirectoryStructure()
        self.scanStatus = False

    def __new__(cls, repository):
        if not os.path.isdir(repository.location): raise DirectoryError("{} is not a directory.".format(repository.location))
        if repository in ManageDirectory.instances:
            return ManageDirectory[ManageDirectory.instances]
        self = object.__new__(cls)
        manager = ManageDirectory.instances[repository] = ManageDirectory.__init__(self, repository)
        manager.scan()
        return manager

    def refreshDirectoryStructure(self):
        self.tagNames = re.findall('%[a-zA-Z]+%', self.repository.directoryStructure)
        self.tagNames = [tagName.replace("%", "").upper() for tagName in \
                         self.tagNames]
        self.defaultMetadata = ExtendedDictionary.fromkeys(self.tagNames, "Unknown")

    def scan(self):
        sqlaSession = SqlaSession()
        if not os.path.exists(self.repository.location):
            raise DirectoryError(self.repository.location, DirectoryError.Nonexistant)
        fileNames = dirtools.search(self.repository.location, SUPPORTED_FILE_TYPES, relative = False)

        self.songs = ExtendedDictionary([(songFile.song, songFile.location) for songFile in \
                           sqlaSession.query(SongFile) \
                           .filter(SongFile.repository == self.repository).all() \
                           if songFile.location in fileNames])
        fileNames.difference_update(set(self.songs.values()))
        signalEmitter.longTask.emit("Scanning Directory", len(fileNames))
        for i, fileName in enumerate(fileNames):
            signalEmitter.longTaskIncrement.emit(i)
            try:
                song = getSongByLocation(fileName)
            except FileError:
                continue
            self.songs[song] = fileName
        signalEmitter.longTaskDone.emit()
        self.scanStatus = True

    def formatPath(self, fileName, metadata, number):
        extension = os.path.splitext(fileName)[1]
        output = os.path.join(self.repository.location, self.repository.directoryStructure)

        metadata += self.defaultMetadata
        for tagName, tagValue in metadata.items():
            tagValue = tagValue.replace("/", "_").strip()
            output = output.replace("%{}%".format(tagName), tagValue)
        if number > 0:
            output += "{}".format(number);
        for invalidCharacter in ("\\", ":", "\"", "<", ">", "|", "*", "?"):
            output = output.replace(invalidCharacter, "_")
        output += extension
        return output

    def fixAllPaths(self, keepOriginal = False):
        for i, (song, location) in enumerate(self.songs.items()):
            signalEmitter.longTaskIncrement.emit(i)
            self.fixFilePath(location, song, keepOriginal)
        signalEmitter.longTaskDone.emit()

        dirtools.cleanup(self.repository.location, [".DS_Store"])

    def fixFilePath(self, fileName, song, keepOriginal = False):
        metadata = song.getMetadata()
        correctPath = self.formatPath(fileName, metadata, 0)

        if fileName == correctPath:
            self.songs[song.id] = fileName
            return fileName
        i = 0
        while os.path.exists(correctPath):
                i += 1
                correctPath = self.formatPath(fileName, metadata, i)

        if not keepOriginal:
            os.renames(fileName, correctPath)
            song.modifyLocation(correctPath)

        else:
            parentDirectory = os.path.dirname(correctPath)
            if not os.path.exists(parentDirectory):
                os.makedirs(parentDirectory)
            shutil.copy(fileName, correctPath)
        self.songs[song.id] = correctPath
        return correctPath

    def __contains__(self, other):
        if isinstance(other, Song):
            return any([other.id == song.id for song in self.getSongs])
        else:
            raise TypeError("{} is not of type Song".format(other))


# Problem #57

def main():
  f = open('test_smoothing_the_weather.txt', 'r')
  f.readline()

  li = list(map(lambda x: float(x), f.readline().strip().split()))
  result = []

  for i in range(0, len(li)):
    if i == 0 or i == len(li) - 1:
      result.append(li[i])
    else:
      result.append((li[i-1] + li[i] + li[i+1]) / 3.0)

  for num in result:
    print(num)

if __name__ == '__main__':
  main()


import numpy as np
import pandas as pd

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.colors as colors

# simulations = ["binary","unary"]
simulations = ["restest"]

with open("../local_config/skirt_data_dir") as f:
    skirt_data_dir = f.readline()

for sim in simulations:
    print(f"Summarising {sim}")
    # fname = f"{skirt_data_dir}/{sim}_lowres_allprobe_cellprops.dat"
    fname = f"{skirt_data_dir}/test_data/big_{sim}_allprobe_cellprops.dat"
    df = pd.read_csv(fname,sep=' ',header=None,skiprows=9,index_col=0,
                     names = ["x","y","z","V","tau","dust","e","H"])
    fname = f"{skirt_data_dir}/test_data/big_{sim}_tdust_T.dat"
    df_tdust = pd.read_csv(fname,sep=' ',header=None,skiprows=3,index_col=0,
                     names = ["Tdust"])
    df = pd.merge(df,df_tdust,left_index=True,right_index=True)

    # print(df)

    for key in df.keys():
        if np.sum(np.nonzero(df[key].values))==0:
            print(f"{key} is zero throughout run")
        else:
            print(f"{key}: Min:{np.nanmin(df[key].values)} Mean:{np.nanmean(df[key].values)} Median:{np.nanmedian(df[key].values)} Max:{np.nanmax(df[key].values)} ")
            print(f"{key}: Non-zero:{np.count_nonzero(df[key].values)/len(df)} Finite:{np.sum(np.isfinite(df[key].values))/len(df)}")

    print("Dust 2D histograms")

    df["r2d"] = np.sqrt(df["x"] ** 2 + df["y"] ** 2)
    for key in ["dust","Tdust","tau"]:
        # dust mean properties plots
        # H,xedges,yedges = np.histogram2d(df["z"],df["r2d"],weights=df[key],bins=1024)

        for coords,prefix in [[["y","x"],"yx"],[["z","r2d"],"rz"]]:

            H,yedges,xedges = np.histogram2d(df[coords[0]],df[coords[1]],weights=df[key],bins=128)
            fig,sp = plt.subplots(figsize=(6,6),dpi=200,constrained_layout=True)
            mp = sp.pcolormesh(xedges,yedges,H,norm=colors.LogNorm())
            fig.colorbar(mp)
            fig.savefig(f"../quick_out/{prefix}_{key}_{sim}.png")
            # fig.savefig(f"../quick_out/xy_{key}_{sim}.png")
            plt.close(fig)


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

from imageio import imread, imwrite
import matplotlib.pyplot as plt
import numpy as np
import scipy.ndimage

Img = imread('C:/Users/fc48286/Downloads/lena.tif')

dim = Img.shape
tipo = Img.dtype
npix = Img.size

plt.figure
plt.imshow(Img)

#Decompor as bandas R,G,B

plt.figure(figsize=(12,3))
plt.subplot(141); plt.imshow(Img)
plt.title('Inicial'); plt.axis('off')
plt.subplot(142); plt.imshow(Img[:,:,0], 'gray')
plt.title('R'); plt.axis('off')
plt.subplot(143); plt.imshow(Img[:,:,1], 'gray')
plt.title('G'); plt.axis('off')
plt.subplot(144); plt.imshow(Img[:,:,2], 'gray')
plt.title('B'); plt.axis('off')

#Gravar Img no disco
imwrite('C:/Users/fc48286/Downloads/' + 'img1_r' + '.tif', Img[:,:, 0])
imwrite('C:/Users/fc48286/Downloads/' + 'img1_g' + '.tif', Img[:,:, 1])
imwrite('C:/Users/fc48286/Downloads/' + 'img1_b' + '.tif', Img[:,:, 2])

val = Img[25,31,0]

linha_cse = 125
coluna_cse = 118
linha_cid = 140
coluna_cid = 180
cropImg = Img[linha_cse:linha_cid,coluna_cse:coluna_cid,:]
plt.figure(); plt.imshow(cropImg);
plt.title('Crop'); plt.axis('off')

Img1 = Img*1
Img1[linha_cse:linha_cid,coluna_cse:coluna_cid,:] = 10
plt.figure(); plt.imshow(Img1)
plt.title(u'Incognita'); plt.axis('off')

#Plot de perfis

#Construir uma linha "num" pontos entre os pontos de coordenadas
#(linha, coluna): (y0, x0) e (y1, x2)
y0, x0 = 100, 50
y1, x1 = 200, 200
num = 100
y, x = np.linspace(y0, y1, num), np.linspace(x0, x1, num)

#Extrair os valores dos pixels ao longo da linha
d = np.vstack((y,x))
perfil = scipy.ndimage.map_coordinates(Img[:,:,0], 'gray')

plt.figure(figsize=(15, 3))
plt.subplot(121); plt.imshow(Img[:,:,0], 'gray')
plt.title['Inicial']; plt.axis('off')
plt.plot([x0, x1], [y0, y1], 'ro=')
plt.subplot(122); plt.plot(perfil, 'ro==')
plt.title('Perfil')


# python
import datetime

# django
from django.core import mail

from djangopeople.unit_tests.testbase import TestCase
from welcome.models import WelcomeEmail

class ModelTestCase(TestCase):

    def test_basic_creation(self):
        user, person = self._create_person('bob', 'bob@example.com')
        welcome_email = WelcomeEmail.objects.create(user=user,
                                                    subject='Test',
                                                    body='Test body')
        self.assertEqual(welcome_email.email, 'bob@example.com')
        self.assertEqual(welcome_email.send_date, None)
        welcome_email.send()

        welcome_email = WelcomeEmail.objects.get(user=user)
        self.assertNotEqual(welcome_email.send_date, None)
        self.assertTrue(welcome_email.send_date <= datetime.datetime.now())


""" Test the cli_simple tool outputs
"""

# All combinations to be tested
#

import argparse
from pathlib import Path

from cmatools.cli_hello_world import cli_parse_args, cli_parser, cli_hello

from cmatools.definitions import ROOT_DIR, PACKAGE

# Define cli filepath
CLI = Path(ROOT_DIR, PACKAGE, "cli_hello_world.py")

def test_cli_parser():

    """Test for cli_parser() function."""

    out = cli_parser()
    # Confirm output object is correct parser type
    assert isinstance(out, argparse.ArgumentParser)
    # Confirm cli tool name is correct
    assert out.prog == "CLI-HELLO"

def test_cli_parse_args():

    """Test for cli_parse_args() function"""

    user_args = ["John"]

    parsed_args = cli_parse_args(user_args)

    print(parsed_args)
    print(type(parsed_args))

    assert isinstance(parsed_args, argparse.Namespace)

    assert parsed_args.user == "John"

def test_cli_hello():
    """Test for cli_hello() function """
    parsed_args = argparse.Namespace(user="John")
    output = cli_hello(parsed_args)
    assert output == "Hello: John"


"""
The :py:mod:`optimization` module contains functions to load the optimizer,
create the starting samples and run the optimization.
"""

import os
import imp
from shutil import copyfile
from pyDOE import lhs
from rheia.CASES.determine_stoch_des_space import load_case, check_dictionary
import numpy as np
import pandas as pd
#######################
# optimizer functions #
#######################

def parse_available_opt():
    """

    Parse all available optimizers. In this version,
    only NSGA-II is available.

    Returns
    -------
    methods : list of tuples
        The available optimizers with the corresponding classes.

    """

    opt_dir = os.path.join(os.path.split(
        os.path.dirname(
            os.path.abspath(__file__)))[0],
        'OPT')

    # get the optimizer modules
    files = [f for f in os.listdir(opt_dir) if f.endswith('algorithms.py')]
    methods = []
    for file in files:
        obj = imp.load_source(file.split('.')[0], os.path.join(opt_dir, file))
        tmp = obj.return_opt_methods()
        for method in tmp:
            methods.append((method, obj))

    return methods

def load_optimizer(optimizer):
    """

    Load the selected optimizer.

    Parameters
    ----------
    optimizer : string
        The name of the optimizer (currently 'NSGA2').

    Returns
    -------
    opt_obj : object
        The optimization object.

    """

    opt_list = parse_available_opt()
    optimizers = [elem[0] for elem in opt_list]

    # check if optimizer exists and load optimization module
    if optimizer not in optimizers:
        raise KeyError('Optimizer is not available!')

    # get object from optimizer class
    for opt in opt_list:
        if optimizer == opt[0]:
            opt_obj = opt[1].return_opt_obj(optimizer)

    return opt_obj

####################
# starting samples #
####################

def check_existing_results(run_dict, space_obj):
    """
    Check if previously generated results exists in the provided results
    directory.

    Parameters
    ----------
    run_dict : dict
        The dictionary with user-defined values for
        the characterization of the optimization.
    space_obj : object
        The object with information on the design space and stochastic space

    Returns
    -------
    start_from_last_gen : bool
        Boolean that indicates if results exist in the considered result
        directory.

    """

    # path of the population file
    pop_file = os.path.join(os.path.split(os.path.dirname(
        os.path.abspath(__file__)))[0],
        'RESULTS',
        space_obj.case,
        list(run_dict['objectives'].keys())[0], run_dict['results dir'],
        'population.csv')

    # path of the fitness file
    fitness_file = os.path.join(os.path.split(os.path.dirname(
        os.path.abspath(__file__)))[0],
        'RESULTS',
        space_obj.case,
        list(run_dict['objectives'].keys())[0], run_dict['results dir'],
        'fitness.csv')

    # check if the population and fitness file exist
    start_from_last_gen = False
    if os.path.isfile(fitness_file) and os.path.isfile(pop_file):

        start_from_last_gen = True

    return start_from_last_gen

def scale_samples_to_design_space(nondim_doe, space_obj):
    """
    Scales the starting sample to the given design space.

    Parameters
    ----------
    nondim_doe : ndarray
        The non-dimensionalized design of experiment.
    space_obj : object
        The object with information on the design space and stochastic space

    Returns
    -------
    dim_doe : ndarray
        The design of experiment scaled up to the design space.

    """

    dim_doe = np.zeros(nondim_doe.shape)
    for j in range(space_obj.n_dim):
        dim_doe[:, j] = ((space_obj.u_b[j] - space_obj.l_b[j]) *
                         nondim_doe[:, j] + space_obj.l_b[j])

    return dim_doe

def write_starting_samples(doe, filename):
    """

    Writes the starting samples to file.

    Parameters
    ----------
    doe : list
        The set of starting samples.
    filename : string
        The path to the filename for the starting samples.

    """
    with open(filename, 'w') as file:
        for x_in in doe:
            if not isinstance(x_in, list):
                x_in = x_in.tolist()
            for item in x_in:
                file.write('%.8f ' % item)
            file.write('\n')

def create_starting_samples(run_dict, space_obj, start_from_last_gen):
    """

    Load the starting samples for the optimization run.

    Parameters
    ----------
    run_dict : dict
        The dictionary with user-defined values for
        the characterization of the optimization.
    space_obj : object
        The object with information on the design space and stochastic space
    start_from_last_gen : bool
        Boolean that determines if the starting samples
        start from a previously generated population.

    """

    # define folder to put starting samples file
    doe_path = os.path.join(os.path.split(
        os.path.dirname(
            os.path.abspath(__file__)))[0],
        'OPT',
        'INPUTS',
        space_obj.case,
        '%iD' %
        space_obj.n_dim)

    # check if folder exists in INPUTS
    # if not, create one
    if not os.path.exists(doe_path):
        os.makedirs(doe_path)

    # create the doe set of samples
    doe_filename = os.path.join(doe_path,
                                'DOE_n%i.csv' % run_dict['population size'])

    if not start_from_last_gen:
        # if the starting population needs to be created
        if 'AUTO' in run_dict['x0'][0]:

            # generate the starting population randomly
            if 'RANDOM' in run_dict['x0'][1]:
                ddoe = np.random.random(
                    (run_dict['population size'], space_obj.n_dim))

            # generate the starting population
            # based on Latin Hypercube Sampling
            if 'LHS' in run_dict['x0'][1]:
                ddoe = lhs(space_obj.n_dim,
                           samples=run_dict['population size'])

            # scale the starting samples
            doe = scale_samples_to_design_space(ddoe, space_obj)

            # Write starting samples to file
            write_starting_samples(doe, doe_filename)

        else:
            # the starting population is provided in a custom file
            doe_custom = os.path.join(os.path.split(
                os.path.dirname(
                    os.path.abspath(__file__)))[0],
                'CASES',
                space_obj.case,
                run_dict['x0'][1])

            # check if the custom file exists
            if not os.path.isfile(doe_custom):
                raise NameError(
                    """The initial population file %s
                    is not found in the case folder.""" %
                    os.path.basename(doe_custom))

            copyfile(doe_custom, doe_filename)

    else:
        # the starting population is the final generation from the existing
        # population in the result directory
        doe_custom = os.path.join(os.path.split(
            os.path.dirname(
                os.path.abspath(__file__)))[0],
            'RESULTS',
            space_obj.case,
            list(run_dict['objectives'].keys())[0],
            run_dict['results dir'],
            'population.csv')

        df = pd.read_csv(doe_custom, header=None)
        df.drop(df.tail(1).index,inplace=True)

        rows = df.tail(run_dict['population size']).to_numpy()

        doe = []
        for line in rows:
            dummy = []
            for l in line:
                dummy.append(float(l))
            doe.append(dummy)

        # check if the size of the existing population matches with the
        # provided one in the optimization dictionary
        if len(doe) == run_dict['population size']:
            write_starting_samples(doe, doe_filename)
        else:
            raise ValueError(""" The defined population size does not match
                                 the population size from a previous run in
                                 this result directory.""")

########################
# run the optimization #
########################

def run_opt(run_dict, design_space='design_space.csv'):
    """
    This function runs the optimization pipeline.
    First, the case, optimizer and configuration
    are loaded. Thereafter, the starting samples
    are created, the specific optimization class
    instantiated and the :py:meth:`run_optimizer`
    method is called.

    Parameters
    ----------
    run_dict : dict
        The dictionary with user-defined values for
        the characterization of the optimization.
    design_space : string, optional
        The design_space filename. The default is 'design_space.csv'.

    """

    # evaluate if the optimization dictionary is properly characterized
    check_dictionary(run_dict)

    # load the object on the design space, the evaluation function
    # and the params provided for each model evaluation
    space_obj, eval_func, params = load_case(run_dict, design_space)

    # add the evaluation function to the optimization dictionary
    run_dict['evaluate'] = eval_func

    # load optimizer class
    opt_class = load_optimizer('NSGA2')

    # check if previous results in this result directory exist
    start_from_last_gen = check_existing_results(run_dict, space_obj)

    # define starting samples
    create_starting_samples(run_dict, space_obj, start_from_last_gen)

    # optimizer object
    res = opt_class(run_dict, space_obj, start_from_last_gen, params)

    # run the optimizer
    res.run_optimizer()


import contextlib
from django.contrib import messages
from django.shortcuts import render, get_object_or_404
from django.http import HttpResponseRedirect
from django.urls import reverse
from libvirt import libvirtError
from computes.models import Compute
from interfaces.forms import AddInterface
from vrtManager.interface import wvmInterface, wvmInterfaces
from admin.decorators import superuser_only

@superuser_only
def interfaces(request, compute_id):
    """
    :param request:
    :param compute_id:
    :return:
    """

    ifaces_all = []
    compute = get_object_or_404(Compute, pk=compute_id)

    try:
        conn = wvmInterfaces(
            compute.hostname,
            compute.login,
            compute.password,
            compute.type
        )
        ifaces = conn.get_ifaces()

        netdevs = ["eth0", "eth1"]
        with contextlib.suppress(Exception):
            netdevs = conn.get_net_devices()

        for iface in ifaces:
            interf = wvmInterface(
                compute.hostname,
                compute.login,
                compute.password,
                compute.type,
                iface
            )
            ifaces_all.append(interf.get_details())

        if request.method == "POST":
            if "create" in request.POST:
                form = AddInterface(request.POST)
                if form.is_valid():
                    data = form.cleaned_data
                    conn.create_iface(
                        data["name"],
                        data["itype"],
                        data["start_mode"],
                        data["netdev"],
                        data["ipv4_type"],
                        data["ipv4_addr"],
                        data["ipv4_gw"],
                        data["ipv6_type"],
                        data["ipv6_addr"],
                        data["ipv6_gw"],
                        data["stp"],
                        data["delay"],
                    )
                    return HttpResponseRedirect(request.get_full_path())
                else:
                    for msg_err in form.errors.values():
                        messages.error(request, msg_err.as_text())
        conn.close()
    except libvirtError as lib_err:
        messages.error(request, lib_err)

    return render(request, "interfaces.html", locals())

@superuser_only
def interface(request, compute_id, iface):
    """
    :param request:
    :param compute_id:
    :param iface:
    :return:
    """

    ifaces_all = []
    compute = get_object_or_404(Compute, pk=compute_id)

    try:
        conn = wvmInterface(
            compute.hostname,
            compute.login,
            compute.password,
            compute.type,
            iface
        )
        start_mode = conn.get_start_mode()
        state = conn.is_active()
        mac = conn.get_mac()
        itype = conn.get_type()
        ipv4 = conn.get_ipv4()
        ipv4_type = conn.get_ipv4_type()
        ipv6 = conn.get_ipv6()
        ipv6_type = conn.get_ipv6_type()
        bridge = conn.get_bridge()
        slave_ifaces = conn.get_bridge_slave_ifaces()

        if request.method == "POST":
            if "stop" in request.POST:
                conn.stop_iface()
                return HttpResponseRedirect(request.get_full_path())
            if "start" in request.POST:
                conn.start_iface()
                return HttpResponseRedirect(request.get_full_path())
            if "delete" in request.POST:
                conn.delete_iface()
                return HttpResponseRedirect(reverse("interfaces", args=[compute_id]))
        conn.close()
    except libvirtError as lib_err:
        messages.error(request, lib_err)

    return render(request, "interface.html", locals())


from PySide6.QtCore import (QCoreApplication,
    QMetaObject, QRect, Slot, QSize, Qt)
from PySide6.QtGui import QFont
from PySide6.QtWidgets import (QApplication, QDialogButtonBox, QDoubleSpinBox,
    QHBoxLayout, QLCDNumber, QLayout,
    QPushButton, QSizePolicy, QSpacerItem, QSpinBox,
    QTextBrowser, QVBoxLayout, QWidget)
import multiprocessing.shared_memory as mpsm
import sys

class Ui_Form(object):
    def setupUi(self, Widget):
        if not Widget.objectName():
            Widget.setObjectName(u"Widget")
        Widget.setWindowModality(Qt.WindowModal)
        Widget.setEnabled(True)
        Widget.resize(700, 200)
        sizePolicy = QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(Widget.sizePolicy().hasHeightForWidth())
        Widget.setSizePolicy(sizePolicy)
        Widget.setMinimumSize(QSize(700, 200))
        Widget.setMaximumSize(QSize(700, 200))
        Widget.setBaseSize(QSize(700, 420))
        self.verticalLayoutWidget_4 = QWidget(Widget)
        self.verticalLayoutWidget_4.setObjectName(u"verticalLayoutWidget_4")
        self.verticalLayoutWidget_4.setGeometry(QRect(20, 10, 661, 171))
        self.verticalLayout_4 = QVBoxLayout(self.verticalLayoutWidget_4)
        self.verticalLayout_4.setSpacing(7)
        self.verticalLayout_4.setObjectName(u"verticalLayout_4")
        self.verticalLayout_4.setSizeConstraint(QLayout.SetDefaultConstraint)
        self.verticalLayout_4.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout_5 = QHBoxLayout()
        self.horizontalLayout_5.setObjectName(u"horizontalLayout_5")
        self.verticalLayout_2 = QVBoxLayout()
        self.verticalLayout_2.setObjectName(u"verticalLayout_2")
        self.verticalLayout_2.setSizeConstraint(QLayout.SetNoConstraint)
        self.horizontalLayout_4 = QHBoxLayout()
        self.horizontalLayout_4.setObjectName(u"horizontalLayout_4")
        self.horizontalSpacer = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)

        self.horizontalLayout_4.addItem(self.horizontalSpacer)

        self.lcdNumber = QLCDNumber(self.verticalLayoutWidget_4)
        self.lcdNumber.setObjectName(u"lcdNumber")
        sizePolicy1 = QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
        sizePolicy1.setHorizontalStretch(0)
        sizePolicy1.setVerticalStretch(0)
        sizePolicy1.setHeightForWidth(self.lcdNumber.sizePolicy().hasHeightForWidth())
        self.lcdNumber.setSizePolicy(sizePolicy1)
        self.lcdNumber.setMinimumSize(QSize(64, 24))
        font = QFont()
        font.setPointSize(12)
        self.lcdNumber.setFont(font)

        self.horizontalLayout_4.addWidget(self.lcdNumber)

        self.horizontalSpacer_2 = QSpacerItem(40, 20, QSizePolicy.Expanding, QSizePolicy.Minimum)

        self.horizontalLayout_4.addItem(self.horizontalSpacer_2)

        self.verticalLayout_2.addLayout(self.horizontalLayout_4)

        self.horizontalLayout_2 = QHBoxLayout()
        self.horizontalLayout_2.setObjectName(u"horizontalLayout_2")
        self.textBrowser = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser.setObjectName(u"textBrowser")
        sizePolicy2 = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.MinimumExpanding)
        sizePolicy2.setHorizontalStretch(0)
        sizePolicy2.setVerticalStretch(0)
        sizePolicy2.setHeightForWidth(self.textBrowser.sizePolicy().hasHeightForWidth())
        self.textBrowser.setSizePolicy(sizePolicy2)
        self.textBrowser.setMinimumSize(QSize(140, 34))

        self.horizontalLayout_2.addWidget(self.textBrowser)

        self.spinBox = QSpinBox(self.verticalLayoutWidget_4)
        self.spinBox.setObjectName(u"spinBox")
        sizePolicy3 = QSizePolicy(QSizePolicy.Minimum, QSizePolicy.MinimumExpanding)
        sizePolicy3.setHorizontalStretch(0)
        sizePolicy3.setVerticalStretch(0)
        sizePolicy3.setHeightForWidth(self.spinBox.sizePolicy().hasHeightForWidth())
        self.spinBox.setSizePolicy(sizePolicy3)
        self.spinBox.setMinimumSize(QSize(46, 29))
        font1 = QFont()
        font1.setPointSize(18)
        self.spinBox.setFont(font1)
        self.spinBox.setMinimum(1)
        self.spinBox.setMaximum(100)

        self.horizontalLayout_2.addWidget(self.spinBox)

        self.verticalLayout_2.addLayout(self.horizontalLayout_2)

        self.horizontalLayout_5.addLayout(self.verticalLayout_2)

        self.verticalLayout = QVBoxLayout()
        self.verticalLayout.setObjectName(u"verticalLayout")
        self.horizontalLayout = QHBoxLayout()
        self.horizontalLayout.setObjectName(u"horizontalLayout")
        self.textBrowser_2 = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser_2.setObjectName(u"textBrowser_2")
        sizePolicy2.setHeightForWidth(self.textBrowser_2.sizePolicy().hasHeightForWidth())
        self.textBrowser_2.setSizePolicy(sizePolicy2)
        self.textBrowser_2.setMinimumSize(QSize(77, 29))

        self.horizontalLayout.addWidget(self.textBrowser_2)

        self.textBrowser_3 = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser_3.setObjectName(u"textBrowser_3")
        sizePolicy2.setHeightForWidth(self.textBrowser_3.sizePolicy().hasHeightForWidth())
        self.textBrowser_3.setSizePolicy(sizePolicy2)
        self.textBrowser_3.setMinimumSize(QSize(77, 29))

        self.horizontalLayout.addWidget(self.textBrowser_3)

        self.textBrowser_4 = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser_4.setObjectName(u"textBrowser_4")
        sizePolicy2.setHeightForWidth(self.textBrowser_4.sizePolicy().hasHeightForWidth())
        self.textBrowser_4.setSizePolicy(sizePolicy2)
        self.textBrowser_4.setMinimumSize(QSize(77, 29))

        self.horizontalLayout.addWidget(self.textBrowser_4)

        self.textBrowser_5 = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser_5.setObjectName(u"textBrowser_5")
        sizePolicy2.setHeightForWidth(self.textBrowser_5.sizePolicy().hasHeightForWidth())
        self.textBrowser_5.setSizePolicy(sizePolicy2)
        self.textBrowser_5.setMinimumSize(QSize(77, 29))

        self.horizontalLayout.addWidget(self.textBrowser_5)

        self.verticalLayout.addLayout(self.horizontalLayout)

        self.horizontalLayout_3 = QHBoxLayout()
        self.horizontalLayout_3.setObjectName(u"horizontalLayout_3")
        self.a_doubleSpinBox = QDoubleSpinBox(self.verticalLayoutWidget_4)
        self.a_doubleSpinBox.setObjectName(u"a_doubleSpinBox")
        sizePolicy3.setHeightForWidth(self.a_doubleSpinBox.sizePolicy().hasHeightForWidth())
        self.a_doubleSpinBox.setSizePolicy(sizePolicy3)
        self.a_doubleSpinBox.setMinimumSize(QSize(77, 20))
        self.a_doubleSpinBox.setMinimum(-360.000000000000000)
        self.a_doubleSpinBox.setMaximum(360.000000000000000)

        self.horizontalLayout_3.addWidget(self.a_doubleSpinBox)

        self.b_doubleSpinBox = QDoubleSpinBox(self.verticalLayoutWidget_4)
        self.b_doubleSpinBox.setObjectName(u"b_doubleSpinBox")
        sizePolicy3.setHeightForWidth(self.b_doubleSpinBox.sizePolicy().hasHeightForWidth())
        self.b_doubleSpinBox.setSizePolicy(sizePolicy3)
        self.b_doubleSpinBox.setMinimumSize(QSize(77, 20))
        self.b_doubleSpinBox.setMinimum(-360.000000000000000)
        self.b_doubleSpinBox.setMaximum(360.000000000000000)

        self.horizontalLayout_3.addWidget(self.b_doubleSpinBox)

        self.dx_doubleSpinBox = QDoubleSpinBox(self.verticalLayoutWidget_4)
        self.dx_doubleSpinBox.setObjectName(u"dx_doubleSpinBox")
        sizePolicy3.setHeightForWidth(self.dx_doubleSpinBox.sizePolicy().hasHeightForWidth())
        self.dx_doubleSpinBox.setSizePolicy(sizePolicy3)
        self.dx_doubleSpinBox.setMinimumSize(QSize(77, 20))
        self.dx_doubleSpinBox.setMinimum(-500.000000000000000)
        self.dx_doubleSpinBox.setMaximum(500.000000000000000)

        self.horizontalLayout_3.addWidget(self.dx_doubleSpinBox)

        self.dy_doubleSpinBox = QDoubleSpinBox(self.verticalLayoutWidget_4)
        self.dy_doubleSpinBox.setObjectName(u"dy_doubleSpinBox")
        sizePolicy3.setHeightForWidth(self.dy_doubleSpinBox.sizePolicy().hasHeightForWidth())
        self.dy_doubleSpinBox.setSizePolicy(sizePolicy3)
        self.dy_doubleSpinBox.setMinimumSize(QSize(77, 20))
        self.dy_doubleSpinBox.setMinimum(-500.000000000000000)
        self.dy_doubleSpinBox.setMaximum(500.000000000000000)

        self.horizontalLayout_3.addWidget(self.dy_doubleSpinBox)

        self.verticalLayout.addLayout(self.horizontalLayout_3)

        self.horizontalLayout_5.addLayout(self.verticalLayout)

        self.verticalLayout_4.addLayout(self.horizontalLayout_5)

        self.verticalSpacer = QSpacerItem(20, 4, QSizePolicy.Minimum, QSizePolicy.MinimumExpanding)

        self.verticalLayout_4.addItem(self.verticalSpacer)

        self.horizontalLayout_6 = QHBoxLayout()
        self.horizontalLayout_6.setObjectName(u"horizontalLayout_6")
        self.horizontalSpacer_4 = QSpacerItem(40, 20, QSizePolicy.MinimumExpanding, QSizePolicy.Minimum)

        self.horizontalLayout_6.addItem(self.horizontalSpacer_4)

        self.pushButton = QPushButton(self.verticalLayoutWidget_4)
        self.pushButton.setObjectName(u"pushButton")
        sizePolicy4 = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
        sizePolicy4.setHorizontalStretch(0)
        sizePolicy4.setVerticalStretch(0)
        sizePolicy4.setHeightForWidth(self.pushButton.sizePolicy().hasHeightForWidth())
        self.pushButton.setSizePolicy(sizePolicy4)
        self.pushButton.setMinimumSize(QSize(102, 52))
        self.pushButton.setMaximumSize(QSize(102, 52))
        self.pushButton.setFont(font1)

        self.horizontalLayout_6.addWidget(self.pushButton)

        self.horizontalSpacer_3 = QSpacerItem(165, 20, QSizePolicy.MinimumExpanding, QSizePolicy.Minimum)

        self.horizontalLayout_6.addItem(self.horizontalSpacer_3)

        self.verticalLayout_3 = QVBoxLayout()
        self.verticalLayout_3.setObjectName(u"verticalLayout_3")
        self.textBrowser_6 = QTextBrowser(self.verticalLayoutWidget_4)
        self.textBrowser_6.setObjectName(u"textBrowser_6")
        sizePolicy2.setHeightForWidth(self.textBrowser_6.sizePolicy().hasHeightForWidth())
        self.textBrowser_6.setSizePolicy(sizePolicy2)
        self.textBrowser_6.setMinimumSize(QSize(150, 34))

        self.verticalLayout_3.addWidget(self.textBrowser_6)

        self.buttonBox = QDialogButtonBox(self.verticalLayoutWidget_4)
        self.buttonBox.setObjectName(u"buttonBox")
        sizePolicy2.setHeightForWidth(self.buttonBox.sizePolicy().hasHeightForWidth())
        self.buttonBox.setSizePolicy(sizePolicy2)
        self.buttonBox.setMinimumSize(QSize(150, 24))
        self.buttonBox.setStandardButtons(QDialogButtonBox.Cancel|QDialogButtonBox.Ok)

        self.verticalLayout_3.addWidget(self.buttonBox)

        self.horizontalLayout_6.addLayout(self.verticalLayout_3)

        self.horizontalSpacer_5 = QSpacerItem(165, 20, QSizePolicy.MinimumExpanding, QSizePolicy.Minimum)

        self.horizontalLayout_6.addItem(self.horizontalSpacer_5)

        self.verticalLayout_4.addLayout(self.horizontalLayout_6)

        self.verticalSpacer_2 = QSpacerItem(20, 4, QSizePolicy.Minimum, QSizePolicy.MinimumExpanding)

        self.verticalLayout_4.addItem(self.verticalSpacer_2)

        self.verticalLayout_4.setStretch(0, 4)
        self.verticalLayout_4.setStretch(1, 7)

        self.retranslateUi(Widget)
        self.pushButton.clicked.connect(Widget.record)
        self.buttonBox.accepted.connect(Widget.load_pos)

        QMetaObject.connectSlotsByName(Widget)
    # setupUi

    def retranslateUi(self, Widget):
        Widget.setWindowTitle(QCoreApplication.translate("Widget", u"温室作物表型信息采集系统", None))
        self.textBrowser.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u6d4b\u91cf\u6b21\u6570
", None))
        self.textBrowser_2.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u89d2\u5ea6a
", None))
        self.textBrowser_3.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u89d2\u5ea6b
", None))
        self.textBrowser_4.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u4f4d\u7f6ex
", None))
        self.textBrowser_5.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u4f4d\u7f6ey
", None))
        self.pushButton.setText(QCoreApplication.translate("Widget", u"\u5f00\u59cb\u5f55\u5236", None))
        self.textBrowser_6.setHtml(QCoreApplication.translate("Widget", u"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
\u8f7d\u5165\u4f4d\u7f6e\u4fe1\u606f
", None))
    # retranslateUi

class MyWidget(QWidget):
    def __init__(self):
        super().__init__()
        self.ui = Ui_Form()
        self.ui.setupUi(self) #这里传递的是MyWidget
        self.__count_flag = True
        self.__count = -1
        self.__counts = -2
        # 录制次数, 录制开关, a, b, x, y, 位置确认开关
        self.__info = mpsm.ShareableList([1, False, 0.0, 0.0, 0.0, 0.0, False], name="info")

    @Slot()
    def record(self):
        if self.__counts == self.__count or not self.__info[1]: return
        if self.__count_flag:
            self.__counts = self.ui.spinBox.value()
            self.__info[0] = self.__counts
            self.__count_flag = False
        self.__count += 1
        self.ui.lcdNumber.display(self.__count)
        self.__info[1] = True

    @Slot()
    def load_pos(self):
        if self.__count != self.__counts:
            self.__info[1], self.__info[2], self.__info[3], self.__info[4] = self.pos
            self.__info[5] = True

    @property
    def pos(self) -> list:
        return [self.ui.a_doubleSpinBox.value(),
                self.ui.b_doubleSpinBox.value(),
                self.ui.dx_doubleSpinBox.value(),
                self.ui.dy_doubleSpinBox.value()]

if __name__ == "__main__":
    app = QApplication([])
    mywidget = MyWidget()
    mywidget.show()
    sys.exit(app.exec())


#
# @lc app=leetcode id=210 lang=python3
#
# [210] Course Schedule II
#

# @lc code=start
class Solution:
    def findOrder(self, numCourses: int, prerequisites: List[List[int]]) -> List[int]:
        graph = [[] for _ in range(numCourses)]
        visit = [0 for _ in range(numCourses)]
        ans = []

        for x, y in prerequisites:
            graph[y].append(x)

        def dfs(i,graph, visit, ans):
            if visit[i] == -1:
                return True
            if visit[i] == 1:
                return False
            visit[i] = -1
            for j in graph[i]:
                if dfs(j, graph, visit, ans):
                    return True
            visit[i] = 1
            ans.append(i)
            return False
        for i in range(numCourses):
            if dfs(i, graph, visit, ans):
                return []
        return ans[::-1]

# @lc code=end


import scrapy
from bankiru_crawl.items import BankiruCrawlItem
# from scrapy.shell import inspect_response

class bankiru_spider(scrapy.Spider):
    handle_httpstatus_list = [404]
    name = "bankiru_spider"
    allowed_domains = ['banki.ru']
    URL = "http://www.banki.ru/services/responses/bank/{}/?page="
    # header = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate',
    #          'Accept': '*/*', 'User-Agent': 'python-requests/2.12.4'}
    xpaths = {'review_text': ('div[@class="responses__item__message '
                              'markup-inside-small '
                              'markup-inside-small--bullet"]/text()'),
              'review_url': 'div/a[@itemprop="url"]/@href',
              'review_header': 'div/a[@itemprop="url"]/text()',
              'review_author_url': 'footer//a[@itemprop="author"]/@href',
              'review_author_name': 'footer//a[@itemprop="author"]/span[@itemprop="name"]/text()',
              'review_datetime': 'footer//time/@datetime',
              'review_rating': 'div//meta[@itemprop="ratingValue"]/@content',
              'bank_answer_datetime': 'div//time/@datetime',
              'bank_answer_text': 'div//div[@class="thread-item__text"]/text()'}
    check_404 = False

    def start_requests(self):
        page_number = int(self.start_page)
        while True:
            if self.check_404:
                break
            page_number += 1
            url = self.URL.format(self.bank_name)
            yield scrapy.Request(url=url + str(page_number), callback=self.parse)

    def parse(self, response):
        def extract_item(article, x_path):
            content = article.xpath(x_path).extract()
            item = content[0] if len(content) > 0 else "empty"
            return item
        # inspect_response(response, self)
        if response.status == 404:
            self.check_404 = True
        for article in response.xpath('//article'):
            item = BankiruCrawlItem()
            item['bank_name'] = self.bank_name
            item["review_text"] = " ".join(article.xpath(self.xpaths['review_text'])
                                           .extract())
            item["review_url"] = extract_item(article, self.xpaths['review_url'])
            item["review_header"] = extract_item(article, self.xpaths['review_header'])
            item["review_author_url"] = extract_item(article,
                                                     self.xpaths['review_author_url'])
            item["review_author_name"] = extract_item(article, self.xpaths['review_author_name'])
            item['review_datetime'] = extract_item(article, self.xpaths['review_datetime'])
            item['review_rating'] = extract_item(article, self.xpaths['review_rating'])
            item['bank_answer_datetime'] = extract_item(article,
                                                        self.xpaths['bank_answer_datetime'])
            item['bank_answer_text'] = " ".join(article.xpath(self.xpaths['bank_answer_text']).extract())
            if item['bank_answer_text'] != '':
                item['bank_answer_check'] = True
            else:
                item['bank_answer_check'] = False
            yield item


import json
import logging
import pandas
import configparser
import sqlite3
from sqlite3 import Error

if __name__ == '__main__':
    config = configparser.ConfigParser()
    config.read('../../config/config.cfg')
    db_filename = config['DEFAULT']['db_filename']

    json_dir = '../../output/repos_with_topics/'
    languages = ['c', 'c++', 'c#', 'go', 'java', 'javascript', 'python', 'php', 'ruby', 'shell']
    languages = ['java']

    log_filename = '../../log/extract_and_load_repos_with_topics_from_json_to_db.log'
    logging.basicConfig(handlers=[logging.FileHandler(log_filename, 'w+', 'utf-8')], level=20)
    logging.getLogger().addHandler(logging.StreamHandler())

    df = pandas.DataFrame(columns=['repo_full_name', 'topics'])
    for l in languages:
        for i in range(1, 11):
            filename = '{0}_{1}.txt'.format(l, i)
            logging.info('Processing {0}'.format(filename))
            with open(json_dir + filename, 'r', encoding='utf-8') as curr_file:
                json_str = curr_file.read()
                json_objects = json.loads(json_str)
                for j_obj in json_objects['items']:
                    repo_full_name = j_obj['full_name']
                    topics = ','.join(j_obj['topics'])
                    df = df.append({'repo_full_name': repo_full_name, 'topics': topics}, ignore_index=True)
    exit()
    conn = sqlite3.connect(db_filename)
    try:
        # drop duplicates (possible if position of one repository shift to next page between queries
        df.drop_duplicates(subset='repo_full_name', keep='first', inplace=True)
        df.to_sql(name='tag_recommender_ground_truth_repo_topics', con=conn, if_exists='replace', index=False)
    except Error as e:
        logging.exception(e)
    except Exception as e:
        logging.exception(e)
    finally:
        conn.close()

    logging.info('Total repositories loaded: {0}'.format(len(df.index)))


def h(m):
    n=len(m)
    visited=[0]*n
    r=0
    def dfs(i):
        for j in range(n):
            if m[i][j] and visited[j]==0:
                visited[j]=1
                dfs(j)
    for i in range(n):
        if visited[i]==0:
            dfs(i)
            r+=1
    print(r)

if __name__ == '__main__':
    h(eval(input()))


import pygame
import random
import queue
import pathfinding

# Initializing graphics
pygame.init()
win = pygame.display.set_mode((1280, 720))
surface_size = (256, 144)
display = pygame.Surface((surface_size))
clock = pygame.time.Clock()

# Variables
running = True  # Just a smol var i use to close the game
mouse_left_button_down = False
mouse_right_button_down = False
tile_size = 8  # I RECOMMEND NOT CHANGING THIS VALUE AS IM NOT SURE WHAT MIGHT END UP HAPPENNING BY DOING SO
fps = 60
surf_x = 0
surf_y = 0
xscroll = 0
yscroll = 0
scrollsp = 1  # Set this to negative to inverse scrolling controls

# Tilemap
tilemap = [
    [' ', ' ', ' '],
    [' ', ' ', ' '],
    [' ', ' ', ' ']
]
current_tile = 'H'
tilemap_key_file = {
    'H': pygame.image.load('sprites/wall.png')
}

def draw_map_from_array(two_dimensional_array, key_file_dict: dict, display_to_draw_to, xpos=0, ypos=0, tile_size=tile_size):
    '''
    two_dimensional_array: This is the 2D array which holds your entire map
    key_file_dict: dictionary that mentions which value in the 2D Array map corrosponds to which image to draw.
    '''
    ty = 0
    for row in two_dimensional_array:
        tx = 0
        for tile in row:
            if tile in key_file_dict:
                display_to_draw_to.blit(key_file_dict[tile], ((tx*tile_size)+xpos, (ty*tile_size)+ypos-(
                    (tilemap_key_file[current_tile].get_rect().size[1])-tile_size)))
            tx += 1
        ty += 1

def add_tile_to_tilemap(two_dimensional_array: list, value_to_insert, xpos_in_array, ypos_in_array):

    # Extending the Array to the point at which it can atleast hold the tile
    while len(two_dimensional_array) <= ypos_in_array:
        two_dimensional_array.append(list())
    while len(two_dimensional_array[ypos_in_array]) <= xpos_in_array:
        two_dimensional_array[ypos_in_array].append(' ')

    two_dimensional_array[ypos_in_array][xpos_in_array] = value_to_insert

def get_2d_array_len(array):
    xlen = 0
    for row in array:
        if len(row) > xlen:
            xlen = len(row)
    ylen = len(array)

    return (xlen, ylen)

# Main Graphics Loop
while running:
    clock.tick(fps)

    # Calculating the position of the tiles
    mx, my = pygame.mouse.get_pos()
    tx, ty = int(round(((mx)/5)-surf_x)), int(round(((my)/5)-surf_y))

    while tx % tile_size != 0:
        tx -= 1
    while ty % tile_size != 0:
        ty -= 1

    # Clearing the screen
    win.fill((0, 0, 0))
    display.fill((0, 0, 50))

    # Input
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            f = open('save.py', 'w')
            f.write(str(tilemap))
            f.close()
            running = False
        if event.type == pygame.MOUSEBUTTONDOWN:
            if event.button == 1:
                mouse_left_button_down = True
            elif event.button == 3:
                mouse_right_button_down = True
        elif event.type == pygame.MOUSEBUTTONUP:
            if event.button == 1:
                mouse_left_button_down = False
            elif event.button == 3:
                mouse_right_button_down = False
        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_UP:
                yscroll = scrollsp
            elif event.key == pygame.K_DOWN:
                yscroll = -scrollsp
            elif event.key == pygame.K_LEFT:
                xscroll = scrollsp
            elif event.key == pygame.K_RIGHT:
                xscroll = -scrollsp
        elif event.type == pygame.KEYUP:
            if event.key == pygame.K_UP:
                yscroll = 0
            elif event.key == pygame.K_DOWN:
                yscroll = 0
            elif event.key == pygame.K_LEFT:
                xscroll = 0
            elif event.key == pygame.K_RIGHT:
                xscroll = 0

    # Scroll
    surf_x += xscroll
    surf_y += yscroll

    # Acting on input
    if not tx < 0 and not ty < 0:
        if mouse_left_button_down:
            add_tile_to_tilemap(tilemap, 'H', int(
                round(tx/tile_size)), int(round(ty/tile_size)))
        if mouse_right_button_down:
            add_tile_to_tilemap(tilemap, ' ', int(
                round(tx/tile_size)), int(round(ty/tile_size)))

        # Drawing tile at the current position
        display.blit(tilemap_key_file[current_tile], (tx+surf_x, ty-(
            (tilemap_key_file[current_tile].get_rect().size[1])-tile_size)+surf_y))

    # Drawing the boundaries
    for line in [
        {'start': [0, 0], 'end':[(get_2d_array_len(tilemap)[0])*tile_size, 0]},
        {'start': [(get_2d_array_len(tilemap)[0])*tile_size, 0],
         'end':[(get_2d_array_len(tilemap)[0])*tile_size, (len(tilemap))*tile_size]},
        {'start': [(get_2d_array_len(tilemap)[0])*tile_size, (len(tilemap))
                   * tile_size], 'end':[0, (len(tilemap))*tile_size]},
        {'start': [0, (len(tilemap))*tile_size], 'end':[0, 0]}
    ]:
        line['start'][0] += surf_x
        line['start'][1] += surf_y
        line['end'][0] += surf_x
        line['end'][1] += surf_y

        pygame.draw.line(display, (200, 200, 200), line['start'], line['end'])

    # Updating the display
    draw_map_from_array(tilemap, tilemap_key_file,
                        display, xpos=surf_x, ypos=surf_y)
    win.blit(pygame.transform.scale(display, (1280, 720)), (0, 0))
    pygame.display.update()


import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import entropy
from scipy.integrate import simps
from sklearn.neighbors import KernelDensity
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import LeaveOneOut, KFold

# error pdfs #
if False:
    #d1 = pd.read_csv('./d1.dat')
    #d2 = pd.read_csv('./collected.csv')
    #d3 = pd.read_csv('./timedsamples.dat')
    #dat = pd.concat([d1, d2, d3])
    hfDs, cvDs = [], []
    dat = pd.read_csv('./collected.csv')
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    f, ax = plt.subplots(2)
    plt.subplots_adjust(hspace=0.5)
    TEST = 50
    LF = 300
    bigBS = int(dat.shape[0] / (LF))
    dats = np.array_split(dat, bigBS)
    samplesx = [125]
    print('*', bigBS)
    while samplesx[-1]*2 < bigBS:
        samplesx.append(samplesx[-1] * 2)
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(samplesx)))
    print(samplesx)
    bss = []
    for ll, BS in enumerate(samplesx):
        bss.append(BS)
        LW = .1
        ALPHA = 0.5
        x = np.array([int(s) for s in np.arange(10, TEST+1, 2)])
        epsh = np.zeros((x.size, BS))
        epscv = np.zeros((x.size, BS))
        for _ in range(BS):
            dat = dats[_]
            def mch(x):
                 return (np.mean(dat.hf[TEST:]) - np.mean(dat.hf.values[:x])) ** 2, np.var(dat.hf.values[:x]) / x, np.mean(dat.hf.values[:x]), np.mean(dat.hf[TEST:])
            mch = np.vectorize(mch)

            def cv2(x):
                alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
                m = (np.mean(dat.hf[TEST:]) - (
                    np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf[:LF]) - np.mean(dat.lf.values[:x])))) ** 2
                std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
                return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
            cv2 = np.vectorize(cv2)

            ev_mch = mch(x)
            ev_cv2 = cv2(x)
            epsh[:, _] = np.sqrt(ev_mch[0])
            epscv[:, _] = np.sqrt(ev_cv2[0])
            if len(ev_cv2[0][np.isnan(ev_cv2[0])]) > 0: hey

        xs, ys = [], []
        y2s = []
        lastpdfhf, lastpdfcv = None, None
        for ii in [np.where(x==TEST)[0][0]]:
            thesex, thesey, vesey = [], [], []
            for jj in range(epsh.shape[1]):
                ys.append(epsh[ii, jj])
                y2s.append(epscv[ii, jj])
                xs.append(x[ii])
                thesey.append(y2s[-1])
                vesey.append(ys[-1])
            np.save('thesey%i'%BS, thesey)
            thesey = np.array(thesey)
            vesey = np.array(vesey)
            xx = np.linspace(np.min(vesey), max(vesey), 100)
            #xx = np.linspace(min(vesey), max(vesey), 100)
            print('---->', min(thesey), max(thesey))
            kde2 = KernelDensity(kernel='gaussian', bandwidth=.05)
            kde2.fit(vesey.reshape(-1, 1))
            pdff = np.exp(kde2.score_samples(xx.reshape(-1, 1)))
            if lastpdfcv: cvDs.append(entropy(pdff, laspdfcv))
            lastpdfcv = pdff.copy()
            ax[0].plot(xx, pdff / simps(pdff, xx), label=BS, c=colors[ll])
            #xx = np.linspace(0, 2 * max(thesey), 1000)
            xx = np.linspace(min(thesey), max(thesey), 100)
            kde = KernelDensity(kernel='gaussian', bandwidth=.01)
            kde.fit(thesey.reshape(-1, 1))
            pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
            if lastpdfhf: hfDs.append(entropy(pdff, laspdfhf))
            lastpdfhf = pdff.copy()
            ax[1].plot(xx, pdff / simps(pdff, xx), label=BS, c=colors[ll])
    ax[0].hist(vesey, 200, alpha=0.3, normed=True)
    ax[1].hist(thesey, 200, alpha=0.3, normed=True)
    ax[0].set_xlabel(r'$\epsilon(Q^{HF})$')
    ax[1].set_xlabel(r'$\epsilon(Q^{CV})$')
    l = ax[0].legend(ncol=3)
    l.set_title('Sample Sets')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    f.suptitle('%i sets, %i HF samples, %i LF samples'%(bigBS, TEST, LF))
    f.savefig('sampleContours.pdf', bbox_inches='tight') ; plt.clf()
    plt.clf()
    plt.close()

    plt.plot(bss[1:], lastpdfhf, label='HF')
    plt.plot(bss[1:], lastpdfcv, label='CV')
    plt.yscale('log')
    plt.xscale('log')
    plt.legend()
    plt.ylabel('D')
    plt.xlabel('n')
    plt.show()

# raw values
if True:
    d1 = pd.read_csv('./d1.dat')
    d2 = pd.read_csv('./collected.csv')
    d3 = pd.read_csv('./timedsamples.dat')
    dat = pd.concat([d1, d2, d3])
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    #dat = dat.convert_objects(convert_numeric=True)
    f, ax = plt.subplots(2, sharex=True)
    #plt.subplots_adjust(bottom=-2, hspace=1)
    SAMPS = [50]
    while SAMPS[-1] < 30000: SAMPS.append(SAMPS[-1] * 2)
    BINS = 100
    HFENTROPIES = []
    LFENTROPIES = []
    lflastp = None
    hflastp = None
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(SAMPS)))
    for ll, n in enumerate(SAMPS):
        HF = dat.hf[:n]
        LF = dat.lf[:n]
        xx = np.linspace(10, 26, 1000)
        #bandwidths = 10 ** np.linspace(-1, 1, 100)
        #print(n)
        #grid = GridSearchCV(KernelDensity(kernel='cosine'),
        #            {'bandwidth': bandwidths},
        #            cv=KFold(5))
        #grid.fit(HF[:, None])
        kde = KernelDensity(kernel='gaussian', bandwidth=.1)
        kde.fit(HF[:, None])
        myfit = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        myfit /= simps(myfit, xx)
        if hflastp is not None: HFENTROPIES.append(entropy(myfit, hflastp))
        hflastp = myfit.copy()
        xx = np.linspace(11, 19, 1000)
        ax[0].plot(xx, np.exp(kde.score_samples(xx.reshape(-1, 1))), c=colors[ll], label=n)
        #xx = np.linspace(min(LF), max(LF), 100)
        kde = KernelDensity(kernel='gaussian', bandwidth=.1)
        kde.fit(LF.reshape(-1, 1))
        pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        pdff /= simps(pdff, xx)
        ax[1].plot(xx, pdff, c=colors[ll])
        if lflastp is not None: LFENTROPIES.append(entropy(pdff, lflastp))
        lflastp = pdff.copy()
    l = ax[0].legend(ncol=4, bbox_to_anchor=[.9, 1.5])
    l.set_title('Number of Samples')
    ax[0].hist(HF, 50, alpha=0.6, normed=True)
    ax[1].hist(LF, 50, alpha=0.6, normed=True)
    ax[1].set_xlabel(r'$\max(\nabla u)_{400 Cells}$')
    ax[0].set_xlabel(r'$\max(\nabla u)_{7000 Cells}$')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    plt.savefig('./rawPDFs.pdf', bbox_inches='tight')
    plt.clf()
    plt.close()

    plt.plot(SAMPS[1:], np.abs(HFENTROPIES))
    plt.plot(SAMPS[1:], np.abs(LFENTROPIES))
    plt.ylabel('D')
    plt.ylabel('n')
    plt.yscale('log')
    plt.xscale('log')
    plt.savefig('./pdfDees.pdf')

# expected values #
if False:

    d1 = pd.read_csv('./d1.dat')
    d2 = pd.read_csv('./collected.csv')
    d3 = pd.read_csv('./timedsamples.dat')
    dat = pd.concat([d1, d2, d3])
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    TEST = 10
    LF = 109
    bigBS = int(dat.shape[0] / (TEST * LF))
    dats = np.array_split(dat, bigBS)
    f, ax = plt.subplots(2, sharex=True)
    SAMPS = [50]
    while SAMPS[-1] * 2 < len(dats): SAMPS.append(SAMPS[-1]*2)
    #plt.subplots_adjust(bottom=-2, hspace=1)
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(SAMPS)))
    for ll, n in enumerate(SAMPS):
        exps = []
        CC = []
        for ii in range(n):
            exps.append(dats[ii].hf[:TEST].mean())
            dat = dats[ii]
            def cv2(x):
                alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
                m = (np.mean(dat.hf[TEST:]) - (
                    np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf[:LF*x]) - np.mean(dat.lf.values[:x])))) ** 2
                std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
                return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
            CC.append(cv2(50)[-1])
        xx = np.linspace(14, 18, 1000)
        #xx = np.linspace(.9 * np.min(exps), 1.1 * np.max(exps), 1000)
        kde = KernelDensity(kernel='gaussian', bandwidth=1e-1)
        kde.fit(np.array(exps).reshape(-1, 1))
        myfit = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        myfit /= simps(myfit, xx)
        ax[0].plot(xx, myfit, c=colors[ll], label=n, zorder=3)
        #xx = np.linspace(min(LF), max(LF), 100)
        kde = KernelDensity(kernel='gaussian', bandwidth=5e-2)
        kde.fit(np.array(CC).reshape(-1, 1))
        pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        ax[1].plot(xx, pdff / simps(pdff, xx), c=colors[ll], zorder=4)
    leg = ax[0].legend(ncol=2, bbox_to_anchor=[1, 1])
    leg.set_title("Number of sample sets")
    ax[0].hist(exps, 30, normed=True, alpha=0.8, zorder=1)
    ax[1].hist(CC, 30, normed=True, alpha=0.8, zorder=2)
    ax[0].set_xlabel(r'$Q^{HF}$')
    ax[1].set_xlabel(r'$Q^{CV}$')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    plt.suptitle('%i sets, %i LF samples'%(bigBS, dat.shape[0]), y=1.)
    plt.savefig('./qPDFs.pdf', bbox_inches='tight')
    plt.clf()
    plt.close()

    exps = []
    CC = []
    NSAMPS = []
    for kk in [10, 20, 50]:
      for ii in range(n):
        exps.append(dats[ii].hf[:kk].mean())
        dat = dats[ii]
        def cv2(x):
            alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
            m = (np.mean(dat.hf[TEST:]) - ( np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x])))) ** 2
            std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
            return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
        CC.append(cv2(kk)[-1])
        NSAMPS.append(kk)

    vdat = pd.DataFrame()
    vdat['NSAMPS'] = NSAMPS
    vdat['HF'] = exps
    vdat['CV'] = CC
    f, ax = plt.subplots(2)
    plt.subplots_adjust(right=1.5)
    sns.violinplot(x='NSAMPS', y='HF', data=vdat, ax=ax[0], color='white')
    sns.violinplot(x='NSAMPS', y='CV', data=vdat, ax=ax[1], color='white')
    for ii in range(2):
        ax[ii].set_xlabel("Number of High Fidelity Samples")
        ax[ii].set_ylim(vdat.HF.min(), vdat.HF.max())
    ax[0].set_ylabel(r'$Q^{HF}$')
    ax[1].set_ylabel(r'$Q^{CV}$')
    plt.suptitle('%i sets, %i LF samples'%(bigBS, dat.shape[0]), y=1.)
    plt.tight_layout()
    plt.savefig('expectedViolins.pdf', bbox_inches='tight')