from bs4 import BeautifulSoup
import requests
import spotipy
from spotipy.oauth2 import SpotifyOAuth

spotify_client_id = "" # Your ID
spotify_client_secret = "" # Your Secret
spotify_url =  'https://accounts.spotify.com/authorize'
redirect_uri = "http://example.com"

time_travel = input("What your would you like to travel to? YYYY-MM-DD: ")

response = requests.get(f"https://www.billboard.com/charts/hot-100/{time_travel}/")
website = response.text
soup = BeautifulSoup(website, "html.parser")
all_titles = soup.find_all(name="h3", id="title-of-a-story", class_="a-no-trucate")
all_artists = soup.find_all(class_="a-truncate-ellipsis-2line")

# Get song titles and artist names
song_titles = [str(title.getText("'[]'").split()) for title in all_titles]
song_artists = [str(artist.getText("'[]'").split()) for artist in all_artists]

songs = []

for i in range(len(song_artists)):
    # get the names from list1 and list2
    name1 = song_titles[i].strip("[]'\"").replace("'", "").replace(",", "")
    name2 = song_artists[i].strip("[]'\"").replace("'", "").replace(",", "")

    # concatenate the names with a hyphen separator
    all_songs = songs.append(f"{name1} {' - '} {name2}")

# Integrating with Spotify

scope = "playlist-modify-private"

oauth_object = spotipy.SpotifyOAuth(client_id = spotify_client_id,
                                   client_secret = spotify_client_secret,
                                   redirect_uri = redirect_uri,
                                   scope = scope)

token_dict = oauth_object.get_access_token()
token = token_dict['access_token']

# Spotify Object
spotify_object = spotipy.Spotify(auth = token)

current = spotify_object.current_user()["id"]

playlists = spotify_object.current_user_playlists()

song_uris_list = []

new_playlist = spotify_object.user_playlist_create(user=current, name=f"{time_travel}", public=False, collaborative=True, description="First Playlist" )

for uris in range(len(song_artists)):
    song = spotify_object.search(q=songs[uris], limit=1, type="track", market="US")
    song_uri = song['tracks']['items'][0]['uri']
    song_uris_list.append(song_uri)
    try:
        spotify_object.playlist_add_items(playlist_id=new_playlist["id"], items=[song_uri], position=None)
    except NameError:
        print(f"Song not found. ID: {song}")

100 Day Python Bootcamp - Intermediate/Day_46_Create-Spotify-Playlist_Musical_Time_Machine/main.py


from pymongo import MongoClient
from pprint import pprint

client = MongoClient()
db = client['employee']
amazon = db['amazon']
# amazon_2=db['amazon_2']

for i, item in enumerate(amazon.find({}), 1):
    print(i, item)

# print('\n')
# print(amazon.find_one({'books':'Python Machine Learning'}))

# db.amazon.copyTo('amazon2')
# db.amazon.drop()
# db.amazon_2.remove()
# db.amazon2.drop()
# client['test_database']['profiles'].drop()
# client['employee']['amazon_python_books'].remove()
print(db['amazon_python_books'].count())
db['amazon'].remove()


#!/usr/bin/env python

import serial
import gtk
import sys

class colorselector(object):
    def __init__(self, port=None):
        if port: self.leds = ledstrip(port=port)
        else: self.leds = ledstrip()

        window = gtk.Window()
        window.connect("destroy", gtk.main_quit)

        self.colorsel = gtk.ColorSelection()
        self.colorsel.connect("color_changed", self.setcolor)

        panel = gtk.VBox()
        panel.pack_start(self.colorsel)

        autofade_panel = gtk.HBox()

        autofade_panel.pack_start(gtk.Label("Delay: "))
        self.delayfield = gtk.Entry()
        autofade_panel.pack_start(self.delayfield, padding=10)

        autofade_panel.pack_start(gtk.Label("Delta: "))
        self.deltafield = gtk.Entry()
        autofade_panel.pack_start(self.deltafield, padding=10)

        quit_button = gtk.Button("Quit")
        quit_button.connect("clicked", gtk.main_quit)
        autofade_panel.pack_end(quit_button, expand=False)

        autofade_button = gtk.Button("Autofade")
        autofade_button.connect("clicked", self.autofade)
        autofade_panel.pack_end(autofade_button, expand=False)

        panel.pack_end(autofade_panel)

        window.add(panel)
        window.show_all()

    def setcolor(self, widget):
        color = self.colorsel.get_current_color()
        self.leds.color(color.red / 65535.0, color.green / 65535.0, color.blue / 65535.0)

    def autofade(self, widget):
        self.leds.autofade(self.delayfield.get_text(), self.deltafield.get_text())

class ledstrip(object):
    def __init__(self, port='/dev/ttyUSB0', baud=9600):
        self.serial = serial.Serial(port, baud)

    def color(self, r, g, b):
        (r, g, b) = (int(r * 255), int(g * 255), int(b * 255))
        self.serial.write('1 %d %d %d\r' % (r, g, b))

    def autofade(self, delay, delta):
        self.serial.write('0 %d %d\r' % (int(delay), int(delta)))

if __name__ == "__main__":
    if len(sys.argv) > 1: colorselector(sys.argv[1])
    else: colorselector()
    gtk.main()


from django.urls import path
from rest_framework.routers import DefaultRouter

from ride_app.views.auth import Get_user, signup, me, update_user, UserDetail, check_email_unique, search_users

router = DefaultRouter()

router.register('users', Get_user)
urlpatterns = [
    path('signup/', signup),
    path('me/', me),
    path('update_user/', update_user),
path('users/<int:pk>/', UserDetail.as_view(), name='user-detail'),
    path('search_users/',search_users),
    path('check-email-unique/', check_email_unique, name='check_email_unique'),

]


import datetime

def salvar(nome):
    with open('DataBase.txt', 'a') as arquivo:
        arquivo.write(f'{nome}\n')

def fazercheckin():
    print('Fazer Check-In')
    varnome = (input('Digite o nome: '))

    while True:
        vartelefone = input('Telefone com DDD: ')
        if not(vartelefone.isnumeric()):

            print('Somente números!')
        else:
            break

    while True:
        varcpf = input('CPF: ')
        #print(cpfValidator(varcpf))

        if cpfValidator(varcpf) != True:
            print(f"CPF inválido, digite certo.".center(50,"="))
        else:
            print(f"Hospede cadastrado com sucesso".center(50,"="))
            dados = {
                'nome': varnome,
                'cpf': varcpf,
                'telefone': vartelefone
            }

            salvar(dados)
        break

def cpfValidator(cpf):
    CPFS = ["00000000000","11111111111", "22222222222", "33333333333", "44444444444",
        "55555555555", "66666666666", "77777777777", "88888888888", "99999999999"]
    hasError = False
    # Limpa o cpf, retira pontos traços e tudo que não seja um número
    cpf = [int(char) for char in cpf if char.isdigit()]

    # Verifica se o cpf são números repetidos
    if ''.join(map(str, cpf)) in CPFS:
        hasError = True

    # verifica se o tamanho está correto
    if len(cpf) != 11:
        hasError = True

    if not(hasError):
        for i in range(9, 11):
            value = sum((cpf[num] * ((i+1) - num) for num in range(0, i)))
            digit = ((value * 10) % 11) % 10
            if digit != cpf[i]:
                hasError = True

    if hasError:
        return False
    else:
        return True

def printmenu():

    funcoes = ['1','2','3','4','5']
    poli = "-"*50
    espaco = " "*10
    espaco1 = " "*2

    print(poli)
    print(f"(1) Fazer check-In\n(2) Relatório Hospedes\n(3) Procurar Hospedes\n(4) Fazer Check-Out\n(5) Finalizar Atendimento")
    print(poli)

def wellcome ():

    atual = datetime.datetime.now()
    hora = atual.hour
    minutos = atual.minute
    poli = "-"*50
    espaco = " "*10
    espaco1 = " "*7

    print(f"{poli}\n{espaco1}SEJA BEM VINDO NO HOTEL TRANSILVÂNIA \n{poli}")
    if hora < 12:
        print(f"{espaco}Bom dia! Horário atual: {hora} horas e {minutos} minutos.{espaco}".center(50,"="))
    elif hora < 18:
        print(f"{espaco}Boa tarde! Horário atual: {hora} horas e {minutos} minutos.{espaco}".center(50,"="))
    else:
        print(f"{espaco}Boa noite! Horário atual: {hora} horas e {minutos} minutos.{espaco}".center(50,"="))

def fazercheckout():
    print('Abra o arquivo Database.txt e delete manualmente seu relaxado!')

def listar():
    nomes = []
    with open('DataBase.txt', 'r') as arquivo:
        for name in arquivo:
            name = name.strip()
            nomes.append(name)
            print(name)


import sys
from pprint import pformat

from pyramid.tweens import EXCVIEW
from pyramid.settings import aslist
from pyramid.settings import asbool
from pyramid.util import DottedNameResolver
from pyramid.httpexceptions import WSGIHTTPException
from pyramid.security import unauthenticated_userid

resolver = DottedNameResolver(None)

PY3 = sys.version_info[0] == 3

if PY3: # pragma: no cover
    import builtins
else:
    import __builtin__ as builtins

def as_globals_list(value):
    L = []
    value = aslist(value)
    for dottedname in value:
        if dottedname in builtins.__dict__:
            if PY3: # pragma: no cover
                dottedname = 'builtins.%s' % dottedname
            else:
                dottedname = '__builtin__.%s' % dottedname
        obj = resolver.resolve(dottedname)
        L.append(obj)
    return L

def _get_url(request):
    try:
        url = request.url
    except UnicodeDecodeError:
        # do the best we can
        url = (request.host_url +
               request.environ.get('SCRIPT_NAME') +
               request.environ.get('PATH_INFO'))
        qs = request.environ.get('QUERY_STRING')
        if qs:
            url += '?' + qs
        url = 'could not decode url: %r' % url
    return url

_MESSAGE_TEMPLATE = """

%(url)s

ENVIRONMENT

%(env)s

PARAMETERS

%(params)s

UNAUTHENTICATED USER

%(usr)s

"""

def _get_message(request):
    url = _get_url(request)
    unauth = unauthenticated_userid(request)

    try:
        params = request.params
    except UnicodeDecodeError:
        params = 'could not decode params'

    return _MESSAGE_TEMPLATE % dict(
            url=url,
            env=pformat(request.environ),
            params=pformat(params),
            usr=unauth if unauth else '')

def _handle_error(request, getLogger, get_message):
    # save the traceback as it may get lost when we get the message.
    # _handle_error is not in the traceback, so calling sys.exc_info
    # does NOT create a circular reference
    exc_info = sys.exc_info()
    try:
        logger = getLogger('exc_logger')
        message = get_message(request)
        logger.error(message, exc_info=exc_info)
    except:
        logger.exception("Exception while logging")
    raise

def exclog_tween_factory(handler, registry):

    get = registry.settings.get

    ignored = get('exclog.ignore', (WSGIHTTPException,))
    get_message = _get_url
    if get('exclog.extra_info', False):
         get_message = _get_message
    get_message = get('exclog.get_message', get_message)

    getLogger = get('exclog.getLogger', 'logging.getLogger')
    getLogger = resolver.resolve(getLogger)

    def exclog_tween(request, getLogger=getLogger):
        # getLogger injected for testing purposes
        try:
            return handler(request)
        except ignored:
            raise
        except:
            _handle_error(request, getLogger, get_message)
            # _handle_error always raises
            raise AssertionError('Should never get here') #pragma NO COVER
    return exclog_tween

def includeme(config):
    """
    Set up am implicit :term:`tween` to log exception information that is
    generated by your Pyramid application.  The logging data will be sent to
    the Python logger named ``exc_logger``.

    This tween configured to be placed 'below' the exception view tween.  It
    will log all exceptions (even those eventually caught by a Pyramid
    exception view) except 'http exceptions' (any exception that derives from
    ``pyramid.httpexceptions.WSGIHTTPException`` such as ``HTTPFound``).  You
    can instruct ``pyramid_exclog`` to ignore custom exception types by using
    the ``excview.ignore`` configuration setting.
    """
    get = config.registry.settings.get
    ignored = as_globals_list(get('exclog.ignore',
                                  'pyramid.httpexceptions.WSGIHTTPException'))
    extra_info = asbool(get('exclog.extra_info', False))
    get_message = get('exclog.get_message', None)
    if get_message is not None:
        get_message = config.maybe_dotted(get_message)
        config.registry.settings['exclog.get_message'] = get_message
    config.registry.settings['exclog.ignore'] = tuple(ignored)
    config.registry.settings['exclog.extra_info'] = extra_info
    config.add_tween('pyramid_exclog.exclog_tween_factory', under=EXCVIEW)


import matplotlib.pyplot as plt
import scipy.ndimage as ndi
import numpy as np
import sys
import os

if 'load' in sys.argv:
    im = plt.imread('remote.jpeg')

target = {'c1': [620,1200],  # [y1, x1]
          'c2': [620,1830],  # [y1, x2]
          'c3': [1096,1830], # [y2, x2]
          'c4': [1096,1200]} # [y2, x1]

c1 = target['c1']
c2 = target['c2']
c3 = target['c3']
c4 = target['c4']

sharpen = [[0,-1,0],[-1,5,-1],[0,-1,0]]

# Create a Field of View
fov = im[c1[0]:c3[0],c1[1]:c3[1]]
fov = ndi.convolve(fov[:,:,2],np.array(sharpen))

edges = 10*fov /ndi.gaussian_laplace(im[c1[0]:c3[0],c1[1]:c3[1]][:,:,2],sigma=2)
f, ax = plt.subplots(1, 2, sharex=True,sharey=True)
#ax[0].imshow(im[c1[0]:c3[0],c1[1]:c3[1]], 'gray_r')
#ax[1].imshow(edges,'gray_r')
#plt.show()

plt.imsave('fov.jpeg', edges)
os.remove('remote.jpeg')


import logging
import sys
from datetime import datetime
import os
#import tensorflow as tf
#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0'
#logging.getLogger("tensorflow").setLevel(logging.INFO)

##
# Try not to initialize many loggers at the same run
# def init_logging(log_file=''):
#     if log_file != '':
#         log_file = datetime.now().strftime(log_file + '_%Y_%m_%d_%H_%M.log')
#     log_format = "%(levelname)s %(asctime)-15s [%(lineno)d] %(funcName)s: %(message)s"
#     if log_file == '':
#         logging.basicConfig(format=log_format, level=logging.INFO, stream=sys.stdout)
#     else:
#         logging.basicConfig(filename=log_file, filemode='w', format=log_format, level=logging.INFO)
#     logger = logging.getLogger()
#     #logger = tf.get_logger()
#     return logger

def init_logging(log_file='', name='log_name', level=logging.DEBUG):
    if len(log_file) == 0:
        return init_logging(log_file)
    """Function setup as many loggers as you want"""
    if log_file != '':
        log_file = datetime.now().strftime(log_file + '_%Y_%m_%d_%H_%M.log')
    formatter = logging.Formatter('%(levelname)s %(asctime)-15s [%(lineno)d] %(funcName)s: %(message)s')
    handler = logging.FileHandler(log_file, 'w')
    handler.setFormatter(formatter)

    logger = logging.getLogger(name)
    #logger = tf.get_logger(name)
    logger.setLevel(level)
    logger.addHandler(handler)

    return logger

if __name__ == '__main__':
    #log_file = "/home/ivan/Research/projects/yhao_cnn_varying/src/python/test"
    #init_logging(log_file)
    #setup_logger(log_file)
    #logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
    logger = setup_logger('')
    #logging.debug('This message should appear on the console')
    logger.info('So should this')
    #logging.warning('And this, too')


import json
import functools
from typing import Callable
from keyword import iskeyword

class FromJson:
    """Объект из JSON"""

    def __init__(self, json_dict: dict):
        """
        Создане объекта из словаря JSON
        ключи словаря - названия атрибутов
            (в случае совпадения с ключевым словом, добавляется символ "_")
        значения словаря - значения атрибутов с возможной вложенностью JSON
        """
        for key, value in json_dict.items():
            if iskeyword(key):
                key += '_'

            if isinstance(value, dict):
                setattr(self, key, FromJson(value))
            elif isinstance(value, list):
                setattr(self, key, self._parse_list(value))
            else:
                setattr(self, key, value)

    def _parse_list(self, arr: list) -> list:
        """
        Обработка списка с учетом возможной вложенности структур JSON
        """
        ans = []
        for item in arr:
            if isinstance(item, list):
                ans.append(self._parse_list(item))
            elif isinstance(item, dict):
                ans.append(FromJson(item))
            else:
                ans.append(item)
        return ans

class ColorizeMixin:
    """
    Изменяет цвет вывода в консоль метода __repr__
    Подменяет функцию __repr__() у класса классе переменной self
    """

    def __init_subclass__(cls, *args, **kwargs):
        super().__init_subclass__(*args, **kwargs)
        cls.__repr__ = cls._replace_repr(cls.__repr__)

    @classmethod
    def _replace_repr(cls, func: Callable) -> Callable:
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            text: str = func(*args, **kwargs)
            color_escape = "\033[1;" + str(cls.repr_color_code) + ";40m"
            drop_color = "\033[1;0;40m"
            return color_escape + text + drop_color
        return wrapper

class Advert(ColorizeMixin, FromJson):
    """Объект рекламное объявление, создаваемый из JSON"""

    repr_color_code = 33  # yellow

    def __init__(self, json_dict: dict):
        super().__init__(json_dict)
        if not hasattr(self, "title"):
            raise ValueError("must have a title attribute")

        if hasattr(self, "price"):
            if self.price < 0:
                raise ValueError("price must be >= 0")
        else:
            self.price = 0

    def __repr__(self):
        return f"{self.title} | {self.price} ₽"

if __name__ == '__main__':
    # проверка атрибута title
    try:
        advert_0 = Advert(
            json.loads(
                """{
                }"""
            )
        )
    except ValueError as e:
        assert str(e) == "must have a title attribute"

    # атрибут title присутствует, атрибута price нет,
    # должен создаеться атрибут price со значением 0
    advert_1 = Advert(
        json.loads(
            """{
                "title": "test"
            }"""
        )
    )
    assert advert_1.title == "test"
    assert advert_1.price == 0

    # корректные атрибуты title, price
    advert_2 = Advert(
        json.loads(
            """{
                "title": "test",
                "price": 100
            }"""
        )
    )
    assert advert_1.title == "test"
    assert advert_2.price == 100

    # некорректный (отрицательный) атрибут price
    try:
        advert_3 = Advert(
            json.loads(
                """{
                    "title": "test",
                    "price": -100
                }"""
            )
        )
    except ValueError as e:
        assert str(e) == "price must be >= 0"

    # глубокая вложенность json
    advert_4 = Advert(
        json.loads(
            """{
                "title": "test",
                "attr_1": {
                    "attr_2": [
                        0, 1, 2, 3, 4,
                        {
                            "attr_3": "deep attribute"
                        }
                    ]
                }
            }"""
        )
    )
    assert advert_4.attr_1.attr_2[5].attr_3 == "deep attribute"

    # выводится адрес при обращении ĸ атрибуту через точĸи: ad.location.address
    advert_5 = Advert(
        json.loads(
            """{
                "title": "python",
                "price": 0,
                "location": {
                    "address": "город Москва, Лесная, 7",
                    "metro_stations": ["Белорусская"]
                }
            }"""
        )
    )
    assert advert_5.location.address == "город Москва, Лесная, 7"

    # выводит ĸатегорию при обращении через точĸу: corgi.class_
    corgi = Advert(
        json.loads(
            """{
                "title": "Вельш-корги",
                "price": 1000,
                "class": "dogs",
                "location": {
                    "address": "город Самара, улица Мориса Тореза, 50"
                }
            }"""
        )
    )
    assert corgi.class_ == "dogs"

    # при выводе обяъвления в ĸонсоли print(corgi) получаем надпись
    # 'Вельш-ĸорги | 1000 ₽' желтым цветом
    print(corgi)


import sys
from PyQt5 import uic
from PyQt5.QtCore import pyqtSignal, QSize
from PyQt5.QtWidgets import QApplication, QFrame, QMessageBox, QLabel, QPushButton, QVBoxLayout, QColorDialog
from PyQt5.Qt import QTest, Qt, QPoint, QRubberBand, QRect
from PyQt5.QtGui import QPixmap, QFont, QIcon, QColor, QImage
from Cliente import Client
from random import choice
import time
import os
import datetime
from manejo_imagenes import crear_idat, leer_imagen, idr, obtner_largos, obtener_matriz, agregar_ceros, transformar

inicio = uic.loadUiType("inicio.ui")
error = uic.loadUiType('error.ui')
main = uic.loadUiType('main.ui')
edicion = uic.loadUiType('edicion.ui')

class MainWindow(main[0], main[1]):
    trigger_salida = pyqtSignal(str)
    trigger_edicion = pyqtSignal(str)
    comment = pyqtSignal(list)
    editt = pyqtSignal()
    trigger_volver_ingreso = pyqtSignal()

    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.cliente = Client(self)
        self.edicion = Edicion(self)
        self.nombres = []
        self.elegidas = []
        self.inicio = Inicio(self)
        self.geometry = [0, 0]
        self.vbox = QVBoxLayout(self.scrollw)
        self.setWindowTitle('Imagenes')
        self.trigger_salida.connect(self.cliente.enviar_inicio)
        self.editar1.clicked.connect(self.ventana_edicion)
        self.editar2.clicked.connect(self.ventana_edicion)
        self.editar3.clicked.connect(self.ventana_edicion)
        self.editar4.clicked.connect(self.ventana_edicion)
        self.editar5.clicked.connect(self.ventana_edicion)
        self.editar6.clicked.connect(self.ventana_edicion)
        self.comment.connect(self.edicion.comentarr)
        self.trigger_edicion.connect(self.edicion.imagen)
        self.editt.connect(self.cliente.enviar_edicion)
        self.cerrar.clicked.connect(self.cerrar_sesion)
        self.trigger_volver_ingreso.connect(self.inicio.volver)
        self.bytess = []

    def ventana_edicion(self):
        if self.sender().objectName() == 'editar1':
            with open(self.nombres[0] + '.png', 'wb') as file:
                file.write(self.bytess[0])
            self.cliente.editando = self.nombres[0]
        elif self.sender().objectName() == 'editar2':
            with open(self.nombres[1] + '.png', 'wb') as file:
                file.write(self.bytess[1])
            self.cliente.editando = self.nombres[1]
        elif self.sender().objectName() == 'editar3':
            with open(self.nombres[2] + '.png', 'wb') as file:
                file.write(self.bytess[2])
            self.cliente.editando = self.nombres[2]
        elif self.sender().objectName() == 'editar4':
            with open(self.nombres[3] + '.png', 'wb') as file:
                file.write(self.bytess[3])
            self.cliente.editando = self.nombres[3]
        elif self.sender().objectName() == 'editar5':
            with open(self.nombres[4] + '.png', 'wb') as file:
                file.write(self.bytess[4])
            self.cliente.editando = self.nombres[4]
        elif self.sender().objectName() == 'editar6':
            with open(self.nombres[5] + '.png', 'wb') as file:
                file.write(self.bytess[5])
            self.cliente.editando = self.nombres[5]
        self.editt.emit()
        self.hide()

    def imagenes(self, imagen):
        self.bytess.append(imagen)

    def cerrar_sesion(self):
        self.trigger_salida.emit('salir')
        self.cliente = Client(self)
        self.edicion = Edicion(self)
        self.nombres = []
        self.elegidas = []
        self.geometry = [0, 0]
        self.vbox = QVBoxLayout(self.scrollw)
        self.hide()
        self.trigger_volver_ingreso.emit()

    def closeEvent(self, event):
        event.accept()
        self.trigger_salida.emit('salir')

    def generar_imagenes(self):
        """imagen1"""

        pixmap = QPixmap()
        pixmap.loadFromData(self.bytess[0])
        self.imag1.setPixmap(pixmap.scaled(121, 161))
        self.imag1.setAlignment(Qt.AlignCenter)
        self.imag1.show()
        """imagen2"""
        pixmap2 = QPixmap()
        pixmap2.loadFromData(self.bytess[1])
        self.imag2.setPixmap(pixmap2.scaled(121, 161))
        self.imag2.setAlignment(Qt.AlignCenter)
        self.imag2.show()

        """imagen3"""
        pixmap3 = QPixmap()
        pixmap3.loadFromData(self.bytess[2])
        self.imag3.setPixmap(pixmap3.scaled(121, 161))
        self.imag3.setAlignment(Qt.AlignCenter)
        self.imag3.show()

        """imagen4"""
        pixmap4 = QPixmap()
        pixmap4.loadFromData(self.bytess[3])
        self.imag4.setPixmap(pixmap4.scaled(121, 161))
        self.imag4.setAlignment(Qt.AlignCenter)
        self.imag4.show()
        self.show()

        """imagen5"""
        pixmap5 = QPixmap()
        pixmap5.loadFromData(self.bytess[4])
        self.imag5.setPixmap(pixmap5.scaled(121, 161))
        self.imag5.setAlignment(Qt.AlignCenter)
        self.imag5.show()
        self.show()

        """imagen6"""
        pixmap6 = QPixmap()
        pixmap6.loadFromData(self.bytess[5])
        self.imag6.setPixmap(pixmap6.scaled(121, 161))
        self.imag6.setAlignment(Qt.AlignCenter)
        self.imag6.show()
        self.show()

    def conect(self, list):
        for nombre in list:
            self.label = QLabel(self.scrollw)
            self.label.move(self.geometry[0], self.geometry[1])
            self.label.setText('- ' + nombre)
            self.vbox.addWidget(self.label)
            self.geometry[1] = self.geometry[1]+25
            self.label.show()

    def nombres_imag(self, nombre):
        self.nombres.append(nombre.replace('.png', ''))

    def comentarios(self, list):
        self.comment.emit(list)

    def change_pag(self, result):
        if result:
            if len(self.nombres) == 6:
                self.inicio.hide()
                self.generar_imagenes()
        else:
            self.inicio.bad_name.error.setText(
                'Nombre ya regristrado!!! \n '
                'Intentalo con otro')
            self.inicio.cuadro_badname()

    def lock(self, bol):
        """blockea botones de edicion, para tener un modo espectador"""
        self.edicion.puede = bol
        if not bol:
            self.edicion.color.hide()
            self.edicion.frame.hide()
            self.edicion.recortar.hide()
            self.edicion.blur.hide()
        self.trigger_edicion.emit(self.cliente.editando)

    def lock2(self, nombre):
        if nombre == self.nombres[0]:
            self.editar1.setText('Espectar')
        elif nombre == self.nombres[1]:
            self.editar2.setText('Espectar')
        elif nombre == self.nombres[2]:
            self.editar3.setText('Espectar')
        elif nombre == self.nombres[3]:
            self.editar4.setText('Espectar')
        elif nombre == self.nombres[4]:
            self.editar5.setText('Espectar')
        elif nombre == self.nombres[5]:
            self.editar6.setText('Espectar')

    def deslock(self, nombre):
        if nombre == self.nombres[0]:
            self.editar1.setText('Editar')
        elif nombre == self.nombres[1]:
            self.editar2.setText('Editar')
        elif nombre == self.nombres[2]:
            self.editar3.setText('Editar')
        elif nombre == self.nombres[3]:
            self.editar4.setText('Editar')
        elif nombre == self.nombres[4]:
            self.editar5.setText('Editar')
        elif nombre == self.nombres[5]:
            self.editar6.setText('Editar')

class Error(error[0], error[1]):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.setWindowTitle('Error')
        self.ok_2.clicked.connect(self.click)

    def click(self):
        self.hide()

class Inicio(inicio[0], inicio[1]):

    trigger = pyqtSignal(str)

    def __init__(self, main):
        super().__init__()
        self.setupUi(self)
        self.main = main
        self.iniciar.clicked.connect(self.iniciar_sesion)
        self.setWindowTitle('Iniciar Sesion')
        self.trigger.connect(self.main.cliente.enviar_inicio)
        self.bad_name = Error()
        self.show()

    def closeEvent(self, event):
        event.accept()
        self.trigger.emit('salir')

    def iniciar_sesion(self):
        acpetados = 'abcdefghijklmnñopqrstuvwxyz1234567890'
        nombre = self.nombre.text()
        if len(nombre) >= 2:
            cont = 0
            for letra in nombre:
                if letra in acpetados or letra in acpetados.upper():
                    cont += 1

            if cont != len(nombre):
                self.bad_name.error.setText(
                    'tu nombre debe tener solo letras  \n y números')
                self.cuadro_badname()
            else:
                self.main.cliente.nombre = nombre
                self.trigger.emit(nombre)
        else:
            self.bad_name.error.setText('tu nombre debe tener un largo mayor '
                                        '\n a 2')
            self.cuadro_badname()

    def cuadro_badname(self):
        self.bad_name.show()

    def volver(self):
        self.show()

class Edicion(edicion[0], edicion[1]):
    trigger_comentario = pyqtSignal(str)
    trigger_deslock = pyqtSignal(str)

    def __init__(self, main):
        super().__init__()
        self.setupUi(self)
        self.pos = [0, 0]
        self.main = main
        self.comentarios.clicked.connect(self.mostrar_comentarios)
        self.comentar.clicked.connect(self.comentar_foto)
        self.trigger_comentario.connect(main.cliente.enviar_comentario)
        self.trigger_deslock.connect(main.cliente.deslock)
        self.veces_click = 0
        self.contscroll.hide()
        self.vbox = QVBoxLayout(self.contscroll)
        self.color.clicked.connect(self.mostrar_colores)
        self.selected_color = QColor(110,20, 50)
        self.frame = QFrame(self)
        self.frame.setGeometry(110 - 20, 25, 15 ,15)
        self.frame.setStyleSheet('QWidget {background-color: %s}'% self.selected_color.name())
        self.puede = None
        self.recortar.clicked.connect(self.recorte)
        self.blur.clicked.connect(self.edit_blur)
        self.origin = QPoint()
        self.imag = None
        self.recortar = []

    def edit_blur(self):
        self.edicion.mousePressEvent = self.blur_press

    def blur_press(self, event):
        position = event.pos()
        """acá deberia inicializar los cambios para la edicion"""

    def mostrar_comentarios(self):
        if self.veces_click % 2 == 0:
            self.contscroll.show()
        else:
            self.contscroll.hide()
        self.veces_click += 1

    def imagen(self, nombre):
        self.imag = nombre
        self.edicion.hide()
        self.edicion.origin = QPoint()
        self.edicion.ruber = QRubberBand(QRubberBand.Rectangle, self.edicion)
        self.pixmap4 = QPixmap(nombre).scaled(781, 621)
        self.pixmap = self.pixmap4.toImage()
        self.edicion.setPixmap(self.pixmap4)
        self.edicion.setAlignment(Qt.AlignCenter)
        self.edicion.show()
        self.show()

    def recorte(self):
        self.edicion.mouseMoveEvent = self.move_event
        self.edicion.mouseReleaseEvent = self.realase_event
        self.edicion.mousePressEvent = self.press_event

    def move_event(self, event):
        if not self.origin.isNull():
            self.edicion.ruber.setGeometry(QRect(self.origin, event.pos()).normalized())

    def realase_event(self, event):
        if event.button() == Qt.LeftButton:
            self.edicion.ruber.hide()
            self.recortar.append([event.pos().x(), event.pos().y()])
            self.realizar_recorte()

    def realizar_recorte(self):

        """obtengo mal los bytes de la imagen que se muestra así que no funciona
        bien uwu """

        byts, ind_ihdr, ind_idat, ind_iend = leer_imagen(self.imag + '.png')
        new_idr = idr(byts, ind_ihdr)
        largo, ancho, largo_ihdr, largo_idat, largo_iend = obtner_largos(byts,
                                                                         ind_ihdr,
                                                                         ind_idat,
                                                                         ind_iend)
        matriz = obtener_matriz(byts, largo, ancho, ind_idat, largo_idat)
        print(self.recortar)

        for i in range(self.recortar[0][0], self.recortar[0][1]):
            for j in range(self.recortar[1][0], self.recortar[1][1]):
                matriz[i][j] = [255, 255, 255]

        m_con0 = agregar_ceros(matriz)
        tr = transformar(m_con0)
        idat = crear_idat(tr, byts, ind_idat)

        foto = byts[0:8] + new_idr + idat + byts[ind_iend - 4: ind_iend + 4 +largo_iend+ 4]
        os.remove(self.imag+'.png')
        time.sleep(0.5)

        with open(self.imag+'.png', 'wb') as fil:
            fil.write(foto)

        self.imagen(self.imag)

    def press_event(self, event):
        if event.button() == Qt.LeftButton:
            self.recortar.append([event.pos().x(), event.pos().y()])
            self.origin = QPoint(event.pos())
            self.edicion.ruber.setGeometry(QRect(self.origin, QSize()))
            self.edicion.ruber.show()

    def pixelSelect(self, event):
        position = QPoint(event.pos().x(), event.pos().y())
        color = QColor.fromRgb(self.pixmap.pixel(position))
        print(color.red(), color.green(), color.blue())

    def comentar_foto(self):
        fecha_hora = datetime.datetime.now()
        fecha = '{}/{}/{}'.format(fecha_hora.day, fecha_hora.month, fecha_hora.year)
        hora = '{}:{}'.format(fecha_hora.hour, fecha_hora.minute)
        self.trigger_comentario.emit(fecha + ' ' + hora + ' ' + ' ' +
                                     self.main.cliente.nombre + ' : ' + self.comentario.text())
        self.comentario.setText('')

    def comentarr(self, list):
        for coment in list:
            self.label = QLabel(self.contscroll)
            self.label.move(self.pos[0], self.pos[1])
            self.label.setText(' ' + coment)
            self.vbox.addWidget(self.label)
            self.label.show()
            self.pos[1] = self.pos[1] + 25

    def closeEvent(self, event):
        event.accept()
        self.main.show()
        self.trigger_deslock.emit(self.imag)
        self.imag = None

    def mostrar_colores(self):
        if self.puede:
            self.edicion.mousePressEvent = self.pixelSelect
            selected_color = QColorDialog.getColor()
            matrix = (selected_color.red(), selected_color.green(), selected_color.blue())
            if selected_color.isValid():
                self.frame.setStyleSheet('QWidget {background-color: %s}'% selected_color.name())

if __name__ == '__main__':
    def hook(type, value, traceback):
        print(type)
        print(traceback)
    sys.__excepthook__ = hook
    app = QApplication([])
    ventana = MainWindow()
    app.exec_()


from django.contrib import admin
from django.urls import path,include

urlpatterns = [

    path('admin/',include('Admin.urls')),
    path('',include('User.urls')),
    path('imageCompression/',include('compression.urls')),
    path('PDFtoJPG/',include('Conversion.urls')),

   ]


import requests
import json

client_id = input("Введите client id: ")
client_secret = input("Введите client secret: ")
code = input("Введите code: ")

req = requests.get(
    f"https://oauth.vk.com/access_token?client_id={client_id}&client_secret={client_secret}&redirect_uri=http://localhost&code={code}")

if not req.ok:
    print(f"Ошибка авторизации: {req.status_code}")
    exit(0)

data = json.loads(req.text)
access_token = data['access_token']

req = requests.get(
    f"https://api.vk.com/method/friends.get?order=hints&count=10&fields=city,domain&access_token={access_token}&v=5.102")

if not req.ok:
    print(f"Ошибка запроса данных: {req.status_code}")
    exit(0)

data = json.loads(req.text)
with open("friends.json", "w", encoding='utf-8') as f:
    json.dump(data, f)


# -*- coding: utf-8 -*-

import sys
import os
import numpy as np
import argparse
import random
import openslide
import PIL.Image as Image
import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import torch.utils.data as data
import torchvision.transforms as transforms
import torchvision.transforms.functional as functional
import torchvision.models as models
import time
from sklearn.metrics import balanced_accuracy_score,recall_score,confusion_matrix,classification_report
import matplotlib.pyplot as plt
from tqdm import tqdm as tqdm
#import pickle
from mark_result import result_excel_origin

def main(parser):
    global args, best_acc
    args = parser.parse_args()
    best_acc = 0

    #cnn
    model = models.resnet34(pretrained = False)
    model_dict = torch.load('/your_dir/resnet34-333f7ec4.pth')
#   如果加载自己模型就改为使用上述两句命令
    model.fc = nn.Linear(model.fc.in_features, 2)
    model.cuda()
    model = nn.DataParallel(model)
    model.load_state_dict(model_dict['state_dict'])

#    if args.weights==0.5:
#        criterion = nn.CrossEntropyLoss().cuda()
#    else:
#        w = torch.Tensor([1-args.weights, args.weights])
#        criterion = nn.CrossEntropyLoss(w).cuda()
#    optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-4)
#
#    cudnn.benchmark = True

    #normalization
    normalize = transforms.Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225])

    val_trans = transforms.Compose([transforms.ToTensor(), normalize])

    #load data
    val_dset = MILdataset(args.val_lib, 0,val_trans)
    val_loader = torch.utils.data.DataLoader(
            val_dset,
            batch_size=args.batch_size, shuffle=False,
            num_workers=args.workers, pin_memory=False)

    time_mark = time.strftime('%Y_%m_%d',time.localtime(time.time()))
#    time_mark = '2020_03_06_'
    #以当前时间作为保存的文件名标识

    #open output file
    fconv = open(os.path.join(args.output, time_mark + 'Test_CNN_convergence_512.csv'), 'w')
    fconv.write('test,,\n')
    fconv.write('acc,recall,fnr')
    fconv.close()
    start_time = time.time()
    val_dset.setmode(1)
    val_probs = inference(val_loader, model, args.batch_size, 'test')
    v_topk = group_argtopk(np.array(val_dset.slideIDX), val_probs[:,1], 1)
    v_pred = group_max(val_probs,v_topk,1)

    metrics_meters = calc_accuracy(v_pred, val_dset.targets)
    str_logs = ['{} - {:.4}'.format(k, v) for k, v in metrics_meters.items()]
    s = ', '.join(str_logs)
    print('\tTest  metrics: ' + s)
    result = '\t'+str(metrics_meters['acc']) + ',' + str(metrics_meters['recall']) + ','\
                 + str(metrics_meters['fnr'])
    fconv = open(os.path.join(args.output, time_mark + 'CNN_convergence_512.csv'), 'a')
    fconv.write(result)
    fconv.close()

    result_excel_origin(val_dset,v_pred,time_mark + '_test')
    np.save('output/numpy_save/test_infer_probs_' + time_mark + '.npy',val_probs)

    print('\test has been finished, needed %.2f sec.' % (time.time() - start_time))

def inference(loader, model, batch_size,phase):
    model.eval()
    probs = np.zeros((1,2))
#    logs = {}
    whole_probably = 0.

    with torch.no_grad():
        with tqdm(loader, desc = phase + '\'s inferencing', \
                  file=sys.stdout, disable = False) as iterator:
            for i, input in enumerate(iterator):
#                print('Inference\tEpoch: [{}/{}]\tBatch: [{}/{}]'.format(run+1, args.nepochs, i+1, len(iterator)))
                input = input.cuda()
                output = F.softmax(model(input), dim=1)
                prob = output.detach().clone()
                prob = prob.cpu().numpy()
                batch_proba = np.mean(prob,axis=0)
                probs = np.row_stack((probs,prob))
                whole_probably = whole_probably + batch_proba
#                temp_log = {'batch proba ': str(batch_proba)}
#                logs.update(temp_log)
#
#                str_logs = ['{} - {:.4}'.format(k, v) for k, v in logs.items()]
#                s = ', '.join(str_logs)
                iterator.set_postfix_str('batch proba :' + str(batch_proba))

            whole_probably = whole_probably / (i+1)
            iterator.set_postfix_str('Whole average probably is ' + str(whole_probably))

    probs = np.delete(probs, 0, axis=0)
    return probs.reshape(-1,2)

def train(run, loader, model, criterion, optimizer):
    model.train()
    whole_loss = 0.
    whole_acc = 0.
    whole_recall = 0.
    whole_fnr = 0.
    logs = {}

    with tqdm(loader, desc = 'Epoch:' + str(run+1) + ' is trainng', \
                  file=sys.stdout, disable= False) as iterator:
        for i, (input, target) in enumerate(iterator):
            input = input.cuda()
            target = target.cuda()
            output = model(input)
            loss = criterion(output, target)
            _, pred = torch.max(output, 1)
            pred = pred.data.cpu().numpy()
            metrics_meters = calc_accuracy(pred, target.cpu().numpy())
            logs.update(metrics_meters)
            logs.update({'loss':loss.item()})
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            str_logs = ['{} - {:.4}'.format(k, v) for k, v in logs.items()]
            s = ', '.join(str_logs)
            iterator.set_postfix_str(s)

            whole_acc += metrics_meters['acc']
            whole_recall += metrics_meters['recall']
            whole_fnr += metrics_meters['fnr']
            whole_loss += loss.item()
    return round(whole_acc/(i+1),3),round(whole_recall/(i+1),3),round(whole_fnr/(i+1),3),round(whole_loss/(i+1),3)

def calc_accuracy(pred,real):
    if str(type(pred)) !="<class 'numpy.ndarray'>":
        pred = np.array(pred)
    if str(type(real)) !="<class 'numpy.ndarray'>":
        real = np.array(real)
    neq = np.not_equal(pred, real)
#    err = float(neq.sum())/pred.shape[0]
#    fpr = float(np.logical_and(pred==1,neq).sum())/(real==0).sum()
    fnr = float(np.logical_and(pred==0,neq).sum())/(real==1).sum()
    balanced_acc = balanced_accuracy_score(real,pred)
    recall = recall_score(real,pred,average='weighted')
    metrics_meters = {'acc': round(balanced_acc,3),'recall':round(recall,3),'fnr':round(fnr,3)}
    label_list = ["MSS","MSI"]
    cr = classification_report(real, pred, target_names = label_list, digits = 2)
    print(cr, "\n")

    cm = confusion_matrix(real, pred)
    plt.imshow(cm, interpolation='nearest', cmap='Pastel1')
    plt.title('Confusion matrix', size=15)
    plt.colorbar()
    tick_marks = np.arange(2)
    plt.xticks(tick_marks, label_list,rotation=45, size=10)
    plt.yticks(tick_marks, label_list,size=10)
    plt.tight_layout()
    plt.ylabel('Actual label',size=15)
    plt.xlabel('Predicted label',size=15)
    width, height=cm.shape
    for x in range(width):
        for y in range(height):
            plt.annotate(str(cm[x][y]),xy=(y,x),horizontalalignment='center',verticalalignment='center')
    plt.show()

    return metrics_meters

def group_argtopk(groups, data,k=1):
    #该方法用于新一轮训练前进行所有图像全图region块预测,提取指定的top k的index
    # 假设group(slideIDX)列表为[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2],以top k=2为例
    # 对应预测label=1的概率为np.array([0,0.32,0.45,0.56,0.1,0.23,0.47,0.55,0.76,0.98,0.99,0.2,0.39,0.6], dtype=np.float64)
    # 通过np.lexsort((data, groups))计算的order为np.array([3,2,0,1,7,8,9,6,10,5,4,11,13,12])
    # 根据下面的index计算过程,最终得出index为np.array([False, False,  True,  True, False, False, False, False, False,\
    #    True,  True, False,  True,  True], dtype=bool),最终得出的结果如下:
    # np.array([ 0,  1,  5,  4, 13, 12], dtype=np.int64),即选出结果概率数组中第3(0.56),2(0.45),8(0.99),7(0.98),12(0.39),13(0.6)
    order = np.lexsort((data, groups))
    groups = groups[order]
    data = data[order]
    index = np.empty(len(groups), 'bool')
    index[-k:] = True
    index[:-k] = groups[k:] != groups[:-k]
    return list(order[index])

def group_max(probs,topk,k):
    #该方法计算过程大致和group_argtopk类似，不过指定top k为1,于Validation过程使用
    # 同样以上述的group(slideIDX)和预测的概率array为例,最终计算到的结果是array([0.56,0.99,0.6], dtype=float64)
    # 即每个WSI中,选择所有region中预测label=1的概率最高值作为当前WSI预测label=1的概率
#    out = np.empty(nmax)
#    out[:] = np.nan
##    whole_mean_proba = np.zeros((1,2))
#    order = np.lexsort((data, groups))
#    groups = groups[order]
#    data = data[order]
#    index = np.empty(len(groups), 'bool')
#    index[-1] = True
#    index[:-1] = groups[1:] != groups[:-1]
#    out[groups[index]] = data[index]
    #######################################
    #上述说明仅针对原来的group_max方法,下面重新编写的validation评估的方法,对于每一张图片来说, \
    # 预测的label取决于所有region截图分类概率相加后最大值者为准
    select_probs = np.array([probs[x,:] for x in topk])
    predict_result = []
#    groups = np.array(groups)
#    for i in np.unique(groups):
##        mean_proba = np.mean(data[groups==i],axis=0)
##        whole_mean_proba = np.row_stack((whole_mean_proba,mean_proba))
##        out[i] = np.argmax(mean_proba)
    for j in range(0,select_probs.shape[0],k):
        if np.sum(np.argmax(select_probs[j:j+k,:],axis=1)) >= k/2 :
            predict_result.append(1)
        else:
            predict_result.append(0)

#    return np.delete(whole_mean_proba, 0, axis=0)
    return np.array(predict_result)

class MILdataset(data.Dataset):
    def __init__(self, libraryfile='', k=0, transform=None):
        lib = torch.load(libraryfile)
        slides = []
        patch_size = []
        for i,name in enumerate(lib['slides']):
            sys.stdout.write('Opening SVS headers: [{}/{}]\r'.format(i+1, len(lib['slides'])))
            sys.stdout.flush()
            slides.append(openslide.OpenSlide(name))
            patch_size.append(int(lib['patch_size'][i]))
            #获取WSI文件对应的切片大小,因为在生成lib时,已经确保lib['slides']和lib['patch_size']顺序是对应的,\
            # 所以可以在一个循环中使用相同的index进行定位
        print('')
        #Flatten grid
        grid = []
        slideIDX = []
        #slideIDX列表存放每个WSI以及其坐标列表的标记,假设有0,1,2号三个WSI图像,分别于grid中记录4,7,3组提取的坐标,\
        # 返回为[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2]
        for i,g in enumerate(lib['grid']):
            if len(g) < k:
                g = g + [(g[x]) for x in np.random.choice(range(len(g)), k-len(g))]
            grid.extend(g)
            slideIDX.extend([i]*len(g))

        print('Number of tiles: {}'.format(len(grid)))
        self.slidenames = lib['slides']
        self.slides = slides
        self.targets = lib['targets']
        self.grid = grid
        self.slideIDX = slideIDX
        self.transform = transform
        self.mode = None
        self.patch_size = patch_size
        self.level = lib['level']
    def setmode(self,mode):
        self.mode = mode
    def maketraindata(self, idxs,repeat=0):
        #repeat这个参数用于是否对采样进行复制,如果进行复制,就会在下面的_getitem_方法中对重复的样本进行不一样的颜色增强
        if abs(repeat) == 0:
            self.t_data = [(self.slideIDX[x],self.grid[x],self.targets[self.slideIDX[x]],0) for x in idxs]
        else:
            repeat = abs(repeat) if repeat % 2 == 1 else abs(repeat) + 1
            self.t_data = [(self.slideIDX[x],self.grid[x],self.targets[self.slideIDX[x]],0) for x in idxs]
            for y in range(-100,int(100 + repeat/2),int(100*2/repeat)):
                self.t_data = self.t_data + [(self.slideIDX[x],self.grid[x],self.targets[self.slideIDX[x]],y/1000) for x in idxs]
    def shuffletraindata(self):
        self.t_data = random.sample(self.t_data, len(self.t_data))
    def __getitem__(self,index):
        if self.mode == 1:
            # mode =1 为预测时使用，会从所有WSI文件中返回全部的region的图像
            slideIDX = self.slideIDX[index]
            coord = self.grid[index]

            img = self.slides[slideIDX].read_region(coord,self.level,(self.patch_size[slideIDX],\
                                                    self.patch_size[slideIDX])).convert('RGB')
            if img.size != (224,224):
                img = img.resize((224,224),Image.BILINEAR)
            if self.transform is not None:
                img = self.transform(img)
            return img

        elif self.mode == 2:
            # mode =2 为训练时使用，只会根据指定的index(经过上一轮MIL过程得出) \
            #   从全部WSI文件中筛选对应的坐标列表,返回相应的训练图像和label
            slideIDX, coord, target,h_value = self.t_data[index]
            img = self.slides[slideIDX].read_region(coord,self.level,(self.patch_size[slideIDX],\
                                                    self.patch_size[slideIDX])).convert('RGB')
            if h_value > 0:
                hue_factor = random.uniform(h_value,0.1)
            elif h_value == 0:
                hue_factor = random.uniform(0,0)
            elif h_value < 0:
                hue_factor = random.uniform(-0.1,h_value)
            img = functional.adjust_hue(img,hue_factor)
            # 只有在训练模式下才进行H通道变换的颜色增强方法
            # 如果在maketraindata方法设置采样复制,那么就会针对h_value的值进行不同方向的hue_factor生成,\
            #    从而达到复制的样本和原来的样本有不一样的增强的效果
            if img.size != (224,224):
                img = img.resize((224,224),Image.BILINEAR)
            if self.transform is not None:
                img = self.transform(img)
            return img, target
    def __len__(self):
        if self.mode == 1:
            return len(self.grid)
        elif self.mode == 2:
            return len(self.t_data)

if __name__ == '__main__':
    os.environ["CUDA_VISIBLE_DEVICES"] = "2"
    # parser = argparse.ArgumentParser(description='MIL-nature-medicine-2019 tile classifier training script')
    # parser.add_argument('--train_lib', type=str, default='', help='path to train MIL library binary')
    # parser.add_argument('--val_lib', type=str, default='', help='path to validation MIL library binary. If present.')
    # parser.add_argument('--output', type=str, default='.', help='name of output file')
    # parser.add_argument('--batch_size', type=int, default=512, help='mini-batch size (default: 512)')
    # parser.add_argument('--nepochs', type=int, default=100, help='number of epochs')
    # parser.add_argument('--workers', default=4, type=int, help='number of data loading workers (default: 4)')
    # parser.add_argument('--test_every', default=10, type=int, help='test on val every (default: 10)')
    # parser.add_argument('--weights', default=0.5, type=float, help='unbalanced positive class weight (default: 0.5, balanced classes)')
    # parser.add_argument('--k', default=1, type=int, help='top k tiles are assumed to be of the same class as the slide (default: 1, standard MIL)')

    parser = argparse.ArgumentParser(description='MIL-nature-medicine-2019 tile classifier training script')
#    parser.add_argument('--train_lib', type=str, default='output/lib/512/cnn_train_data_lib.db', help='path to train MIL library binary')
    parser.add_argument('--val_lib', type=str, default='output/lib/512/cnn_val_data_lib.db', help='path to validation MIL library binary. If present.')
    parser.add_argument('--output', type=str, default='output/', help='name of output file')
    parser.add_argument('--batch_size', type=int, default=256, help='mini-batch size (default: 512)')
    parser.add_argument('--nepochs', type=int, default=14, help='number of epochs')
    parser.add_argument('--workers', default=0, type=int, help='number of data loading workers (default: 4)')
    # 如果是在docker中运行时需注意,因为容器设定的shm内存不够会出现相关报错,此时将num_workers设为0则可
    #parser.add_argument('--test_every', default=10, type=int, help='test on val every (default: 10)')
    parser.add_argument('--weights', default=0.79, type=float, help='unbalanced positive class weight (default: 0.5, balanced classes)')
    parser.add_argument('--k', default=10, type=int, help='top k tiles are assumed to be of the same class as the slide (default: 1, standard MIL)')
    #parser.add_argument('--tqdm_visible',default = True, type=bool,help='keep the processing of tqdm visible or not, default: True')

    main(parser)


from contextlib import contextmanager
from typing import ContextManager
from typing import List

from sqlalchemy import Column, Integer, String, create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session

DB_URI = 'sqlite:///weather.db'

Base = declarative_base()

class City(Base):
    __tablename__ = 'City'

    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False, unique=True)

    def __repr__(self) -> str:
        return f"City(id={self.id}, name={self.name})"

engine = create_engine(DB_URI, encoding='utf-8')
Base.metadata.create_all(engine)

@contextmanager
def session(**kwargs) -> ContextManager[Session]:
    new_session = Session(**kwargs, bind=engine, expire_on_commit=False)
    try:
        yield new_session
        new_session.commit()
    except:
        new_session.rollback()
        raise
    finally:
        new_session.close()

def get_cities(*,
               ids: List[int] = None) -> List[City]:
    with session() as s:
        if ids:
            return s.query(City).filter(
                City.id.in_(ids)
            ).all()

        return s.query(City).all()

def add_cities(*, name: str) -> None:
    if not name:
        return

    with session() as s:
        city = City(name=name)
        s.add(city)

def delete_cities(*, city_id: int) -> None:
    with session() as s:
        city = s.query(City).filter_by(id=city_id).first()
        s.delete(city)


import argparse
import requests

def get_args():
    """all the argparse stuff.
    params  : -
    returns : namespace
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("input")
    parser.add_argument("--show-malformed", dest="show_malformed", action="store_true",
                        help="show instruction lines that were rejected.")
    parser.set_defaults(show_malformed=False)
    return parser.parse_args()

def read_file(filename):
    """Reads the contents of a local file and returns the lines in a list.
    params  : filename (str)
    returns : list(str) or None
    """
    try:
        f = open(filename, "rt")
    except FileNotFoundError:
        return None
    with f:
        return [l.rstrip("\n") for l in f.readlines()]

def read_remote_file(url):
    """Reads the contents of a remote file and returns the lines in a list.
    params  : url (str)
    returns : list(str) or None
    """
    try:
        r = requests.get(url)
    except requests.ConnectionError:
        return None
    if r.ok:
        return r.text.split("\n")[:-1]

def parse_line(line):
    """Parses an instruction string and returns a dictionary containing the instruction. d
    params  : line (str), strict (bool)
    returns : {"action": str, "start": [int, int], "stop": [int, int]}
    """
    # lowercase line, then split on whitespace, and again on commas.
    l = []
    for c in line.lower().split():
        if isinstance(c, str) and "," in c:
            for i in c.split(','):
                if i:
                    l.append(i)
        else:
            l.append(c)

    # if the first word is "turn", remove it
    if l[0] == "turn":
        l = l[1:]

    action = None
    for a in ["switch", "on", "off"]:
        if l[0] == a:
            action = a
    if not action:
        return None

    coords = [int(i) for i in [l[1], l[2], l[4], l[5]]]

    # if any coordinate is negative, set to 0
    for i, c in enumerate(coords):
        coords[i] = 0 if c < 0 else c

    # if the first set of coordinates are higher than the second, return none
    if coords[0] > coords[2] or coords[1] > coords[3]:
        return None

    start = coords[:2]
    stop = coords[2:]
    return {"action": action, "start": start, "stop": stop}

def execute(board, instruction):
    """executes an instruction. Returns new board.
    params  : board [[bool]], instruction (dict)
    returns : [[bool]]
    """
    start = [n if n < len(board) else None for n in instruction["start"]]
    stop = [n + 1 if n < len(board) else len(board) for n in instruction["stop"]]

    # if you "start" after the board, there's nothing to do
    if None in start:
        return board

    if instruction["action"] == "on":
        for x in range(start[0], stop[0]):
            for y in range(start[1], stop[1]):
                board[x][y] = True

    elif instruction["action"] == "off":
        for x in range(start[0], stop[0]):
            for y in range(start[1], stop[1]):
                board[x][y] = False

    elif instruction["action"] == "switch":
        for x in range(start[0], stop[0]):
            for y in range(start[1], stop[1]):
                board[x][y] = not board[x][y]
    return board

def count_true(board):
    """Returns the number of True values in a 2d list
    params  : board [[bool]]
    returns : int
    """
    result = 0
    for row in board:
        for value in row:
            result += value
    return result

def main():
    args = get_args()
    malformed_lines = []

    if args.input.startswith("http://") or args.input.startswith("https://"):
        lines = read_remote_file(args.input)
    else:
        lines = read_file(args.input)
    if not lines:
        raise SystemExit("Error reading input. Exiting...")

    try:
        size = int(lines[0])
    except TypeError:
        raise SystemExit(
            "Error in instruction[0]: Invalid display board size. Exiting...")

    instructions = []
    for l in lines[1:]:
        instr = parse_line(l)
        if instr:
            instructions.append(instr)
        else:
            malformed_lines.append(l)

    board = [[False for _ in range(size)] for _ in range(size)]
    for instr in instructions:
        board = execute(board, instr)

    if args.show_malformed:
        print('{} malformed lines not executed.'.format(len(malformed_lines)))
        for l in malformed_lines:
            print(l)
    print("{} {}".format(args.input, count_true(board)))

if __name__ == "__main__":
    main()


#!python

# import useful Flask libs
from flask import request, Response, jsonify, json

# import broker libs that will interface with the DB
from broker import SommelierBroker

# import the Sommelier recommender
from recommender import SommelierPearsonCFRecommender, SommelierRecsysSVDRecommender

class Sommelier():

    def __init__(self, b=SommelierBroker(), r=SommelierRecsysSVDRecommender()):
        self.broker = b
        self.recommender = r

    # Takes dict of content and generates JSON response with
    # appropriate MIME type, HTTP status code etc.
    def http_success_json(self, content):
        response = json.dumps(content, encoding="utf-8")
        response = ''.join(response.decode('unicode-escape').splitlines())
        return response, { 'status': 200, 'mimetype': 'application/json; charset=utf-8' }

    def http_not_found_json(self):
        response = json.dumps("404: Not found", encoding="utf-8")
        return response, { 'status': 404, 'mimetype': 'application/json; charset=utf-8' }

    def index(self):
        return self.http_success_json({
            'type': 'list',
            'self': {
                'title': 'Sommelier',
                'link': '/'
            },
            'list': [
                {
                'title': 'All Wines',
                'link': '/wines/1'
                },
                {
                'title': 'All Authors',
                'link': '/authors/1'
                }
            ]
        })

    def wine_page(self, page_num):
        records = self.broker.get_wine_page(page_num)
        if not records:
            return self.http_not_found_json()
        num_pages = self.broker.get_num_wine_pages()
        wines_list = []
        for wine in records:
            wines_list.append({
                'title': '{} {}'.format(wine['name'], wine['vintage']),
                'link': '/wine/{}'.format(wine['id'])
            })
        return self.http_success_json({
            'type': 'list',
            'self': {
                'title': 'Wines, Page {}'.format(page_num),
                'link': '/wines/{}'.format(page_num)
            },
            'list': wines_list
        })

    def wine(self, wine_id):
        record = self.broker.get_wine(wine_id)
        if not record:
            return self.http_not_found_json()
        tastings_list = []
        for tasting in record['tastings']:
            tastings_list.append({
                'rating': tasting['rating'],
                'notes': tasting['notes'],
                'tasting_date': tasting['tasting_date'],
                'author': {
                    'title': tasting['author'],
                    'link': '/author/{}'.format(tasting['author_id'])
                }
            })
        wines = self.recommender.wines_for_wine(wine_id)
        wines_list = []
        for wine in wines:
            wines_list.append({
                'title': '{} {}'.format(wine['name'], wine['vintage']),
                'link': '/wine/{}'.format(wine['id'])
            })
        return self.http_success_json({
            'type': 'wine',
            'self': {
                'title': '{} {}'.format(record['name'], record['vintage']),
                'name': record['name'],
                'vintage': record['vintage'],
                'grape_variety': record['grape_variety'],
                'appellation': record['appellation'],
                'sub_region': record['sub_region'],
                'region': record['region'],
                'country': record['country'],
                'producer': record['producer'],
                'tastings': tastings_list,
                'link': '/wine/{}'.format(record['id'])
            },
            'related_content': {
                'similar_wines': wines_list
            }
        })

    def author_page(self, page_num):
        records = self.broker.get_author_page(page_num)
        if not records:
            return self.http_not_found_json()
        num_pages = self.broker.get_num_author_pages()
        authors_list = []
        for author in records:
            authors_list.append({
                'title': author['name'],
                'link': '/author/{}'.format(author['id'])
            })
        return self.http_success_json({
            'type': 'list',
            'self': {
                'title': 'Authors, Page {}'.format(page_num),
                'link': '/authors/{}'.format(page_num)
            },
            'list': authors_list
        })

    def author(self, author_id):
        author_id = int(author_id)
        record = self.broker.get_author(author_id)
        if not record:
            return self.http_not_found_json()
        tastings_list = []
        if 'tastings' in record:
            for tasting in record['tastings']:
                tastings_list.append({
                    'rating': tasting['rating'],
                    'notes': tasting['notes'],
                    'tasting_date': tasting['tasting_date'],
                    'wine': {
                        'title': '{} {}'.format(tasting['wine'], tasting['vintage']),
                        'link': '/wine/{}'.format(tasting['wine_id'])
                    }
                })
        wines = self.recommender.wines_for_author(author_id)
        wines_list = []
        for wine in wines:
            wines_list.append({
                'title': '{} {}'.format(wine['name'], wine['vintage']),
                'link': '/wine/{}'.format(wine['id'])
            })
        authors = self.recommender.authors_for_author(author_id)
        authors_list = []
        for author in authors:
            authors_list.append({
                'title': author['name'],
                'link': '/author/{}'.format(author['id'])
            })
        return self.http_success_json({
            'type': 'author',
            'self': {
                'title': record['name'],
                'name': record['name'],
                'tastings': tastings_list,
                'link': '/author/{}'.format(record['id'])
            },
            'related_content': {
                'recommended_wines': wines_list,
                'similar_authors': authors_list
            }
        })


from __future__ import print_function
import importlib.util
import xml.etree.ElementTree as et
import pandas as pd
from shutil import copyfile
import pickle
import os

"""Curation/Utils
"""
spec = importlib.util.spec_from_file_location(
    "__main__", os.path.abspath("./metaboverse_cli/curate/__main__.py"))
curate = importlib.util.module_from_spec(spec)
spec.loader.exec_module(curate)

spec = importlib.util.spec_from_file_location(
    "__main__", os.path.abspath("./metaboverse_cli/curate/load_reactions_db.py"))
load_reactions = importlib.util.module_from_spec(spec)
spec.loader.exec_module(load_reactions)

spec = importlib.util.spec_from_file_location(
    "__main__", os.path.abspath("./metaboverse_cli/curate/load_complexes_db.py"))
load_complexes = importlib.util.module_from_spec(spec)
spec.loader.exec_module(load_complexes)

spec = importlib.util.spec_from_file_location(
    "__main__", os.path.abspath("./metaboverse_cli/curate/fetch_species.py"))
fetch_species = importlib.util.module_from_spec(spec)
spec.loader.exec_module(fetch_species)

spec = importlib.util.spec_from_file_location(
    "get_table", os.path.abspath("./metaboverse_cli/curate/utils.py"))
get_table = importlib.util.module_from_spec(spec)
spec.loader.exec_module(get_table)

spec = importlib.util.spec_from_file_location(
    "unpack_table", os.path.abspath("./metaboverse_cli/curate/utils.py"))
unpack_table = importlib.util.module_from_spec(spec)
spec.loader.exec_module(unpack_table)

spec = importlib.util.spec_from_file_location(
    "", os.path.abspath("./metaboverse_cli/curate/load_reactions_db.py"))
load_reactions_db = importlib.util.module_from_spec(spec)
spec.loader.exec_module(load_reactions_db)

# test __main__() -- functional test
args_dict = {
    'organism_id': 'SCE',
    'output': os.path.abspath(
        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep}

# load_reactions_db.py
args_dict, pathway_database, reaction_database, species_database, \
name_database, compartment_dictionary, \
components_database = load_reactions.__main__(
        species_id=args_dict['organism_id'],
        output_dir=args_dict['output'],
        database_source='reactome',
        sbml_url='None',
        args_dict=args_dict)

assert type(pathway_database) == dict, "load_reactions_db.py failed"
assert type(pathway_database[list(pathway_database.keys())[
            0]]) == dict, "load_reactions_db.py failed"

assert type(reaction_database) == dict, "load_reactions_db.py failed"
assert type(reaction_database[list(reaction_database.keys())[
            0]]) == dict, "load_reactions_db.py failed"

assert type(species_database) == dict, "load_reactions_db.py failed"
assert type(species_database[list(species_database.keys())[
            0]]) == str, "load_reactions_db.py failed"

assert type(name_database) == dict, "load_reactions_db.py failed"
assert type(name_database[list(name_database.keys())[0]]
            ) == str, "load_reactions_db.py failed"

assert type(compartment_dictionary) == dict, "load_reactions_db.py failed"
assert type(compartment_dictionary[list(compartment_dictionary.keys())[
            0]]) == str, "load_reactions_db.py failed"

assert type(components_database) == dict, "load_reactions_db.py failed"
assert type(components_database[list(components_database.keys())[
            0]]) == dict, "load_reactions_db.py failed"

# load_complexes_db.py
d = load_complexes.__main__(
    output_dir=args_dict['output'])
assert type(d['complex_participants']
            ) == pd.DataFrame, "load_complexes_db.py failed"
assert type(d['complex_pathway']
            ) == pd.DataFrame, "load_complexes_db.py failed"

# fetch_species.py
organisms = fetch_species.__main__()
assert type(organisms) == list, "fetch_species.py failed"

# write_database()
args_dict = {
    'output': os.path.abspath(
        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep,
    'file': 'test.mvdb'}
database = {
    'chebi_reference': {
        'R-HSA-0000': {
            'analyte_id': 'R-ANA-1017',
            'source_id': '0017',
            'other': None
        }
    },
    'uniprot_reference': {
        'R-HSA-0000': {
            'analyte_id': 'R-ANA-1015',
            'source_id': '0015',
            'other': None
        }
    },
    'ensembl_reference': {
        'R-HSA-0001': {
            'analyte_id': 'R-ANA-1014',
            'source_id': '0014',
            'other': None
        }
    },
    'ncbi_reference': {
        'R-HSA-0000': {
            'analyte_id': 'R-ANA-1013',
            'source_id': '0013',
            'other': None
        }
    },
    'mirbase_reference': {
        'R-HSA-0000': {
            'analyte_id': 'R-ANA-1012',
            'source_id': '0012',
            'other': None
        }
    }
}
curate.write_database(
    output=args_dict['output'],
    file=args_dict['file'],
    database=database)

assert os.path.exists(args_dict['output'] + args_dict['file']
                      ) == True, 'Failed to write database to pickle file'
os.remove(args_dict['output'] + args_dict['file'])

# unpack_table()
url = 'https://reactome.org/download/current/'
test_file = 'models2pathways.tsv'
file = unpack_table.unpack_table(
    url + test_file,
    output_dir=args_dict['output'])
assert file == args_dict['output'] + test_file, 'unpack_table() failed'
os.remove(args_dict['output'] + test_file)

# get_table()
url = 'https://reactome.org/download/current/'
test_file = 'models2pathways.tsv'
table = get_table.get_table(
    args_dict['output'],
    url + test_file,
    column_names=[
        '1',
        '2',
        '3',
        '4',
        '5',
        '6',
        '7'],
    organism='Homo sapiens',
    organism_key='7')  # Key for testing purposes only
assert 'Homo sapiens' in table['7'].tolist(), 'Problem getting table download'
os.remove(args_dict['output'] + test_file)

# parse_table()

def run_checks(
        ref_dict):

    if ref_dict['R-ALL-00000']['analyte_id'] != 'R-ALL-00000' \
            or ref_dict['R-ALL-00000']['reaction_id'] != 'R-HSA-00000' \
            or ref_dict['R-ALL-00000']['reaction_name'] != 'fake_reaction' \
            or ref_dict['R-ALL-00000']['analyte'] != 'fake_molecule' \
            or ref_dict['R-ALL-00000']['compartment'] != 'organelle':
        return False

    else:
        return True

data_1 = [
    ['analyte_id', 'analyte_name', 'reaction_id',
        'reaction_name', 'source_id', 'extra_column'],
    ['R-ALL-00000', 'fake_molecule [organelle]',
        'R-HSA-00000', 'fake_reaction', '0101', '????']
]
table = pd.DataFrame(data_1, columns=data_1[0])
table = table.drop(0, axis=0)
reference = {
    'this_one': table,
    'not_this_one': False}
ref_dict = curate.parse_table(
    reference=reference,
    key='this_one')
assert run_checks(ref_dict) == True, 'Problem parsing Reactome table'

# Test unpacking of a reaction file
species_id = 'HSA'
path = args_dict['output']
file = 'R-HSA-realtest.sbml'
test_db = args_dict['output'] + file
sbml_namespace = '{{http://www.sbml.org/sbml/level{0}/version{1}/core}}'
rdf_namespace = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}'
bqbiol_namespace = '{http://biomodels.net/biology-qualifiers/}'
sbml_level = '3'
sbml_version = '1'

path_list = load_reactions_db.get_pathways(
    species_id=species_id,
    pathways_dir=path)
assert path_list == ['R-HSA-realtest'], 'get_pathways() failed'

contents = load_reactions_db.get_database(
    pathways_dir=path,
    pathway_name=path_list[0])
assert type(contents) == et.Element, 'get_database() failed'

args_dict, pathway_database, reaction_database, species_database, \
name_database, compartment_database, compartment_dictionary, \
components_database = load_reactions_db.process_components(
        output_dir=args_dict['output'],
        pathways_dir=args_dict['output'],
        pathways_list=path_list,
        species_id=species_id,
        args_dict=args_dict,
        bqbiol_namespace=bqbiol_namespace,
        rdf_namespace=rdf_namespace)

assert 'R-HSA-realtest' in pathway_database.keys(), 'process_components() failed'
assert pathway_database['R-HSA-realtest']['id'] == 'pathway_168268', 'process_components() failed'
assert pathway_database['R-HSA-realtest']['reactome'] == 'R-HSA-realtest', 'process_components() failed'
assert pathway_database['R-HSA-realtest']['name'] == 'Virus Assembly and Release', 'process_components() failed'
assert len(pathway_database['R-HSA-realtest']
           ['reactions']) == 21, 'process_components() failed'
truth_d = set([
    'reaction_168884',
    'reaction_168875',
    'reaction_168882',
    'reaction_195733',
    'reaction_169919',
    'reaction_195739',
    'reaction_168894',
    'reaction_168870',
    'reaction_168871',
    'reaction_195734',
    'reaction_168858',
    'reaction_168862',
    'reaction_168895',
    'reaction_168860',
    'reaction_195726',
    'reaction_169921',
    'reaction_195926',
    'reaction_195730',
    'reaction_168869',
    'reaction_188544',
    'reaction_169847'])
assert set(pathway_database['R-HSA-realtest']['reactions']
           ) == truth_d, 'process_components() failed'

assert reaction_database['reaction_195733']['compartment'] == 'compartment_17957', 'process_components() failed'
assert reaction_database['reaction_195733']['id'] == 'reaction_195733', 'process_components() failed'
assert reaction_database['reaction_195733']['name'] == 'M2 protein synthesis', 'process_components() failed'
assert reaction_database['reaction_195733']['reversible'] == 'false', 'process_components() failed'
assert type(reaction_database['reaction_195733']
            ['notes']) == str, 'process_components() failed'
assert reaction_database['reaction_195733']['reactants'] == [
], 'process_components() failed'
assert reaction_database['reaction_195733']['products'] == [
    'species_195756'], 'process_components() failed'
assert reaction_database['reaction_195733']['modifiers'] == [
], 'process_components() failed'
assert reaction_database['reaction_168870']['reactants'] == [
    'species_195945'], 'process_components() failed'
assert reaction_database['reaction_168870']['products'] == [
    'species_195941'], 'process_components() failed'
assert reaction_database['reaction_168870']['modifiers'] == [
    ['species_195915', 'catalyst']], 'process_components() failed'

assert species_database['species_195752'] == 'NP', 'process_components() failed'
assert species_database['species_195915'] == 'NA', 'process_components() failed'

assert name_database['NP'] == 'species_195752', 'process_components() failed'
assert name_database['NA'] == 'species_195915', 'process_components() failed'

assert compartment_database['species_195752'] == 'compartment_876', 'process_components() failed'
assert compartment_database['species_195915'] == 'compartment_984', 'process_components() failed'

assert compartment_dictionary['compartment_876'] == 'plasma membrane', 'process_components() failed'
assert compartment_dictionary['compartment_984'] == 'extracellular region', 'process_components() failed'

assert components_database['species_195752']['id'] == 'species_195752', 'process_components() failed'
assert components_database['species_195752']['reactome_id'] == 'R-FLU-195752', 'process_components() failed'
assert components_database['species_195752']['name'] == 'NP', 'process_components() failed'
assert components_database['species_195752']['is'] == 'P03466', 'process_components() failed'
assert components_database['species_195752']['hasPart'] == [
], 'process_components() failed'
print(components_database['species_195752'])
assert components_database['species_195752']['type'] == 'protein_component', 'process_components() failed'
assert components_database['species_195752']['compartment'] == 'compartment_876', 'process_components() failed'

print(components_database['species_195941'])

assert set([
    'P03466',
    'P03428',
    'P03433',
    'P03431',
    'P03485',
    'P03508',
    'P03468',
    'P03452',
    'P06821']).issubset(set(components_database['species_195941']['hasPart'])), 'process_components() failed'

# BioModels tests
src = os.path.abspath(os.path.join(".", "metaboverse_cli",
                                   "curate", "test", "test_session_data.json"))
dst = os.path.abspath(os.path.join(".", "metaboverse_cli",
                                   "curate", "test", "test_session_data_copy.json"))
s_out = copyfile(src, dst)

args_dict = {
    'organism_id': 'find',
    'output': os.path.abspath(
        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep,
    'session_data': os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))}

args_dict, pathway_database, reaction_database, species_database, \
name_database, compartment_dictionary, \
components_database = load_reactions.__main__(
    species_id=args_dict['organism_id'],
    output_dir=args_dict['output'],
    database_source='biomodels/bigg',
    sbml_url=os.path.abspath(os.path.join(
        ".", "metaboverse_cli", "curate", "test", "iIS312.xml")),
    args_dict=args_dict)

if 'All' in pathway_database:
    print("Pass")
else:
    raise Exception("BioModels/BiGG database curation failed")
assert reaction_database['R_PROt2r_copy2']['reactants'] == [
    'M_h_e', 'M_pro__L_e'], "BioModels/BiGG database curation failed"
assert species_database['M_pro__L_e'] == 'L-Proline', "BioModels/BiGG database curation failed"
assert name_database['M_pro__L_e'] == 'M_pro__L_e', "BioModels/BiGG database curation failed"
assert name_database['L-Proline'] == 'M_pro__L_e', "BioModels/BiGG database curation failed"
assert compartment_dictionary['c'] == 'cytosol', "BioModels/BiGG database curation failed"
assert components_database['M_pro__L_e']['is'] == 'D00035', "BioModels/BiGG database curation failed"

os.remove(dst)

# BiGG tests
src = os.path.abspath(os.path.join(".", "metaboverse_cli",
                                   "curate", "test", "test_session_data.json"))
dst = os.path.abspath(os.path.join(".", "metaboverse_cli",
                                   "curate", "test", "test_session_data_copy.json"))
s_out = copyfile(src, dst)

args_dict = {
    'organism_id': 'find',
    'database_source': 'biomodels/bigg',
    'output': os.path.abspath(
        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep,
    'session_data': os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))}

args_dict, pathway_database, reaction_database, species_database, \
name_database, compartment_dictionary, \
components_database = load_reactions.__main__(
    species_id=args_dict['organism_id'],
    output_dir=args_dict['output'],
    database_source='biomodels/bigg',
    sbml_url=os.path.abspath(os.path.join(
        ".", "metaboverse_cli", "curate", "test", "BMID000000141967_url.xml")),
    args_dict=args_dict)

if 'All' in pathway_database:
    print("Pass")
else:
    raise Exception("BioModels/BiGG database curation failed")
assert reaction_database['MNXR42496_i']['reactants'] == [
    'MNXM2885_i', 'bigg_gthrd_i'], "BioModels/BiGG database curation failed"
assert species_database['MNXM2885_i'] == '(1S,2R)-Naphthalene 1,2-oxide', "BioModels/BiGG database curation failed"
assert species_database['MNXM2885_e'] == '(1S,2R)-Naphthalene 1,2-oxide', "BioModels/BiGG database curation failed"
assert name_database['(1S,2R)-Naphthalene 1,2-oxide'] == 'MNXM2885_e', "BioModels/BiGG database curation failed"
assert compartment_dictionary['i'] == 'intracellular', "BioModels/BiGG database curation failed"
assert components_database['MNXM2885_e']['is'] == 'MNXM2885_e', "BioModels/BiGG database curation failed"

os.remove(dst)

# test all
"""
src = os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data.json"))
dst = os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))
s_out = copyfile(src, dst)

args_dict = {
    'organism_id': 'SCE',
    'database_source': 'reactome',
    'organism_curation': "None",
    'output': os.path.abspath(
        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep,
    'session_data':os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))}
curate.__main__(
    args_dict=args_dict
)
os.remove(dst)
os.remove(args_dict['output'] + "SCE.mvdb")
"""

# test all bigg
#src = os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data.json"))
#dst = os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))
#s_out = copyfile(src, dst)
# args_dict = {
#    'organism_id': 'find',
#    'database_source': 'biomodels/bigg',
#    'organism_curation': os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "BMID000000141967_url.xml")),
#    'output': os.path.abspath(
#        os.path.join(".", "metaboverse_cli", "curate", "test")) + os.path.sep,
#    'session_data':os.path.abspath(os.path.join(".", "metaboverse_cli", "curate", "test", "test_session_data_copy.json"))}
# curate.__main__(
#    args_dict=args_dict
# )
# os.remove(dst)
#os.remove(args_dict['output'] + "BMID000000141967.mvdb")

print('Tests completed')


from __future__ import print_function
from Model.e2e import End2EndMPNet
import Model.mlp as mlp
import Model.CAE as CAE_2d
import numpy as np
import argparse
import os
import torch
from data_loader import data_loader_2d
from utility import *
from torch.autograd import Variable
import copy
import os
import random
import progressbar
from Model.complete_model import MLPComplete
import psutil
import datetime
from tqdm import tqdm
from torch.optim.lr_scheduler import StepLR
from torch.optim.lr_scheduler import ReduceLROnPlateau

from tensorboardX import SummaryWriter

def main(args):

    timestamp = datetime.datetime.now().strftime("%d_%H%M%S")
    print('timestamp: ', timestamp)

    # set seed
    torch_seed = np.random.randint(low=0, high=1000)
    np_seed = np.random.randint(low=0, high=1000)
    py_seed = np.random.randint(low=0, high=1000)
    torch.manual_seed(torch_seed)
    np.random.seed(np_seed)
    random.seed(py_seed)
    # Build the models
    if torch.cuda.is_available():
        torch.cuda.set_device(args.device)
    # decide dataloader, MLP, AE based on env_type
    if args.env_type == '2d':
        AE_output_size = 28
        if args.with_start:
            total_input_size = 2800+6
            AE_input_size = 2800
            mlp_input_size = 28+6
        else:
            total_input_size = 2800+4
            AE_input_size = 2800
            mlp_input_size = 28+4

        if not args.point_cloud:
            total_input_size -= 2800 - 28
            AE_input_size -= 2800 - 28

        output_size = 1
        load_train_dataset = data_loader_2d

        CAE = CAE_2d
        MLP = mlp.MLP
        # mpc = MLPComplete
    elif args.env_type == '3d':

        AE_output_size = 21

        output_size = 3
        # load_train_dataset = data_loader_2d
        CAE = CAE_2d
        MLP = mlp.MLP
        if args.with_start:
            total_input_size = 1400 * 3 + 9
            AE_input_size = 1400 * 3
            AE_output_size = 21
            mlp_input_size = 21 + 9
        else:
            total_input_size = 1400 * 3 + 6
            AE_input_size = 1400 * 3
            mlp_input_size = 21 + 6
            AE_output_size = 21

        if not args.point_cloud:
            total_input_size -= 1400 * 3 - 21
            AE_input_size -= 1400 * 3 - 21

        output_size = 1
        load_train_dataset = data_loader_2d

        CAE = CAE_2d
        MLP = mlp.MLP

    if args.activation == 'relu':
        activation_f = torch.nn.ReLU
    elif args.activation == 'tanh':
        activation_f = torch.nn.Tanh
    elif args.activation == 'selu':
        activation_f = torch.nn.SELU
    elif args.activation == 'elu':
        activation_f = torch.nn.ELU
    elif args.activation == 'leaky_relu':
        activation_f = torch.nn.LeakyReLU
    elif args.activation == 'prelu':
        activation_f = torch.nn.PReLU

    model = End2EndMPNet(total_input_size, AE_input_size, mlp_input_size,
                         output_size, CAE, MLP, activation_f=activation_f, dropout=args.dropout, AE_output_size=AE_output_size)

    if args.env_type == '2d' or args.env_type == '3d':
        loss_f = model.loss

    if not os.path.exists(args.model_path):
        os.makedirs(args.model_path)

    model_path = 'model_env_{}_epoch_{}.pkl'.format(
        args.env_type, args.start_epoch)

    if args.start_epoch > 0:
        print('Loading model from {}'.format(model_path))
        load_net_state(model, os.path.join(args.model_path, model_path))
        torch_seed, np_seed, py_seed = load_seed(
            os.path.join(args.model_path, model_path))
        # set seed after loading
        torch.manual_seed(torch_seed)
        np.random.seed(np_seed)
        random.seed(py_seed)

    process = psutil.Process(os.getpid())
    print('Physical Memory Usage: ', process.memory_info().rss/1024**3, 'GB')

    print('psutil.virtual_memory().total : ',
          psutil.virtual_memory().total/1024**3, 'GB')

    if torch.cuda.is_available():
        print('Available devices ', torch.cuda.device_count())
        print('Current cuda device ', torch.cuda.current_device())
        print('Current cuda device name ',
              torch.cuda.get_device_name(torch.cuda.current_device()))

        model.cuda()

        if hasattr(model, 'encoder'):
            model.encoder.cuda()

    if args.opt == 'Adagrad':
        optimizer = torch.optim.Adagrad(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
    elif args.opt == 'Adam':
        optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
    elif args.opt == 'SGD':
        optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, momentum=0.9, weight_decay=args.weight_decay)
    elif args.opt == 'ASGD':
        optimizer = torch.optim.ASGD(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)
    elif args.opt == 'RMSprop':
        optimizer = torch.optim.RMSprop(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay)

    model.opt = optimizer

    # scheduler = StepLR(model.opt, step_size=args.decay_step, gamma=args.decay_rate, verbose=True)
    # scheduler = ReduceLROnPlateau(model.opt, mode='min', factor=args.decay_rate, patience=args.decay_step, verbose=True)

    if args.start_epoch > 0:
        load_opt_state(model, os.path.join(args.model_path, model_path))

    # load train and test data
    print('loading...')
    obstacles, dataset, targets, env_indices = data_loader_2d(
        N=args.N, with_start=args.with_start, samples=args.samples,
        get_together=False, point_cloud=args.point_cloud, env_type=args.env_type)

    print('obstacles.shape', np.array(obstacles).shape, flush=True)
    print('dataset.shape', np.array(dataset).shape, flush=True)
    print('target.shape', np.array(targets).shape, flush=True)
    print('env_indices.shape', np.array(env_indices).shape, flush=True)

    # 0.9 train, val split
    val_size = int(0.1*len(dataset))

    # val_dataset = dataset[:-val_size]
    # val_targets = targets[:-val_size]
    # val_env_indices = env_indices[:-val_size]

    dataset = dataset[:-val_size]
    targets = targets[:-val_size]
    env_indices = env_indices[:-val_size]

    val_dataset = dataset[-val_size:]
    val_targets = targets[-val_size:]
    val_env_indices = env_indices[-val_size:]

    print('dataset.shape', np.array(dataset).shape, flush=True)
    print('val_dataset.shape', np.array(val_dataset).shape, flush=True)

    # Train the Models
    print('training...')
    writer_fname = '%s_%f_%s' % (args.env_type, args.learning_rate, args.opt)
    writer = SummaryWriter('./runs/'+writer_fname)
    record_loss = 0.
    record_i = 0
    val_record_loss = 0.
    val_record_i = 0

    print(model)
    print(model.opt)
    model.train()
    # print number of parameters
    total_params = sum(
        param.numel() for param in model.parameters()
    )

    trainable_params = sum(
        param.numel() for param in model.parameters() if param.requires_grad
    )
    print('Total number of trainable parameters: {}'.format(trainable_params))

    print('Total number of parameters: {}'.format(total_params))

    for epoch in range(args.start_epoch, args.epochs+1):

        sum_train_loss = 0.0
        sum_val_loss = 0.0

        # widgets = [f'epoch{epoch} ',
        #            progressbar.Bar(),
        #            ' (batch ', progressbar.SimpleProgress(), ') ',
        #            progressbar.ETA(),
        #            ' train/val loss ',
        #            progressbar.Variable(
        #                'train_loss', format='{formatted_value}'), '/',
        #            progressbar.Variable(
        #                'val_loss', format='{formatted_value}'),
        #            ]
        # bar = progressbar.ProgressBar(widgets=widgets)
        for i in range(0, len(dataset), args.batch_size):
            # randomly pick 100 data
            #bi = np.concatenate( (obstacles[env_indices[i:i+100]], dataset[i:i+100]), axis=1).astype(np.float32)
            bi = np.array(dataset[i:i+args.batch_size]).astype(np.float32)

            batch_data = np.array(
                dataset[i:i+args.batch_size]).astype(np.float32)
            batch_target = np.array(
                targets[i:i+args.batch_size]).astype(np.float32)
            batch_env_indices = np.array(
                env_indices[i:i+args.batch_size]).astype(np.float32)
            batch_obstacles = np.array(
                obstacles[env_indices[i:i+args.batch_size]]).astype(np.float32)

            batch_data = torch.FloatTensor(batch_data)
            batch_target = np.array(batch_target).astype(np.float32)
            batch_target = torch.FloatTensor(batch_target)
            batch_env_indices = torch.FloatTensor(batch_env_indices)
            batch_obstacles = torch.FloatTensor(batch_obstacles)

            model.zero_grad()

            batch_data = to_var(batch_data)
            batch_target = to_var(batch_target)
            batch_env_indices = to_var(batch_env_indices)
            batch_obstacles = to_var(batch_obstacles)

            # print('batch_data.shape', batch_data.shape)
            # print('batch_target.shape', batch_target.shape)
            # print('batch_env_indices.shape', batch_env_indices.shape)
            # print('batch_obstacles.shape', batch_obstacles.shape)

            model.zero_grad()
            # print('batch_data.shape', batch_data.shape)
            # print('batch_obstacles.shape', batch_obstacles.shape)
            loss = loss_f(model(batch_data, batch_obstacles), batch_target)
            loss.backward()
            loss = loss.item()
            optimizer.step()

            # model.step(batch_data, batch_target)

            # loss = loss_f(model(batch_data),
            #               batch_target).item()

            sum_train_loss += loss * args.batch_size
            record_loss += loss
            record_i += 1
            if record_i % 100 == 0:
                writer.add_scalar('train_loss', record_loss / 100, record_i)
                record_loss = 0.

            ######################################
            # loss on validation set
            idx = i % val_size

            batch_data = np.array(
                val_dataset[idx:idx+args.batch_size]).astype(np.float32)
            batch_target = np.array(
                val_targets[idx:idx+args.batch_size]).astype(np.float32)
            batch_env_indices = np.array(
                val_env_indices[idx:idx+args.batch_size]).astype(np.float32)
            batch_obstacles = np.array(
                obstacles[val_env_indices[idx:idx+args.batch_size]]).astype(np.float32)

            batch_data = torch.FloatTensor(batch_data)
            batch_target = np.array(batch_target).astype(np.float32)
            batch_target = torch.FloatTensor(batch_target)
            batch_env_indices = torch.FloatTensor(batch_env_indices)
            batch_obstacles = torch.FloatTensor(batch_obstacles)

            model.zero_grad()
            batch_data = to_var(batch_data)
            batch_target = to_var(batch_target)
            batch_env_indices = to_var(batch_env_indices)
            batch_obstacles = to_var(batch_obstacles)

            # temp = model(batch_data, batch_obstacles)

            loss = loss_f(model(batch_data, batch_obstacles),
                          batch_target).item()
            sum_val_loss += loss * args.batch_size

            val_record_loss += loss
            val_record_i += 1
            if val_record_i % 100 == 0:
                writer.add_scalar(
                    'val_loss', val_record_loss / 100, val_record_i)
                val_record_loss = 0.

            # if i % 500 == 0:
            #     bar.update(train_loss=sum_train_loss / (i + args.batch_size),
            #            val_loss=sum_val_loss / (i + args.batch_size))

            # if i % 250 == 0:
            #     print('epoch {}, batch: {}/{}, train loss {}, val loss {}'.format(epoch, i / args.batch_size, len(dataset) / args.batch_size, sum_train_loss / (i + args.batch_size), sum_val_loss / (i + args.batch_size)), flush=True)

        print('epoch {}/{}, train loss {}, val loss {}'.format(epoch, args.epochs,
              sum_train_loss * 40 / len(dataset), sum_val_loss * 40 / len(dataset)), flush=True)

        # scheduler.step()

        # Save the models every 50 epochs
        if epoch % 50 == 0:

            # create a time stamp folder
            if not os.path.exists('models/{}'.format(timestamp)):
                os.makedirs('models/{}'.format(timestamp))

            model_path = "{}/model_env_{}_epoch_{}.pkl".format(
                timestamp, args.env_type, epoch)
            print(model_path)
            # model_path='mpnet_epoch_%d.pkl' %(epoch)
            save_state(model, torch_seed, np_seed, py_seed,
                       os.path.join(args.model_path, model_path))

            entire_model_path = "{}/entire_model_env_{}_epoch_{}.pt".format(
                timestamp, args.env_type, epoch)

            torch.save(model, os.path.join(args.model_path, entire_model_path))
            # test
    writer.export_scalars_to_json("./all_scalars.json")
    writer.close()

parser = argparse.ArgumentParser()

parser.add_argument('--model-path', type=str,
                    default='./models/', help='path for saving trained models')
parser.add_argument('--N', type=int, default=10,
                    help='number of environments')

parser.add_argument('--batch-size', type=int, default=100)
parser.add_argument('--learning-rate', type=float, default=0.01)
parser.add_argument('--epochs', type=int, default=500)

parser.add_argument('--start-epoch', type=int, default=0)
parser.add_argument('--device', type=int, default=0, help='cuda device')

parser.add_argument('--env-type', type=str, default='2d',
                    help='2d')
parser.add_argument('--world-size', nargs='+', type=float,
                    default=20., help='boundary of world')
parser.add_argument('--decay-step', type=int, default=20)
parser.add_argument('--decay-rate', type=float, default=0.5)
parser.add_argument('--opt', type=str, default='Adagrad')
parser.add_argument('--with-start', action='store_true')
parser.add_argument('--samples', type=int, default=10000)
parser.add_argument('--point-cloud', action='store_true')
parser.add_argument('--activation', type=str, default='relu')
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--weight-decay', type=float, default=0.0001)

args = parser.parse_args()
print(args)
main(args)