import sys
import unittest

import numpy as np
import pandas as pd
import itertools
from pathlib import Path

sys.path.append('../src/')
from sensorutils.datasets.dsads import DSADS, load, load_raw, reformat

class DSADSTest(unittest.TestCase):
    path = None
    activities = list(range(19))
    subjects = list(range(1, 9))

    @classmethod
    def setUpClass(cls) -> None:
        if cls.path is None:
            raise RuntimeError('dataset path is not specified')

    def setUp(self):
        self.loader = DSADS(self.path)

    def tearDown(self):
        pass

    def test_load_fn(self):
        data, meta = load(self.path)

        # compare between data and meta
        self.assertEqual(len(data), len(meta))

        # check meta
        ## type check
        self.assertIsInstance(meta, list)
        self.assertTrue(all(isinstance(m, pd.DataFrame) for m in meta))

        ## shape and column check
        self.assertTrue(all([list(m.columns) == ['activity', 'subject'] for m in meta]))

        ## data type check
        self.assertTrue(all([
            m.dtypes['activity'] == np.dtype(np.int8) and \
            m.dtypes['subject'] == np.dtype(np.int8) \
            for m in meta
        ]))

        ## data check
        self.assertTrue(all([
            set(np.unique(m['subject'])) == set([m['subject'].iloc[0]]) for m in meta
        ]))
        self.assertTrue(all([
            set(np.unique(m['activity'])) == set([m['activity'].iloc[0]]) for m in meta
        ]))

        # data
        ## type check
        self.assertIsInstance(data, list)
        self.assertTrue(all(isinstance(d, pd.DataFrame) for d in data))

        ## shape and column check
        self.assertTrue(all([
            list(d.columns) == \
            ['T_xacc', 'T_yacc', 'T_zacc', 'T_xgyro', 'T_ygyro', 'T_zgyro', 'T_xmag', 'T_ymag', 'T_zmag', 'RA_xacc', 'RA_yacc', 'RA_zacc', 'RA_xgyro', 'RA_ygyro', 'RA_zgyro', 'RA_xmag', 'RA_ymag', 'RA_zmag', 'LA_xacc', 'LA_yacc', 'LA_zacc', 'LA_xgyro', 'LA_ygyro', 'LA_zgyro', 'LA_xmag', 'LA_ymag', 'LA_zmag', 'RL_xacc', 'RL_yacc', 'RL_zacc', 'RL_xgyro', 'RL_ygyro', 'RL_zgyro', 'RL_xmag', 'RL_ymag', 'RL_zmag', 'LL_xacc', 'LL_yacc', 'LL_zacc', 'LL_xgyro', 'LL_ygyro', 'LL_zgyro', 'LL_xmag', 'LL_ymag', 'LL_zmag']
            for d in data
        ]))

        ## data type check
        self.assertTrue(all([
            d.dtypes[c] == np.dtype(np.float64) \
            for d in data for c in data[0].columns
        ]))

    def test_load_raw_fn(self):
        raw = load_raw(self.path)

        # raw
        ## type check
        self.assertIsInstance(raw, list)
        self.assertTrue(all(isinstance(r, pd.DataFrame) for r in raw))

        ## shape and column check
        tgt = tuple([
            'T_xacc', 'T_yacc', 'T_zacc', 'T_xgyro', 'T_ygyro', 'T_zgyro', 'T_xmag', 'T_ymag', 'T_zmag',
            'RA_xacc', 'RA_yacc', 'RA_zacc', 'RA_xgyro', 'RA_ygyro', 'RA_zgyro', 'RA_xmag', 'RA_ymag', 'RA_zmag',
            'LA_xacc', 'LA_yacc', 'LA_zacc', 'LA_xgyro', 'LA_ygyro', 'LA_zgyro', 'LA_xmag', 'LA_ymag', 'LA_zmag',
            'RL_xacc', 'RL_yacc', 'RL_zacc', 'RL_xgyro', 'RL_ygyro', 'RL_zgyro', 'RL_xmag', 'RL_ymag', 'RL_zmag',
            'LL_xacc', 'LL_yacc', 'LL_zacc', 'LL_xgyro', 'LL_ygyro', 'LL_zgyro', 'LL_xmag', 'LL_ymag', 'LL_zmag',
            'activity', 'subject',
        ])
        self.assertTrue(all([
            tuple(r.columns) == tgt for r in raw
        ]))

        ## data type check
        flgs = []
        for r in raw:
            for col in r.columns:
                if col in ['activity', 'subject']:
                    flgs += [r.dtypes[col] == np.dtype(np.int8)]
                else:
                    flgs += [r.dtypes[col] == np.dtype(np.float64)]
        self.assertTrue(all(flgs))

        ## data check
        self.assertTrue(all([
            set(np.unique(r['subject'])) == set([r['subject'].iloc[0]]) for r in raw
        ]))
        self.assertSetEqual(
            set(itertools.chain(*[np.unique(r['activity']).tolist() for r in raw])),
            set(self.activities)    # performed activity(protocol) + others
        )

    def test_reformat_fn(self):
        raw = load_raw(self.path)
        data, meta = reformat(raw)

        # compare between data and meta
        self.assertEqual(len(data), len(meta))

        # check meta
        ## type check
        self.assertIsInstance(meta, list)
        self.assertTrue(all(isinstance(m, pd.DataFrame) for m in meta))

        ## shape and column check
        self.assertTrue(all([list(m.columns) == ['activity', 'subject'] for m in meta]))

        ## data type check
        self.assertTrue(all([
            m.dtypes['activity'] == np.dtype(np.int8) and \
            m.dtypes['subject'] == np.dtype(np.int8) \
            for m in meta
        ]))

        ## data check
        self.assertTrue(all([
            set(np.unique(m['subject'])) == set([m['subject'].iloc[0]]) for m in meta
        ]))
        self.assertTrue(all([
            set(np.unique(m['activity'])) == set([m['activity'].iloc[0]]) for m in meta
        ]))

        # data
        ## type check
        self.assertIsInstance(data, list)
        self.assertTrue(all(isinstance(d, pd.DataFrame) for d in data))

        ## shape and column check
        self.assertTrue(all([
            list(d.columns) == \
            ['T_xacc', 'T_yacc', 'T_zacc', 'T_xgyro', 'T_ygyro', 'T_zgyro', 'T_xmag', 'T_ymag', 'T_zmag', 'RA_xacc', 'RA_yacc', 'RA_zacc', 'RA_xgyro', 'RA_ygyro', 'RA_zgyro', 'RA_xmag', 'RA_ymag', 'RA_zmag', 'LA_xacc', 'LA_yacc', 'LA_zacc', 'LA_xgyro', 'LA_ygyro', 'LA_zgyro', 'LA_xmag', 'LA_ymag', 'LA_zmag', 'RL_xacc', 'RL_yacc', 'RL_zacc', 'RL_xgyro', 'RL_ygyro', 'RL_zgyro', 'RL_xmag', 'RL_ymag', 'RL_zmag', 'LL_xacc', 'LL_yacc', 'LL_zacc', 'LL_xgyro', 'LL_ygyro', 'LL_zgyro', 'LL_xmag', 'LL_ymag', 'LL_zmag']
            for d in data
        ]))

        ## data type check
        self.assertTrue(all([
            d.dtypes[c] == np.dtype(np.float64) \
            for d in data for c in data[0].columns
        ]))

    def test_dsads_load_method_filed_labels(self):
        # check processing for exceptions
        x_labels = [
            'T_xacc', 'T_yacc', 'T_zacc',
            'T_xgyro', 'T_ygyro', 'T_zgyro',
            'T_xmag', 'T_ymag', 'T_zmag',
            'RA_xacc', 'RA_yacc', 'RA_zacc',
            'RA_xgyro', 'RA_ygyro', 'RA_zgyro',
            'RA_xmag', 'RA_ymag', 'RA_zmag',
            'LA_xacc', 'LA_yacc', 'LA_zacc',
            'LA_xgyro', 'LA_ygyro', 'LA_zgyro',
            'LA_xmag', 'LA_ymag', 'LA_zmag',
            'RL_xacc', 'RL_yacc', 'RL_zacc',
            'RL_xgyro', 'RL_ygyro', 'RL_zgyro',
            'RL_xmag', 'RL_ymag', 'RL_zmag',
            'LL_xacc', 'LL_yacc', 'LL_zacc',
            'LL_xgyro', 'LL_ygyro', 'LL_zgyro',
            'LL_xmag', 'LL_ymag', 'LL_zmag'
        ]
        y_labels = ['activity', 'subject']

        with self.assertRaises(ValueError):
            _, _ = self.loader.load(subjects=None, x_labels=y_labels)

        with self.assertRaises(ValueError):
            _, _ = self.loader.load(subjects=None, y_labels=x_labels)

        # shape check
        ## x
        for n in range(10, len(x_labels), 10):
            x, _ = self.loader.load(subjects=None, x_labels=np.random.choice(x_labels, n, replace=False).tolist())
            self.assertTupleEqual(x.shape[1:], (n, 125))
        ## y
        _, y = self.loader.load(subjects=None, y_labels=['activity'])
        self.assertEqual(y.shape[1], 1)
        _, y = self.loader.load(subjects=None, y_labels=['subject'])
        self.assertEqual(y.shape[1], 1)
        _, y = self.loader.load(subjects=None, y_labels=['activity', 'subject'])
        self.assertEqual(y.shape[1], 2)

    def test_dsads_load_method_filed_subjects(self):
        patterns = [
            [6, 2],
            [4, 3, 1, 2, 5],
            [5, 6, 4, 2],
            [8, 3, 4, 2, 1],
            [2, 5, 1, 7, 8, 4, 6, 3],
            [8],
            [5, 7, 2, 1, 8],
            [4],
            [8, 3, 6, 2, 1, 7, 5, 4],
            [2, 7, 1, 8, 4, 6, 5],
        ]

        for i, subjects in enumerate(patterns):
            print(f'pattern {i}')
            with self.subTest(f'pattern {i}'):
                _, y = self.loader.load(subjects=subjects)
                self.assertSetEqual(set(np.unique(y[:, 1])), set(subjects))

if __name__ == '__main__':
    args = sys.argv
    if len(args) != 2:
        sys.stderr.write('Usage: {} <dataset path>'.format(args[0]))
        sys.exit(1)

    ds_path = Path(args[1])

    DSADSTest.path = ds_path

    unittest.main(verbosity=2, argv=args[0:1])


import json
import os

def report_file():
	if not os.path.exists("reports"):
		os.makedirs("reports")
	return "reports/keys_list.txt"

def get_keys(start=0, end=49):
	key_set = []
	with open(report_file(), "w") as output:
		for idir in range(start, end):
			dir_name = "../repos/part{}".format(idir)
			for ifile in range(0,100):
				file_number = idir*100+ifile
				file_name = "repos_{}.json".format(file_number)
				with open(dir_name+"/"+file_name) as f:
					dicts = json.loads(f.read())
					for d in dicts:
						for key in d.keys():
							if key not in key_set:
								key_set.append(key)
								print >> output, "{} {}".format(ifile, key)

get_keys()


import pygame
import random
import sys

class Game:
    def __init__(self):
        pygame.init()
        self.screen = pygame.display.set_mode((900, 750))

        self.count_of_mines = 40

        self.field = []
        self.fill_empty_field()
        self.fill_mines()
        self.fill_other_types()

    def start(self):
        self.draw()
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()
                elif event.type == pygame.MOUSEBUTTONDOWN:
                    if event.button == 1:
                        pos = event.pos
                        self.open_cell(pos[0]//50,
                                       pos[1]//50 - 1)
                    elif event.button == 3:
                        pos = event.pos
                        self.set_flag(pos[1]//50 - 1,
                                      pos[0]//50)

                self.draw()
            pygame.display.flip()

    def set_flag(self, x, y):
        if not self.field[x][y].is_open:
            self.field[x][y].sprite = pygame.image.load('image/flag.png')
            self.field[x][y].is_open = True
            self.field[x][y].is_flag = True
        elif self.field[x][y].is_flag:
            self.field[x][y].sprite = pygame.image.load('image/closen_empty.png')
            self.field[x][y].is_flag = False
            self.field[x][y].is_open = False

    def open_cell(self, y, x):
        if x < 0 or y < 0 or \
           x > 13 or y > 17:
            return
        #print(self.field[x][y].type)
        for i in range(1, 9):
            if self.field[x][y].type == str(i):
                self.field[x][y].sprite = pygame.image.load(f'image/{i}.png')
                self.field[x][y].is_open = True
                self.field[x][y].is_flag = False
        if self.field[x][y].type == 'mine':
            self.field[x][y].sprite = pygame.image.load('image/lose_mine.png')
            self.field[x][y].is_open = True
            self.field[x][y].is_flag = False
        elif self.field[x][y].type == '0':
            self.field[x][y].sprite = pygame.image.load('image/empty.png')
            self.field[x][y].is_open = True
            self.field[x][y].is_flag = False
        #print(self.field[x][y].type)

    def fill_empty_field(self):
        x = 0
        y = 50
        for i in range(14):
            self.field.append([])
            for j in range(18):
                self.field[i].append(Cell(x, y))
                x += 50
            x = 0
            y += 50

    def fill_mines(self):
        for i in range(self.count_of_mines):
            while True:
                x = random.randint(0, 13)
                y = random.randint(0, 17)
                if self.field[x][y].type == None:
                    self.field[x][y].type = 'mine'
                    break

    def fill_other_types(self):
        for i in range(len(self.field)):
            for j in range(len(self.field[0])):
                if self.field[i][j].type == None:
                    self.field[i][j].type = self.get_type(i, j)
                    #print(self.field[i][j].type)

    def get_type(self, x, y):
        res = 0
        for dx in range(-1, 2):
            for dy in range(-1, 2):
                new_x = dx + x
                new_y = dy + y
                if new_x < 0 or new_y < 0 or \
                   new_x > 13 or new_y > 17:
                    continue
                #print(x, y, new_x, new_y)
                if self.field[new_x][new_y].type == 'mine':
                    res += 1
        return str(res)

    def draw(self):
        for i in self.field:
            for j in i:
                self.screen.blit(j.sprite, (j.x, j.y))

class Cell:
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.type = None
        self.sprite = pygame.image.load('image/closen_empty.png')
        self.width = 50
        self.height = 50
        self.is_open = False
        self.is_flag = False

game = Game()
game.start()


from django.shortcuts import render
from .models import Book, Author, BookInstance, Genre
from django.views import generic
from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import get_object_or_404
from django.http import HttpResponseRedirect
from django.urls import reverse
import datetime
from .forms import RenewBookForm
from django.contrib.auth.decorators import permission_required
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.urls import reverse_lazy

# Create your views here.

def index(request):
    """
    Функция отображения для домашней страницы сайта.
    """
    # Генерация "количеств" некоторых главных объектов
    num_books = Book.objects.all().count()
    num_instances = BookInstance.objects.all().count()
    # Доступные книги (статус = 'a')
    num_instances_available = BookInstance.objects.filter(status__exact='a').count()
    num_authors = Author.objects.count()  # Метод 'all()' применён по умолчанию.
    num_genre = Genre.objects.count()
    num_book = Book.objects.filter(title__icontains='Тарас Бульба').count()
    # Количество посещений этого представления, подсчитанное в переменной сеанса.
    num_visits = request.session.get('num_visits', 0)
    request.session['num_visits'] = num_visits + 1

    # Визуализация HTML-шаблона index.html с данными в контекстной переменной.
    return render(
        request,
        'index.html',
        context={'num_books': num_books, 'num_instances': num_instances,
                 'num_instances_available': num_instances_available, 'num_authors': num_authors,
                 'num_genre': num_genre, 'num_book': num_book, 'num_visits': num_visits},  # num_visits appended
    )

class BookListView(generic.ListView):
    model = Book
    paginate_by = 10
    context_object_name = 'book_list'  # ваше собственное имя переменной контекста в шаблоне
    template_name = 'books/book_list.html'  # Определение имени вашего шаблона и его расположения

    def get_queryset(self):
        return Book.objects.filter(title__icontains='')  # [:5]  # Получить 5 книг, содержащих 'war' в заголовке

    def get_context_data(self, **kwargs):
        # В первую очередь получаем базовую реализацию контекста
        context = super(BookListView, self).get_context_data(**kwargs)
        # Добавляем новую переменную к контексту и инициализируем её некоторым значением
        context['some_data'] = 'This is just some data'
        return context

class BookDetailView(generic.DetailView):
    model = Book

class AuthorListView(generic.ListView):
    model = Author
    paginate_by = 10
    context_object_name = 'author_list'  # ваше собственное имя переменной контекста в шаблоне
    template_name = 'authors/author_list.html'  # Определение имени вашего шаблона и его расположения

    def get_queryset(self):
        # return Author.objects.filter(last_name__icontains='')  # [:5]  # Получить 5 книг, содержащих 'war' в заголовке
        return Author.objects.all()

    def get_context_data(self, **kwargs):
        # В первую очередь получаем базовую реализацию контекста
        context = super(AuthorListView, self).get_context_data(**kwargs)
        # Добавляем новую переменную к контексту и инициализируем её некоторым значением
        context['some_data'] = 'This is just some data'
        return context

class AuthorDetailView(generic.DetailView):
    model = Author

class LoanedBooksByUserListView(LoginRequiredMixin, generic.ListView):
    """
    Общее представление на основе класса, в котором перечислены книги, предоставленные текущему пользователю.
    """
    model = BookInstance
    template_name = 'catalog/bookinstance_list_borrowed_user.html'
    paginate_by = 10

    def get_queryset(self):
        return BookInstance.objects.filter(borrower=self.request.user).filter(status__exact='o').order_by('due_back')

class LoanedBooksAllListView(PermissionRequiredMixin, generic.ListView):
    """Общее представление на основе классов, в котором перечислены все книги, предоставленные взаймы. Видно только
    пользователям с разрешением can_mark_returned. """
    model = BookInstance
    permission_required = 'catalog.can_mark_returned'
    template_name = 'catalog/bookinstance_list_borrowed_all.html'
    paginate_by = 10

    def get_queryset(self):
        return BookInstance.objects.filter(status__exact='o').order_by('due_back')

@permission_required('catalog.can_mark_returned')
def renew_book_librarian(request, pk):
    #  Функция просмотра для обновления определенного BookInstance библиотекарем
    book_inst = get_object_or_404(BookInstance, pk=pk)

    # Если данный запрос типа POST, тогда
    if request.method == 'POST':

        # Создаём экземпляр формы и заполняем данными из запроса (связывание, binding):
        form = RenewBookForm(request.POST)

        # Проверка валидности данных формы:
        if form.is_valid():
            # Обработка данных из form.cleaned_data
            # (здесь мы просто присваиваем их полю due_back)
            book_inst.due_back = form.cleaned_data['renewal_date']
            book_inst.save()

            # Переход по адресу 'all-borrowed':
            return HttpResponseRedirect(reverse('all-borrowed'))

    # Если это GET (или какой-либо ещё), создать форму по умолчанию.
    else:
        proposed_renewal_date = datetime.date.today() + datetime.timedelta(weeks=3)
        form = RenewBookForm(initial={'renewal_date': proposed_renewal_date, })

    return render(request, 'catalog/book_renew_librarian.html', {'form': form, 'bookinst': book_inst})

class AuthorCreate(CreateView):
    model = Author
    fields = '__all__'
    initial = {'date_of_death': '12/10/2016', }
    permission_required = 'catalog.can_mark_returned'

class AuthorUpdate(UpdateView):
    model = Author
    fields = ['first_name', 'last_name', 'date_of_birth', 'date_of_death']
    permission_required = 'catalog.can_mark_returned'

class AuthorDelete(DeleteView):
    model = Author
    success_url = reverse_lazy('authors')
    permission_required = 'catalog.can_mark_returned'

class BookCreate(PermissionRequiredMixin, CreateView):
    model = Book
    fields = ['title', 'author', 'summary', 'isbn', 'genre']
    permission_required = 'catalog.can_mark_returned'

class BookUpdate(PermissionRequiredMixin, UpdateView):
    model = Book
    fields = ['title', 'author', 'summary', 'isbn', 'genre']
    permission_required = 'catalog.can_mark_returned'

class BookDelete(PermissionRequiredMixin, DeleteView):
    model = Book
    success_url = reverse_lazy('books')
    permission_required = 'catalog.can_mark_returned'


from iqoptionapi.stable_api import IQ_Option
from datetime import datetime, date, timedelta
from dateutil import tz
import time, json

# função open config.json
def config():

    with open("config.json") as f:
        return json.load(f)

    return False

# tornar config GLOBAL
config = config()

# função converter time em data
def timestamp_converter(x, retorno = 1):

    timestamp = datetime.fromtimestamp(x).strftime('%Y-%m-%d')

    return timestamp

API = IQ_Option(config['user_email'], config['password'])

# realizar conexão
API.connect()
API.change_balance(config['change_balance'])

# checar conexão e reconnectar caso não esteja logado
tentativas = 0
while not API.check_connect():
    if tentativas < config['max_reconnetion']:
        print("Conectando ao servidor IQ Option, tentativa "+str(tentativas))
        API.connect()
        tentativas = tentativas+1
    else:
        print("Erro ao realizr conexão, tentativas excedidas!")
        break

    time.sleep(1)

print("Usuário logada na API IQOptions com sucesso!")

def get_list_position():

    list_id_trader = []
    ranking = API.get_leader_board('Worldwide', 1, 100, 5)

    for i in ranking['result']['positional']:

        id = ranking['result']['positional'][i]['user_id']

        list_id_trader.append(id)

    return list_id_trader

# carregar list ranking
print("Carregando list do ranking")
list_ranking = get_list_position()

print(list_ranking)
print("======================================================")
print("Aguardando capturar traders ranking")
API.subscribe_live_deal(config['type_par'], config['par_change'], config['timeframe'], 10)

trader_old = 0
while True:
    traders = API.get_live_deal(config['type_par'], config['par_change'], config['timeframe'])

    if len(traders) > 0 and traders[0]['user_id'] != trader_old:

         #print(json.dumps(traders[0], indent=1))

         id = traders[0]['user_id']
         if id in list_ranking:
            print("O Trader "+str(traders[0]['name'])+" estar operando")

            status, id = API.buy(traders[0]['amount_enrolled'], config['par_change'], traders[0]['instrument_dir'], 1)

            if status:
                print("[!] Ordem enviada com sucesso. Detalhes: DIR -> "+str(traders[0]['instrument_dir']))
         trader_old = traders[0]['user_id']

    time.sleep(1)


### --- ARCHIVOS FUNCIONES AUXILIARES --- ###

# Librerias

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.gridspec import GridSpec
from matplotlib.lines import Line2D
from pylab import rcParams
from scipy import stats

# Funcion Correlacion entre atributos y vector objetivo

def fetch_features (df, vec_ob):
    columns = df.columns #extraemos los nombres de las columnas en la base de datos

    # generamos 3 arrays vacíos para guardar los valores
    attr_name = [] # nombre de la variable
    pearson_r = [] # correlación de pearson
    abs_pearson_r = [] # valor absoluto de la correlación

    # para cada columna en el array de columnas
    for col in columns:
        # si la columna no es la dependiente
        if col != vec_ob:
            # adjuntar el nombre de la variable en attr_name
            attr_name.append(col)
            # adjuntar la correlación de pearson
            pearson_r.append(df[col].corr(df[vec_ob]))
            # adjuntar el absoluto de la correlación de pearson
            abs_pearson_r.append(abs(df[col].corr(df[vec_ob])))

    # transformamos los arrays en un DataFrame
    features = pd.DataFrame({'attribute': attr_name,'corr':pearson_r,'abs_corr':abs_pearson_r})
    # generamos el index con los nombres de las variables
    features = features.set_index('attribute')
    # ordenamos los valores de forma descendiente
    feat_sort = features.sort_values(by=['abs_corr'], ascending=False)

    return feat_sort

# Función para la importancia de atributos

def plot_feature_importance(fit_model, feat_names):
    """
    Plot relative importance of a feature subset given a fitted model.
    """

    # Seteamos el tama{o de nuestro plot
    rcParams['figure.figsize'] = 10, 5
    # Guardamos las columnas de nuestro conjunto de entrenamiento
    features = feat_names
    # Obtenemos la importancia de nuestros atributos desde el modelo entrenado
    importances = fit_model.feature_importances_
    # Ordenamos de maoyr a menor la importancia de nustro atributos
    indices = np.argsort(importances)

    # Graficamos
    plt.title('Feature Importance')
    plt.barh(range(len(indices)), importances[indices], color='b', align='center')
    plt.yticks(range(len(indices)), [features[i] for i in indices])
    plt.xlabel('porcentaje de importancia relativa')
    plt.show()

# Función para gráficar metricas de desempeño

def plot_metrics (metrics, models, title):

    length = len(metrics)
    x_labels = models

    # Set plot parameters
    rcParams['figure.figsize'] = 10, 6
    fig, ax = plt.subplots()
    width = 0.2 # width of bar
    x = np.arange(length)

    rects1 = ax.bar(x, metrics[:,0], width, color='#000080', label='MAE s/imputar')
    rects2 = ax.bar(x + width, metrics[:,1], width, color='#0F52BA', label='MAE imputados')
    rects3 = ax.bar(x + (2 * width), metrics[:,2], width, color='#6593F5', label='RMSE s/imputar')
    rects4 = ax.bar(x + (3 * width), metrics[:,3], width, color='#73C2FB', label='RMSE imputados')

    ax.set_ylabel('Minutos')
    ax.set_ylim(0,75)
    ax.set_xticks(x + width + width/2)
    ax.set_xticklabels(x_labels)
    ax.set_xlabel('Modelos')
    ax.set_title(title)
    ax.legend()
    plt.grid(True, 'major', 'y', ls='--', lw=.9, c='k', alpha=.3)
    fig.tight_layout()
    plt.show()


import time
import json
from datetime import date
from dynaconf import settings
from requests.exceptions import HTTPError
from investagram_data_loader.logger import logging
from investagram_data_loader.repository.ig_api import InvestagramApi
from investagram_data_loader.constants.mode import DateMode
from investagram_data_loader.date.date_handler import generate_date_range
from investagram_data_loader.repository.sqlite_dao import SqliteDao
from investagram_data_loader.reader.data_reader import get_stocks
from investagram_data_loader.exceptions import NoDataFoundException

# global settings here
SLEEP_TIME = 10
FROM_DATE, TO_DATE = date(2020, 1, 1), date(2021, 2, 20)

def load_transactions_by_stock(api: InvestagramApi, stock_code: str, mode: DateMode, from_date: date,
                               to_date: date) -> list:
    transactions = []
    logging.info(f'Loading transaction by stock code {stock_code} from {from_date} to {to_date} on a {mode} basis...')

    stock = api.get_stock_info(stock_code)

    for range_start, range_end in generate_date_range(mode, from_date, to_date):
        transactions.extend(api.get_stock_transaction_by_stock_id_and_date(stock.stock_id, range_start, range_end))

    return transactions

def load_transactions_by_broker(api: InvestagramApi, broker_code: str, mode: DateMode, from_date: date, to_date: date):
    transactions = []
    logging.info(f'Loading transaction by broker code {broker_code} from {from_date} to {to_date} on a {mode} basis...')

    broker = api.get_broker_info(broker_code)

    for range_start, range_end in generate_date_range(mode, from_date, to_date):
        transactions.extend(api.get_stock_transaction_by_broker_id_and_date(
            broker.stock_broker_id, range_start, range_end)
        )

    return transactions

def main():
    logging.info("Downloading data from investagram...")
    # replace with 'size=-1' to download all the stocks data
    stocks, unprocessed = get_stocks(size=-1), []

    with SqliteDao() as dao:
        with InvestagramApi() as api:
            for stock in stocks:
                try:
                    transactions = load_transactions_by_stock(api, stock, DateMode.DAILY, FROM_DATE, TO_DATE)
                    dao.bulk_insert_transactions(transactions)

                    logging.info(f'Sleeping for {SLEEP_TIME} seconds...')
                    time.sleep(SLEEP_TIME)

                except NoDataFoundException as exp:
                    logging.error(f'No data found for stock code {stock}.')
                    unprocessed.append(stock)

                except HTTPError as exp:
                    logging.error(f'Non-zero http status found {exp}.')
                    unprocessed.append(stock)

    if len(unprocessed) != 0:
        logging.error(f'Found {len(unprocessed)} unprocessed stock codes: {unprocessed}. Dumping to a file.')

        with open(settings.UNPROCESSED_CODES_FILE, 'w') as fd:
            fd.writelines(unprocessed)

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
import requests, json

# Create your views here.
def home(request):
    return HttpResponse("Hello, Django!")

def taskstring(request):
    result = 'Rest API string!'
    return HttpResponse(result, content_type="text/plain")

def taskxml(request):
    result = '''<employees>
                <employee><firstName>John</firstName> <lastName>Doe</lastName></employee>
                <employee><firstName>Anna</firstName> <lastName>Smith</lastName></employee>
                </employees>'''
    return HttpResponse(result, content_type='text/xml')

def taskjson(request):
    result = {"employees":[{"firstName":"John", "lastName":"Doe"},
                           {"firstName":"Anna", "lastName":"Smith"},
                           {"firstName":"Peter", "lastName":"Jones"}]}
    return JsonResponse(result)

def openweatherget(request):
    url = 'http://api.openweathermap.org/data/2.5/weather?q={}&appid=fd3121df45fa165283b9f3fe57b82b19'
    cities = ['London', "Seoul"]
    for city in cities:
        return city
    c = url.format(city)
    headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36'}
    res = requests.get(url=c, headers=headers)
    res.raise_for_status()
    rt_dict = json.loads(res.content)
    return JsonResponse(rt_dict)

    #depts = {'name':[{'f':'moon'},{'s':'seob'}]}
    #print(depts['name'][0]['f'])
    #output(moon)


from unittest import TestCase

import textdistance
from gensim import matutils
from numpy.core.multiarray import dot

from model_util.entity_vector.entity_vector_model import EntityVectorModel
from script.knowledgeFusion.affinity_propagation import pre_process

class TestAffinityPropagation(TestCase):
    def test_pre_process(self):
        data_list = [{"id": 1, "sentence_id": 1, "paragraph_id": 1, "sentence": "just a test sentence",
                    "merge_np_id": 1, "noun_phrase": "test", "sub_nodes": []},
                     {"id": 2, "sentence_id": 2, "paragraph_id": 2, "sentence": "just a test sentence",
                      "merge_np_id": 1, "noun_phrase": "test", "sub_nodes": []},
                     {"id": 3, "sentence_id": 3, "paragraph_id": 3, "sentence": "just a test sentence",
                      "merge_np_id": 3, "noun_phrase": "test2", "sub_nodes": []},
                     {"id": 4, "sentence_id": 4, "paragraph_id": 4, "sentence": "test case",
                      "merge_np_id": 4, "noun_phrase": "test4", "sub_nodes": []},
                     {"id": 5, "sentence_id": 5, "paragraph_id": 5, "sentence": "just a test sentence",
                      "merge_np_id": 3, "noun_phrase": "test5", "sub_nodes": []}]
        result = pre_process(data_list)
        for each in result:
            print(each)
            print(len(each["sub_nodes"]))

    def test_similarity_calculation(self):
        str1 = "AbstractInputMethodService provides a abstract base class for inut methods."
        str2 = "The default implementation in this abstract class returns 1.0 for all components."

        vector_map = EntityVectorModel.load("mean_vector_api_paragraph.plain.txt", binary=False)

        vector1 = vector_map.compute_mean_vector(str1)
        vector2 = vector_map.compute_mean_vector(str2)
        semantic_similarity = dot(matutils.unitvec(vector1), matutils.unitvec(vector2))
        print("semantic similarity is " + semantic_similarity)
        structure_similarity = textdistance.jaccard(str1, str2)
        print("structure similarity is " + structure_similarity)


from flask.testing import FlaskClient

from main.models.movie import MovieModel
from main.models.phone import PhoneModel

sample_phone_uuid = "518361D5-B7DA-42D6-9CA6-025CB35BEF80"

class CustomClient(FlaskClient):
    def __init__(self, *args, **kwargs):
        self._headers = {
            "Authorization": f'Bearer {kwargs.pop("token")}',
        }

        super().__init__(*args, **kwargs)

    def _prepare_kwargs(self, kwargs):
        if "headers" in kwargs:
            kwargs["headers"].update(self._headers)
        else:
            kwargs["headers"] = self._headers

        return kwargs

    def get(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().get(*args, **kwargs)

    def post(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().post(*args, **kwargs)

    def put(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().put(*args, **kwargs)

    def delete(self, *args, **kwargs):
        kwargs = self._prepare_kwargs(kwargs)

        return super().delete(*args, **kwargs)

def create_sample_phone(session) -> PhoneModel:
    phone = PhoneModel(uuid=sample_phone_uuid)
    session.add(phone)
    session.commit()
    return phone

def create_sample_movie(title: str, uuid: str, session) -> MovieModel:
    movie = MovieModel(
        title=title, description="a", uuid=uuid, release_year=1970, is_a_series=False
    )
    session.add(movie)
    session.commit()
    return movie


import multiprocessing
import numpy as np
import pandas as pd
from pathlib import Path
import pickle

import replays_to_database

def process_replay_folder(replay_folder):
    replay_paths = list(replay_folder.glob('*[0-9].json'))
    sub_id = int(replay_folder.name)
    print(f'Converting {len(replay_paths)} replays to visualization format for submission {sub_id} in {replay_folder}')

    n_replays_processed = 0
    for rp in replay_paths:
        # if not Path(f'{replay_folder}/{rp.stem}_visualization_dict.pkl').exists() or FORCE:
        replay = replays_to_database.read_json(rp)
        replay_info = replays_to_database.read_json(rp.parent / (rp.stem + '_info.json'))
        if replay_info['agents'][0]['submissionId'] == sub_id:
            agent_idx = 0
        elif replay_info['agents'][1]['submissionId'] == sub_id:
            agent_idx = 1
        else:
            raise RuntimeError(f'{rp}: Neither agent has the correct submissionId')
        replay_viz_dict = replays_to_database.convert_replay_for_visualization(replay, agent_idx)

        if replay_viz_dict is not None:
            with open(f'{replay_folder}/{rp.stem}_visualization_dict.pkl', 'wb') as f:
                pickle.dump(replay_viz_dict, f, pickle.HIGHEST_PROTOCOL)
            n_replays_processed += 1

    return n_replays_processed

if __name__ == '__main__':
    DESTINATION = 'episode_scraping/replays_by_sub_id'
    sub_id_to_model_name_df = pd.read_csv(f'{DESTINATION}/sub_id_to_model_name.csv', index_col='sub_id')
    nan_mask = sub_id_to_model_name_df.index.isna() | np.any(sub_id_to_model_name_df.isna(), axis=1)
    if nan_mask.any():
        print('Ignoring the following nan-valued rows:')
        sub_id_to_model_name_df = sub_id_to_model_name_df[~nan_mask]
        sub_id_to_model_name_df.index = sub_id_to_model_name_df.index.astype(int)
    sub_id_to_model_names_dict = sub_id_to_model_name_df.to_dict()['model_name']
    assert len(np.unique(list(sub_id_to_model_name_df.index))) == len(sub_id_to_model_name_df)

    replay_folders = [Path(f'{DESTINATION}/{sub_id}') for sub_id in sub_id_to_model_names_dict.keys()]
    with multiprocessing.Pool(processes=multiprocessing.cpu_count() - 1) as pool:
        n_replays = pool.map(process_replay_folder, replay_folders)

    print(f'Finished processing {np.sum(n_replays)} replays from {len(replay_folders)} submissions')


from tkinter import *

import numpy as np
from PIL import ImageTk, Image
from tkinter import filedialog
import cv2

imgX = Image.open("Q2.png")
x = None

# open image function
def open_img():
    # Select the ImageName from a folder
    global x
    x = openfilename()
    # opens the image
    img = Image.open(x)
    # resize the image and apply a high-quality down sampling filter
    img = img.resize((640, 480), Image.ANTIALIAS)
    # PhotoImage class is used to add image to widgets, icons etc
    img = ImageTk.PhotoImage(img)
    # create a label
    panel = Label(root, image=img)
    # set the image as img
    panel.image = img
    panel.grid(row=2, column=1)

# function to get file name
def openfilename():
    # open file dialog box to select image
    # The dialogue box has a title "Open"
    filename = filedialog.askopenfilename(title='Open')
    return filename

# transform to grayscale using PIL
def gray():
    global imgX
    imgX = Image.open(x)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS).convert('L')
    imgX1 = ImageTk.PhotoImage(imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# invert image using cv2
def invert():
    global imgX
    # read & process image using cv2
    imgX1 = cv2.imread(x)
    imgX1 = cv2.bitwise_not(imgX1)

    # convert cv2 image to imageTk
    b, g, r = cv2.split(imgX1)
    imgX1 = cv2.merge((r, g, b))
    print(imgX1)
    imgX = Image.fromarray(imgX1)

    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)

def grayslice():
    global imgX
    # read & process image using cv2
    imgX1 = cv2.imread(x, 0)
    # Find width and height of image
    row, column = imgX1.shape

    # Specify the min and max range
    min_range = 10
    max_range = 100

    # Loop over the input image and if pixel value lies in desired range set it to 255 otherwise set it to 0.
    for i in range(row):
        for j in range(column):
            if min_range < imgX1[i, j] < max_range:
                imgX1[i, j] = 255
            else:
                imgX1[i, j] = 0

    # convert cv2 image to imageTk
    imgX = Image.fromarray(imgX1)

    imgX = imgX.resize((640, 480))
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# Bit plane slicing
def bitPlaneSlice():
    global imgX
    imgX = cv2.imread(x, 0)
    # Iterate over each pixel and change pixel value to binary using np.binary_repr() and store it in a list.
    lst = []
    for i in range(imgX.shape[0]):
        for j in range(imgX.shape[1]):
            lst.append(np.binary_repr(imgX[i][j], width=8))  # width = no. of bits

    # We have a list of strings where each string represents binary pixel value. To extract bit planes we need to
    # iterate over the strings and store the characters corresponding to bit planes into lists. Multiply with 2^(n-1)
    # and reshape to reconstruct the bit image.
    eight_bit_img = (np.array([int(i[0]) for i in lst], dtype=np.uint8) * 128).reshape(imgX.shape[0], imgX.shape[1])
    seven_bit_img = (np.array([int(i[1]) for i in lst], dtype=np.uint8) * 64).reshape(imgX.shape[0], imgX.shape[1])
    six_bit_img = (np.array([int(i[2]) for i in lst], dtype=np.uint8) * 32).reshape(imgX.shape[0], imgX.shape[1])
    five_bit_img = (np.array([int(i[3]) for i in lst], dtype=np.uint8) * 16).reshape(imgX.shape[0], imgX.shape[1])
    four_bit_img = (np.array([int(i[4]) for i in lst], dtype=np.uint8) * 8).reshape(imgX.shape[0], imgX.shape[1])
    three_bit_img = (np.array([int(i[5]) for i in lst], dtype=np.uint8) * 4).reshape(imgX.shape[0], imgX.shape[1])
    two_bit_img = (np.array([int(i[6]) for i in lst], dtype=np.uint8) * 2).reshape(imgX.shape[0], imgX.shape[1])
    one_bit_img = (np.array([int(i[7]) for i in lst], dtype=np.uint8) * 1).reshape(imgX.shape[0], imgX.shape[1])

    # Concatenate these images for ease of display using cv2.hconcat()
    finalR = cv2.hconcat([eight_bit_img, seven_bit_img, six_bit_img, five_bit_img])
    finalV = cv2.hconcat([four_bit_img, three_bit_img, two_bit_img, one_bit_img])

    # Vertically concatenate
    final = cv2.vconcat([finalR, finalV])

    # display edited image.
    imgX = Image.fromarray(final)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# Contrast Stretching
def contrastStretch():
    global imgX
    # Open the image.
    imgX1 = cv2.imread(x, 0)
    # Define parameters.
    r1 = 100
    s1 = 0
    r2 = 200
    s2 = 255
    row, column = imgX1.shape
    for i in range(row):
        for j in range(column):
            if 0 <= imgX1[i, j] <= r1:
                imgX1[i, j] = (s1 / r1) * imgX1[i, j]
            elif r1 < imgX1[i, j] <= r2:
                imgX1[i, j] = ((s2 - s1) * (imgX1[i, j] - r1) / (r2 - r1)) + s1
            else:
                imgX1[i, j] = ((255 - s2) * (imgX1[i, j] - r2) / (255 - r2)) + s2

    # display edited image.
    imgX = Image.fromarray(imgX1)
    imgX = imgX.resize((640, 480), Image.ANTIALIAS)
    imgX1 = ImageTk.PhotoImage(image=imgX)
    panel = Label(root, image=imgX1)
    panel.image = imgX1
    panel.grid(row=2, column=2)
    return

# save image to directory
def save():
    image = imgX
    image.save("Q2.png")
    return

# Create a window
root = Tk()
# Set Title as Image Loader
root.title("Image Processing")
# Set the resolution of window
root.geometry("1920x1080")
# Allow Window to be resizable
root.resizable(width=True, height=True)
# Create a button and place it into the window using grid layout
btnOp = Button(root, text='Open', command=open_img).grid(row=1, column=1, padx=300, pady=10)
btnSave = Button(root, text='save', command=save).grid(row=1, column=2, padx=300, pady=10)

imag = Image.open('Q.jpg')
imag = imag.resize((640, 480), Image.ANTIALIAS)
imag = ImageTk.PhotoImage(imag)
panel1 = Label(root, image=imag)
panel2 = Label(root, image=imag)
panel1.image = imag
panel2.image = imag
panel1.grid(row=2, column=1)
panel2.grid(row=2, column=2)

# btnGray = Button(root, text='grayscale', command=gray).grid(row=3, column=1, padx=300, pady=10)
# btnInv = Button(root, text='Invert', command=invert).grid(row=4, column=1, padx=300, pady=10)
# btnGSlice = Button(root, text='Intensity Slicing', command=grayslice).grid(row=5, column=1, padx=300, pady=10)
# btnBPSlice = Button(root, text='Bit Plane Slicing', command=bitPlaneSlice).grid(row=5, column=1, padx=300, pady=10)
btnCS = Button(root, text='Contrast Stretch', command=contrastStretch).grid(row=6, column=1, padx=300, pady=10)
root.mainloop()


from flask import render_template, request, jsonify

from . import storage, VERSION

def todo_page():
    data = {"VERSION": VERSION}
    return render_template("service/todo.html", **data)

def todo_json_get():
    items = storage.Todo.select()

    data = []
    for item in items:
        row = {
            "id": item.id,
            "title": item.title,
            "description": item.description,
            "created": str(item.created),
            "updated": str(item.updated),
            "status": item.status,
            "priority": item.priority
        }
        data.append(row)

    return jsonify(items=data)

def todo_json_add():

    in_data = request.get_json()
    if in_data is None:
        raise Exception('no input data')

    storage.Todo.create(title=in_data["title"],
                        description=in_data["description"],
                        status=in_data["status"],
                        priority=in_data["priority"])
    last_record = storage.Todo.select().order_by(storage.Todo.id.desc()).get()

    in_data["id"] = last_record.id
    in_data["created"] = str(last_record.created)
    in_data["updated"] = str(last_record.updated)
    return jsonify(item=in_data)

def todo_json_update():

    in_data = request.get_json()

    if in_data is None:
        raise Exception('no input data')

    item = storage.Todo.get(storage.Todo.id == in_data["id"])
    item.title = in_data["title"]
    item.description = in_data["description"]
    item.status = in_data["status"]
    item.priority = in_data["priority"]
    item.updated = storage.sql_date()
    item.save()

    in_data["updated"] = str(item.updated)

    return jsonify(item=in_data)

def todo_json_remove():

    in_data = request.get_json()

    if in_data is None:
        raise Exception('no input data')

    item = storage.Todo.get(storage.Todo.id == in_data["id"])
    item.delete_instance()

    return jsonify(item=in_data)


from typing import *

import os
import sys
import tempfile
import subprocess

import enum
import attrs
import hydra.types
import hydra.utils
import hydra.core.config_store
from omegaconf import OmegaConf, MISSING, DictConfig

import torch

from denoised_mdp import utils
from denoised_mdp.envs import EnvConfig
from denoised_mdp.memory import ExperienceReplayConfig
from denoised_mdp.agents import (
    DenoisedMDP, DynamicsBackpropagateActorCritic, SoftActorCritic,
    BasePolicyLearning, VariationalModelLearning
)

@attrs.define(kw_only=True, auto_attribs=True)
class DeviceConfig:
    _target_: str = 'torch.device'

    type: str = 'cuda'
    index: Optional[int] = 0

def get_dotted_name(cls: Type):
    return f"{cls.__module__}.{cls.__qualname__}"

cs = hydra.core.config_store.ConfigStore.instance()

@attrs.define(kw_only=True, auto_attribs=True)
class Config:
    defaults: List = attrs.Factory(lambda: [
        {'learning/policy_learning': 'dynamics_backprop'},
        {'override hydra/hydra_logging': 'disabled'},  # we handle ourselves
        {'override hydra/job_logging': 'disabled'},  # we handle ourselves
        '_self_',
    ])
    _target_: str = attrs.Factory(lambda: get_dotted_name(InstantiatedConfig))

    # Disable hydra working directory creation
    hydra: Dict = dict(
        output_subdir=None,
        job=dict(chdir=False),
        run=dict(dir=tempfile.TemporaryDirectory().name),  # can't disable https://github.com/facebookresearch/hydra/issues/1937
        mode=hydra.types.RunMode.RUN,  # sigh: https://github.com/facebookresearch/hydra/issues/2262
    )

    # Directories: e.g., output

    base_git_dir: str = subprocess.check_output(
        r'git rev-parse --show-toplevel'.split(),
        cwd=os.path.dirname(__file__), encoding='utf-8').strip()

    def fmtdir(self, dir):
        return f"$GIT/{os.path.relpath(dir, self.base_git_dir)}"

    overwrite_output: bool = False
    output_base_dir: str = attrs.field(default=os.path.join(base_git_dir, 'results'))

    @output_base_dir.validator
    def exists_path(self, attribute, value):
        assert os.path.exists(value)

    output_folder: Optional[str] = None

    # Seeding
    seed: int = 60912
    test_seed: int = 1841
    device: DeviceConfig = DeviceConfig()

    # Env
    env: EnvConfig = EnvConfig()

    @attrs.define(kw_only=True, auto_attribs=True)
    class LearningConfig:
        _target_: str = attrs.Factory(lambda: get_dotted_name(InstantiatedConfig.LearningConfig))

        experience_replay: ExperienceReplayConfig = ExperienceReplayConfig()

        @attrs.define(kw_only=True, auto_attribs=True)
        class OptimizationConfig:
            _target_: str = attrs.Factory(lambda: get_dotted_name(InstantiatedConfig.LearningConfig.OptimizationConfig))

            adam_eps: float = attrs.field(default=1e-7, validator=attrs.validators.gt(0))
            train_interval: int = attrs.field(default=1000, validator=attrs.validators.gt(0))  # num steps between training
            train_iterations: int = attrs.field(default=100, validator=attrs.validators.gt(0))  # num gd steps per training
            model_every: int = attrs.field(default=1, validator=attrs.validators.gt(0))  # during train_iterations steps, frequency of model updatres
            policy_every: int = attrs.field(default=1, validator=attrs.validators.gt(0))  # during train_iterations steps, frequency of policy updatres

        optimization: OptimizationConfig = attrs.Factory(OptimizationConfig)

        model: DenoisedMDP.Config = DenoisedMDP.Config()
        model_learning: VariationalModelLearning.Config = VariationalModelLearning.Config()
        # See `defaults` list above. Related: https://github.com/facebookresearch/hydra/issues/2227
        policy_learning: BasePolicyLearning.Config = MISSING

        @attrs.define(kw_only=True, auto_attribs=True)
        class ExplorationConfig:
            _target_: str = attrs.Factory(lambda: get_dotted_name(InstantiatedConfig.LearningConfig.ExplorationConfig))

            action_noise: float = attrs.field(default=0.3, validator=attrs.validators.gt(0))
            prefill_steps: int = attrs.field(default=5000, validator=attrs.validators.gt(0))
            total_steps: int = attrs.field(default=int(1e6), validator=attrs.validators.gt(0))

        exploration: ExplorationConfig = attrs.Factory(ExplorationConfig)

        batch_size: int = attrs.field(default=50, validator=attrs.validators.gt(0))
        chunk_length: int = attrs.field(default=50, validator=attrs.validators.gt(0))

        @attrs.define(kw_only=True, auto_attribs=True)
        class TestConfig:
            _target_: str = attrs.Factory(lambda: get_dotted_name(InstantiatedConfig.LearningConfig.TestConfig))

            num_episodes: int = attrs.field(default=10, validator=attrs.validators.gt(0))
            visualize_num_episodes: int = attrs.field(default=3, validator=attrs.validators.gt(0))
            test_interval: int = attrs.field(default=10000, validator=attrs.validators.gt(0))
            visualize_interval: int = attrs.field(default=20000, validator=attrs.validators.gt(0))

        test: TestConfig = attrs.Factory(TestConfig)

        checkpoint_interval: int = attrs.field(default=250000, validator=attrs.validators.gt(0))

    learning: LearningConfig = attrs.Factory(LearningConfig)

global_config: Optional[Config] = None

@attrs.define(kw_only=True, auto_attribs=True)
class InstantiatedConfig(Config):
    config: Config = attrs.field(init=False)

    output_dir: str = attrs.field(init=False)

    # NOTE [ Pre-emption ]
    #
    # SIGUSR1 is our mechanism to pre-empt. Unpon SIGUSR1, we catch the signal,
    # start checkpointing (incl. replay buffer) via `ModelTrainer.save_resumable_state_if_possible`,
    # and finally kill the training job.
    #
    # We make 3 files/dirs for handling preemption.
    # 1. a COMPLETE file which signals that the jobs is done, so, upon SIGUSR1, no-op
    # 2. a temp CKPT dir, which we will write ckpt to upon SIGUSR1
    # 3. an actual CKPT dir, which we will copy/rename temp CKPT to, after finish writing to
    #    the temp CKPT, in case the job gets killed when we write to temp CKPT dir, leaving
    #    the ckpt in an inconsistent state.
    #
    # When we start running with the same `output_dir`, we will attempt to resume via `ModelTrainer.resume_from_saved_state_if_exists`.

    job_complete_file: str = attrs.field(init=False)
    tmp_preempt_ckpt_dir: str = attrs.field(init=False)
    preempt_ckpt_dir: str = attrs.field(init=False)
    received_SIGUSR1: bool = attrs.field(init=False)

    # logging
    logging_file: bool = attrs.field(init=False)

    def __attrs_post_init__(self):
        # Validate several entries
        assert global_config is not None
        self.config = global_config
        # Set output directory
        if self.output_folder is None:
            specs = [
                f'alpha={self.config.learning.model_learning.kl.alpha:g}',
                f'betaX={self.config.learning.model_learning.kl.beta_x:g}',
                f'betaY={self.config.learning.model_learning.kl.beta_y:g}',
                f'betaZ={self.config.learning.model_learning.kl.beta_z:g}',
                self.config.learning.policy_learning._kind_,
                f'seed={self.seed}',
            ]
            if self.config.learning.model_learning.kl.beta_free_nats_x is not None:
                specs.insert(2, f'betaFreeNatsX={self.config.learning.model_learning.kl.beta_free_nats_x:g}')
            self.output_folder = os.path.join(
                f'{self.config.env.kind.name}_{self.config.env.spec}',
                '_'.join(specs),
            )
        self.output_dir = os.path.join(self.output_base_dir, self.output_folder)
        os.makedirs(self.output_dir, exist_ok=True)
        # See NOTE [ Pre-emption ]
        self.job_complete_file = os.path.join(self.output_dir, 'COMPLETE')
        if os.path.exists(self.job_complete_file) and not self.overwrite_output:
            print(f"Complete run found at {self.output_dir} and overwrite_output=False, exiting.")
            sys.exit(0)
        self.tmp_preempt_ckpt_dir = os.path.join(self.output_dir, 'preempt.tmp')
        if os.path.exists(self.tmp_preempt_ckpt_dir):
            utils.rm_if_exists(self.tmp_preempt_ckpt_dir, maybe_dir=True)
        self.preempt_ckpt_dir = os.path.join(self.output_dir, 'preempt')
        self.received_SIGUSR1 = False
        # Logging
        self.logging_file = os.path.join(self.output_dir, 'output.log')
        utils.logging.configure(self.logging_file)

    device: torch.device
    env: EnvConfig.InstantiatedT

    @attrs.define(kw_only=True, auto_attribs=True)
    class LearningConfig(Config.LearningConfig):
        experience_replay: ExperienceReplayConfig.InstantiatedT
        model: DenoisedMDP.Config.InstantiatedT
        model_learning: VariationalModelLearning.Config.InstantiatedT
        policy_learning: BasePolicyLearning.Config.InstantiatedT

    learning: LearningConfig

cs.store(name='config', node=Config())
cs.store(group='learning/policy_learning', name='dynamics_backprop', node=DynamicsBackpropagateActorCritic.Config())
cs.store(group='learning/policy_learning', name='sac', node=SoftActorCritic.Config())

def to_config_and_instantiate(dict_cfg: DictConfig) -> Tuple[Config, InstantiatedConfig]:
    global global_config

    def convert(v: Any, desired_ty: Type):
        if isinstance(v, DictConfig):
            ty = OmegaConf.get_type(v)
            assert issubclass(ty, desired_ty)
            if attrs.has(ty):
                fields = attrs.fields_dict(ty)
                kwargs = {}
                for k, subv in v.items():
                    # sign, attrs auto-strip leading underscore so we have to manually do this
                    # rather than using `OmegaConf.to_object`
                    kwargs[k.lstrip('_')] = convert(subv, fields[k].type)
                v = ty(**kwargs)
        elif isinstance(desired_ty, type) and issubclass(desired_ty, enum.Enum):
            if isinstance(v, str) and v != MISSING:
                v = desired_ty[v]
        return v

    global_config = convert(dict_cfg, Config)
    return global_config, hydra.utils.instantiate(global_config)


import torch
from scripts.utils import *
from tqdm import tqdm

def train_model(model, optimizer, loss_function,
                data_dict, batch_size, word_indexer,
                tag_indexer, training_metrics, validation_metrics, num_epochs=5):
    val_words, val_tags = data_dict['dev']

    def validate_model():
        with torch.no_grad():
            inputs = torch.tensor(word_indexer.elements_to_index(val_words), dtype=torch.long)
            true_vals = tag_indexer.elements_to_index(val_tags)
            tag_scores = model(inputs)
            prediction = get_tag_indexes_from_scores(tag_scores)
        validation_metrics.update(prediction, true_vals)
        validation_metrics.collect()
        for metric in validation_metrics.metrics_dict.keys():
            print(f"{metric} - {validation_metrics.metrics_dict[metric][-1]}")
        print()

    words, tags = data_dict['train']

    batches = list(get_batched(words, tags, batch_size))

    losses = []

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch + 1, num_epochs))
        print('-' * 10)

        running_loss = 0.0

        for sentence, tags in tqdm(batches):
            model.zero_grad()

            sentence_in = torch.tensor(word_indexer.elements_to_index(sentence), dtype=torch.long)
            targets = torch.tensor(tag_indexer.elements_to_index(tags), dtype=torch.long)

            tag_scores = model(sentence_in)

            loss = loss_function(tag_scores, targets)
            loss.backward()
            optimizer.step()

            prediction = get_tag_indexes_from_scores(tag_scores.detach().numpy())
            training_metrics.update(prediction, targets)

            running_loss += loss.item() * sentence_in.size(0)

        training_metrics.collect()
        epoch_loss = running_loss / len(batches)

        losses.append(epoch_loss)
        print(f"Loss per epoch - {epoch_loss}")
        for metric in training_metrics.metrics_dict.keys():
            print(f"{metric} - {training_metrics.metrics_dict[metric][-1]}")
        print()
        print("Validating on dev test: ")
        validate_model()
        print()

    return model, training_metrics, validation_metrics, losses


#!/usr/bin/env python
import math
import matplotlib.pyplot as plt

def distance(p1, p2):
    return math.sqrt((p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2)

def distance2(p1, p2):
    return (p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2

def k_means(seeds, points, distance_fn):
    results = [[] for _ in range(len(seeds))]

    for point in points:
        min_dist = math.inf
        min_element = 0
        for i in range(len(seeds)):
            new_dist = distance_fn(seeds[i], point)
            if new_dist < min_dist:
                min_dist = new_dist
                min_element = i
        (results[min_element]).append(point)

    new_seeds = seeds[:]
    for i in range(len(seeds)):
        cluster = results[i]
        if len(cluster) < 1:
            continue
        center = seeds[i]
        distance_sum = (0, 0)
        for point in cluster:
            dist = (center[0] - point[0], center[1] - point[1])
            distance_sum = (distance_sum[0] + dist[0], distance_sum[1] + dist[1])
        mean = (distance_sum[0] / len(cluster), distance_sum[1] / len(cluster))
        new_seeds[i] = (new_seeds[i][0] - mean[0], new_seeds[i][1] - mean[1])

    return results, new_seeds

def plot_result(result, seeds):
    x_list = [list()] * len(result)
    y_list = [list()] * len(result)

    for i in range(len(x_list)):
        x_list[i] = [p[0] for p in result[i]]
        y_list[i] = [p[1] for p in result[i]]

    for i in range(len(x_list)):
        plt.plot(x_list[i], y_list[i], 'o')

    for seed in seeds:
        plt.plot(seed[0], seed[1], 'o')

    plt.xlim((-1, 6))
    plt.ylim((0, 3))
    plt.show()

seeds = [(1, 1.5), (3, 1)]
points = [
    (0, 0.8), (0, 0.5), (1.2, 0.4), (1.5, 0.8), (1, 1), (1.5, 1),
    (2.5, 1), (3, 1), (3, 2), (4, 1.5), (4, 2.5), (5, 2)
]

regular_result, new_seeds = k_means(seeds, points, distance)
plot_result(regular_result, seeds)

regular_result, _ = k_means(new_seeds, points, distance)
plot_result(regular_result, new_seeds)


import pytest
from click.testing import CliRunner

import time
import contextlib
import os
import traceback
import multiprocessing
from datetime import datetime, timedelta

import yaml

import requests
from fastapi import FastAPI
from alchemiscale.tests.integration.utils import running_service

from alchemiscale.cli import get_settings_from_options, cli
from alchemiscale.cli_utils import ApiApplication
from alchemiscale.models import Scope
from alchemiscale.security.auth import hash_key, authenticate, AuthenticationError
from alchemiscale.security.models import (
    CredentialedUserIdentity,
    CredentialedComputeIdentity,
)
from alchemiscale.settings import Neo4jStoreSettings
from alchemiscale.storage.statestore import Neo4JStoreError

def click_success(result):
    if result.exit_code != 0:  # -no-cov-  (only occurs on test error)
        print(result.output)
        traceback.print_tb(result.exc_info[2])
        print(result.exc_info[0], result.exc_info[1])
    return result.exit_code == 0

# based on https://stackoverflow.com/a/34333710
@contextlib.contextmanager
def set_env_vars(env):
    old_env = dict(os.environ)
    try:
        os.environ.update(env)
        yield
    finally:
        os.environ.clear()
        os.environ.update(old_env)

toyapp = FastAPI()

@toyapp.get("/ping")
def read_root():
    return {"Hello": "World"}

def test_api_application():
    # this checks that the gunicorn BaseApplication subclass works correctly
    # with a FastAPI app
    workers = 1
    host = "127.0.0.1"
    port = 50100
    app = ApiApplication.from_parameters(toyapp, workers, host, port)

    expected_ping = {"Hello": "World"}
    with running_service(app.run, port, args=tuple()):
        response = requests.get(f"http://{host}:{port}/ping")

    assert response.status_code == 200
    assert response.json() == expected_ping

def test_api(n4js, s3os):
    workers = 2
    host = "127.0.0.1"
    port = 50100
    command = ["api"]
    api_opts = ["--workers", workers, "--host", host, "--port", port]
    db_opts = [
        "--url",
        "bolt://localhost:7687",
        "--user",
        "neo4j",
        "--password",
        "password",
    ]
    s3_opts = [
        "--access-key-id",
        "test-key-id",
        "--secret-access-key",
        "test-key",
        "--session-token",
        "test-session-token",
        "--s3-bucket",
        "test-bucket",
        "--s3-prefix",
        "test-prefix",
        "--default-region",
        "us-east-1",
    ]
    jwt_opts = []  # leaving empty, we have default behavior for these

    expected_ping = {"api": "AlchemiscaleAPI"}

    runner = CliRunner()
    with running_service(
        runner.invoke, port, (cli, command + api_opts + db_opts + s3_opts + jwt_opts)
    ):
        response = requests.get(f"http://{host}:{port}/ping")

    assert response.status_code == 200
    assert response.json() == expected_ping

@pytest.fixture
def compute_api_args():
    workers = 2
    host = "127.0.0.1"
    port = 50100
    command = ["compute", "api"]
    api_opts = ["--workers", workers, "--host", host, "--port", port]
    db_opts = [
        "--url",
        "bolt://localhost:7687",
        "--user",
        "neo4j",
        "--password",
        "password",
    ]
    s3_opts = [
        "--access-key-id",
        "test-key-id",
        "--secret-access-key",
        "test-key",
        "--session-token",
        "test-session-token",
        "--s3-bucket",
        "test-bucket",
        "--s3-prefix",
        "test-prefix",
        "--default-region",
        "us-east-1",
    ]
    jwt_opts = []  # leaving empty, we have default behavior for these

    return host, port, (command + api_opts + db_opts + s3_opts + jwt_opts)

@pytest.fixture
def compute_service_config(compute_api_args):
    host, port, _ = compute_api_args

    config = {
        "init": {
            "api_url": f"http://{host}:{port}",
            "identifier": "test-compute-user",
            "key": "test-comute-user-key",
            "name": "test-compute-service",
            "shared_basedir": "./shared",
            "scratch_basedir": "./scratch",
            "loglevel": "INFO",
        },
        "start": {"max_time": None},
    }

    return config

def test_compute_api(n4js, s3os, compute_api_args):
    host, port, args = compute_api_args

    expected_ping = {"api": "AlchemiscaleComputeAPI"}

    runner = CliRunner()
    with running_service(runner.invoke, port, (cli, args)):
        response = requests.get(f"http://{host}:{port}/ping")

    assert response.status_code == 200
    assert response.json() == expected_ping

def test_compute_synchronous(
    n4js_fresh, s3os, compute_api_args, compute_service_config, tmpdir
):
    host, port, args = compute_api_args
    n4js = n4js_fresh

    # create compute identity; add all scope access
    identity = CredentialedComputeIdentity(
        identifier=compute_service_config["init"]["identifier"],
        hashed_key=hash_key(compute_service_config["init"]["key"]),
    )

    n4js.create_credentialed_entity(identity)
    n4js.add_scope(identity.identifier, CredentialedComputeIdentity, Scope())

    # start up compute API
    runner = CliRunner()
    with running_service(runner.invoke, port, (cli, args)):
        # start up compute service
        with tmpdir.as_cwd():
            command = ["compute", "synchronous"]
            opts = ["--config-file", "config.yaml"]

            with open("config.yaml", "w") as f:
                yaml.dump(compute_service_config, f)

            multiprocessing.set_start_method("fork", force=True)
            proc = multiprocessing.Process(
                target=runner.invoke, args=(cli, command + opts), daemon=True
            )
            proc.start()

            q = f"""
            match (csreg:ComputeServiceRegistration)
            where csreg.identifier =~ "{compute_service_config['init']['name']}.*"
            return csreg
            """
            while True:
                csreg = n4js.graph.run(q).to_subgraph()
                if csreg is None:
                    time.sleep(1)
                else:
                    break

            assert csreg["registered"] > datetime.utcnow() - timedelta(seconds=30)

            proc.terminate()
            proc.join()

@pytest.mark.parametrize(
    "cli_vars",
    [
        {},  # no cli options; all from env
        {
            "NEO4J_URL": "https://foo",
            "NEO4J_USER": "me",
            "NEO4J_PASS": "correct-horse-battery-staple",
        },  # all CLI options given (test without env vars)
        {
            "NEO4J_URL": "https://baz",
        },  # some CLI options given (test that we override)
    ],
)
def test_get_settings_from_options(cli_vars):
    env_vars = {
        "NEO4J_URL": "https://bar",
        "NEO4J_USER": "you",
        "NEO4J_PASS": "Tr0ub4dor&3",
    }
    # if we give all by CLI, we don't need the env_vars
    context_vars = {} if len(cli_vars) == 4 else env_vars

    expected = env_vars | cli_vars
    expected["NEO4J_DBNAME"] = "neo4j"  # leave as default

    kwargs = {k: cli_vars[k] if k in cli_vars else None for k in expected}
    with set_env_vars(context_vars):
        settings = get_settings_from_options(kwargs, Neo4jStoreSettings)
        settings_dict = settings.dict()

    for key in expected:
        assert expected[key] == settings_dict[key]

def test_database_init(n4js_fresh):
    n4js = n4js_fresh
    # ensure the database is empty
    n4js.reset()

    with pytest.raises(Neo4JStoreError):
        n4js.check()

    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }

    # run the CLI
    runner = CliRunner()
    with set_env_vars(env_vars):
        result = runner.invoke(cli, ["database", "init"])
        assert click_success(result)

    assert n4js.check() is None

def test_database_check(n4js_fresh, network_tyk2, scope_test):
    n4js = n4js_fresh

    # set starting contents
    n4js.create_network(network_tyk2, scope_test)

    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }

    # run the CLI
    runner = CliRunner()
    with set_env_vars(env_vars):
        result = runner.invoke(cli, ["database", "check"])
        assert click_success(result)

        n4js.reset()

        result = runner.invoke(cli, ["database", "check"])
        assert not click_success(result)

        n4js.initialize()

        result = runner.invoke(cli, ["database", "check"])
        assert click_success(result)

def test_database_reset(n4js_fresh, network_tyk2, scope_test):
    n4js = n4js_fresh

    # set starting contents
    n4js.create_network(network_tyk2, scope_test)

    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }

    # run the CLI
    runner = CliRunner()
    with set_env_vars(env_vars):
        result = runner.invoke(cli, ["database", "reset"])
        assert click_success(result)

    assert n4js.graph.run("MATCH (n) WHERE NOT n:NOPE RETURN n").to_subgraph() is None

    with pytest.raises(Neo4JStoreError):
        n4js.check()

@pytest.mark.parametrize(
    "identity_type",
    [("user", CredentialedUserIdentity), ("compute", CredentialedComputeIdentity)],
)
def test_identity_add(n4js_fresh, identity_type):
    n4js = n4js_fresh
    identity_type_str, identity_type_cls = identity_type
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        ident = "bill"
        key = "and ted"
        result = runner.invoke(
            cli,
            [
                "identity",
                "add",
                "--identity-type",
                identity_type_str,
                "--identifier",
                ident,
                "--key",
                key,
            ],
        )
        assert click_success(result)

        cred = authenticate(n4js, identity_type_cls, ident, key)
        assert cred

@pytest.mark.parametrize(
    "identity_type",
    [("user", CredentialedUserIdentity), ("compute", CredentialedComputeIdentity)],
)
def test_identity_remove(n4js_fresh, identity_type):
    n4js = n4js_fresh
    identity_type_str, identity_type_cls = identity_type
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        ident = "bill"
        key = "and ted"

        identity = identity_type_cls(
            identifier=ident,
            hashed_key=hash_key(key),
        )

        n4js.create_credentialed_entity(identity)

        result = runner.invoke(
            cli,
            [
                "identity",
                "remove",
                "--identity-type",
                identity_type_str,
                "--identifier",
                ident,
            ],
        )
        assert click_success(result)

        with pytest.raises(KeyError, match="No such object in database"):
            cred = n4js.get_credentialed_entity(ident, identity_type_cls)

def test_identity_list(n4js_fresh):
    n4js = n4js_fresh
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        identities = ("bill", "ted", "napoleon")
        for ident in identities:
            key = "a string for a key"

            identity = CredentialedUserIdentity(
                identifier=ident,
                hashed_key=hash_key(key),
            )

            n4js.create_credentialed_entity(identity)

        result = runner.invoke(
            cli,
            [
                "identity",
                "list",
                "--identity-type",
                "user",
            ],
        )
        assert click_success(result)
        for ident in identities:
            assert ident in result.output

def test_scope_list(n4js_fresh):
    n4js = n4js_fresh
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        ident = "bill"
        key = "a string for a key"

        identity = CredentialedUserIdentity(
            identifier=ident,
            hashed_key=hash_key(key),
        )

        n4js.create_credentialed_entity(identity)
        n4js.add_scope(
            "bill", CredentialedUserIdentity, Scope.from_str("org1-campaign2-project3")
        )
        n4js.add_scope(
            "bill", CredentialedUserIdentity, Scope.from_str("org4-campaign5-project6")
        )
        n4js.add_scope("bill", CredentialedUserIdentity, Scope.from_str("org7-*-*"))
        result = runner.invoke(
            cli,
            [
                "identity",
                "list-scope",
                "--identity-type",
                "user",
                "--identifier",
                ident,
            ],
        )
        assert click_success(result)
        assert "org1-campaign2-project3" in result.output
        assert "org4-campaign5-project6" in result.output
        assert "org7-*-*" in result.output

@pytest.mark.parametrize(
    "scopes",
    [
        ("org1-campaign2-project3",),
        (
            "org1-campaign2-project3",
            "org1-campaign2-project4",
        ),
    ],
)
def test_scope_add(n4js_fresh, scopes):
    n4js = n4js_fresh
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        ident = "bill"
        key = "a string for a key"

        identity = CredentialedUserIdentity(
            identifier=ident,
            hashed_key=hash_key(key),
        )

        n4js.create_credentialed_entity(identity)

        scopes_cli = []
        for scope in scopes:
            scopes_cli.append("--scope")
            scopes_cli.append(scope)

        result = runner.invoke(
            cli,
            [
                "identity",
                "add-scope",
                "--identity-type",
                "user",
                "--identifier",
                ident,
            ]
            + scopes_cli,
        )
        assert click_success(result)
        scopes_ = n4js.list_scopes("bill", CredentialedUserIdentity)
        assert len(scopes_) == len(scopes)
        assert set(scopes_) == set([Scope.from_str(scope) for scope in scopes])

@pytest.mark.parametrize(
    "scopes_remove",
    [
        ("org1-campaign2-project3",),
        (
            "org1-campaign2-project3",
            "org1-campaign2-project4",
        ),
    ],
)
def test_scope_remove(n4js_fresh, scopes_remove):
    n4js = n4js_fresh
    env_vars = {
        "NEO4J_URL": n4js.graph.service.uri,
        "NEO4J_USER": "neo4j",
        "NEO4J_PASS": "password",
    }
    runner = CliRunner()
    with set_env_vars(env_vars):
        ident = "bill"
        key = "a string for a key"

        identity = CredentialedUserIdentity(
            identifier=ident,
            hashed_key=hash_key(key),
        )
        n4js.create_credentialed_entity(identity)

        n4js.add_scope(
            ident, CredentialedUserIdentity, Scope.from_str("org1-campaign2-project3")
        )
        n4js.add_scope(
            ident, CredentialedUserIdentity, Scope.from_str("org1-campaign2-project4")
        )
        n4js.add_scope(
            ident, CredentialedUserIdentity, Scope.from_str("org4-campaign5-project6")
        )

        scopes_cli = []
        for scope in scopes_remove:
            scopes_cli.append("--scope")
            scopes_cli.append(scope)

        result = runner.invoke(
            cli,
            [
                "identity",
                "remove-scope",
                "--identity-type",
                "user",
                "--identifier",
                ident,
            ]
            + scopes_cli,
        )
        assert click_success(result)
        scopes = n4js.list_scopes("bill", CredentialedUserIdentity)
        scope_strs = [str(s) for s in scopes]
        for scope in scopes_remove:
            assert scope not in scope_strs
        assert "org4-campaign5-project6" in scope_strs


"""
460. LFU Cache
Hard

1959

151

Add to List

Share
Design and implement a data structure for a Least Frequently Used (LFU) cache.

Implement the LFUCache class:

LFUCache(int capacity) Initializes the object with the capacity of the data structure.
int get(int key) Gets the value of the key if the key exists in the cache. Otherwise, returns -1.
void put(int key, int value) Update the value of the key if present, or inserts the key if not already present. When the cache reaches its capacity, it should invalidate and remove the least frequently used key before inserting a new item. For this problem, when there is a tie (i.e., two or more keys with the same frequency), the least recently used key would be invalidated.
To determine the least frequently used key, a use counter is maintained for each key in the cache. The key with the smallest use counter is the least frequently used key.

When a key is first inserted into the cache, its use counter is set to 1 (due to the put operation). The use counter for a key in the cache is incremented either a get or put operation is called on it.

Example 1:

Input
["LFUCache", "put", "put", "get", "put", "get", "get", "put", "get", "get", "get"]
[[2], [1, 1], [2, 2], [1], [3, 3], [2], [3], [4, 4], [1], [3], [4]]
Output
[null, null, null, 1, null, -1, 3, null, -1, 3, 4]

Explanation
// cnt(x) = the use counter for key x
// cache=[] will show the last used order for tiebreakers (leftmost element is  most recent)
LFUCache lfu = new LFUCache(2);
lfu.put(1, 1);   // cache=[1,_], cnt(1)=1
lfu.put(2, 2);   // cache=[2,1], cnt(2)=1, cnt(1)=1
lfu.get(1);      // return 1
                 // cache=[1,2], cnt(2)=1, cnt(1)=2
lfu.put(3, 3);   // 2 is the LFU key because cnt(2)=1 is the smallest, invalidate 2.
                 // cache=[3,1], cnt(3)=1, cnt(1)=2
lfu.get(2);      // return -1 (not found)
lfu.get(3);      // return 3
                 // cache=[3,1], cnt(3)=2, cnt(1)=2
lfu.put(4, 4);   // Both 1 and 3 have the same cnt, but 1 is LRU, invalidate 1.
                 // cache=[4,3], cnt(4)=1, cnt(3)=2
lfu.get(1);      // return -1 (not found)
lfu.get(3);      // return 3
                 // cache=[3,4], cnt(4)=1, cnt(3)=3
lfu.get(4);      // return 4
                 // cache=[3,4], cnt(4)=2, cnt(3)=3

Constraints:

0 <= capacity, key, value <= 104
At most 105 calls will be made to get and put.

Follow up: Could you do both operations in O(1) time complexity?

"""
"""
Design

use nums: {key: freq}
use freqs: {freq: OrderedDict {key: value}}

key points:
1. OrderedDict orders item by insert order
2. OrderedDict.move_to_end(key) moves an key to end (re-insert)
3. OrderedDict.popitem(last=False) removes oldest value (head)

"""
from collections import defaultdict, OrderedDict
class LFUCache:

    def __init__(self, capacity: int):
        self.size = 0  # total number of caches
        self.capacity = capacity
        self.nums = {} # key to freq
        self.freqs = defaultdict(OrderedDict)
        self.minfreq = 0  # the minimum freq, to help identify which dll to evict an cached item

    def _update(self, key, value=None):
        """
        update node when add(key) or put(key) and the key exists already
        1. remove node from self.freq[node.freq] doublylinkedlist
        2. update node.freq node.freq += 1
        3. add node to self.freq[node.freq] doublylinkedlist (this is old freq+1)
        4. if minfreq = old freq, and self.freqs[old freq].dllsize == 0: update minfreq to new freq = old freq + 1
        """
        freq = self.nums[key]
        # remove key from self.freqs[freq]
        self.freqs[freq].move_to_end(key)
        item = self.freqs[freq].popitem(last=True)
        newfreq = freq + 1
        # check minfreq
        if freq == self.minfreq and len(self.freqs[freq]) == 0:  # minfreq list is now empty
            self.minfreq = newfreq  # we know this node would move from freq to freq+1, so minfreq += 1
        # add key into self.freqs[newfreq]
        self.nums[key] = newfreq
        if value is not None:
            self.freqs[newfreq][key] = value # update to new value
        else:
            self.freqs[newfreq][key] = item[1] # dont' update value

    def get(self, key: int) -> int:
        # print('get key=%s self.nums=%s self.freqs=%s' % (key, self.nums, self.freqs))
        if key not in self.nums:
            # print('get key=%s returning %s' % (key, -1))
            return -1
        self._update(key, value=None)
        # print('get key=%s returning %s' % (key, self.freqs[self.nums[key]][key]))
        return self.freqs[self.nums[key]][key]

    def put(self, key: int, value: int) -> None:
        """
        if key exist, update freq and value in OrderedDict
        """
        # print('put key=%s value=%s self.nums=%s self.freqs=%s' % (key, value, self.nums, self.freqs))
        if self.capacity == 0:
            return
        if key in self.nums:
            self._update(key, value)
        else:
            # new key, evict one if cache full
            if self.size == self.capacity:
                # remove least frequently used item in self.freqs[self.minfreq]
                item = self.freqs[self.minfreq].popitem(last=False)
                del self.nums[item[0]]
                self.size -= 1
            self.freqs[1][key] = value
            self.size += 1
            self.nums[key] = 1
            self.minfreq = 1

"""
Design

use map to store key to node
dict of frequency {freq: doublylinkedlist of LRU of same freq}

key points:
1. self.minfreq keep track of minfreq, so when evicting least frequenty used, we can get it via self.freqs[self.minfreq]
2. keep freq inside node
3. keep doublylinkedlist size inside doublylinkedlist
4. keep total cache size in LFUCache self.size (to check capacity easily)
5. check capacity before adding new node
6. when updating node, move it from old freq list to new list, we need to update minfreq

mistakes:
1. use LFUCache.size instead of len(self.nums) to keep track of exceeding capacity or not

"""

class DLinkedNode:
    def __init__(self, key=-1, val=-1, prev=None, nxt=None):
        # use key=-1 and val=-1 to indicate dummy node
        self.key = key
        self.val = val
        self.prev = prev
        self.nxt = nxt
        self.freq = 1

    def __repr__(self):
        return '{%s : %s (%s)}' % (self.key, self.val, self.freq)

class DLinkedList:
    """
    represent a Least Recently Used (LRU) list, excluding dummy head/tail, the first element is the most recently used
    when expiring least recently used, remove this.tail.prev
    """

    def __init__(self):
        self.head = DLinkedNode()
        self.tail = DLinkedNode()
        self.head.nxt = self.tail
        self.tail.prev = self.head
        self.dllsize = 0

    def add_node(self, node):
        """
        add node to right after dummy head
        """
        node.prev, node.nxt = self.head, self.head.nxt
        self.head.nxt.prev = node
        self.head.nxt = node
        self.dllsize += 1

    def remove_node(self, node):
        node_prev = node.prev
        node_nxt = node.nxt
        node_prev.nxt = node_nxt
        node_nxt.prev = node_prev
        self.dllsize -= 1

        return node

    def evict_lru(self):
        """
        evict the item with least recent use time for given freq
        """
        return self.remove_node(self.tail.prev)

    def move_to_head(self, node):
        """
        move a node to head (just used, updating to most recently used)
        """
        node = self.remove_node(node)
        self.add_node(node)

    def __repr__(self):
        ans = ''
        cur = self.head.nxt
        while cur != self.tail:
            ans +=  str(cur)
            cur = cur.nxt

        return ans

class LFUCache1:

    def __init__(self, capacity: int):
        self.size = 0  # total number of caches
        self.capacity = capacity
        self.nums = {}
        self.freqs = defaultdict(DLinkedList)
        self.minfreq = 0  # the minimum freq, to help identify which dll to evict an cached item

    def _update(self, node):
        """
        update node when add(key) or put(key) and the key exists already
        1. remove node from self.freq[node.freq] doublylinkedlist
        2. update node.freq node.freq += 1
        3. add node to self.freq[node.freq] doublylinkedlist (this is old freq+1)
        4. if minfreq = old freq, and self.freqs[old freq].dllsize == 0: update minfreq to new freq = old freq + 1
        """
        freq = node.freq
        self.freqs[freq].remove_node(node)
        newfreq = freq + 1
        node.freq = newfreq
        # check minfreq
        if freq == self.minfreq and self.freqs[freq].dllsize == 0:  # minfreq list is now empty
            self.minfreq = newfreq  # we know this node would move from freq to freq+1, so minfreq += 1
        if newfreq not in self.freqs:
            self.freqs[newfreq] = DLinkedList()
        self.freqs[newfreq].add_node(node)

    def get(self, key: int) -> int:
        # print('get key=%s self.nums=%s self.freqs=%s' % (key, self.nums, self.freqs))
        if key not in self.nums:
            print('get key=%s returning %s' % (key, -1))
            return -1
        node = self.nums[key]
        self._update(node)
        print('get key=%s returning %s' % (key, node.val))
        return node.val

    def put(self, key: int, value: int) -> None:
        """
        if key exist, update node value, and update node
        """
        # print('put key=%s value=%s self.nums=%s self.freqs=%s' % (key, value, self.nums, self.freqs))
        if self.capacity == 0:
            return
        if key in self.nums:
            node = self.nums[key]
            node.val = value
            self._update(node)
        else:
            # new key, evict one if cache full
            if self.size == self.capacity:
                # remove last in doublylinkedlist in the smallest frequency
                node = self.freqs[self.minfreq].evict_lru()
                del self.nums[node.key]
                self.size -= 1
            node = DLinkedNode(key=key, val=value)
            if 1 not in self.freqs:
                self.freqs[1] = DLinkedList()
            self.freqs[1].add_node(node)
            self.size += 1
            self.nums[key] = node
            self.minfreq = 1

# Your LFUCache object will be instantiated and called as such:
# obj = LFUCache(capacity)
# param_1 = obj.get(key)
# obj.put(key,value)

def main():
    obj = LFUCache(2)
    obj.put(1,1)
    obj.put(2,2)
    assert obj.get(1) == 1, 'fails'
    obj.put(3,3)
    assert obj.get(2) == -1, 'fails'
    assert obj.get(3) == 3, 'fails'
    obj.put(4,4)
    assert obj.get(1) == -1, 'fails'
    assert obj.get(3) == 3, 'fails'
    assert obj.get(4) == 4, 'fails'

    # obj = LFUCache(0)
    # obj.put(0,0)
    # assert obj.get(0) == -1, 'fails'

if __name__ == '__main__':
   main()


from keras import layers
from keras import regularizers
from keras import models
from keras import optimizers

def conv2d_bn(filters, kernel_size, padding='same', strides=1):
    def f(x):
        x = layers.Conv2D(filters=filters, kernel_size=kernel_size,
                          strides=strides, padding=padding,
                          kernel_regularizer=regularizers.l2(1e-4))(x)
        x = layers.Activation('relu')(x)
        x = layers.BatchNormalization()(x)
        return x

    return f

def OscarNet(input_size: int, n_classes: int, initial_filters=32, repetitions=(2, 2, 3)):
    inputs = layers.Input(shape=(input_size, input_size, 3))

    x = conv2d_bn(initial_filters, kernel_size=3)(inputs)
    x = layers.MaxPooling2D(pool_size=2)(x)

    filters = initial_filters
    for r in repetitions:
        for i in range(r):
            x = conv2d_bn(filters, kernel_size=3)(x)
        x = layers.MaxPooling2D(pool_size=2)(x)
        filters *= 2

    x = layers.Dropout(0.5)(x)
    x = conv2d_bn(n_classes, kernel_size=1, padding='valid')(x)
    x = layers.GlobalAveragePooling2D()(x)
    x = layers.Activation('softmax')(x)

    model = models.Model(inputs, x)

    opt = optimizers.Adam()
    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

    return model


# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details
# http://www.gnu.org/licenses/gpl-3.0.txt

import asyncio
from collections import abc

from ... import objects
from ...completion import candidates
from .. import CmdError, CommandMeta
from . import _mixin as mixin
from ._common import (make_COLUMNS_doc, make_SCRIPTING_doc, make_SORT_ORDERS_doc,
                      make_X_FILTER_spec)

from ...logging import make_logger  # isort:skip
log = make_logger(__name__)

class ListTrackersCmdbase(mixin.get_tracker_sorter, mixin.get_tracker_columns,
                          mixin.get_tracker_filter, metaclass=CommandMeta):
    name = 'trackerlist'
    aliases = ('trkls', 'lstrk')
    provides = set()
    category = 'tracker'
    description = 'List tracker(s) of torrent(s)'
    usage = ('trackerlist [<OPTIONS>]',
             'trackerlist [<OPTIONS>] <TORRENT FILTER>',
             'trackerlist [<OPTIONS>] <TORRENT FILTER> <TRACKER FILTER>')
    examples = ("trackerlist 'name~debian'",
                "trackerlist tracker=my.tracker.local error")
    argspecs = (
        make_X_FILTER_spec('TORRENT', or_focused=True, nargs='?'),
        make_X_FILTER_spec('TRACKER', or_focused=True, nargs='?'),

        {'names': ('--sort', '-s'),
         'default_description': "current value of 'sort.trackers' setting",
         'description': ('Comma-separated list of sort orders '
                         "(see SORT ORDERS section)")},

        {'names': ('--columns', '-c'),
         'default_description': "current value of 'columns.trackers' setting",
         'description': ('Comma-separated list of column names '
                         "(see COLUMNS section)")},
    )

    from ...client.sorters import TrackerSorter
    from ...views.tracker import COLUMNS
    more_sections = {
        'COLUMNS': make_COLUMNS_doc(COLUMNS, '--columns', 'columns.trackers', append=(
            '',
            'The "torrent" column is added automatically if multiple '
            'torrents could be listed potentially.')),
        'SORT ORDERS': make_SORT_ORDERS_doc(TrackerSorter, '--sort', 'sort.trackers'),
        'SCRIPTING': make_SCRIPTING_doc(name),
    }

    async def run(self, TORRENT_FILTER, TRACKER_FILTER, sort, columns):
        columns = objects.localcfg['columns.trackers'] if columns is None else columns
        sort = objects.localcfg['sort.trackers'] if sort is None else sort
        try:
            torfilter = self.select_torrents(TORRENT_FILTER,
                                             allow_no_filter=True,
                                             discover_torrent=True)
            trkfilter = self.get_tracker_filter(TRACKER_FILTER)
            sort      = self.get_tracker_sorter(sort)
            columns   = self.get_tracker_columns(columns)
        except ValueError as e:
            raise CmdError(e)

        # Unless we're listing trackers of exactly one torrent, specified by its
        # ID, automatically add the 'torrent' column.
        if 'torrent' not in columns and \
           (not isinstance(torfilter, abc.Sequence) or len(torfilter) != 1):
            columns += ('torrent',)

        log.debug('Listing %s trackers of %s torrents', trkfilter, torfilter)

        if asyncio.iscoroutinefunction(self.make_tracker_list):
            await self.make_tracker_list(torfilter, trkfilter, sort, columns)
        else:
            self.make_tracker_list(torfilter, trkfilter, sort, columns)

    @classmethod
    def completion_candidates_posargs(cls, args):
        """Complete positional arguments"""
        posargs = args.posargs({('--columns', '-c'): 1,
                                ('--sort', '-s'): 1})
        if posargs.curarg_index == 1:
            return candidates.torrent_filter(args.curarg)
        elif posargs.curarg_index == 2:
            torrent_filter = posargs[1]
            return candidates.tracker_filter(args.curarg, torrent_filter)

    @classmethod
    def completion_candidates_params(cls, option, args):
        """Complete parameters (e.g. --option parameter1,parameter2)"""
        if option == '--columns':
            return candidates.column_names('trackers')
        elif option == '--sort':
            return candidates.sort_orders('TrackerSorter')

class AnnounceCmdbase(metaclass=CommandMeta):
    name = 'announce'
    aliases = ('an',)
    provides = set()
    category = 'tracker'
    description = 'Announce torrents to their trackers now if possible'
    usage = ('announce',
             'announce <TORRENT FILTER> <TORRENT FILTER> ...')
    examples = ('announce tracker~example.org',)
    argspecs = (
        make_X_FILTER_spec('TORRENT', or_focused=True, nargs='*'),
    )

    async def run(self, TORRENT_FILTER):
        try:
            tfilter = self.select_torrents(TORRENT_FILTER,
                                           allow_no_filter=False,
                                           discover_torrent=True)
        except ValueError as e:
            raise CmdError(e)
        else:
            response = await self.make_request(
                objects.srvapi.torrent.announce(tfilter),
                polling_frenzy=False)
            if not response.success:
                raise CmdError()

    @classmethod
    def completion_candidates_posargs(cls, args):
        """Complete positional arguments"""
        return candidates.torrent_filter(args.curarg)

class TrackerCmdbase(metaclass=CommandMeta):
    name = 'tracker'
    aliases = ('trk',)
    provides = set()
    category = 'tracker'
    description = 'Add/Remove trackers to/from torrents'
    _ADD_ACTIONS = ('add',)
    _REMOVE_ACTIONS = ('remove', 'rm')
    _ALL_ACTIONS = _ADD_ACTIONS + _REMOVE_ACTIONS
    usage = ('tracker %s <URL>' % '|'.join(_ALL_ACTIONS),
             'tracker %s <TORRENT FILTER> <URL> <URL> ...' % '|'.join(_ALL_ACTIONS))

    examples = ('tracker add !tracker http://tracker3.example.org:12345/announce',
                'tracker remove all tracker1.example tracker2.example ')
    argspecs = (
        {'names': ('ACTION',)},

        make_X_FILTER_spec('TORRENT', or_focused=True, nargs='?'),

        {'names': ('URL',), 'nargs': '+',
         'description': ('Announce URL to add to or remove from matching torrents; '
                         'may be partial (e.g. domain name) when removing trackers')},
    )

    async def run(self, ACTION, TORRENT_FILTER, URL):
        urls = tuple(URL)
        try:
            tfilter = self.select_torrents(TORRENT_FILTER,
                                           allow_no_filter=False,
                                           discover_torrent=True)
        except ValueError as e:
            raise CmdError(e)

        if any(ACTION == action for action in self._ADD_ACTIONS):
            request = objects.srvapi.torrent.tracker_add(tfilter, urls)
            log.debug('Adding trackers to %s torrents: %s', tfilter, ', '.join(urls))
        elif any(ACTION == action for action in self._REMOVE_ACTIONS):
            request = objects.srvapi.torrent.tracker_remove(tfilter, urls, partial_match=True)
            log.debug('Removing trackers from %s torrents: %s', tfilter, ', '.join(urls))
        else:
            raise CmdError('Invalid ACTION: %r' % (ACTION,))

        response = await self.make_request(request, polling_frenzy=True)
        if not response.success:
            raise CmdError()

    @classmethod
    def completion_candidates_posargs(cls, args):
        """Complete positional arguments"""
        if args.curarg_index == 1:
            return candidates.Candidates(('add', 'remove'), label='Action')
        elif args.curarg_index == 2:
            return candidates.torrent_filter(args.curarg)