import sys
from copy import deepcopy
import glob

import matplotlib.pyplot as plt
import numpy as np

from gomokuapp.board import Board
from neuralnetwork.training_data import parse_training_file, simulate
from treesearch.monte_carlo import Neural_Network

FILE_LOCATION = "resources/training/training_data_max/2x1-31(98).psq"

# Generate network input, determine network output, create heatmap
def main(file_path=FILE_LOCATION, game_number=16):
	network = Neural_Network()

	board = parse_training_file(file_path)
	inp = simulate(board)
	instance = inp[0][int(game_number)]  # Some move within a game
	b = Board(instance)

	# Gather neural network outputs for each possible move on board
	outputs = gather_outputs(b, network)
	b.print_board()

	# Call with additional parameter False to prevent PDF output
	draw_graph(outputs)

# Allow the graph generation code to be called elsewhere within the application,
# enabling graphs to be generated as a game plays out
def use_graph_gen(board, network):
	b = deepcopy(board)
	outputs = gather_outputs(b, network)
	draw_graph(outputs)

# For a given board, determine the network's output for all adjacent states
def gather_outputs(board, network):
	nn_outputs = []
	moves = board.get_possible_moves()
	played = board.get_played_moves()
	total = moves + played
	new = sorted(total)
	for move in new:
		if move not in played:
			new_board = deepcopy(board)
			valid = new_board.move(move[0], move[1], board.get_next_player())
			if valid:
				nn_outputs.append(network.nn(new_board, new_board.get_next_player()))
			else:
				# This should never happen
				assert False
		else:
			nn_outputs.append(-1)

	return nn_outputs

# Draw the heatmap
def draw_graph(outputs, should_save=True):
	# Generate the coordinates
	x = []
	y = []
	for i in range(0, 20):
		for j in range(0, 20):
			x.append(j)
			y.append(i)

	heatmap, _, _ = np.histogram2d(x, y, weights=outputs, bins=20)

	f = plt.figure()

	plt.clf()
	plt.imshow(heatmap)

	# Style
	plt.set_cmap("hot")
	plt.title("Heatmap of Neural Network Output")
	plt.xlabel("X Coordinate on Board")
	plt.ylabel("Y Coordinate on Board")
	plt.colorbar().set_label("Neural Network Output")

	# Bespoke config
	plt.gca().invert_yaxis()
	plt.xticks(np.arange(min(x), max(x) + 1, 2.0))
	plt.yticks(np.arange(min(y), max(y) + 1, 2.0))
	# Uncomment to have the graph displayed on-screen when generated
	# plt.show()

	if should_save:
		pdf_list = glob.glob("resources/heatmaps/*.pdf")
		pdf_count = len(pdf_list)
		f.savefig("resources/heatmaps/heatmap%s.pdf" % (pdf_count))

if __name__ == "__main__":
	# The user can supply a file location and game count to generate a heatmap for
	# Otherwise, default values will be used
	if len(sys.argv) == 3:
		path = sys.argv[1]
		game = sys.argv[2]
		try:
			main(path, game)
		except IOError as fnf:
			print("%s is not a valid file location." % path)
		except Exception as e:
			print("Please enter a valid game count.")
			print(e)
	else:
		main()


#!/usr/bin/env python3
# author:Alnk(李成果)
import socket
from socket import AF_INET, SOCK_STREAM

# 1 创建套接字对象  family=AF_INET, type=SOCK_STREAM
# AF_INET:表示ipv4,  AF_INET6:表示ipv6
# SOCK_STREAM:表示tcp协议,  SOCK_DGRAM:表示udp协议
sock = socket.socket(family=AF_INET, type=SOCK_STREAM)  # 看源码，括号里参数也可以不写

# 2 绑定IP和端口
sock.bind(("127.0.0.1", 8899))

# 3 创建一个监听数
# 表示可以等待5个请求连接，如果算上已经连接的请求，那么第7个才会报错
# 排队5个，连接1个，所以第7个才会报错
sock.listen(5)

# 4 等待连接，一旦客户端请求过来，会返回两个值：客户端的套接字对象，客户端的地址
print('等待连接...')
conn, addr = sock.accept()  # 阻塞状态，不占CPU资源
print('conn', conn)
print('addr', addr)

# 5 接受客户端信息
data = conn.recv(1024)
print(data.decode())

# 6 给客户端返回一个welcome字符串
conn.send('welcom! 苑!'.encode())

# 7 关闭与客户端连接
conn.close()
# 关闭服务器的socket
sock.close()

09 day09/02 笔记整理/02 socket的简单使用/server.py


# Using the same corpus of 10,000 common English words as before, create a new file that consists of
# each consecutive non-overlapping sequence of five lines merged into one line. Here are the first 10 lines:

# the of and to a
# in for is on that
# by this with i you
# it not or be are
# from at as your all
# have new more an was
# we will home can us
# about if page my has
# search free but our one
# other do no information time

# If the last group has less than five at the end, just write out the last group.

# you can use this bit of code to download the words from the corpus
import urllib2

u = 'https://raw.githubusercontent.com/first20hours/google-10000-english/master/google-10000-english-no-swears.txt'
response = urllib2.urlopen(u)
words = [i.strip() for i in response.readlines()]

def write_chunks_of_five(words,fname):
    '''
    Given a list of words, create a new file that consists of each consecutive non-overlapping
    sequence of five lines merged into one line.
    :param: words
    :type: list
    :param: fname
    :type: str
    '''
    f = open(fname, 'w')  # write mode
    cc = 0
    for i in words:
        cc = cc + 1
        if cc % 5 == 0:
            f.write(i + '\n')
        else:
            f.write(i + ' ')

    f.close()

fname = 'output_by_5.txt'
write_chunks_of_five(words,fname)


from django.db import models
from django.urls import reverse
from django.utils import timezone
from django.contrib.auth.models import User
from PIL import Image
from pathlib import Path
import os.path
from .myutilits import Photoes
from parler.models import TranslatableModel, TranslatedFields
from django.contrib.sites.models import Site

class Pictures(TranslatableModel):
    translations = TranslatedFields(
    alt=models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    )

    image = models.ImageField(blank=True)
    width = models.IntegerField(blank=True, verbose_name='ширина', help_text='название раздела', default=600)
    height = models.IntegerField(blank=True, verbose_name='высота', help_text='название раздела', default=600)

    class Meta:
        verbose_name = "Картинки страницы"
        verbose_name_plural = "Картинки страницы"

    def save(self, force_insert=False, force_update=False):
        img = Image.open(self.image)
        if (self.width,self.height)==img.size:
            img.close
            super(Pictures, self).save(force_insert, force_update)

        else:
            if (self.width,self.height)==(0,0) or (self.width,self.height)==(None, None):
                (self.width, self.height) = img.size
            upload_path='material-kit-master/products/'
            if str(self.image).find(upload_path)==0:
                os.remove(self.image.path)
                self.image = str(self.image)[str(self.image).rfind('/') + 1:]
            baseurl = str(os.path.join(Path(__file__).resolve().parent.parent, ''))
            stroka0 = str(self.image)[:str(self.image).rfind('/') + 1] + upload_path + str(self.width) + 'x' + str(self.height) + '/'
            stroka1 = str(self.image)[str(self.image).rfind('/') + 1:]
            stroka2 = stroka1[:stroka1.find('.')]
            folder = baseurl + str(self.image)[:str(self.image).rfind('/') + 1] + upload_path + str(self.width) + 'x' + str(self.height) + '/'
            output_image_path = baseurl + stroka0 + stroka2 + '.png'
            Photoes(self.width, self.height, img, output_image_path,folder)

            self.image=upload_path + str(self.width) + 'x' + str(self.height) + '/'+ stroka2 + '.png'
            # print(dir(self.image))
            super(Pictures, self).save(force_insert, force_update)

    def __str__(self):
        return self.alt

class Post(models.Model):
    STATUS_CHOICES = (
        ('draft', 'Draft'),
        ('published', 'Published'),
    )
    title = models.CharField(max_length=250)
    slug = models.SlugField(max_length=250, unique_for_date='publish')
    author = models.ForeignKey(User, related_name='blog_posts', on_delete=models.CASCADE)
    body = models.TextField()
    publish = models.DateTimeField(default=timezone.now)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='draft')

    def get_absolute_url(self):
        return reverse('standart:post_detail',
                       args=[self.publish.year,
                             self.publish.strftime('%m'),
                             self.publish.strftime('%d'),
                             self.slug])

    class Meta:
        ordering = ('-publish',)

    def __str__(self):
        return self.title

class Category(TranslatableModel):
    translations = TranslatedFields(
        name=models.CharField(max_length=200, db_index=True),
        slug=models.SlugField(max_length=200, db_index=True, unique=True)
    )

    class Meta:
        verbose_name = 'category'
        verbose_name_plural = 'categories'

class Templatecategory(TranslatableModel):
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
    )
    translations = TranslatedFields(
        name=models.CharField(max_length=200, db_index=True),
        slug=models.SlugField(max_length=200, db_index=True, unique=True)
    )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    class Meta:
        verbose_name = 'Название категории'
        verbose_name_plural = 'Категории меню'
    def __str__(self):
        return self.slug

class Navconstruct(TranslatableModel):
    MENU_CHOICES = (
        ('first', 'First'),
        ('second', 'Second'),

    )
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
        ('example', 'Example'),
        ('login', 'Login'),
    )
    translations = TranslatedFields(
    name = models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    slug = models.SlugField(max_length=200,null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    hreflogo = models.TextField(max_length=100, blank=True, verbose_name='Текст блока', help_text='текст блока'),
    alt = models.TextField(max_length=300, blank=True, verbose_name='Текст блока', help_text='текст блока'),
    title = models.CharField(max_length=300, blank=True, verbose_name='Заголовок', help_text='описание заголовка в строке браузера'),
    descrtionmeta = models.TextField(max_length=300, blank=True, verbose_name='Описание страницы', help_text='описание страницы в поискаовика'),
    keywordsmeta = models.TextField(max_length=300, blank=True, verbose_name='Ключевые слова', help_text='ключевые слова для поисковика'),

    )
    order= models.IntegerField(null=True, blank=True, verbose_name='Порядок в меню', help_text='последователность отображения в меню')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="navtemp", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    example = models.BooleanField(verbose_name='Наличие примера', help_text='наличие примеров в разделе',default=False)
    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="navpict", )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    bar = models.CharField(max_length=10, choices=MENU_CHOICES, default='first')
    newslug = models.ForeignKey(Templatecategory,on_delete=models.CASCADE, null=True, blank=True, related_name="newslug", verbose_name='Первое меню',
                            )
    newsslug = models.ForeignKey(Templatecategory,on_delete=models.CASCADE, null=True, blank=True, related_name="newsslug", verbose_name='Второе меню',
                            )
    site = models.ManyToManyField(Site, blank=True, related_name="site", )
    class Meta:
        verbose_name = "Раздел"
        verbose_name_plural = "Основное меню"
        ordering = [
            "order",

        ]

    def get_absolute_url(self):
        if self.bar=='first':
            return reverse('standart:navigator',
                               args=[self.newslug])
        elif self.bar=='second':
            return reverse('standart:submain',
                           args=[self.newslug, self.newsslug])
    def __str__(self):
        return self.name

class Subnavigator(TranslatableModel):
    STATUS_CHOICES = (
        ('odinary', 'Odinary'),
        ('form', 'Form'),
        ('call', 'Call'),
        ('service', 'Service'),
        ('example', 'Example'),
        ('login', 'Login'),
    )
    translations = TranslatedFields(
    name = models.CharField(max_length=200, verbose_name='Название раздела', help_text='название основного раздела'),
    slug = models.SlugField(max_length=200,null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    hreflogo = models.CharField(max_length=100, blank=True, verbose_name='URL картинка', help_text='URL картинка'),
    alt = models.CharField(max_length=150, blank=True, verbose_name='Alt картинка', help_text='описание картинки'),
    title = models.CharField(max_length=300, blank=True, verbose_name='Заголовок', help_text='описание заголовка в строке браузера'),
    descrtionmeta = models.TextField(max_length=300, blank=True, verbose_name='Описание страницы', help_text='описание страницы в поискаовика'),
    keywordsmeta = models.TextField(max_length=300, blank=True, verbose_name='Ключевые слова', help_text='ключевые слова для поисковика'),

    )
    subname = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="sub", null=True, blank=True, verbose_name='Основное меню', help_text='привязка к основному меню')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="subtemp", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="subpict", )
    status = models.CharField(max_length=10, choices=STATUS_CHOICES, default='odinary')
    class Meta:
        verbose_name = "Подраздел"
        verbose_name_plural = "Подменю"
    def get_absolute_url(self):
        return reverse('standart:submain',
            args=[self.subname.slug, self.slug])
    def __str__(self):
        return self.name

class Templates(models.Model):

    class Meta:
        verbose_name = "Template"
        verbose_name_plural = "Template"

    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название темплейта')
    description = models.TextField(blank=True, null=True, verbose_name='Текст', help_text='описание')
    def __str__(self):
        return self.title

class Metka(TranslatableModel):
    class Meta:
        verbose_name = "Категория проекта"
        verbose_name_plural = "Категории проектов"
        ordering = [
            "translations__title",

        ]

    translations = TranslatedFields(
    title = models.CharField(max_length=100, blank=True, verbose_name='Категория проекта', help_text='категория проекта'),
    )
    def __str__(self):
        return self.title

class Examples(TranslatableModel):
    class Meta:
        verbose_name = "Пример"
        verbose_name_plural = "Примеры"

    name = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="nexamples", null=True, blank=True,
                                verbose_name='Основное меню', help_text='привязка к основному меню')
    marks = models.ManyToManyField('Metka', blank=True,related_name="marks",)

    translations = TranslatedFields(
    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела'),
    slug=models.SlugField(max_length=200, null=True, unique=True, blank=True, verbose_name='URL адрес', help_text='название url адреса'),
    )
    def __str__(self):
        return self.title
    def get_absolute_url(self):
        return reverse('standart:submain',
            args=[self.name.slug, self.slug])

class Container(models.Model):
    class Meta:
        verbose_name = "Тип блока"
        verbose_name_plural = "Тип блока"
        ordering = [
            "order",

        ]

    name = models.ForeignKey('Navconstruct', on_delete=models.CASCADE, related_name="ncont", null=True, blank=True,
                                verbose_name='Основное меню', help_text='привязка к основному меню')
    examples = models.ForeignKey('Examples', on_delete=models.CASCADE, related_name="examples", null=True, blank=True,
                                verbose_name='Пример', help_text='привязка к примеру')

    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела')
    template_name = models.ForeignKey('Templates', on_delete=models.CASCADE, related_name="template", null=True, blank=True,
                                verbose_name='Templates', help_text='привязка к Templates')
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    def __str__(self):
        return self.title

class Text(TranslatableModel):
    class Meta:
        verbose_name = "Информация страницы"
        verbose_name_plural = "Информация страницы"
        ordering = [
            "order",
        ]

    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE,null=True,blank=True,related_name="picture", )
    container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name="container", null=True,
                                      blank=True,
                                      verbose_name='Templates', help_text='привязка к Templates')
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    translations = TranslatedFields(
    title = models.CharField(max_length=30, blank=True, verbose_name='Наименование', help_text='название раздела'),
    text = models.TextField(blank=True, verbose_name='Текст', help_text='описание'))
    examples = models.ForeignKey('Examples', on_delete=models.CASCADE, related_name="example", null=True, blank=True,
                                 verbose_name='Пример', help_text='привязка к примеру')
    def __str__(self):
        return self.title
    def get_absolute_url(self):
        return reverse('standart:example',
                       args=[self.title])

class Table(TranslatableModel):
    class Meta:
        verbose_name = "Информация таблицы"
        verbose_name_plural = "Информация таблицы"
        ordering = [
            "order",
        ]

    pictures = models.ForeignKey(Pictures, on_delete=models.CASCADE,null=True,blank=True,related_name="table", )
    picturesvs = models.ForeignKey(Pictures, on_delete=models.CASCADE, null=True, blank=True, related_name="tablevs", )
    order = models.IntegerField(null=True, blank=True, verbose_name='Порядок на странице',
                                help_text='последователность отображения на странице')
    container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name="contable", null=True,
                                      blank=True,
                                      verbose_name='Templates', help_text='привязка к Templates')
    translations = TranslatedFields(
    description = models.TextField(blank=True, verbose_name='Текст', help_text='описание'),
    one = models.TextField(blank=True, verbose_name='Текст', help_text='описание'),
    two=  models.TextField(blank=True, verbose_name='Текст', help_text='описание'))
    def __str__(self):
        return self.description


from sanic.response import json
from sanic import Blueprint
from sanic.exceptions import ServerError

bp_profiles = Blueprint('profiles')

@bp_profiles.route('/profiles/<username>', methods=["GET"])
async def profile(request, username):
    mongo_profile = await request.app.config['db'].profiles.find({'profile.username': username}).to_list(None)
    if (len(mongo_profile) == 0):
        raise ServerError("There is no profile with username like {}".format(username) , status_code=404)
    return json({'profile': mongo_profile[0]['profile']})

@bp_profiles.route('/profiles/<username>/follow', methods=["POST", "DELETE"])
async def follow_profile(request, username):

    try:
        follow = True if request.method == 'POST' else False
        await request.app.config['db'].profiles.update_one(
            {'profile.username': username},
            {'$set': {"profile.following":follow}}
        )
        mongo_profile = await request.app.config['db'].profiles.find({'profile.username': username}).to_list(None)
        return json({'profile': mongo_profile[0]['profile']})
    except Exception as e:
        print(e)
        raise ServerError("There is no profile with username like {}".format(username) , status_code=404)


import rapidjson
import sys
from collections import OrderedDict

if sys.version_info[0] < 3:
    from future import range
if sys.version_info[0] == 3 and sys.version_info[1] < 5:
    from .is_close import isclose
else:
    from math import isclose

class Stack:
    def __init__(self):
        self.stack_items = []

    def append(self, stack_item):
        self.stack_items.append(stack_item)
        return self

    def __repr__(self):
        stack_dump = ''
        for item in self.stack_items:
            stack_dump += str(item)
        return stack_dump

    def __str__(self):
        stack_dump = ''
        for item in self.stack_items:
            stack_dump += str(item)
        return stack_dump

class StackItem:
    def __init__(self, reason, expected, actual):
        self.reason = reason
        self.expected = expected
        self.actual = actual

    def __repr__(self):
        return 'Reason: {0}\nExpected:\n{1}\nActual:\n{2}'.format(self.reason, _format_value(self.expected),
                                                                  _format_value(self.actual))

    def __str__(self):
        return '\n\nReason: {0}\nExpected:\n{1}\nActual:\n{2}'.format(self.reason, _format_value(self.expected),
                                                                      _format_value(self.actual))

def _indent(s):
    return '\n'.join('  ' + line for line in s.splitlines())

def _format_value(value):
    return _indent(_generate_pprint_json(value))

def _generate_pprint_json(value):
    return rapidjson.dumps(value, indent=4)

def _is_dict_same(expected, actual, ignore_value_of_keys, greater_than=1, less_than=1, compare_ints_floats=False, ignore_missing_keys=False):
    # DAN - I had to flip flop this
    for key in expected:
        if key not in actual:
            return False, Stack().append(
                StackItem('Expected key "{0}" Missing from Actual'.format(key), expected, actual))

        if key not in ignore_value_of_keys:
            # have to change order
            # are_same_flag, stack = _are_same(actual[key], expected[key], ignore_value_of_keys)
            are_same_flag, stack = _are_same(expected[key], actual[key], ignore_value_of_keys, ignore_missing_keys,
                                             greater_than, less_than, compare_ints_floats)
            if not are_same_flag:
                return False, stack.append(StackItem('Different values', expected[key], actual[key]))
    return True, Stack()

def _is_list_same(expected, actual, ignore_value_of_keys, greater_than=1, less_than=1, compare_ints_floats=False, ignore_missing_keys=False):
    for i in range(len(expected)):
        are_same_flag, stack = _are_same(expected[i], actual[i], ignore_value_of_keys, ignore_missing_keys, greater_than, less_than,
                                         compare_ints_floats)
        if not are_same_flag:
            return False, stack.append(StackItem('Different values (Check order)', expected[i], actual[i]))
    return True, Stack()

def _bottom_up_sort(unsorted_json):
    if isinstance(unsorted_json, dict):
        return sorted((k, _bottom_up_sort(v)) for k, v in unsorted_json.items())
    if isinstance(unsorted_json, list):
        return sorted(_bottom_up_sort(x) for x in unsorted_json)
    else:
        return unsorted_json

def _to_ordered_dict(sorted_json):
    if isinstance(sorted_json, list):
        ordered = OrderedDict()
        if sorted_json and isinstance(sorted_json[0], list):
            ordered = []
            for i in sorted_json:
                ordered.append(_to_ordered_dict(i))
            return ordered
        else:
            for i in sorted_json:
                if isinstance(i, list):
                    ordered.update(_to_ordered_dict(i))
                elif isinstance(i, tuple):
                    ordered[i[0]] = _to_ordered_dict(i[1])
            return ordered
    else:
        return sorted_json

def _are_same(expected, actual, ignore_value_of_keys, ignore_missing_keys=False, rel_tolerance=0, abs_tolerance=0,
              compare_ints_floats=False):
    # Check for None
    if expected is None:
        return expected == actual, Stack()

    # Ensure they are of same type
    if type(expected) != type(actual):
        if compare_ints_floats and type(expected) in (int, float) and type(actual) in (int, float):
            pass
        else:
            return False, Stack().append(
                StackItem('Type Mismatch: Expected Type: {0}, Actual Type: {1}'.format(type(expected), type(actual)),
                          expected, actual))

    # Compare primitive types immediately
    if type(expected) in (str, bool):
        return expected == actual, Stack()
    elif type(expected) in (int, float):
        if rel_tolerance != 1 or abs_tolerance != 1:
            return isclose(expected, actual, rel_tol=rel_tolerance, abs_tol=abs_tolerance), Stack()
        else:
            return expected == actual, Stack()

    # Ensure collections have the same length (if applicable)
    if ignore_missing_keys:
        # Ensure collections has minimum length (if applicable)
        # This is a short-circuit condition because (b contains a)
        if len(expected) > len(actual):
            return False, Stack().append(StackItem(
                'Length Mismatch: Minimum Expected Length: {0}, Actual Length: {1}'.format(len(expected), len(actual)),
                expected, actual))

    else:
        # Ensure collections has same length
        if len(expected) != len(actual):
            return False, Stack().append(StackItem(
                'Length Mismatch: Expected Length: {0}, Actual Length: {1}'.format(len(expected), len(actual)),
                expected, actual))

    if isinstance(expected, dict) or isinstance(expected, OrderedDict):
        return _is_dict_same(expected, actual, ignore_value_of_keys, rel_tolerance, abs_tolerance, compare_ints_floats, ignore_missing_keys)

    if isinstance(expected, list):
        return _is_list_same(expected, actual, ignore_value_of_keys, rel_tolerance, abs_tolerance, compare_ints_floats, ignore_missing_keys)

    return False, Stack().append(StackItem('Unhandled Type: {0}'.format(type(expected)), expected, actual))

def are_same(original_a, original_b, ignore_list_order_recursively=False, ignore_value_of_keys=[], rel_tolerance=1,
             abs_tolerance=1, compare_ints_floats=False):
    if ignore_list_order_recursively:
        a = _to_ordered_dict(_bottom_up_sort(original_a))
        b = _to_ordered_dict(_bottom_up_sort(original_b))
    else:
        a = original_a
        b = original_b
    return _are_same(a, b, ignore_value_of_keys, False, rel_tolerance, abs_tolerance, compare_ints_floats)

def contains(expected_original, actual_original, ignore_list_order_recursively=False, ignore_value_of_keys=[],
             compare_ints_floats=False):
    if ignore_list_order_recursively:
        actual = _to_ordered_dict(_bottom_up_sort(actual_original))
        expected = _to_ordered_dict(_bottom_up_sort(expected_original))
    else:
        actual = actual_original
        expected = expected_original
    return _are_same(expected, actual, ignore_value_of_keys, True, compare_ints_floats=compare_ints_floats)

def json_are_same(a, b, ignore_list_order_recursively=False, ignore_value_of_keys=[], rel_tolerance=1, abs_tolerance=1,
                  compare_ints_floats=False):
    return are_same(rapidjson.loads(a), rapidjson.loads(b), ignore_list_order_recursively, ignore_value_of_keys,
                    rel_tolerance,
                    abs_tolerance, compare_ints_floats)


# encoding: UTF-8

from datetime import datetime, timedelta
from collections import OrderedDict

from ctaBase import *
from vtConstant import *
from vtGateway import VtOrderData, VtTradeData

import pandas as pd

########################################################################
class BacktestingEngine(object):

    '''
    CTA回测引擎
    '''

    TICK_MODE = 'tick'
    BAR_MODE = 'bar'

    #----------------------------------------------------------------------
    def __init__(self):
        """Constructor"""

        # 回测相关
        self.strategy = None        # 回测策略
        self.mode = self.BAR_MODE   # 回测模式，默认为K线

        self.slippage = 0           # 回测时假设的滑点
        self.rate = 0               # 回测时假设的佣金比例（适用于百分比佣金）
        self.size = 1               # 合约大小，默认为1

        self.dataStartDate = None       # 回测数据开始日期，datetime对象
        self.dataEndDate = None         # 回测数据结束日期，datetime对象

        self.limitOrderDict = OrderedDict()         # 限价单字典
        self.workingLimitOrderDict = OrderedDict()  # 活动限价单字典，用于进行撮合用
        self.limitOrderCount = 0                    # 限价单编号

        self.tradeCount = 0             # 成交编号
        self.tradeDict = OrderedDict()  # 成交字典

        self.logList = []               # 日志记录

        # 当前最新数据，用于模拟成交用
        self.tick = None
        self.bar = None
        self.dt = None      # 最新的时间

    #----------------------------------------------------------------------
    def setStartDate(self, startDate='20100416'):
        """设置回测的启动日期"""
        self.dataStartDate = datetime.strptime(startDate, '%Y%m%d')

    #----------------------------------------------------------------------
    def setEndDate(self, endDate=''):
        """设置回测的结束日期"""
        if endDate:
            self.dataEndDate= datetime.strptime(endDate, '%Y%m%d')

    #----------------------------------------------------------------------
    def setBacktestingMode(self, mode):
        """设置回测模式"""
        self.mode = mode

    #----------------------------------------------------------------------
    def runBacktesting(self, strategyClass):

        """初始化策略"""

        self.strategy = strategyClass(self)
        self.strategy.name = self.strategy.className

        """运行回测"""
        # 首先根据回测模式，确认要使用的数据类
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
            func = self.newBar
        else:
            dataClass = CtaTickData
            func = self.newTick

        self.output(u'开始回测')

        df = pd.read_csv('IF0000_1min.csv')
        dict_bar = df.to_dict(outtype='records')
        self.output(u'开始回放数据')
        for d in dict_bar:
            bar = CtaBarData()
            bar.vtSymbol = 'IF0000'
            bar.symbol = 'IF0000'
            bar.open = float(d['Open'])
            bar.high = float(d['High'])
            bar.low = float(d['Low'])
            bar.close = float(d['Close'])
            bar.date = datetime.strptime(d['Date'], '%Y/%m/%d').strftime('%Y%m%d')
            bar.time = d['Time']
            bar.datetime = datetime.strptime(bar.date + ' ' + bar.time, '%Y%m%d %H:%M:%S')
            bar.volume = d['TotalVolume']
            func(bar)

        self.output(u'数据回放结束')

    #----------------------------------------------------------------------
    def newBar(self, bar):
        """新的K线"""
        self.bar = bar
        self.dt = bar.datetime
        self.crossLimitOrder()      # 先撮合限价单
        self.strategy.onBar(bar)    # 推送K线到策略中

    #----------------------------------------------------------------------
    def newTick(self, tick):
        """新的Tick"""
        self.tick = tick
        self.dt = tick.datetime
        self.crossLimitOrder()
        self.strategy.onTick(tick)

    #----------------------------------------------------------------------
    def sendOrder(self, vtSymbol, orderType, price, volume, strategy):
        """发单"""
        self.limitOrderCount += 1
        orderID = str(self.limitOrderCount)

        order = VtOrderData()
        order.vtSymbol = vtSymbol
        order.price = price
        order.totalVolume = volume
        order.status = STATUS_NOTTRADED     # 刚提交尚未成交
        order.orderID = orderID
        order.vtOrderID = orderID
        order.orderTime = str(self.dt)

        # CTA委托类型映射
        if orderType == CTAORDER_BUY:
            order.direction = DIRECTION_LONG
            order.offset = OFFSET_OPEN
        elif orderType == CTAORDER_SELL:
            order.direction = DIRECTION_SHORT
            order.offset = OFFSET_CLOSE
        elif orderType == CTAORDER_SHORT:
            order.direction = DIRECTION_SHORT
            order.offset = OFFSET_OPEN
        elif orderType == CTAORDER_COVER:
            order.direction = DIRECTION_LONG
            order.offset = OFFSET_CLOSE

        # 保存到限价单字典中
        self.workingLimitOrderDict[orderID] = order
        self.limitOrderDict[orderID] = order

        return orderID
    #----------------------------------------------------------------------
    def crossLimitOrder(self):
        """基于最新数据撮合限价单"""
        # 先确定会撮合成交的价格
        if self.mode == self.BAR_MODE:
            buyCrossPrice = self.bar.low    # 若买入方向限价单价格高于该价格，则会成交
            sellCrossPrice = self.bar.high  # 若卖出方向限价单价格低于该价格，则会成交
            bestCrossPrice = self.bar.open  # 在当前时间点前发出的委托可能的最优成交价
        else:
            buyCrossPrice = self.tick.lastPrice
            sellCrossPrice = self.tick.lastPrice
            bestCrossPrice = self.tick.lastPrice

        # 遍历限价单字典中的所有限价单
        for orderID, order in self.workingLimitOrderDict.copy().items():
            # 判断是否会成交
            buyCross = order.direction==DIRECTION_LONG and order.price>=buyCrossPrice
            sellCross = order.direction==DIRECTION_SHORT and order.price<=sellCrossPrice

            # 如果发生了成交
            if buyCross or sellCross:
                # 推送成交数据
                self.tradeCount += 1            # 成交编号自增1
                tradeID = str(self.tradeCount)
                trade = VtTradeData()
                trade.vtSymbol = order.vtSymbol
                trade.tradeID = tradeID
                trade.vtTradeID = tradeID
                trade.orderID = order.orderID
                trade.vtOrderID = order.orderID
                trade.direction = order.direction
                trade.offset = order.offset

                # 以买入为例：
                # 1. 假设当根K线的OHLC分别为：100, 125, 90, 110
                # 2. 假设在上一根K线结束(也是当前K线开始)的时刻，策略发出的委托为限价105
                # 3. 则在实际中的成交价会是100而不是105，因为委托发出时市场的最优价格是100
                if buyCross:
                    trade.price = min(order.price, bestCrossPrice)
                    # 更新持仓
                    self.strategy.pos += order.totalVolume
                else:
                    trade.price = max(order.price, bestCrossPrice)
                    # 更新持仓
                    self.strategy.pos -= order.totalVolume

                trade.volume = order.totalVolume
                trade.tradeTime = str(self.dt)
                trade.dt = self.dt
                self.strategy.onTrade(trade)

                self.tradeDict[tradeID] = trade

                # 从字典中删除该限价单
                del self.workingLimitOrderDict[orderID]

    #----------------------------------------------------------------------
    def output(self, content):
        """输出内容"""
        print (content)

    #----------------------------------------------------------------------
    def showBacktestingResult(self):
        """
        显示回测结果
        """
        self.output(u'显示回测结果')

        # 首先基于回测后的成交记录，计算每笔交易的盈亏
        pnlDict = OrderedDict()     # 每笔盈亏的记录
        longTrade = []              # 未平仓的多头交易
        shortTrade = []             # 未平仓的空头交易

        # 计算滑点，一个来回包括两次
        totalSlippage = self.slippage * 2

        for trade in self.tradeDict.values():
            # 多头交易
            if trade.direction == DIRECTION_LONG:
                # 如果尚无空头交易
                if not shortTrade:
                    longTrade.append(trade)
                # 当前多头交易为平空
                else:
                    entryTrade = shortTrade.pop(0)
                    # 计算比例佣金
                    commission = (trade.price+entryTrade.price) * self.rate
                    # 计算盈亏
                    pnl = ((trade.price - entryTrade.price)*(-1) - totalSlippage - commission) \
                        * trade.volume * self.size
                    pnlDict[trade.dt] = pnl
            # 空头交易
            else:
                # 如果尚无多头交易
                if not longTrade:
                    shortTrade.append(trade)
                # 当前空头交易为平多
                else:
                    entryTrade = longTrade.pop(0)
                    # 计算比例佣金
                    commission = (trade.price+entryTrade.price) * self.rate
                    # 计算盈亏
                    pnl = ((trade.price - entryTrade.price) - totalSlippage - commission) \
                        * trade.volume * self.size
                    pnlDict[trade.dt] = pnl

        # 然后基于每笔交易的结果，我们可以计算具体的盈亏曲线和最大回撤等
        timeList = list(pnlDict.keys())
        pnlList = list(pnlDict.values())

        capital = 0
        maxCapital = 0
        drawdown = 0

        capitalList = []        # 盈亏汇总的时间序列
        maxCapitalList = []     # 最高盈利的时间序列
        drawdownList = []       # 回撤的时间序列

        for pnl in pnlList:
            capital += pnl
            maxCapital = max(capital, maxCapital)
            drawdown = capital - maxCapital

            capitalList.append(capital)
            maxCapitalList.append(maxCapital)
            drawdownList.append(drawdown)

        # 输出
        self.output('-' * 50)
        self.output(u'第一笔交易时间：%s' % timeList[0])
        self.output(u'最后一笔交易时间：%s' % timeList[-1])
        self.output(u'总交易次数：%s' % len(pnlList))
        self.output(u'总盈亏：%s' % capitalList[-1])
        self.output(u'最大回撤: %s' % min(drawdownList))

        # 绘图
        import matplotlib.pyplot as plt
        plt.figure(figsize=(20,10))
        pCapital = plt.subplot(3, 1, 1)
        pCapital.set_ylabel("capital")
        pCapital.plot(timeList, capitalList)

        pDD = plt.subplot(3, 1, 2)
        pDD.set_ylabel("DD")
        pDD.bar(range(len(drawdownList)), drawdownList)

        pPnl = plt.subplot(3, 1, 3)
        pPnl.set_ylabel("pnl")
        pPnl.hist(pnlList, bins=20)

        plt.show()
    #----------------------------------------------------------------------
    def setSlippage(self, slippage):
        """设置滑点"""
        self.slippage = slippage

    #----------------------------------------------------------------------
    def setSize(self, size):
        """设置合约大小"""
        self.size = size

    #----------------------------------------------------------------------
    def setRate(self, rate):
        """设置佣金比例"""
        self.rate = rate

if __name__ == '__main__':
    # 以下内容是一段回测脚本的演示，用户可以根据自己的需求修改
    # 建议使用ipython notebook或者spyder来做回测
    # 同样可以在命令模式下进行回测（一行一行输入运行）
    from ctaDemo import *

    # 创建回测引擎
    engine = BacktestingEngine()

    # 设置引擎的回测模式为K线
    engine.setBacktestingMode(engine.BAR_MODE)

    # 设置回测用的数据起始日期
    engine.setStartDate('20120101')

    # 设置产品相关参数
    engine.setSlippage(0.2)     # 股指1跳
    engine.setRate(0.3/10000)   # 万0.3
    engine.setSize(300)         # 股指合约大小

    # 在引擎中创建策略对象
    engine.initStrategy(DoubleEmaDemo)

    # 开始跑回测
    engine.runBacktesting()

    # 显示回测结果
    # spyder或者ipython notebook中运行时，会弹出盈亏曲线图
    # 直接在cmd中回测则只会打印一些回测数值
    engine.showBacktestingResult()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################
import os as OS
from bson.objectid import ObjectId

from girder_client import GirderClient

__version__ = '0.0.1'
__license__ = 'Apache 2.0'

appName = 'Slicer'

class Constant:
    """
    A bunch of utilities constant, as to handle ``Error`` or set default parameters.
    """
    # Errors
    ERROR_ALREADY_EXIST = 1
    ERROR_APP_NOT_EXIST = 2
    ERROR_RELEASE_NOT_EXIST = 3
    ERROR_EXT_NOT_EXIST = 4
    ERROR_EXT_NO_FILE = 5

    # Default
    # TODO: Find a good default path
    DEFAULT_DOWNLOAD_PATH = '/Users/pierre.assemat/slicer-extension-manager/extensions'
    DEFAULT_RELEASE = 'Nightly'
    DEFAULT_LIMIT = 50

class SlicerExtensionClient(GirderClient):
    """
    The SlicerExtensionClient allow to us the GirderClient with specific functions to use the
    slicer_extension_manager plugin of Girder. This allow to manage 3 top level entity:

        * Application
        * Release
        * Extension

    To know more about the it : TODO: LINK to README
    """

    def __init__(self, host=None, port=None, apiRoot=None, scheme=None, apiUrl=None,
                 progressReporterCls=None):
        super(SlicerExtensionClient, self).__init__(
            host=host, port=port, apiRoot=apiRoot, scheme=scheme, apiUrl=apiUrl,
            progressReporterCls=progressReporterCls)

    def createApp(self, name, desc=None):
        """
        Create a new application in the default collection ``Applications``.
        The application will contain a ``Nightly`` folder.
        A template of the name of each future uploaded extension will be set as a metadata of
        this new application.

        :param name: name of the new application
        :param desc: Optional description of the application
        :return:  The new application
        """
        apps = self.listApp(name=name)
        if apps:
            return Constant.ERROR_ALREADY_EXIST
        return self.post('/app', parameters={
            'name': name,
            'app_description': desc
        })

    def listApp(self, name=None):
        """
        1. List all the applications within the default collection ``Applications``.
        2. Get the application by name.

        :param name: application mame
        :return:  A list of applications
        """
        apps = self.get('/app', parameters={
            'name': name
        })
        return apps

    def deleteApp(self, name):
        """
        Delete the application by ID.

        :param name: application name
        :return: The deleted application
        """
        apps = self.listApp(name=name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]
        self.delete('/app/%s' % app['_id'])
        return app

    def createRelease(self, app_name, name, revision, desc=None):
        """
        Create a new release within the application corresponding to ``app_name``.

        :param app_name: Name of the application
        :param name: Name of the release
        :param revision: Revision of the application
        :param desc: Description of the release
        :return: The new release
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]

        releases = self.listRelease(app_name=app_name)
        for release in releases:
            if release['name'] == name:
                return Constant.ERROR_ALREADY_EXIST
        return self.post('/app/%s/release' % app['_id'], parameters={
            'name': name,
            'app_revision': revision,
            'description': desc
        })

    def listRelease(self, app_name, name=None):
        """
        1. List all the release within an application.
        2. Get the release by name.

        :param app_name: Name of the application
        :param name: Name of the release
        :return: A list of all the release within the application
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]
        if name:
            releases = self.get('/app/%s/release/%s' % (app['_id'], name))
        else:
            releases = self.get('/app/%s/release' % app['_id'])
        return releases

    def deleteRelease(self, app_name, name):
        """
        Delete a release within an application.

        :param app_name: Name of the application
        :param name: Name of the release
        :return: The deleted release
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]
        release = self.listRelease(app_name, name)
        if not release:
            return Constant.ERROR_RELEASE_NOT_EXIST
        self.delete('/app/%s/release/%s' % (app['_id'], name))
        return release

    def uploadExtension(self, filepath, app_name, os, arch, name, repo_type, repo_url, revision, app_revision,
                        packagetype, codebase, desc):
        """
        Upload an extension by providing a path to the file.

        :param filepath: The path to the file
        :param app_name: The name of the application
        :param os: The target operating system of the package
        :param arch: The os chip architecture
        :param name: The basename of the extension
        :param repo_type:
        :param repo_url:
        :param revision: The revision of the extension
        :param app_revision: The revision of the application supported by the extension
        :param packagetype:
        :param codebase:
        :param desc: The description of the extension
        :return: The uploaded extension
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]

        # Create the extension into Girder hierarchy
        extension = self.post('/app/%s/extension' % app['_id'], parameters={
            'os': os,
            'arch': arch,
            'baseName': name,
            'repository_type': repo_type,
            'repository_url': repo_url,
            'revision': revision,
            'app_revision': app_revision,
            'packagetype': packagetype,
            'codebase': codebase,
            'description': desc
        })

        # Upload the extension
        def _displayProgress(*args, **kwargs):
            pass

        filename = OS.path.basename(filepath)  # extract full name of file
        self.uploadFileToItem(
            extension['_id'],
            filepath,
            reference='',
            mimeType='application/octet-stream',
            filename=filename,
            progressCallback=_displayProgress)

        return extension

    def downloadExtension(self, app_name, id_or_name, dir_path=Constant.DEFAULT_DOWNLOAD_PATH):
        """
        Download an extension by ID and store it in the given option ``dir_path``.

        :param app_name: Name of the application
        :param id_or_name: ID or name of the extension
        :param dir_path: Path of the directory when the extension has to be downloaded
        :return: The downloaded extension
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]

        if ObjectId.is_valid(id_or_name):
            ext = self.get('/resource/%s' % id_or_name, parameters={'type': 'item'})
        else:
            ext = self.get('/app/%s/extension/%s' % (app['_id'], id_or_name))
        if not ext:
            return Constant.ERROR_EXT_NOT_EXIST
        files = self.get('/item/%s/files' % ext['_id'])
        if not files:
            return Constant.ERROR_EXT_NO_FILE
        file = files[0]
        self.downloadFile(
            file['_id'],
            OS.path.join(dir_path, '%s.%s' % (ext['name'], file['name'].split('.')[1])))
        return ext

    def listExtension(self, app_name, os, arch, app_revision, release=Constant.DEFAULT_RELEASE,
                      limit=Constant.DEFAULT_LIMIT, all=False, fullname=None, id=False):
        """
        List all the extension for a specific release and filter them with some option (os, arch, ...).
        By default the extensions within ``Nightly`` release are listed.
        It's also possible to specify the ``--all`` option to list all the extensions from all
        the release of an application.

        :param app_name: Name of the application
        :param os: The target operating system of the package
        :param arch: The os chip architecture
        :param app_revision: Revision of the application
        :param release: Name of the release
        :param limit: Limit of the number of extensions listed
        :param all: Boolean that allow to list extensions from all the release
        :param fullname: The full name of the extension
        :param id: Boolean to return only the extension ID
        :return: A list of extensions filtered by optional parameters
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]

        if fullname:
            extension = self.get('app/%s/extension/%s' % (app['_id'], fullname))
            if not extension:
                return Constant.ERROR_EXT_NOT_EXIST
            if id:
                return extension['_id']
            return extension

        if all:
            release_id = None
        else:
            release_folder = self.listRelease(app_name, release)
            if release_folder:
                release_id = release_folder['_id']
            else:
                return Constant.ERROR_RELEASE_NOT_EXIST

        extensions = self.get('/app/%s/extension' % app['_id'], parameters={
            'os': os,
            'arch': arch,
            'app_revision': app_revision,
            'release_id': release_id,
            'limit': limit,
            'sort': 'created',
            'sortDir': -1
        })
        return extensions

    def deleteExtension(self, app_name, id_or_name):
        """
        Delete an extension within an application.

        :param app_name: Name of the application
        :param id_or_name: Extension ID or name
        :return: The deleted extension
        """
        apps = self.listApp(app_name)
        if not apps:
            return Constant.ERROR_APP_NOT_EXIST
        app = apps[0]

        if ObjectId.is_valid(id_or_name):
            ext = self.get('/resource/%s' % id_or_name, parameters={'type': 'item'})
        else:
            ext = self.get('/app/%s/extension/%s' % (app['_id'], id_or_name))
        if not ext:
            return Constant.ERROR_EXT_NOT_EXIST
        self.delete('/app/%s/extension/%s' % (app['_id'], ext['_id']))
        return ext


import os
from pathlib import Path
import cv2
import chainer
import numpy as np
from tools.detector_hyperface import HyperFaceModel, apply_hyperface
import tools.detector_hyperface as detector_hyperface

def create_lists(img_path, train_file, test_file, train_output='lists/train_list.txt', test_output='lists/test_list.txt'):
    f1 = open(train_file,'r')
    f2 = open(test_file,'r')

    train_list = f1.readlines()
    test_list = f2.readlines()

    f3 = open(train_output, 'w')
    f4 = open(test_output, 'w')

    clip_length = 16

    for line in train_list:
        name = line.split(' ')[0]
        image_path = img_path+name
        label = line.split(' ')[-1]
        images = os.listdir(image_path)
        nb = len(images) // clip_length
        for i in range(nb):
            f3.write(name+' '+ str(i*clip_length+1)+' '+label)

    for line in test_list:
        name = line.split(' ')[0]
        image_path = img_path+name
        label = line.split(' ')[-1]
        images = os.listdir(image_path)
        nb = len(images) // clip_length
        for i in range(nb):
            f4.write(name+' '+ str(i*clip_length+1)+' '+label)

    f1.close()
    f2.close()
    f3.close()
    f4.close()

def create_ucf101_list():

    img_path = '/home/tianz/datsets/ucfimgs/'
    f1 = 'ucfTrainTestlist/train_file.txt'
    f2 = 'ucfTrainTestlist/test_file.txt'
    create_lists(img_path, f1, f2)

def create_lr2_vid_list():

    img_path = ''
    f1 = 'lrs2TrainTestlist/train_file.txt'
    f2 = 'lrs2TrainTestlist/test_file.txt'
    create_lists(img_path, f1, f2)

def create_afew():

    img_path = ''
    f1 = 'afew-va/train_file.txt'
    f2 = 'afew-va/test_file.txt'
    create_lists(img_path, f1, f2)

def make_label(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage = 9999):

    # dirName = '/home/varun/Downloads'

    print("****************")

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0:# or '_45D' in dirpath or '_00D' in dirpath:
            continue
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):
            if count % 4:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + label]
            else:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + label]

        count += 1

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

def make_label_ibugs(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage=9999, max_seq=9999):

    # dirName = '/home/varun/Downloads'

    print("****************")

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0
    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0: # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        if not '-' in label:
            if '_00D' in dirpath:
                l = label
            elif '_45D' in dirpath:
                l = str(int(label) + 1)
            elif '_90D' in dirpath:
                l = str(int(label) + 2)
            elif '_270D' in dirpath:
                l = str(int(label) + 3)
            elif '_315D' in dirpath:
                l = str(int(label) + 4)
        else:
            l = str(abs(int(label)))
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)
        filenames = [k for k in filenames if '.wav' not in k and 'flow' not in k]
        n_seq += 1

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):
            if count % 4:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            else:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]

            count += 1
            # if count > max_seq:
            #     break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

    return  n_seq, count

def make_label_amicorpus(dirName, label = '#', train_output = 'lists/train_list.txt', test_output = 'lists/test_list.txt', maximage=9999, max_seq=9999):
    print("****************")

    # Load hyperface
    hyperface_module = HyperFaceModel()
    hyperface_module.train = False
    hyperface_module.report = False
    hyperface_module.backward = False
    chainer.serializers.load_npz("/local/users/jfmadrig/ownCloud/LinTo/src/tools/weights/model_epoch_190", hyperface_module)
    # Setup GPU
    config_gpu = 0
    if config_gpu >= 0:
        chainer.cuda.check_cuda_available()
        chainer.cuda.get_device(config_gpu).use()
        hyperface_module.to_gpu()
        xp = chainer.cuda.cupy
    else:
        xp = np

    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0
    IMG_SIZE = (227, 227)

    for (dirpath, dirnames, filenames) in os.walk(dirName):
        if len(filenames) == 0: # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        if not '-' in label:
            if '_00D' in dirpath:
                l = label
            elif '_45D' in dirpath:
                l = str(int(label) + 1)
            elif '_90D' in dirpath:
                l = str(int(label) + 2)
            elif '_270D' in dirpath:
                l = str(int(label) + 3)
            elif '_315D' in dirpath:
                l = str(int(label) + 4)
        else:
            l = str(abs(int(label)))
        # listOfFiles += [dirpath]
        # listOfFiles += [os.path.join(dirpath, file) for file in filenames]

        # filenames.sort(key=str.lower)
        filenames = [k for k in filenames if '.wav' not in k and 'flow' not in k]
        n_seq += 1

        filenames = filenames.sort()

        nb = len(filenames) // clip_length
        nb = min(nb,maximage)
        for i in range(nb):

            imgs = []
            for j in range(clip_length):
                frame = cv2.imread(dirpath + '/' + filenames[clip_length*i + j])
                img = frame.astype(np.float32) / 255.0  # [0:1]
                img = cv2.resize(img, IMG_SIZE)
                img = cv2.normalize(img, None, -0.5, 0.5, cv2.NORM_MINMAX)
                img = np.transpose(img, (2, 0, 1))
                imgs.append(img)

            for j in range(clip_length):
                frame = cv2.imread(dirpath + '/' + filenames[clip_length*i + j])
                poses = apply_hyperface(frame, None, None, hyperface_module, bh, xp)

            if 'CV5' in dirpath:
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            else:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + l]
            # if count > max_seq:
            #     break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    with open(test_output, 'w') as f:
        for item in testFiles:
            f.write("%s\n" % item)

    return n_seq, count

def make_label_ict3d(dir_path, gt_label_path='#', train_output='lists/train_list.txt', test_output='lists/test_list.txt',
                     maximage=9999, max_seq=9999):
    from numpy import genfromtxt
    from tools.utils import rotation_matrix, project_plane_yz, get_xpose
    import numpy as np
    import cv2

    def _draw_line(img, pt1, pt2, color, thickness=2):
        pt1 = (int(pt1[0]), int(pt1[1]))
        pt2 = (int(pt2[0]), int(pt2[1]))
        cv2.line(img, pt1, pt2, color, int(thickness))

    print("****************")
    clip_length = 16

    # Get the list of all files in directory tree at given path
    trainFiles = list()
    testFiles = list()
    count = 1
    n_seq = 0

    for (dirpath, dirnames, filenames) in os.walk(dir_path):
        if len(filenames) == 0:  # or '_45D' in dirpath or '_00D' in dirpath:
            continue
        seq = dirpath.split('/')[-1]
        gt_path = gt_label_path + seq + '/' + "polhemusNorm.csv"
        gt_data = genfromtxt(gt_path, delimiter=',')

        init_pose = np.loadtxt(gt_label_path + seq + '/' + "initpose.txt", delimiter=',')

        n_seq += 1

        nb = len(filenames) // clip_length
        nb = min(nb, maximage)
        for i in range(nb):

            img_path = dirpath + '/{:06d}'.format(i * clip_length) + '.jpg'
            if not os.path.isfile(img_path):
                continue
            # img = cv2.imread(img_path)

            pose = gt_data[i*clip_length][3:]
            rotpose = rotation_matrix(-pose[2], pose[0], -pose[1])
            # rotinit = rotation_matrix(-init_pose[2], init_pose[0], -init_pose[1])
            # rotmat = np.transpose(rotpose).dot(rotinit)

            rotmat = rotpose
            pred = get_xpose(rotmat)
            l = np.argmax(pred)
            # print(pred)

            zvec = np.array([0, 0, 1], np.float32)
            yvec = np.array([0, 1, 0], np.float32)
            xvec = np.array([1, 0, 0], np.float32)
            zvec = project_plane_yz(rotmat.dot(zvec))
            yvec = project_plane_yz(rotmat.dot(yvec))
            xvec = project_plane_yz(rotmat.dot(xvec))

            # # Lower left
            # size = 30
            # idx = 0
            # org_pt = ((size + 5) * (2 * idx + 1), img.shape[0] - size - 5)
            # _draw_line(img, org_pt, org_pt + zvec * size, (255, 0, 0), 3)
            # _draw_line(img, org_pt, org_pt + yvec * size, (0, 255, 0), 3)
            # _draw_line(img, org_pt, org_pt + xvec * size, (0, 0, 255), 3)
            # cv2.imshow('result', img)
            # cv2.waitKey(1)
            if '02' in dirpath and test_output is not '':
                testFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + str(l)]
            else:
                trainFiles += [dirpath + ' ' + str(i * clip_length + 1) + ' ' + str(l)]
            if count > max_seq:
                break

        # if count > max_seq:
        #     break

    dirpath = Path(train_output).parent
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

    with open(train_output, 'w') as f:
        for item in trainFiles:
            f.write("%s\n" % item)

    if test_output is not '':
        with open(test_output, 'w') as f:
            for item in testFiles:
                f.write("%s\n" % item)

    return n_seq, count

if __name__ == '__main__':
    # create_ucf101_list()
    # create_lr2_vid_list()
    # create_afew()
    # make_label('/data/jfmadrig/mvlrs_v1/pretrain_heads', '0',
    #            train_output = 'lrs2TrainTestlist/train_heads_list.txt',
    #            test_output = 'lrs2TrainTestlist/test_heads_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_heads', '1',
    #            train_output='lists/vid/train_heads_list.txt',
    #            test_output='lists/vid/test_heads_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_heads_slow', '1',
    #            train_output='lists/vid/train_heads_slow_short_list.txt',
    #            test_output='lists/vid/test_heads_slow_short_list.txt',
    #            maximage=100)
    # make_label('/data/jfmadrig/VidTIMIT_lips', '1',
    #            train_output='lists/vid/train_lips_list.txt',
    #            test_output='lists/vid/test_lips_list.txt')
    # make_label('/data/jfmadrig/mvlrs_v1/pretrain_lips', '0',
    #            train_output = 'lists/lrs2TrainTestlist/train_lips_list.txt',
    #            test_output = 'lists/lrs2TrainTestlist/test_lips_list.txt')
    # make_label('/data/jfmadrig/VidTIMIT_lips_slow', '1',
    #            train_output='lists/vid/train_lips_list_slow.txt',
    #            test_output='lists/vid/test_lips_list_slow.txt',
    #            maximage=200)
    #
    # make_label('/data/jfmadrig/hpedatasets/ict3DHP_lips', '1',
    #            train_output='lists/ict3DHP/train_lips_list.txt',
    #            test_output='lists/ict3DHP/test_lips_list.txt')

    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_lips', '0',
    #            train_output='lists/ibug-avs/Silence/train_lips_list_angle.txt',
    #            test_output='lists/ibug-avs/Silence/test_lips_list_angle.txt')
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_lips', '0',
    #            train_output='lists/ibug-avs/Voice/train_lips_list_angle.txt',
    #            test_output='lists/ibug-avs/Voice/test_lips_list_angle.txt')
    # silence_train_output = 'lists/ibug-avs/Silence/train_lips_list_angle_c6.txt'
    # silence_test_output = 'lists/ibug-avs/Silence/test_lips_list_angle_c6.txt'
    # speaker_train_output='lists/ibug-avs/Voice/train_lips_list_angle_c6.txt'
    # speaker_test_output='lists/ibug-avs/Voice/test_lips_list_angle_c6.txt'

    # silence_train_output = 'lists/ibug-avs/Silence/train_heads_list_angle_c5.txt'
    # silence_test_output = 'lists/ibug-avs/Silence/test_heads_list_angle_c5.txt'
    # speaker_train_output='lists/ibug-avs/Voice/train_heads_list_angle_c5.txt'
    # speaker_test_output='lists/ibug-avs/Voice/test_heads_list_angle_c5.txt'
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_heads', '5',
    #            train_output=silence_train_output,
    #            test_output=silence_test_output)
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_heads', '0',
    #            train_output=speaker_train_output,
    #            test_output=speaker_test_output)

    dataset = 'ibugs'
    dataset = 'amicorpus'
    # dataset = 'ict3d'

    if dataset == 'ibugs':
        silence_train_output = 'lists/ibug-avs/Silence_v2/train_heads_list_angle_c5.txt'
        silence_test_output = 'lists/ibug-avs/Silence_v2/test_heads_list_angle_c5.txt'
        silence_dir_path = '/data/jfmadrig/ibug-avs/Silence_Digits_v2_head/'
        # silence_init_label = '-1' # Binary
        silence_init_label = '5'
        speaker_train_output='lists/ibug-avs/Voice_v2/train_heads_list_angle_c5.txt'
        speaker_test_output='lists/ibug-avs/Voice_v2/test_heads_list_angle_c5.txt'
        speaker_dir_path = '/data/jfmadrig/ibug-avs/Voice_Digits_v2_head/'
        # speaker_init_label = '-0' # binary
        speaker_init_label = '0'

        train_outpu_file = 'lists/ibug-avs/train_heads_list_angle_c10_v2.txt'
        test_outpu_file = 'lists/ibug-avs/test_heads_list_angle_c10_v2.txt'

    elif dataset == 'amicorpus':
        # silence_train_output = 'lists/amicorpus/Silence/train_heads_list_angle_binary.txt'
        # silence_test_output  = 'lists/amicorpus/Silence/test_heads_list_angle_binary.txt'
        # silence_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_Silence/CV1/IS1002b/video/Closeup1/'
        # speaker_train_output = 'lists/amicorpus/Speaking/train_heads_list_angle_binary.txt'
        # speaker_test_output  = 'lists/amicorpus/Speaking/test_heads_list_angle_binary.txt'
        # speaker_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_Speaking/CV1/IS1002b/video/Closeup1/'
        # train_outpu_file = 'lists/amicorpus/train_heads_list_binary.txt'
        # test_outpu_file  = 'lists/amicorpus/test_heads_list_binary.txt'
        silence_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_wholeshort_v2_Silence/'
        speaker_dir_path   = '/data/jfmadrig/amicorpus/5fold_valid_wholeshort_v2_Speaking/'

        silence_train_output = 'lists/amicorpus/Silence/train_orientation_head_list_cv5.txt'
        silence_test_output  = 'lists/amicorpus/Silence/test_orientation_head_list_cv5.txt'
        speaker_train_output = 'lists/amicorpus/Speaking/train_orientation_head_list_cv5.txt'
        speaker_test_output  = 'lists/amicorpus/Speaking/test_orientation_head_list_cv5.txt'
        train_outpu_file = 'lists/amicorpus/train_orientation_head_list_cv5.txt'
        test_outpu_file  = 'lists/amicorpus/test_orientation_head_list_cv5.txt'

        # silence_init_label = '5'
        # speaker_init_label = '0'
        silence_init_label = '-1' # Binary
        speaker_init_label = '-0' # binary
    elif dataset == 'ict3d':
        imgs_path   = '/data/jfmadrig/hpedatasets/ict3DHP_rgb_of_heads/'
        gt_path = '/data/jfmadrig/hpedatasets/ict3DHP_data/'
        train_output = 'lists/ict3DHP/orientation/train_heads_list.txt'
        test_output = ''
        make_label_ict3d(imgs_path, gt_path, train_output, test_output)
        exit()

    max_seq = 5000
    maximage= 100
    n_seq, max_seq = \
    make_label_amicorpus(speaker_dir_path, speaker_init_label,
                     train_output=speaker_train_output,
                     test_output=speaker_test_output,
                     maximage=maximage,
                     max_seq=max_seq)

    make_label_amicorpus(silence_dir_path, silence_init_label,
                    train_output=silence_train_output,
                    test_output=silence_test_output,
                     maximage=maximage,
                    max_seq=max_seq )
    ## Binary
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Silence_Digits_v2_head/', '-1',
    #            train_output=silence_train_output,
    #            test_output=silence_test_output)
    #
    # make_label_ibugs('/data/jfmadrig/ibug-avs/Voice_Digits_v2_head/', '-0',
    #            train_output=speaker_train_output,
    #            test_output=speaker_test_output)

    filenames = [silence_train_output, speaker_train_output]
    with open(train_outpu_file, 'w') as outfile:
        for fname in filenames:
            with open(fname) as infile:
                for line in infile:
                    outfile.write(line)
    filenames = [silence_test_output, speaker_test_output]
    with open(test_outpu_file, 'w') as outfile:
        for fname in filenames:
            with open(fname) as infile:
                for line in infile:
                    outfile.write(line)

    # make_label_ibugs('/data/jfmadrig/ibug-avs/Static_Digits_heads', '0',
    #            train_output='lists/ibug-avs/Static/train_heads_list_angle.txt',
    #            test_output='lists/ibug-avs/Static/test_heads_list_angle.txt')


import threading
import re
from funcy import memoize, post_processing, ContextDecorator, decorator
from django.db import DEFAULT_DB_ALIAS
from django.db.models.expressions import F, Expression

from cacheops.conf import settings
from cacheops.sharding import get_prefix
from cacheops.redis import redis_client, handle_connection_failure
from cacheops.signals import cache_invalidated
from cacheops.transaction import queue_when_in_transaction

from cacheops.cluster.key_crafter import craft_scheme_key, craft_invalidator_key

__all__ = ('invalidate_obj', 'invalidate_model', 'invalidate_all', 'no_invalidation')

@decorator
def skip_on_no_invalidation(call):
    if not settings.CACHEOPS_ENABLED or no_invalidation.active:
        return

    if not settings.CACHEOPS_CLUSTER_ENABLED:
        return

    return call()

# ================== lua script functions ==================

def craft_conj_dict(scheme, old_data):
    conj = {}

    for field in re.split(r'[,]+', scheme):
        field_value = old_data.get(field)

        conj[field] = field_value

    return conj

def calculate_conj_key(prefix, db_table, old_data):
    conj_keys = []
    # return a set of binarty
    schemes = redis_client.smembers(craft_scheme_key(prefix, db_table))
    schemes = map(lambda s: s.decode('utf-8'), schemes)

    for scheme in schemes:
        conj = craft_conj_dict(scheme, old_data)
        conj_keys.append(
            craft_invalidator_key(prefix, db_table, conj)
        )

    return conj_keys

def remove_chunk(client, data_keys):
    for i in range(0, len(data_keys), 500):
        client.delete(*data_keys[i: i + 500])

def invalidation_dict_transaction(prefix, db_table, obj):
    # not a part of transaction, since getting key may take a long time
    # hence, we shouldn't include it

    # step 1: create the pattern to get invalidation keys
    conj_keys = calculate_conj_key(prefix, db_table, obj)
    if not conj_keys:
        return

    # step 2: get all the invalidation keys
    invalidation_keys = conj_keys
    if not invalidation_keys:
        return

    # step 3: get all the data keys from invalidation keys
    # each invalidation keys will hold a list of data keys
    # simple diagram: invalidation-key -> data-key -> data
    # so once we have a invalidation key,
    # we will get all the data key it is holding and invalidate them
    data_keys = list(redis_client.sunion(invalidation_keys))

    # step 4: removal

    # remove all data key and its data
    remove_chunk(client=redis_client, data_keys=data_keys)

    # remove all invalidation keys
    redis_client.delete(*invalidation_keys)

    # end invalidation transactions

# =========================================

@skip_on_no_invalidation
@queue_when_in_transaction
@handle_connection_failure
def invalidate_dict(model, obj_dict, using=DEFAULT_DB_ALIAS):
    """
    This function is patched to remove the LUA Script from being used.
    Since the Lua script can only run on 1 node

    However, we are using redis cluster for a better scaling
    -> This lead to some keys are not able to invalidate because of lua script limitation

    To solve this,
    I converted the lua script into python so that we can use RedisCluster client,
    to perform invalidate action on multiple nodes

    The logic of this function will be as close to the lua script as possible

    Some code from lua script that only work with single node,
    will be modified to work with multiple nodes
    """
    if (
        no_invalidation.active or
        not settings.CACHEOPS_ENABLED
    ):
        return

    model = model._meta.concrete_model
    prefix = get_prefix(
        _cond_dnfs=[(model._meta.db_table,list(obj_dict.items()))],
        dbs=[using]
    )

    # INFO: this part is removed since it only works on single redis cluster node
    # load_script('invalidate')(keys=[prefix], args=[
    #     model._meta.db_table,
    #     json.dumps(obj_dict, default=str)
    # ])

    # move to use redis client = redis cluster client
    # so that we can invalidate all the keys across multiple clusters
    invalidation_dict_transaction(
        prefix=prefix,
        db_table=model._meta.db_table,
        obj=obj_dict,
    )

    cache_invalidated.send(sender=model, obj_dict=obj_dict)

@skip_on_no_invalidation
def invalidate_obj(obj, using=DEFAULT_DB_ALIAS):
    """
    Invalidates caches that can possibly be influenced by object
    """
    model = obj.__class__._meta.concrete_model
    invalidate_dict(model, get_obj_dict(model, obj), using=using)

@skip_on_no_invalidation
@queue_when_in_transaction
@handle_connection_failure
def invalidate_model(model, using=DEFAULT_DB_ALIAS):
    """
    Invalidates all caches for given model.
    NOTE: This is a heavy artillery which uses redis KEYS request,
          which could be relatively slow on large datasets.
    """
    model = model._meta.concrete_model
    # NOTE: if we use sharding dependent on DNF then this will fail,
    #       which is ok, since it's hard/impossible to predict all the shards
    # !WARNING: performance issue on large dataset due to KEYS command search with pattern
    conjs_keys = redis_client.keys("conj:%s:*" % (model._meta.db_table))
    if conjs_keys:
        cache_keys = redis_client.sunion(conjs_keys)
        keys = list(cache_keys) + conjs_keys
        redis_client.delete(*keys)
    cache_invalidated.send(sender=model, obj_dict=None)

@skip_on_no_invalidation
@handle_connection_failure
def invalidate_all():
    redis_client.flushdb()
    cache_invalidated.send(sender=None, obj_dict=None)

class InvalidationState(threading.local):
    def __init__(self):
        self.depth = 0

class _no_invalidation(ContextDecorator):
    state = InvalidationState()

    def __enter__(self):
        self.state.depth += 1

    def __exit__(self, type, value, traceback):
        self.state.depth -= 1

    @property
    def active(self):
        return self.state.depth

no_invalidation = _no_invalidation()

### ORM instance serialization

@memoize
def serializable_fields(model):
    return {f for f in model._meta.fields
              if f.get_internal_type() not in settings.CACHEOPS_SKIP_FIELDS}

@post_processing(dict)
def get_obj_dict(model, obj):
    for field in serializable_fields(model):
        # Skip deferred fields, in post_delete trying to fetch them results in error anyway.
        # In post_save we rely on deferred values be the same as in pre_save.
        if field.attname not in obj.__dict__:
            continue

        value = getattr(obj, field.attname)
        if value is None:
            yield field.attname, None
        elif isinstance(value, (F, Expression)):
            continue
        else:
            yield field.attname, field.get_prep_value(value)


from lunch import place
from lunch.vehicles import Car, Motocycle
from lunch.people import Person, PersonPool

pool = PersonPool([
    Person('burak.donmez', vehicle=Motocycle()),
    Person('yasin.ozel', vehicle=Motocycle(allow_twoup=False)),

    Person('baran.demirci', vehicle=Motocycle()),
    Person('fatma.tosun'),

    Person('hande.sert'),
    Person('tarkan.dikmen'),

    # Person('hakan.ozay', vehicle=Motocycle()),
    Person('can.cilingir', vehicle=Car()),

    Person('emre.yilmaz', vehicle=Car()),
    Person('halil.kaya'),
    Person('altay.inci'),
    Person('gurkan.kizil'),
])

result = place(pool)

for i in result:
    print(i)


"""
Model free algorithm for solving MDPs
"""
#import sys
import numpy as np

#sys.path.insert(0, "/home/paolo/Scrivania/aicourse/ai-lab/session2/mdps")
#from planning import value_iteration as val_iter

def value_iteration(problem, maxiters, gamma, delta):
    """
    Performs the value iteration algorithm for a specific environment.
    :param problem: problem
    :param maxiters: max iterations allowed
    :param gamma: gamma value
    :param delta: delta value
    :return: policy
    """

    #return val_iter(problem, maxiters, gamma, delta)

    p = np.zeros(problem.observation_space.n, dtype="int8")  # Initial policy
    #p length of |S|

    #V length of |S|
    V = np.zeros(problem.observation_space.n, dtype="float_")
    viter = 0

    while True:

        Vp = V.copy()
        viter += 1
        #for each state s
        for s in range(problem.observation_space.n):
            #value_per_a remembers the value obtained for that action
            value_per_a = np.zeros(max(problem.actions) + 1, dtype="float_")
            #for every action a
            #for a in range(problem.action_space.n):
            for a in problem.actions:
                sum_sp = 0
                #towards every state
                for sp in range(problem.observation_space.n):
                    #problem.T[s, a, sp], probability to reach sp from s with action a
                    #problem.R[s, a, sp], reward from s to sp with a
                    #computation of a value:
                    #problem.T[s, a, sp] * (problem.R[s, a, sp] + gamma * V[sp])
                    value = problem.T[s, a, sp] * (problem.R[s, a, sp] + gamma * V[sp])
                    #value sum towards each state
                    sum_sp += value
                #value_per_a[a] stores the obtained value
                #from the sum of reachable states with that action
                value_per_a[a] = sum_sp
            #computing the max value that depends from the action
            V[s] = max(value_per_a)
        #np.absolute(np.subtract(V, Vp))
        #np.absolute(V - Vp)
        if (max(np.absolute(V - Vp)) < delta) or (viter == maxiters):
            #print("\tNumber of iteractions: ", viter)
            break

    for s in range(problem.observation_space.n):
        value_per_a = np.zeros(max(problem.actions) + 1, dtype="float_")
        for a in problem.actions:
            sum_sp = 0
            for sp in range(problem.observation_space.n):
                value = problem.T[s, a, sp] * (problem.R[s, a, sp] + gamma * V[sp])
                sum_sp += value
            value_per_a[a] = sum_sp
            #actions_value.append(value_per_a[a])
            #argmax cycle on actions
        p[s] = np.argmax(value_per_a)
        #p[s] = argmax(a) sum(sp) (problem.T[s, a, sp] * (problem.R[s, a, sp] + gamma * V[sp]))

    return np.asarray(p)

def model_based(problem, episodes, ep_limit, vmaxiters, gamma, delta):
    """
    Performs the model-based algorithm for a specific environment
    :param problem: problem
    :param episodes: number of episodes for training
    :param ep_limit: limit to episode length
    :param vmaxiters: max iterations allowed for VI
    :param gamma: gamma value
    :param delta: delta value
    :return: (policy, rews, ep_lengths): final policy, rewards for each episode [array], length of each episode [array]
    """
    pass


import pandas as pd
from pymongo import MongoClient
from tqdm import tqdm

import config

def build_test_ids():
    print('Building test ids...')
    client = MongoClient(connect=False)
    db = client.cdiscount['test']
    data = list()
    for product in tqdm(db.find(), total=config.TEST_PRODUCTS_COUNT):
        assert 1 <= len(product['imgs']) <= 4
        for i in range(len(product['imgs'])):
            data.append((product['_id'], i))
    df = pd.DataFrame(data=data, columns=['id', 'image_numb'])
    df.to_csv('test_ids.csv')
    print('Generated dataframe with shape', df.shape)
    print('Done.')

if __name__ == '__main__':
    build_test_ids()


# minh_duc = ["Minh Duc","Son La",19,2,1,10]
# dictionary/object/map

# key : value
# create new dictionary
person = {
    "name": "Minh Duc",
    "age" : 19,
    "ex" : 2,
    "IQ" : 1

}
# print (person)
# # add/update new key - value
# person["hometown"] = "Son La"
# print (person)
# person['ex'] = 3
# print(person)

# read
# print(person["age"])
# for key in person.keys() :
#     print(key,end = "\t")

# for value in person.values() :
#     print(value)

key = "school"
if key in person :

    print(person[key])
else :
    print("Not found")


import os
import csv

MAIN_DIR_NAMES = ["pdnn-lfv", "work"]

def get_valid_cfg_path(path):
    """Finds valid path for cfg file in /share folder.

  If path is already valid:
    Nothing will be done and original path will be returned.
  If path is not valid:
    Try to add share folder path before to see whether we can get a valid path.
    Otherwise, raise error to ask configuration correction.

  """
    # Check path:
    if os.path.isfile(path):
        return path
    # Check try add share folder prefix
    current_dir = os.getcwd()
    main_dirs = []
    for main_dir_name in MAIN_DIR_NAMES:
        try:
            found_dirs = re.findall(".*" + main_dir_name, current_dir)
            main_dirs += found_dirs
        except:
            pass
    share_dir = None
    for temp in main_dirs:
        share_dir_temp = temp + "/share"
        if os.path.isdir(share_dir_temp):
            share_dir = share_dir_temp
            break
    if share_dir is None:
        raise ValueError(
            "No valid path found for {}, please check .ini file.".format(share_dir)
        )
    if os.path.isfile(share_dir + "/train/" + path):
        return share_dir + "/train/" + path
    elif os.path.isfile(share_dir + "/" + path):
        return share_dir + "/" + path
    else:
        raise ValueError("No valid path found, please check .ini file.")

def make_table(data, save_dir, num_para=1):
    """Makes table for scan meta data and so on.

    Input example:
        data = [
            ["col-1", "col-2", "col-3", "col-4" ],
            [1,2,3,4],
            ["a", "b", "c", "d"]
        ]
    """
    # save csv format
    save_path = save_dir + "/scan_meta_report.csv"
    with open(save_path, "w", newline="") as myfile:
        wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
        for single_list in data:
            wr.writerow(single_list)


import csv
import math
from random import randint

with open('people.csv', 'r') as f:
  reader = csv.reader(f)
  your_list = list(reader)

# for i in range(1,len(your_list)):
    # print(your_list[i][2], your_list[i][3])

def formatear_cvs(cvs):
    lista_parcial = []
    for i in range(1,len(cvs)):
        nuevo_elemento = str(cvs[i][2]) + " " + str(cvs[i][3])
        lista_parcial.append(nuevo_elemento)
    return lista_parcial

def crear_grupos(lista_participantes, integrantes, tipo):
    contador = 1
    organizacion = []
    while len(lista_participantes) >= integrantes:
        grupo = []
        for i in range(integrantes):
            largo = len(lista_participantes) - 1
            index = randint(0, largo)
            integ = lista_participantes[index]
            grupo.append(integ)
            lista_participantes.pop(index)
        organizacion.append(grupo)
    if len(lista_participantes) == 0:
        return organizacion
    elif tipo == "1":
        for i in range(len(lista_participantes)):
            participante = lista_participantes[i]
            var = len(organizacion) - 1
            organizacion[var].append(participante)
        return organizacion
    else:
        nuevo_grupo = []
        for i in range(len(lista_participantes)):
            nuevo_grupo.append(lista_participantes[i])
        organizacion.append(nuevo_grupo)
        return organizacion

def imprimir_grupos(lista_grupos):
    for i in range(len(lista_grupos)):
        print("Grupo " + str(i+1))
        print(lista_grupos[i])

cant_por_grupo = input("Inserte la cantidad de integrantes por grupo: ")
int_grupo = int(cant_por_grupo)
tipo = input("La organizacion debe ser por (1) exceso o por (2) defecto? ")
participantes_format = formatear_cvs(your_list)
agrupacion = crear_grupos(participantes_format, int_grupo, tipo)
imprimir_grupos(agrupacion)


print()
print("Fibonacci Sequence")
n = int(input("Enter n for how many times you want add the preceding numbers. n = "))
a = 0
b = 1
count = 0
max_count = n
while count < max_count:
	count = count + 1
	a2 = a
	b2 = b
	a = b2
	b = a2 + b2
	print(a2)


import tensorflow as tf
import warnings
warnings.filterwarnings('ignore')
#定义神经网络结构相关的参数
input_node = 784
out_node = 10
layer_node = 500
#通过tf.get_Variable函数来获取变量。在训练神经网络时会创建这些变量；在测试是会通过保存的模型加载这些变量的取值。
#而且更加高方便的是，因为可以在变量加载时将滑动平均变量重命名，所以可以直接通过同样的名字在训练时使用变量自身，而
#在测试时使用变量的滑动平均值，在这个函数中也会将变量的正则化损失加入损失集合。
def get_weight_variable(shape,regularizer):
    weights = tf.get_variable('weights',shape,initializer=tf.truncated_normal_initializer(stddev=0.1))
    #当给出了正则化生成函数时，将当前变量的正则化损失加入名字为losses的集合，在这里使用了add_to_collection函数
    #将一个张量加入一个集合，而这个集合名称为losses，这是自定义的集合，不在TF自动管理的集合列表中
    if regularizer != None:
        tf.add_to_collection('losses',regularizer(weights))
    return weights
#定义神经网络的前向传播过程
def inference(input_tensor,regularizer):
    with tf.variable_scope('layer1'):
        #这里通过tf.get_variable或tf.Variable没有本质区别，因为在训练或者是测试没有在同一个程序中多次调用这个函数
        #如果在一个程序中多次调用这个函数，在第一次调用这之后需要将reuse参数设置为True
        weights = get_weight_variable([input_node,layer_node],regularizer)
        biases = tf.get_variable('biases',[layer_node],initializer=tf.constant_initializer(0.0))
        layer1 = tf.nn.relu(tf.matmul(input_tensor,weights)+biases)
        #类似的声明第二层神经网络的变量并完成向前传播过程
    with tf.variable_scope('layer2'):
        weights = get_weight_variable([layer_node,out_node],regularizer)
        biases = tf.get_variable('biases',[out_node],initializer=tf.constant_initializer(0.0))
        layer2 = tf.matmul(layer1,weights)+biases
    return layer2


# _*_ coding: utf-8 _*_

import update_data
from update_views import update_views

from data_source import data_source
from dash.dependencies import Input, Output

from tb_volumeData import tb_volumeData
from tb_valueData import tb_valueData
from tb_priceData import tb_priceData

from cht_priceShare import cht_priceShare
from tb_companyRank import tb_companyRank
from cht_topFiveAnnualTrend import cht_topFiveAnnualTrend
from cht_topFiveMonthlyTrend import cht_topFiveMonthlyTrend
from cht_companyPositioning import cht_companyPositioning

def callbacks(app, data_source):

    @app.callback(
        [
            ####### Key Indicator s#######
            ### Volume ###
            Output(component_id = "kiVolume", component_property = "children"),
            Output(component_id = "kiVolShare", component_property = "children"),
            Output(component_id = "kiVolYoY", component_property = "children"),
            ### Price ###
            Output(component_id = "kiPrice", component_property = "children"),
            Output(component_id = "kiPriceShare", component_property = "children"),
            Output(component_id = "kiPriceYoY", component_property = "children"),
            ### Value ###
            Output(component_id = "kiValue", component_property = "children"),
            Output(component_id = "kiValueShare", component_property = "children"),
            Output(component_id = "kiValueYoY", component_property = "children"),
            ### Share of Price Section ###
            Output(component_id = "priceShare", component_property = "children"),

            ####### Competition Summary s#######
            ### Company Rank ###
            Output(component_id = "tb_companyRank", component_property = "children"),
            ### Company Annual & Monthly Trend ###
            Output(component_id = "cht_topFiveAnnualTrend", component_property = "children"),
            Output(component_id = "cht_topFiveMonthlyTrend", component_property = "children"),
            ### Company Positioning ###
            Output(component_id = "cht_companyPositioning", component_property = "children")
        ],
        [Input(component_id = "company", component_property = "value"),
        Input(component_id = "country", component_property = "value"),
        Input(component_id = "year", component_property = "value"),
        Input(component_id = "month", component_property = "value"),
        Input(component_id = "indicator", component_property = "value")]
    )
    def update_condition(company, cn, year, month, indicator, data_source = data_source):

        update_data.updateData_company(data_source, company)
        update_data.updateData_cn(data_source, cn)
        update_data.updateData_year(data_source, year)
        update_data.updateData_period(data_source, month)
        update_data.updateData_indicator(data_source, indicator)

        print(data_source["company"])
        print(data_source["country"])
        print(data_source["year"])
        print(data_source["last_year"])
        print(data_source["period"])
        print(data_source["indicator"])
        print("\n")

        update_views(data_source)

        Volume = tb_volumeData(data_source)["volume"]
        VolShare = tb_volumeData(data_source)["volume_share"]
        VolYoY = tb_volumeData(data_source)["volume_YoY"]

        Price = tb_priceData(data_source)["price"]
        PriceShare = tb_priceData(data_source)["price_diff"]
        PriceYoY = tb_priceData(data_source)["price_YoY"]

        Value = tb_valueData(data_source)["value"]
        ValShare = tb_valueData(data_source)["value_share"]
        ValYoY = tb_valueData(data_source)["value_YoY"]

        priceShare = cht_priceShare(data_source)
        table = tb_companyRank(data_source)
        chartAnnual = cht_topFiveAnnualTrend(data_source)
        chartMonthly = cht_topFiveMonthlyTrend(data_source)
        chartPositioning = cht_companyPositioning(data_source)

        return Volume, VolShare, VolYoY, Price, PriceShare, PriceYoY, Value, ValShare, ValYoY, priceShare, table, chartAnnual, chartMonthly, chartPositioning


if __name__ == '__main__':
    N = int(raw_input())
ls = []
s = []
for i in range(N):
    s = raw_input().split()
    if s[0]=='insert':
        ls.insert(int(s[1]),int(s[2]))
    if s[0]=='print':
        print(ls)
    if s[0]=='remove':
        ls.remove(int(s[1]))
    if s[0]=='append':
        ls.append(int(s[1]))
    if s[0]=='sort':
        ls.sort()
    if s[0]=='pop':
        ls.pop()
    if s[0]=='reverse':
        ls.reverse()