from part4.models import *

def buildComputerObj(spec):
    obj = {}
    obj['id'] = spec.computer.pk
    obj['name'] = spec.computer.name
    obj['price'] = spec.computer.price
    obj['components'] = {
        'cpu': spec.cpu.pk,
        'graphics': spec.graphics_card.pk,
        'hardDrive': spec.hard_drive.pk,
        'ram': spec.ram.pk,
        'os': spec.os.pk,
        'display': spec.display.pk,
        'soundCard': spec.sound_card.pk
    }
    return obj

def get_computers():
    specs = ComputerSpecs.objects.filter(computer__default=True)
    res = []
    for spec in specs:
        res.append(buildComputerObj(spec))
    return res

def get_computer(id):
    spec = ComputerSpecs.objects.filter(computer__id=id)[0]
    return buildComputerObj(spec)

def get_components():
    res = {
        'cpu': list(Cpu.objects.values()),
        'graphics': list(GraphicsCard.objects.values()),
        'hardDrive': list(HardDrive.objects.values()),
        'ram': list(Ram.objects.values()),
        'os': list(Os.objects.values()),
        'display': list(Display.objects.values()),
        'soundCard': list(SoundCard.objects.values()),
    }
    return res

def get_components_for_computer(id):
    computer = get_computer(id)
    all_components = get_components()
    components = {}
    for key in all_components.keys():
        for component in all_components[key]:
            if component['id'] == computer['components'][key]:
                components[key] = component
    return components

def get_component_details(components):
    all_components = get_components()
    component_details = {}
    for key in components.keys():
        for candidate in all_components[key]:
            if candidate['id'] == components[key]:
                component_details[key] = candidate

    return component_details

def create_new_computer(selected):
    computer_name = selected['name']
    computer_price = selected['price']
    components = selected['components']

    #1. Create the computer
    computer = Computer.objects.create(
        name=computer_name,
        price=computer_price,
    )

    #2. Create the spec
    cpu_id = components['cpu']
    graphics_id = components['graphics']
    hard_drive_id = components['hardDrive']
    ram_id = components['ram']
    os_id = components['os']
    display_id = components['display']
    sound_card_id = components['soundCard']

    ComputerSpecs.objects.create(
        computer=computer,
        cpu=Cpu.objects.filter(pk=cpu_id)[0],
        graphics_card=GraphicsCard.objects.filter(pk=graphics_id)[0],
        hard_drive=HardDrive.objects.filter(pk=hard_drive_id)[0],
        ram=Ram.objects.filter(pk=ram_id)[0],
        os=Os.objects.filter(pk=os_id)[0],
        display=Display.objects.filter(pk=display_id)[0],
        sound_card=SoundCard.objects.filter(pk=sound_card_id)[0],
    )

    return computer

def assign_computer_user_cart(user, computer):
    su = StoreUser.objects.filter(user__id=user.id)[0]
    cart = Cart.objects.filter(store_user__id=su.id)[0]
    computer.cart = cart
    computer.save()

    cart.total_price = round(cart.total_price + computer.price, 2)
    cart.save()

def delete_computer_from_cart(computer_id):
    to_delete = Computer.objects.filter(pk=computer_id)[0]
    cart = Cart.objects.filter(pk=to_delete.cart.id)[0]
    cart.total_price =  round(cart.total_price - to_delete.price, 2)

    if (cart.total_price <= 1):
        cart.total_price = 0

    cart.save()
    to_delete.delete()

def get_user_cart(user):
    su = StoreUser.objects.filter(user__id=user.id)[0]
    cart = Cart.objects.filter(store_user__id=su.id)[0]
    computers = Computer.objects.filter(cart__id=cart.id)
    formatted_cart = []
    for computer in computers:
        components = get_components_for_computer(computer.id)
        formatted_cart.append({
            'id': computer.id,
            'name': computer.name,
            'price': computer.price,
            'components': components
        })

    return formatted_cart, cart.total_price

def create_user_order(user):
    su = StoreUser.objects.filter(user__id=user.id)[0]
    computers = Computer.objects.filter(cart__store_user__id=su.id)
    cart = Cart.objects.filter(store_user__id=su.id)[0]
    order = Order.objects.filter(store_user__id=su.id)[0]
    for computer in computers:
        computer.cart = None
        computer.order = order
        computer.save()

    order.total_price += cart.total_price
    cart.total_price = 0
    order.save()
    cart.save()

def delete_computer_from_order(computer_id):
    to_delete = Computer.objects.filter(pk=computer_id)[0]
    order = Order.objects.filter(pk=to_delete.order.id)[0]
    order.total_price =  round(order.total_price - to_delete.price, 2)

    if (order.total_price <= 1):
        order.total_price = 0

    order.save()
    to_delete.delete()

def get_user_orders(user):
    su = StoreUser.objects.filter(user__id=user.id)[0]
    orders = Order.objects.filter(store_user__id=su.id)[0]
    computers = Computer.objects.filter(order__id=orders.id)
    formatted_orders = []
    for computer in computers:
        components = get_components_for_computer(computer.id)
        formatted_orders.append({
            'id': computer.id,
            'name': computer.name,
            'price': computer.price,
            'components': components
        })

    return formatted_orders, orders.total_price


from xai.brain.wordbase.verbs._film import _FILM

#calss header
class _FILMED(_FILM, ):
	def __init__(self,):
		_FILM.__init__(self)
		self.name = "FILMED"
		self.specie = 'verbs'
		self.basic = "film"
		self.jsondata = {}


import re

file = open("dnsrecords.txt", "r")
text = file.read()

# select main domain (I need the newline character)
regEx = re.compile(r'\w+.cu'
                   r'.\s+IN\sSOA') # zoo.cmg.minag.cu.  IN  SOA
print(regEx.findall(text))

# select domain and IP address
regEx = re.compile(r'(\w+.cu.)\s+IN\s+A\s+(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})') # minag.cu.  IN  A 172.17.25.2
#print(regEx.findall(text))

# select serial number
regEx = re.compile(r'(\d{10})\s+;\s+serial') # (2018060700) ; serial
#print(regEx.findall(text))

# print the whole text
#print(text)


'''
2013/3/7
choose a proxy from USER_AGENT_LIST randomly
'''
from scrapy.exceptions import IgnoreRequest
from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware

class RedirectMiddleware(RedirectMiddleware):
    def process_response(self, request, response, spider):
        if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
            if 'www.HTTP_FORBIDDEN.net' in response.headers['Location']: #or '' in response.headers['Location']:
                spider.switch_proxy_reason = 'it is redirected to %s' % response.headers['Location']
                spider.switch_proxy = True
                redirected = request.replace(url=response.url, method='GET', body='')
                redirected.headers.pop('Content-Type', None)
                redirected.headers.pop('Content-Length', None)
                return redirected
        return super(RedirectMiddleware, self).process_response(request, response, spider)

    def process_exception(self, request, exception, spider):
        spider.switch_proxy = True
        spider.switch_proxy_reason = str(exception)
        redirected = request.replace(method='GET', body='')
        redirected.headers.pop('Content-Type', None)
        redirected.headers.pop('Content-Length', None)
        return redirected


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 25 12:05:48 2021

@author: acer
"""

###############################################################################
###############################################################################
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize
import numpy as np
###############################################################################

sentences = ["I ate dinner.",
       "We had a three-course meal.",
       "Brad came to dinner with us.",
       "He loves fish tacos.",
       "In the end, we all felt like we ate too much.",
       "We all agreed; it was a magnificent evening."]

def cosine(u, v):
    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))

###############################################################################
###############################################################################
#SentenceBERT

from sentence_transformers import SentenceTransformer
sbert_model = SentenceTransformer('bert-base-nli-mean-tokens')

sentence_embeddings = sbert_model.encode(sentences)

#print('Sample BERT embedding vector - length', len(sentence_embeddings[0]))
#print('Sample BERT embedding vector - note includes negative values', sentence_embeddings[0])

query = "I had pizza and pasta"
query_vec = sbert_model.encode([query])[0]

for sent in sentences:
  sim = cosine(query_vec, sbert_model.encode([sent])[0])
  print("Sentence = ", sent, "; similarity = ", sim)


from twisted.trial.unittest import SynchronousTestCase

from conveyer.conveyer import Conveyer, CreateLogCmd, AppendLogCmd

class TestConveyer(SynchronousTestCase):
    def test_first_log_post(self):
        """
        The first time we send a log message/event to the conveyer, it
        needs to create an empty log file, then append to it.
        """
        self.conveyer_config = {
            "log_file": "testfile.dat",
        }
        self.conveyer = Conveyer(config=self.conveyer_config)
        commands = self.conveyer.log("{message: \"test\"}")
        self.assertEquals(len(commands), 2)
        creator, appender = commands
        self.assertEquals(type(creator), CreateLogCmd)
        self.assertEquals(type(appender), AppendLogCmd)
        self.assertEquals(creator.filename, "testfile.dat")


import math

def build_map(file_name, horizontal):
    """
    Purpose:
        Build the hill with trees (#) required by the challenge
    Precond:
        :param file_name: name of .txt file
        :param horizontal: The number of horizontal increments for the provided map
    Return:
        Returns an arrayed list where each item is a horizontal strip of the map
    """
    in_file = open(file_name, "r")
    final_map = []
    for line in in_file:
        new_line = ""
        for n in range(horizontal):
            new_line += line.strip()
        final_map.append(new_line)

    return final_map

def challenge_1(tree_map, y_move, x_move):
    """
    Purpose:
        Process a map variable and count the number of times you hit a tree (#) when moving y=1 and x=3
    Pre:
        :param tree_map: Arrayed list where each list is a horizontal row of the map
        :param y_move: y-movement in the map each step
        :param x_move: x-position in the map each step
    Return:
        Number of times a tree is hit.
    """
    x, y = 0, 0
    tree_hits = 0
    while y < len(tree_map):
        if tree_map[y][x] == "#":
            tree_hits += 1
        y += y_move
        x += x_move
    return tree_hits

def challenge_2():
    """
    Purpose:
        Calculates the product of the number of trees hit for 5 different directional patterns provided by the advent challenge
    Return:
        The product of all the trees hit for 5 different movement patterns
    """
    answers = []
    x_moves = [1, 3, 5, 7, 1]
    y_moves = [1, 1, 1, 1, 2]
    for n in range(len(x_moves)):
        answers.append(challenge_1(build_map("d3/day3_input.txt", 100), y_moves[n], x_moves[n]))
    return math.prod(answers)

if __name__ == '__main__':
    print(challenge_1(build_map("d3/day3_example.txt", 15), 1, 3))
    print(challenge_1(build_map("d3/day3_input.txt", 35), 1, 3))
    print(challenge_2())


import pandas as pd
import numpy as np

def read_DB(Code=[], start= None, end= None, sampling= 'D'):
    """
    Read stock raw information from DB (PC)
    """
    from Stock_Class import Stocka

    if start is None:
        start = '2015-1-1'
    if end is None:
        end = '2017-12-31'

    StockRaw = pd.read_table(joinpath(r'C:\Users\Administrator\PycharmProjects\Stock\DB', Code))
    StockRaw['Date'] = pd.to_datetime(StockRaw['Date'])
    StockRaw.set_index(['Date'], inplace=True)

    StockRaw = StockRaw.ix[start:end]

    return Stock(StockRaw)

def joinpath(*args):
    from os.path import join

    for i in range(len(args)):
        if i == 0:
            path =args[i]
        else:
            path = join(path, args[i])

    return path

def excel(Df, index=True):

    """
    Save DataFrame to excel format

    :param Df: DataFrame to save
    :param index: index field is included or not
    """

    writer = pd.ExcelWriter(r'C:\Users\Administrator\PycharmProjects\Stock\Output\Output.xlsx', engine='xlsxwriter')
    Df.to_excel(writer, sheet_name='Sheet1', index= index)

def csv(Df, index= True):

    """
    Save DataFrame to csv format

    :param Df: DataFrame to save
    :param index: index field is included or not
    """
    Df.to_csv(r'C:\Users\Administrator\PycharmProjects\Stock\Output\Output.txt', index= index, sep= '\t')

def volume_check(Df, long_periods= 60, short_periods= 1):

    """
    Stock Volume comparison between short periods over long periods

    :param Df: DataFrame which contains stock volume information
    :param long_periods: long period [int]
    :param short_periods: short period [int]
    :return:
    """

    today = Df.index.max()

    _short_periods = pd.bdate_range(end= today, periods= short_periods)
    _long_periods = pd.bdate_range(end= today, periods= long_periods)

    sample = Df[Df.index.isin(_long_periods)]

    _long_periods = _long_periods - _short_periods

    sample.loc[sample.index.isin(_short_periods), 'Section'] = 'New'
    sample.loc[sample.index.isin(_long_periods), 'Section'] = 'Old'

    pivot = pd.pivot_table(sample, index= ['Code','Name'], columns= ['Section'], values= 'Volume', aggfunc= np.mean)
    pivot['Date Duration [Day]'] = str(long_periods) + ' vs. ' + str(short_periods)
    pivot['Volume PCT'] = pivot['New'] / pivot['Old'] * 100

    pivot['Rank'] = pivot['Volume PCT'].rank(method= 'first', ascending= False)
    pivot.sort_values(by = ['Rank'], ascending= True, inplace= True)

    return pivot

def profit_margin(Df, period_range= 20):

    Summary = pd.DataFrame(columns= ['Name','Date','Period','Min','Max','Min pct','Max pct','Price start','Price end','Price pct','Min-Max Range'])

    for i, st in enumerate(Df['Name'].unique()):
        try:
            temp = Df[Df['Name'] == st]

            end = temp.index[len(temp) -1]
            start = temp.index[len(temp) -1 -period_range]

            Sample = temp.ix[start: end]
            Sample.set_index([Sample.index,'Code','Name','Volume'], inplace= True)

            Sample = Sample.stack().reset_index()
            Sample.rename(columns= {'level_3':'Category', 0:'Price'}, inplace= True)

            Summary.set_value(i, 'Name', Sample['Name'].ix[0])
            Summary.set_value(i, 'Date', str(start)[:10] + '->' + str(end)[:10])
            Summary.set_value(i, 'Period', period_range)
            Summary.set_value(i, 'Min', Sample['Price'].min())
            Summary.set_value(i, 'Max', Sample['Price'].max())
            Summary.set_value(i, 'Min pct', round((- temp.get_value(start, 'Close') + Sample['Price'].min()) / temp.get_value(start, 'Close') * 100, 1))
            Summary.set_value(i, 'Max pct', round((- temp.get_value(start, 'Close') + Sample['Price'].max()) / temp.get_value(start, 'Close') * 100, 1))
            Summary.set_value(i, 'Price start', temp.get_value(start, 'Close'))
            Summary.set_value(i, 'Price end', temp.get_value(end, 'Close'))
            Summary.set_value(i, 'Price pct', round((temp.get_value(end, 'Close') - temp.get_value(start, 'Close')) / temp.get_value(start, 'Close') *100, 1))
            Summary['Min-Max Range'] = Summary['Max pct'] - Summary['Min pct']

        except:
            pass

    return Summary

def read_stock(code= 'all', engine= 'yahoo',start= None, end= None, sampling= 'D'):
    """
    Read stock raw information from server

    :param code: stock names to extract from KOSPI excel file
    :param engine: DB server to extract. Ex) 'yahoo', 'google'
    :param start: start day. Ex) '2013-1-1'
    :param end: end day. Ex) '2016-5-23'
    :param sampling: Sampling date interval. Ex) D: day, W: week, y: year
    :return: DataFrame of stock information. Valid columns are Open,High,Low,Close,Volume,Adj_Close
    """

    from Stock_Class import Stock
    import pandas.io.data as web

    if start is None:
        start = '2015-1-1'
        end = '2016-12-31'

    if code == 'all':
        StockList = pd.read_excel(r'C:\Users\Administrator\PycharmProjects\Stock\Input\KOSPI Code.xlsx', sheetname= 'Code', dtypes= {'Code':str})
        code = StockList['Code'].values
        name = StockList['Name'].values

    Df = pd.DataFrame()

    for i, st in enumerate(code):
        try:
            print(i,'-',st)
            if engine == 'google':
                st = str(st).replace('.KS','')
            temp = web.DataReader(st, engine, start, end)
            temp = temp[temp['Volume'] != 0]
            temp['Code'] = str(st)
            temp['Name'] = name[i]

            Df = zipping(Df, temp)

        except:
            pass

    if sampling != 'D':
        Df = Df.resample(sampling)

    return Stock(Df)

def zipping(Df1, Df2):

    """
    Two DataFrane concatenate along vertical direction

    :param Df1: Original DataFrame to join
    :param Df2: Concatenated DataFrame
    :return: Concatenated DataFrame including Df1, Df2
    """
    if Df1.empty:
        return Df2
    else:
        return pd.concat([Df1,Df2])

def beep(freq=400, duration=1000):

    """
    Make sound from given frequency & duration
    """
    from winsound import Beep
    Beep(freq,duration)


import maxify.modals.constants as constants
import os
import re

class CLICampaign(object):
    def __init__(self, name="Default", variants=1, debug=True, is_booking_path=True, conditions=[], full_campaign_name=None):
        self.name = name
        self.elements_folders = []
        self.variant_script_files = []
        self.site_scripts = []
        self.variants = variants
        self.debug = debug
        self.is_booking_path = is_booking_path
        self.conditions = conditions
        self.folder_path = None
        self.campaign_name = self.name
        self.destination_folder = constants.ORIGINAL_FOLDER_PATH

        if self.debug:
            self.destination_folder = constants.DEBUG_FOLDER_PATH
        self.name_prefix = "QA - {}"
        if not self.is_booking_path:
            self.name_prefix = self.name_prefix.format("")
        else:
            self.name_prefix = self.name_prefix.format("BookingPath_")

        if full_campaign_name:
            self.full_campaign_name = full_campaign_name
        else:
            self.full_campaign_name = self.name_prefix + self.campaign_name

        #Folders

        self.campaign_path = os.path.join(
            self.destination_folder, self.full_campaign_name)
        self.requirment_folder = os.path.join(
            self.campaign_path, "Requirement")
        self.source_folder = os.path.join(self.campaign_path, "Source File")
        self.test_suite_folder = os.path.join(
            self.campaign_path, "Dev test suite")
        self.campaign_scripts_folder = os.path.join(
            self.source_folder, "campaign scripts")
        self.variant_scripts_folder = os.path.join(
            self.source_folder, "variant script")

        self.vscode_folder = os.path.join(self.source_folder,".vscode")

        #Files

        self.configuration_file = os.path.join(
            self.source_folder, "maxymiser.json")

        self.common_script = os.path.join(
            self.campaign_scripts_folder, "common.js")
        self.qualification_script = os.path.join(
            self.campaign_scripts_folder, "qualification.js")
        self.analytics_script = os.path.join(
            self.campaign_scripts_folder, "analytics.js")
        self.variant_script_file = os.path.join(
            self.variant_scripts_folder, "variant$.html")

        self.test_suit_name = self.campaign_name + " test suits.xlsx"
        self.test_suite_file = os.path.join(
            self.test_suite_folder, self.test_suit_name)

        self.vscode_task_file = os.path.join(self.vscode_folder,"tasks.json")

        self.is_folder_setup = False

class MaxymiserCampaign(CLICampaign):
    def __init__(self, configuration):
        super(MaxymiserCampaign, self).__init__(
            full_campaign_name=configuration["name"], is_booking_path=False)

        self.configurations = configuration

        self.script_content_mapping = {}

        for element in configuration["elements"]:
            element_name = "{}-{}".format(element["name"], element["id"])
            element_path = os.path.join(self.source_folder, element_name)

            for variant in element["variants"]:

                if variant["name"].find("variant") != -1:
                    variant_name = "{}-{}-.html".format(
                        variant["name"], variant["id"])
                    variant_path = os.path.join(element_path, variant_name)
                    self.variant_script_files.append(variant_path)
                    self.script_content_mapping[variant_path] = variant["content"]
            self.elements_folders.append(element_path)

        for site_script in configuration["scripts"]:

            script_name = "{}-{}-.js".format(
                site_script["name"], site_script["id"])
            script_path = os.path.join(
                self.campaign_scripts_folder, script_name)
            self.script_content_mapping[script_path] = site_script["content"]
            if re.search(r'^qualification-.*-\.js$', script_name, re.IGNORECASE):
                self.qualification_script = script_path
            elif re.search('analytics', script_name, re.IGNORECASE):
                self.analytics_script = script_path
            else:
                self.site_scripts.append(script_path)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from six import text_type as str

import unittest
from MyCapytain.common.utils import *
from copy import copy

class TestUtils(unittest.TestCase):

    def test_clean_xpath(self):
        """ Cleaning XPATH and normalizing them """
        l = ['tei:text', 'tei:body', 'tei:div', "tei:div[@n='1']", "tei:div[@n='pr']", "tei:l[@n='2']"]
        self.assertEqual(normalizeXpath(l), l)

        l = ['tei:text', 'tei:body', 'tei:div', "tei:div[@n='1']", "", "tei:div[@n='pr']", "tei:l[@n='2']"]
        self.assertEqual(
            normalizeXpath(l),
            ['tei:text', 'tei:body', 'tei:div', "tei:div[@n='1']", "/tei:div[@n='pr']", "tei:l[@n='2']"],
            "Empty list element should be replaced with / in the next element"
        )

    def test_copy_node_without_children(self):
        node = xmlparser("Mc")

        no_text = copy(node)
        no_text.text = None  # Remove text
        [no_text.remove(a) for a in no_text]  # Remove nodes
        copied_node = copyNode(node)
        self.assertEqual(
            etree.tostring(copied_node),
            etree.tostring(no_text),
            "Text without children should have no text nor xml nodes"
        )
        self.assertNotIn(
            "",
            etree.tostring(copied_node, encoding=str),
            "Text without children should have no text nor xml nodes"
        )
        self.assertNotIn(
            "M",
            etree.tostring(copied_node, encoding=str),
            "Text without children should have no text nor xml nodes"
        )

    def test_copy_node_with_children(self):
        node = xmlparser("Mc")
        comparison = copy(node)

        copied_node = copyNode(node, children=True)
        self.assertEqual(
            etree.tostring(copied_node),
            etree.tostring(comparison),
            "Text without children should have no text nor xml nodes"
        )
        self.assertIn(
            "",
            etree.tostring(copied_node, encoding=str),
            "Text without children should have no text nor xml nodes"
        )


# -*- coding: utf-8 -*-

import numpy as np
from sklearn import datasets

from ztlearn.utils import *
from ztlearn.dl.models import Sequential
from ztlearn.dl.optimizers import register_opt
from ztlearn.dl.layers import BatchNormalization, Dense

data = datasets.load_digits()
plot_digits_img_samples(data)

img_rows = 8
img_cols = 8
img_dim = 64  # img_rows * img_cols
latent_dim = 4
init_type = 'he_normal'

def stack_encoder_layers(init):
    model = Sequential(init_method = init)
    model.add(Dense(256, activation = 'relu', input_shape = (img_dim,)))
    model.add(BatchNormalization())
    model.add(Dense(128, activation = 'relu'))
    model.add(BatchNormalization())
    model.add(Dense(latent_dim, activation = 'relu'))

    return model

def stack_decoder_layers(init):
    model = Sequential(init_method = init)
    model.add(Dense(128, activation = 'relu', input_shape = (latent_dim,)))
    model.add(BatchNormalization())
    model.add(Dense(256, activation = 'relu'))
    model.add(BatchNormalization())
    model.add(Dense(img_dim, activation = 'sigmoid'))

    return model

encoder = stack_encoder_layers(init = init_type)
decoder = stack_decoder_layers(init = init_type)

opt = register_opt(optimizer_name = 'adam', momentum = 0.01, learning_rate = 0.0001)

autoencoder = Sequential(init_method = init_type)
autoencoder.layers.extend(encoder.layers)
autoencoder.layers.extend(decoder.layers)
autoencoder.compile(loss = 'categorical_crossentropy', optimizer = opt)

images = range_normalize(data.data.astype(np.float32), 0, 1)  # rescale to range [0, 1]
train_data, test_data, train_label, test_label = train_test_split(images,
                                                                  images,
                                                                  test_size = 0.2,
                                                                  random_seed = 15)

model_epochs = 500
fit_stats = autoencoder.fit(train_data,
                            train_label,
                            batch_size = 64,
                            epochs = model_epochs,
                            validation_data = (test_data, test_label),
                            shuffle_data = True)

# generate non rescaled test labels for use in generated digits plot
_, _, _, test_label = train_test_split(data.data, data.target, test_size = 0.2, random_seed = 15)
predictions = autoencoder.predict(test_data).reshape((-1, img_rows, img_cols))

plot_generated_digits_samples(unhot(one_hot(test_label)), predictions)
plot_metric('Loss', model_epochs, fit_stats['train_loss'], fit_stats['valid_loss'])
plot_metric('Accuracy', model_epochs, fit_stats['train_acc'], fit_stats['valid_acc'])


import json
import time
from os import path
from pathlib import Path
import pandas as pd
from scraper import scrape_urls
from vars import Paths

def preprocess():
    url_path = Paths.URLS.value

    if path.isfile(url_path):
        df = pd.read_csv(url_path).drop_duplicates(subset=['url'])
        df.to_csv(url_path, index=False)
    else:
        try:
            with open(Path('./data', 'urls.json'), 'r') as json_data:
                data = json.load(json_data)

            urls = []

            for category in data:
                for url in data[category]:
                    urls.append([url, category])

            scrapped_urls = scrape_urls(urls)

            labeled_urls = []

            for url in scrapped_urls:
                if url[2] == 'Adult':
                    labeled_urls.append([*url, 0, 0, 0, 1])
                elif url[2] == 'Games' or url[2] == 'Entertainment':
                    labeled_urls.append([*url, 1, 0, 0, 0])
                elif url[2] == 'Shopping':
                    labeled_urls.append([*url, 1, 1, 0, 0])
                else:
                    labeled_urls.append([*url, 1, 1, 1, 1])

            header = ['url', 'content', 'category',
                      'underage', 'office', 'student', 'custom']

            url_path = Path('./data', 'urls.csv')
            pd.DataFrame(labeled_urls).to_csv(
                url_path, header=header, index=False)
        except Exception as e:
            print(e)

if __name__ == '__main__':
    tic = time.perf_counter()
    preprocess()
    print(f'Preprocessed in {round(time.perf_counter() - tic, 2)} seconds')


import json

from django.shortcuts import render, redirect

from tournament.api.tournaments import (
    delete_api_tournament,
    get_tournament_matches,
    get_tournament_stages,
    refusal_api_participate,
    upsert_api_tournament,
    upsert_api_participate,
)
from tournament.forms import UpsertTournamentForm, ParticipateTournamentForm
from service_api.models.tournaments import Match, MatchTeam, Stage, Tournament, Participate
from service_api.models.teams import Team
from service_api.models.disciplines import Game

def tournament_list(request):
    """
    大会リストを新しい順番で返す
    :param request:
    :rtype render:
    """
    return render(request, 'cms/tournament/tournament_list.html', context={
        'tournaments': Tournament.objects.order_by('-date_start').all()
    })

def upsert_tournament(request, tournament_id=None):
    """
    大会の新規作成・修正を行う
    :param request:
    :param tournament_id:
    :rtype render|redirect:
    """
    if request.method == 'POST':
        # 編集時
        if tournament_id:
            form = UpsertTournamentForm(request.POST, request.FILES, instance=Tournament.objects.get(pk=tournament_id))
            if form.is_valid():
                # Toornament APIにPOSTする
                api_tournament_id = upsert_api_tournament(
                    form.instance, api_tournament_id=Tournament.objects.get(pk=tournament_id).api_tournament_id)
                form.instance.api_tournament_id = api_tournament_id
                form.save()
                return redirect('/competition/tournament/')
            return render(request, 'cms/tournament/upsert_tournament.html', context={
                'tournament_id': tournament_id,
                'tournament_api_id': Tournament.objects.get(pk=tournament_id).api_tournament_id,
                'tournament_bracket': _get_tournament_bracket(tournament_id) if tournament_id else None,
                'participate_teams': Participate.objects.select_related('tournament', 'team').filter(
                    tournament_id=tournament_id).all(),
                'matches': _get_match_list(tournament_id),
                'form': form
            })
        # 新規作成
        form = UpsertTournamentForm(request.POST, request.FILES)
        if form.is_valid():
            # Toornament APIにPOSTする
            api_tournament_id = upsert_api_tournament(form.instance)
            form.instance.api_tournament_id = api_tournament_id
            form.save()
            return redirect('/competition/tournament/')
        return render(request, 'cms/tournament/upsert_tournament.html', context={
            'form': form
        })
    return render(request, 'cms/tournament/upsert_tournament.html', context={
        'tournament_id': tournament_id,
        'tournament_api_id': Tournament.objects.get(pk=tournament_id).api_tournament_id if tournament_id else None,
        'tournament_bracket': _get_tournament_bracket(tournament_id) if tournament_id else None,
        'participate_teams': Participate.objects.select_related('tournament', 'team').filter(
            tournament_id=tournament_id).all(),
        'matches': _get_match_list(tournament_id),
        'form': UpsertTournamentForm(
            instance=Tournament.objects.get(pk=tournament_id)) if tournament_id else UpsertTournamentForm()
    })

def participate_tournament(request, team_id):
    """
    大会に参加する
    :param request:
    :param int team_id:
    :rtype render:
    """
    if request.method == 'POST':
        form = ParticipateTournamentForm(request.POST,
                                         initial={'team': Team.objects.get(pk=team_id)})
        # 既に登録済みの大会は除く
        form.fields['tournament'].queryset = Tournament.objects.filter(is_active=True).exclude(
            participate__team__id=team_id)
        if form.is_valid():
            form.instance.team = Team.objects.get(pk=team_id)
            # Toornament APIの方にも登録しておく
            api_participate_id = upsert_api_participate(form.instance.team,
                                                        api_tournament_id=form.instance.tournament.api_tournament_id)
            form.instance.api_participate_id = api_participate_id
            form.save()
            return redirect('/competition/team/edit/{}/'.format(team_id))
        return render(request, 'cms/tournament/participate_tournament.html', context={
            'form': form
        })
    form = ParticipateTournamentForm()
    # 既に登録済みの大会は除く
    form.fields['tournament'].queryset = Tournament.objects.filter(is_active=True).exclude(
        participate__team__id=team_id)
    return render(request, 'cms/tournament/participate_tournament.html', context={
        'form': form
    })

def refusal_tournament(request, team_id, tournament_id):
    """
    大会参加を辞退する
    :param request:
    :param int team_id:
    :param int tournament_id:
    :rtype redirect:
    """
    team = Team.objects.get(pk=team_id)
    tournament = Tournament.objects.get(pk=tournament_id)
    if team and tournament:
        participate = Participate.objects.filter(team=team, tournament=tournament).first()
        refusal_api_participate(api_tournament_id=participate.tournament.api_tournament_id,
                                api_participate_id=participate.api_participate_id)
        participate.delete()
    return redirect('/competition/tournament/edit/{}/'.format(tournament_id))

def delete_tournament(request, tournament_id):
    """
    大会を削除する
    :param request:
    :param int tournament_id:
    :rtype redirect:
    """
    tournament = Tournament.objects.get(pk=tournament_id)
    if tournament:
        delete_api_tournament(api_tournament_id=tournament.api_tournament_id)
        tournament.delete()
    return redirect('/competition/tournament')

def get_and_upsert_matches(request, tournament_id):
    """
    Toonament APIからマッチ情報を取得し, DBに新規登録or更新をする
    :param request:
    :param tournament_id:
    :rtype redirect:
    """
    tournament = Tournament.objects.get(pk=tournament_id)
    match_entities = get_tournament_matches(api_tournament_id=tournament.api_tournament_id)
    stage_entities = get_tournament_stages(api_tournament_id=tournament.api_tournament_id)
    for stage_entity in stage_entities:
        # ステージ情報を登録・更新
        Stage.objects.update_or_create(
            tournament=tournament,
            api_stage_id=stage_entity.number(),
            defaults={
                'name': stage_entity.name(),
                'type': stage_entity.type()
            }
        )
    for match_entity in match_entities:
        # マッチ情報を登録・更新
        match = Match.objects.update_or_create(
            api_match_id=match_entity.id(),
            game=Game.objects.get(pk=tournament.game.id),
            tournament=tournament,
            defaults={
                'stage': Stage.objects.get(tournament_id=tournament_id, api_stage_id=match_entity.stage_number()),
                'group_number': match_entity.group_number(),
                'round_number': match_entity.round_number(),
                'match_format': tournament.match_format,
                'status': match_entity.status(),
            })
        for opponent in match_entity.opponents():
            # マッチ詳細情報を登録・更新
            participate_id = opponent.participant().id()
            if not participate_id:
                # 全戦分のマッチ情報が帰ってくるので勝ち上がりチームがまだ分からない(participate_id=None)時は処理を抜ける
                continue
            MatchTeam.objects.update_or_create(
                match=match[0],
                team=Participate.objects.get(api_participate_id=participate_id).team,
                defaults={
                    'api_opponent_id': opponent.number(),
                    'result': opponent.result(),
                    'score': opponent.score()
                })
    return redirect('/competition/tournament/edit/{}/'.format(tournament_id))

def _get_tournament_bracket(tournament_id):
    """
    トーナメント表生成用のdictを返す
    :param tournament_id:
    :rtype json:
    """
    round_numbers = sorted(set(Match.objects.values_list('round_number', flat=True)
                               .filter(tournament_id=tournament_id)))
    teams = []
    results = []
    for round_number in round_numbers:
        match_teams = MatchTeam.objects.select_related('match', 'team')\
            .filter(match__tournament__id=tournament_id)\
            .filter(match__round_number=round_number)
        match_ids = sorted(set(mt.match.id for mt in match_teams))
        for match_id in match_ids:
            _teams = []
            _results = []
            _match_teams = [mt for mt in match_teams if mt.match.id == match_id]
            for match_team in _match_teams:
                _teams.append(match_team.team.name)
                _results.append(match_team.score)
            if round_number == 1:
                teams.append(_teams)
            results.append(_results)

    # こちらを参考に頑張りましょう http://www.aropupu.fi/bracket/
    brackets = json.dumps({
        'teams': teams,
        'results': [
            [
                results
            ]
        ]
    })
    return brackets

def _get_match_list(tournament_id):
    """
    指定されたトーナメントのマッチリストを返す
    :param int tournament_id:
    :rtype List[Match]:
    """
    matches = Match.objects.filter(tournament_id=tournament_id)
    match_ids = list(match.id for match in matches)
    match_teams = MatchTeam.objects.select_related('team', 'match').filter(match_id__in=match_ids)\
        .order_by('match__stage__id', 'match__group_number', 'match__round_number')
    # Matchオブジェクトにマッチ情報をListで入れておく
    # FIXME: Matchに直接入れるのは微妙
    for match in matches:
        match.detail = ' vs '.join([mt.team.name + '(' + str(mt.score) + ')'
                                    for mt in match_teams if mt.match_id == match.id])
    return matches


import unittest
from examen import *

class Pruebas(unittest.TestCase):

    def test_libros(self):
        libros = [Libro(Autor(1, "pepe", "pepon"), "Pepe aventuras", 1920), Libro(Autor(1, "pepe", "pepon"), "Aventuras de pepo", 1922), Libro(Autor(2, "pepa", "pepon"), "Pepa's Adventure", 1922)]
        self.assertEqual(mas_antiguos(libros, 1901), [])
        self.assertEqual(mas_antiguos(libros, 1920), ["Pepe aventuras"])
        self.assertEqual(mas_antiguos(libros, 1922), ["Aventuras de pepo","Pepa's Adventure"])

class Suite(unittest.TestSuite):
    def __init__(self):
        super(Suite, self).__init__()
        self.addTest(Pruebas('test_libros'))

if __name__ == "__main__":
    runner = unittest.TextTestRunner()
    my_suite = Suite()
    runner.run(my_suite)


# Forest Fire Sim, by Al Sweigart al@inventwithpython.com
# A simulation of fires spreading in a growing forest.
# Inspired by Nicky Case's Emoji Sim http://ncase.me/simulating/model/

import random, time, sys

try:
    import bext
except ImportError:
    print("""This program requires the bext module, which you can install by
opening a Terminal window (on macOS & Linux) and running:

    python3 -m pip install --user bext

or a Command Prompt window (on Windows) and running:

    python -m pip install --user bext""")
    sys.exit()

WIDTH = 79
HEIGHT = 21

INITIAL_TREE_DENSITY = 20 # Percentage of board that starts with trees.
GROW_CHANCE = 1.0         # % chance a blank spot turns into a tree.
LIGHTNING_CHANCE = 0.1    # % chance a tree is hit by lightning & burns.

NUMBER_OF_FIREWALLS = 4
FIREWALL_LENGTH = 20
WALL_CHAR = chr(9608)

PAUSE_LENGTH = 0.05

# Create a new board data structure.
board = {'width': WIDTH, 'height': HEIGHT}
for x in range(WIDTH):
    for y in range(HEIGHT):
        if (random.randint(1, 10000) / 100) <= INITIAL_TREE_DENSITY:
            board[(x, y)] = 'A' # Start as a tree.
        else:
            board[(x, y)] = ' ' # Start as an empty space.

# Create firewalls
for i in range(NUMBER_OF_FIREWALLS):
    if random.randint(0, 1) == 0:
        # Make a horizontal firewall:
        x = random.randint(0, max(WIDTH - FIREWALL_LENGTH - 1, 0))
        y = random.randint(0, HEIGHT)
        for ix in range(FIREWALL_LENGTH):
            board[(x + ix, y)] = WALL_CHAR
    else:
        # Make a vertical firewall:
        x = random.randint(0, WIDTH)
        y = random.randint(0, max(HEIGHT - (FIREWALL_LENGTH // 2) - 1, 0))
        for iy in range(FIREWALL_LENGTH // 2):
            board[(x, y + iy)] = WALL_CHAR

bext.clear()
try:
    while True:
        # Draw the board data structure.
        bext.goto(0, 0)
        for y in range(board['height']):
            for x in range(board['width']):
                if board[(x, y)] == 'A':
                    bext.fg('green')
                    print('A', end='')
                elif board[(x, y)] == 'W':
                    bext.fg('red')
                    print('W', end='')
                else:
                    bext.fg('reset')
                    print(board[(x, y)], end='')
            print()
        bext.fg('reset') # Use the default font color.
        print('Grow chance: {}%  Lightning chance: {}%'.format(GROW_CHANCE, LIGHTNING_CHANCE))
        print('Press Ctrl-C to quit.')

        # Run a single simulation step:
        nextBoard = {'width': board['width'], 'height': board['height']}

        for x in range(board['width']):
            for y in range(board['height']):
                if (x, y) in nextBoard:
                    # If we've already set nextBoard[(x, y)] on a previous iteration, just do nothing here.
                    continue

                if board[(x, y)] == ' ' and (random.randint(1, 10000) / 100) <= GROW_CHANCE:
                    # Grow a tree in this empty space:
                    nextBoard[(x, y)] = 'A' # The letter 'A' sort of looks like a tree.
                elif board[(x, y)] == 'A' and (random.randint(1, 10000) / 100) <= LIGHTNING_CHANCE:
                    # Lightning sets this tree on fire:
                    nextBoard[(x, y)] = 'W'
                elif board[(x, y)] == 'W':
                    # Fire spreads to neighboring trees.
                    for ix in range(-1, 2):
                        for iy in range(-1, 2):
                            #if (0 <= x + ix < board['width']) and (0 <= y + iy < board['height']):
                            if (x + ix, y + iy) in board:
                                if board[(x + ix, y + iy)] == 'A':
                                    nextBoard[(x + ix, y + iy)] = 'W'
                                else:
                                    nextBoard[(x + ix, y + iy)] = board[(x + ix, y + iy)]
                    nextBoard[(x, y)] = ' ' # The original tree has burned down now.
                else:
                    # Just copy the existing object.
                    nextBoard[(x, y)] = board[(x, y)]
        board = nextBoard

        time.sleep(PAUSE_LENGTH)
except KeyboardInterrupt:
    pass # When Ctrl-C is pressed, stop looping.


#!/usr/bin/python
# -*- coding: utf-8 -*-
import xdrlib ,sys, os
import xlrd
import collections
import re

class ExcelToDict:
    def __init__(self, excelPath, ignoreList):
        self.excelPath = excelPath
        # 需要忽略的tag
        self.ignoreTagList = ignoreList

    #打开excel并返回数据
    def openExcel(self):
        try:
            data = xlrd.open_workbook(self.excelPath)
            return data
        except Exception as e:
            print("Exception")

    #解析excel
    def parseExcel(self):
        data = self.openExcel()
        sheetNames = data.sheet_names()
        typeDict = {}
        valueDict = {}
        for sheetName in sheetNames:
            needIgnore, newKey = self.checkNeedIgnoreKey(sheetName)
            if not needIgnore:
                sheet = data.sheet_by_name(sheetName)
                #sheet 名字 统一转换为小写 再存储
                newSheetName = newKey.lower()
                (typeDict[newSheetName],valueDict[newSheetName]) = self.parseSheet(sheet)
        return typeDict,valueDict

    #解析sheet返回dict
    def parseSheet(self, sheet):
        #总行数
        nRows = sheet.nrows
        #总列数
        nCols = sheet.ncols

        #第一行 字段名
        fieldName = sheet.row_values(0)
        #第二行 字段类型
        fieldValueType = sheet.row_values(1)
        #第三行 字段简短解释
        comment = sheet.row_values(2)
        #第四行 字段详细描述
        #第五行 字段赋值

        #第一列 存储key
        keyCol = sheet.col_values(0)

        typeDict = collections.OrderedDict()
        for i in range(0,nCols):
            #检查是否有过滤字段
            tempKey = fieldName[i]
            needIgnore, newKey = self.checkNeedIgnoreKey(tempKey)
            if not needIgnore:
                tempKey = newKey
                typeDict[tempKey] = (fieldValueType[i], comment[i])

        valueDict = {}
        if "client" in self.ignoreTagList:
            valueDict = []
        else:
            valueDict = collections.OrderedDict()
        for i in range(4,nRows):
            row = sheet.row_values(i)
            tempDict = collections.OrderedDict()
            for j in range(nCols):
                tempKey = fieldName[j]
                #检查是否有过滤字段
                needIgnore, newKey = self.checkNeedIgnoreKey(tempKey)
                if not needIgnore:
                    tempKey = newKey
                    tempDict[tempKey] = self.parseValueByType(row[j],fieldValueType[j])
            if len(tempDict) > 0:
                if isinstance(valueDict, list):
                    valueDict.append(tempDict)
                else:
                    keyValue = self.parseValueByType(keyCol[i],fieldValueType[0])
                    valueDict[str(keyValue)] = tempDict

        return typeDict,valueDict

    #根据类型对字段进行重新处理 支持 int string int[] string[] float float[]
    def parseValueByType(self, value, type):
        if type == "int":
            if value == "":
                return 0
            return int(value)
        elif type == "float":
            if value == "":
                return 0
            if float(value) == int(float(value)):
                return int(value)
            return float(value)
        elif type == "int[]":
            newValue = str(value).split(',')
            if len(newValue) > 0 and newValue[0] == "":
                return []
            #print(newValue)
            for i in range(0, len(newValue)):
                newValue[i] = int(float(newValue[i]))
            return newValue
        elif type == "float[]":
            newValue = str(value).split(',')
            for i in range(0, len(newValue)):
                newValue[i] = float(newValue[i])
                if newValue[i] == int(newValue[i]):
                    newValue[i] = int(newValue[i])
            return newValue
        elif type == "string[]":
            return value.split(',')
        return value

    # 是否需要忽略此key, 返回忽略 以及处理过的key
    def checkNeedIgnoreKey(self, oldKey):
        pattern = r"\((.*?)\)"
        # 搜索"(Tag)" 并提取出其中的Tag
        matchObj = re.search(pattern, oldKey)
        if matchObj is None:
            return False, oldKey
        # 如果tag在忽略列表中，则返回 True
        matchTag = matchObj.groups()[0]
        if matchTag.lower() in self.ignoreTagList:
            return True, oldKey
        # 否则将tag处理掉，返回False + 新的key
        newKey = re.sub(pattern, "", oldKey)
        return False, newKey


#!/usr/bin/python2.7

""" Samples price, adjprice, vol, vol for first trading day of each month.
    (The last column is supposed to be adjvol, but we don't have that for
    yahoo data so we use vol instead).
    EDIT: actually yahoo may not have unadjusted vol (ie, vol is actually
    adjvol).  We don't rely on (adj)vol much so this is okay for now.

    For price and adjprice, it's just the closing value of the day.
    For vol and adjvol, it's the average value across the previous
    trading month.
"""

import argparse
import os
import util

EXTENSION = '.csv'
HEADER = 'Date,Open,High,Low,Close,Volume,Adj Close'

def getVolume(v):
  if v == '': return 0.0
  return float(v)

def processYahoo(raw_dir, processed_dir):
  raw_files = os.listdir(raw_dir)
  for raw_file in raw_files:
    assert raw_file.endswith(EXTENSION)
    ticker = raw_file[:-len(EXTENSION)]
    input_file = '%s/%s' % (raw_dir, raw_file)
    with open(input_file, 'r') as fp:
      lines = fp.read().splitlines()
    assert len(lines) > 0
    assert lines[0] == HEADER
    processed_file = '%s/%s' % (processed_dir, ticker)
    with open(processed_file, 'w') as fp:
      previous_ymd = None
      previous_ym = None
      previous_vo = 0.0
      previous_days = 0
      for i in range(len(lines) - 1, 0, -1):
        ymd, op, hi, lo, cl, vo, acl = lines[i].split(',')
        if previous_ymd is not None:
          assert ymd > previous_ymd
        previous_ymd = ymd
        ym = util.ymdToYm(ymd)
        if ym != previous_ym:
          previous_ym = ym
          if previous_days > 0:
            previous_vo /= previous_days
          print >> fp, '%s\t%s\t%s\t%s\t%s' % (
              ymd, cl, acl, previous_vo, previous_vo)
          previous_vo = 0.0
          previous_days = 0
        previous_vo += getVolume(vo)
        previous_days += 1

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('--raw_dir', required=True)
  parser.add_argument('--processed_dir', required=True)
  args = parser.parse_args()
  processYahoo(args.raw_dir, args.processed_dir)

if __name__ == '__main__':
  main()


from __future__ import division, absolute_import, print_function
from past.builtins import xrange

import fitsio
import esutil
import numpy as np
from scipy import interpolate

from .chisq_dist import ChisqDist
from .chisq_dist import compute_chisq
from .catalog import Catalog
from .utilities import CubicSpline
from .utilities import MStar

class RedSequenceColorPar(object):
    """
    Class which contains a red-sequence parametrization

    parameters
    ----------
    filename: string
        red sequence parameter file
    zbinsize: float, optional
        interpolation bin size in redshift
    minsig: float, optional
        minimum intrinsic scatter.  Default 0.01
    fine: bool, optional
        finest binning, set in filename header.  Default False
    zrange: float*2, optional
        redshift range [zlo,zhi].  Default from filename header
        (maximum range)

    """
    def __init__(self, filename, zbinsize=None, minsig=0.01, fine=False, zrange=None, config=None, limmag=None):

        if filename is None:
            if config is None:
                raise ValueError("Must have either filename or config")
            if limmag is None:
                limmag = config.limmag_catalog
            else:
                limmag = limmag
            if zrange is None:
                zrange = config.zrange
            alpha = config.calib_lumfunc_alpha
            mstar_survey = config.mstar_survey
            mstar_band = config.mstar_band
            ncol = config.nmag - 1

            # A compromize zbinsize
            zbinsize = 0.001

            has_file = False
        else:
            pars,hdr=fitsio.read(filename, ext=1, header=True, upper=True)
            try:
                if limmag is None:
                    limmag = hdr['LIMMAG']
                else:
                    limmag = limmag
                if (zrange is None):
                    zrange = np.array([hdr['ZRANGE0'],hdr['ZRANGE1']])
                alpha = hdr['ALPHA']
                mstar_survey = hdr['MSTARSUR']
                mstar_band = hdr['MSTARBAN']
                ncol = hdr['NCOL']
            except:
                raise ValueError("Missing field from parameter header.")
            has_file = True

        if len(zrange) != 2:
            raise ValueError("zrange must have 2 elements");

        if zbinsize is None:
            try:
                if fine:
                    zbinsize=hdr['ZBINFINE']
                else:
                    zbinsize=hdr['ZBINCOAR']
            except:
                raise ValueError("Missing field from parameter header.")

        try:
            lowzmode=hdr['LOWZMODE']
        except:
            lowzmode = 0

        nmag = ncol + 1
        self.nmag = nmag

        if has_file:
            bvalues=np.zeros(nmag)
            do_lupcorr = False
            try:
                for i in xrange(nmag):
                    bvalues[i] = hdr['BVALUE%1d' % (i+1)]
                do_lupcorr = True
            except:
                bvalues[:] = 0.0

            try:
                ref_ind = hdr['REF_IND']
            except:
                try:
                    ref_ind = hdr['I_IND']
                except:
                    raise ValueError("Need REF_IND or I_IND")

        nz = np.round((zrange[1]-zrange[0])/zbinsize).astype('i4') #nr of bins
        self.z = zbinsize*np.arange(nz) + zrange[0]                #z bins

        # append a high-end overflow bin
        # for computation this will be the same as the top bin, and at the end
        # we set it to some absurdly large number.
        self.z = np.append(self.z,self.z[self.z.size-1])
        nz=nz+1

        self.zbinsize=zbinsize
        self.zbinscale=int(1./zbinsize)

        ms=MStar(mstar_survey,mstar_band)

        refmagbinsize=0.01
        if (lowzmode):
            refmagrange=np.array([10.0,limmag],dtype='f4')
            # FIXME
            lumrefmagrange=np.array([10.0,ms(zrange[1])-2.5*np.log10(0.1)])
        else:
            refmagrange=np.array([12.0,limmag],dtype='f4')
            lumrefmagrange=np.array([12.0,ms(zrange[1])-2.5*np.log10(0.1)])
        self.refmagbins = np.arange(refmagrange[0], refmagrange[1], refmagbinsize, dtype='f8')
        self.lumrefmagbins=np.arange(lumrefmagrange[0],lumrefmagrange[1],refmagbinsize,dtype='f8')

        # and for fast look-ups...
        self.refmagbins = np.append(self.refmagbins,self.refmagbins[self.refmagbins.size-1])
        self.lumrefmagbins = np.append(self.lumrefmagbins,self.lumrefmagbins[self.lumrefmagbins.size-1])

        self.refmagbinsize = refmagbinsize
        self.refmagbinscale = int(1./refmagbinsize)
        self.refmaginteger = (self.refmagbins*self.refmagbinscale).astype(np.int64)
        self.lumrefmaginteger = (self.lumrefmagbins*self.refmagbinscale).astype(np.int64)

        if has_file:
            # is this an old or new structure?
            if 'PIVOTMAG_Z' in pars.dtype.names:
                refmag_name = 'REFMAG'
                pivotmag_name = 'PIVOTMAG'
            else :
                refmag_name = 'IMAG'
                pivotmag_name = 'REFMAG'

            # mark the extrapolated values
            self.extrapolated = np.zeros(nz,dtype=np.bool_)
            loz,=np.where(self.z < np.min(pars[pivotmag_name+'_Z']))
            hiz,=np.where(self.z > np.max(pars[pivotmag_name+'_Z']))
            if (loz.size > 0) : self.extrapolated[loz] = True
            if (hiz.size > 0) : self.extrapolated[hiz] = True

            # set the pivotmag
            spl=CubicSpline(pars[0][pivotmag_name+'_Z'],pars[0][pivotmag_name])
            self.pivotmag = spl(self.z)

            # and the max/min refmag
            spl=CubicSpline(pars[0][pivotmag_name+'_Z'], pars[0]['MAX'+refmag_name])
            self.maxrefmag = spl(self.z)
            spl=CubicSpline(pars[0][pivotmag_name+'_Z'], pars[0]['MIN'+refmag_name])
            self.minrefmag = spl(self.z)

            # c/slope
            self.c = np.zeros((nz,ncol),dtype=np.float64)
            self.slope = np.zeros((nz,ncol),dtype=np.float64)
            for j in xrange(ncol):
                jstring='%02d' % (j)
                spl=CubicSpline(pars[0]['Z'+jstring],pars[0]['C'+jstring])
                self.c[:,j] = spl(self.z)
                spl=CubicSpline(pars[0]['ZS'+jstring],pars[0]['SLOPE'+jstring])
                self.slope[:,j] = spl(self.z)

            # sigma/covmat
            self.sigma = np.zeros((ncol,ncol,nz),dtype=np.float64)
            self.covmat = np.zeros((ncol,ncol,nz),dtype=np.float64)

            # diagonals
            for j in xrange(ncol):
                spl=CubicSpline(pars[0]['COVMAT_Z'],pars[0]['SIGMA'][j,j,:])
                self.sigma[j,j,:] = spl(self.z)

                self.covmat[j,j,:] = self.sigma[j,j,:]*self.sigma[j,j,:]

            # off-diagonals
            for j in xrange(ncol):
                for k in xrange(j+1,ncol):
                    spl=CubicSpline(pars[0]['COVMAT_Z'],pars[0]['SIGMA'][j,k,:])
                    self.sigma[j,k,:] = spl(self.z)

                    too_high,=np.where(self.sigma[j,k,:] > 0.9)
                    if (too_high.size > 0):
                        self.sigma[j,k,too_high] = 0.9
                    too_low,=np.where(self.sigma[j,k,:] < -0.9)
                    if (too_low.size > 0):
                        self.sigma[j,k,too_low] = -0.9

                    self.sigma[k,j,:] = self.sigma[j,k,:]

                    self.covmat[j,k,:] = self.sigma[k,j,:] * self.sigma[j,j,:] * self.sigma[k,k,:]
                    self.covmat[k,j,:] = self.covmat[j,k,:]

            # volume factor
            spl=CubicSpline(pars[0]['VOLUME_FACTOR_Z'],pars[0]['VOLUME_FACTOR'])
            self.volume_factor = spl(self.z)

            # corrections
            spl=CubicSpline(pars[0]['CORR_Z'],pars[0]['CORR'])
            self.corr = spl(self.z)
            spl=CubicSpline(pars[0]['CORR_SLOPE_Z'],pars[0]['CORR_SLOPE'])
            self.corr_slope = spl(self.z)

            spl=CubicSpline(pars[0]['CORR_Z'],pars[0]['CORR2'])
            self.corr2 = spl(self.z)
            spl=CubicSpline(pars[0]['CORR_SLOPE_Z'],pars[0]['CORR2_SLOPE'])
            self.corr2_slope = spl(self.z)

            if 'CORR_R' in pars.dtype.names:
                # protect against stupidity
                if (pars[0]['CORR_R'][0] <= 0.0) :
                    self.corr_r = np.ones(nz)
                else:
                    spl=CubicSpline(pars[0]['CORR_SLOPE_Z'],pars[0]['CORR_R'])
                    self.corr_r = spl(self.z)

                test,=np.where(self.corr_r < 0.5)
                if (test.size > 0) : self.corr_r[test] = 0.5

                if (pars[0]['CORR2_R'][0] <= 0.0):
                    self.corr2_r = np.ones(nz)
                else:
                    spl=CubicSpline(pars[0]['CORR_SLOPE_Z'],pars[0]['CORR2_R'])
                    self.corr2_r = spl(self.z)

                test,=np.where(self.corr2_r < 0.5)
                if (test.size > 0) : self.corr2_r[test] = 0.5

            else :
                self.corr_r = np.ones(nz)
                self.corr2_r = np.ones(nz)

        # mstar
        # create LUT
        self._mstar = ms(self.z)

        # luminosity function integrations
        self.lumnorm = np.zeros((self.lumrefmagbins.size,nz))
        self.alpha = alpha
        for i in xrange(nz):
            f=10.**(0.4*(self.alpha+1.0)*(self._mstar[i]-self.lumrefmagbins))*np.exp(-10.**(0.4*(self._mstar[i]-self.lumrefmagbins)))
            self.lumnorm[:,i] = refmagbinsize*np.cumsum(f)

        if has_file:
            # lupcorr (annoying!)
            self.lupcorr = np.zeros((self.refmagbins.size,nz,ncol),dtype='f8')
            if (do_lupcorr):
                bnmgy = bvalues*1e9

                for i in xrange(nz):
                    mags = np.zeros((self.refmagbins.size,nmag))
                    lups = np.zeros((self.refmagbins.size,nmag))

                    mags[:,ref_ind] = self.refmagbins

                    # go redward
                    for j in xrange(ref_ind+1,nmag):
                        mags[:,j] = mags[:,j-1] - (self.c[i,j-1]+self.slope[i,j-1]*(mags[:,ref_ind]-self.pivotmag[i]))
                    # blueward
                    for j in xrange(ref_ind-1,-1,-1):
                        mags[:,j] = mags[:,j+1] + (self.c[i,j]+self.slope[i,j]*(mags[:,ref_ind]-self.pivotmag[i]))

                    # and the luptitude conversion
                    for j in xrange(nmag):
                        flux = 10.**((mags[:,j]-22.5)/(-2.5))
                        lups[:,j] = 2.5*np.log10(1.0/bvalues[j]) - np.arcsinh(0.5*flux/bnmgy[j])/(0.4*np.log(10.0))

                    magcol = mags[:,0:ncol] - mags[:,1:ncol+1]
                    lupcol = lups[:,0:ncol] - lups[:,1:ncol+1]

                    self.lupcorr[:,i,:] = lupcol - magcol

        # set top overflow bins to very large number
        self.z[self.z.size-1] = 1000.0
        self.zinteger=np.round(self.z*self.zbinscale).astype(np.int64)
        self.refmagbins[self.refmagbins.size-1] = 1000.0
        self.refmaginteger = np.round(self.refmagbins*self.refmagbinscale).astype(np.int64)
        self.lumrefmagbins[self.lumrefmagbins.size-1] = 1000.0
        self.lumrefmaginteger = np.round(self.lumrefmagbins*self.refmagbinscale).astype(np.int64)
        self.ncol = ncol
        self.alpha = alpha
        self.mstar_survey = mstar_survey
        self.mstar_band = mstar_band
        self.limmag = limmag

        # don't make this into a catalog
        #super(RedSequenceColorPar, self).__init__(zredstr)

    def mstar(self,z):
        """
        M_star lookup

        parameters
        ----------
        z: array of floats
           redshift

        returns
        -------
        mstar: array of floats

        """
        # lookup and return mstar...awesome.
        zind = self.zindex(z)
        return self._mstar[zind]

    def zindex(self,z):
        """
        redshift index lookup

        parameters
        ----------
        z: array of floats

        returns
        -------
        indices: array of integers
            redshift indices

        """
        # return the z index/indices with rounding.

        zind = np.searchsorted(self.zinteger,np.round(np.atleast_1d(z)*self.zbinscale).astype(np.int64))
        #zind = np.searchsorted(self.zinteger,(np.atleast_1d(z)*self.zbinscale).astype(np.int64))
        if (zind.size == 1):
            return np.asscalar(zind)
        else:
            return zind

        # and check for top overflows.  Minimum is always 0
        #test,=np.where(zind == self.z.size)
        #if (test.size > 0): zind[test] = self.z.size-1

    def refmagindex(self,refmag):
        """
        reference magnitude index lookup

        parameters
        ----------
        refmag: array of floats

        returns
        -------
        indices: array of integers
            refmag indices
        """
        # return the refmag index/indices with rounding

        refmagind = np.searchsorted(self.refmaginteger,np.round(np.atleast_1d(refmag)*self.refmagbinscale).astype(np.int64))
        if (refmagind.size == 1):
            return np.asscalar(refmagind)
        else:
            return refmagind

    def lumrefmagindex(self,lumrefmag):
        """
        luminosity reference magnitude index lookup

        parameters
        ----------
        lumrefmag: array of floats

        returns
        -------
        indices: array of integers
            lumrefmag indices

        """

        lumrefmagind = np.searchsorted(self.lumrefmaginteger,np.round(np.atleast_1d(lumrefmag)*self.refmagbinscale).astype(np.int64))
        if (lumrefmagind.size == 1):
            return np.asscalar(lumrefmagind)
        else:
            return lumrefmagind

    def calculate_chisq(self, galaxies, z, calc_lkhd=False, z_is_index=False):
        """
        compute chisq for a set of galaxies at redshift z OR a galaxy at many redshifts OR
          many galaxies at many redshifts

        parameters
        ----------
        galaxies: GalaxyCatalog
            galaxies which need chisq values
        z: redshift
            redshift to compute chisq
        calc_lkhd: bool, optional
            compute likelihood rather than chisq (default False)
        z_is_index: bool, optional
            The input redshift is already a redshift index

        returns
        -------
        chisqs: array of floats
        """
        if z_is_index:
            zind = z
        else:
            zind = self.zindex(z)
        magind = self.refmagindex(galaxies.refmag)
        #magind = self.refmagindex(galaxies._ndarray['REFMAG'])
        galcolor = galaxies.galcol
        #chisq_dist = ChisqDist(self.covmat[:,:,zind], self.c[zind,:],
        #                       self.slope[zind,:], self.pivotmag[zind],
        #                       galaxies.refmag, galaxies.mag_err,
        #                       galcolor, refmagerr=galaxies.refmag_err,
        #                       lupcorr=self.lupcorr[magind,zind,:])
        #chisq_dist = ChisqDist(self.covmat[:, :, zind], self.c[zind, :],
        #                       self.slope[zind, :], self.pivotmag[zind],
        #                       galaxies._ndarray['REFMAG'], galaxies._ndarray['MAG_ERR'][0],
        #                       galcolor, refmagerr=galaxies._ndarray['REFMAG_ERR'],
        #                       lupcorr=self.lupcorr[magind, zind, :])
        #if calc_lkhd:
        #    return chisq_dist.compute_chisq(chisq_mode=False)
        #return chisq_dist.compute_chisq(chisq_mode=True)
        if calc_lkhd:
            calc_chisq = False
        else:
            calc_chisq = True

        return compute_chisq(self.covmat[:,:,zind], self.c[zind,:],
                             self.slope[zind,:], self.pivotmag[zind],
                             galaxies.refmag, galaxies.mag_err,
                             galcolor, refmagerr=galaxies.refmag_err,
                             lupcorr=self.lupcorr[magind,zind,:],
                             calc_chisq=calc_chisq, calc_lkhd=calc_lkhd)

    def calculate_chisq_redshifts(self, galaxy, zs, calc_lkhd=False):
        """
        Compute chisq for a galaxy at a set of redshifts zs

        parameters
        ----------
        galaxy: element of GalaxyCatalog
           Galaxy which needs chisq values
        zs: numpy array
           redshifts to compute chisq
        calc_lkhd: bool, optional
           compute likelihood rather than chisq (default False)

        returns
        -------
        chisqs: array of floats
        """

        zind = self.zindex(zs)
        magind = self.refmagindex(galaxy.refmag)
        galcolor = galaxy.galcol
        chisq_dist = ChisqDist(self.covmat[:, :, zind], zredstr.c[zind, :], zredstr.slope[zind, :],
                               self.pivotmag[zind], galaxy.refmag, galaxy.mag_err,
                               galcolor, refmagerr=galaxy.refmag_err,
                               lupcorr=self.lupcorr[magind, zind, :])
        if calc_lkhd:
            return chisq_dist.compute_chisq(chisq_mode=False)
        return chisq_dist.compute_chisq(chisq_mode=True)

    def plot_redsequence_diag(self, fig, ind, bands):
        """
        """

        not_extrap, = np.where(~self.extrapolated)

        ax = fig.add_subplot(221)
        ax.plot(self.z[: -1], self.c[: -1, ind], 'r--')
        ax.plot(self.z[not_extrap], self.c[not_extrap, ind], 'r-')
        ax.set_xlabel('Redshift')
        ax.set_ylabel('<%s - %s>' % (bands[ind], bands[ind + 1]))

        ax = fig.add_subplot(222)
        ax.plot(self.z[: -1], self.slope[: -1, ind], 'r--')
        ax.plot(self.z[not_extrap], self.slope[not_extrap, ind], 'r-')
        ax.set_xlabel('Redshift')
        ax.set_ylabel('(%s - %s) slope' % (bands[ind], bands[ind + 1]))

        ax = fig.add_subplot(223)
        ax.plot(self.z[: -1], self.sigma[ind, ind, : -1], 'r--')
        ax.plot(self.z[not_extrap], self.sigma[ind, ind, not_extrap], 'r-')
        ax.set_xlabel('Redshift')
        ax.set_ylabel('(%s - %s) sigma' % (bands[ind], bands[ind + 1]))

        fig.tight_layout()

    def plot_redsequence_offdiags(self, fig, bands):
        """
        """

        noff = (self.ncol * self.ncol - self.ncol) / 2

        nrow = (noff + 1) / 2

        not_extrap, = np.where(~self.extrapolated)

        ctr = 1
        for j in xrange(self.ncol):
            for k in xrange(j + 1, self.ncol):
                ax = fig.add_subplot(nrow, 2, ctr)
                ax.plot(self.z[: -1], self.sigma[j, k, : -1], 'r--')
                ax.plot(self.z[not_extrap], self.sigma[j, k, not_extrap], 'r-')
                ax.set_xlabel('Redshift')
                ax.set_ylabel('Corr %s-%s / %s-%s' % (bands[j], bands[j + 1], bands[k], bands[k + 1]))
                ctr += 1

        fig.tight_layout()

    def __repr__(self):
        return "Representation here."


import config
import cv2 as cv
import numpy as np
import os
from detectors import SkeletonDetector, BODY_MODEL
from stereo import DisparityCalculator, StereoCapture, StereoParams, prepare_for_vis
import time
import random

ROOT_DIR = os.path.dirname(__file__)

colors = [
    (11, 138, 236),
    (136, 7, 105),
    (200, 4, 4),
    (200, 4, 200),
    (242, 113, 211),
    (4, 4, 200),
    (4, 200, 200),
    (4, 200, 4),
]

def draw_skeleton(image, people_joints):
    rendered = image.copy()
    if rendered.ndim == 2 or rendered.shape[-1] == 1:
        rendered = cv.cvtColor(rendered, cv.COLOR_GRAY2BGR)

    for n, person in enumerate(people_joints):
        color = colors[n % len(colors)][::-1]
        for i in range(len(person)):
            if person[i] is not None:
                for j in BODY_MODEL[i][1]:
                    if person[j] is not None:
                        cv.line(rendered, (int(person[i][0]), int(person[i][1])),
                                (int(person[j][0]), int(person[j][1])), color, 1)

        joints = [p for p in person if p is not None]
        joints = np.reshape(joints, [len(joints), 1, 2])
        joints = np.int32(joints)
        cv.polylines(rendered, joints, True, (0, 0, 255), 2)

    return rendered

def min_max_norm(x):
    mn, mx = x.min(), x.max()
    return (x - mn) / (mx - mn)

def get_iou(bbox1, bbox2):
    x1, y1, w1, h1 = bbox1
    x2, y2, w2, h2 = bbox2

    xy1 = np.array([x1, y1], np.float32)
    wh1 = np.array([w1, h1], np.float32)

    xy2 = np.array([x2, y2], np.float32)
    wh2 = np.array([w2, h2], np.float32)

    mins1 = xy1
    maxes1 = xy1 + wh1

    mins2 = xy2
    maxes2 = xy2 + wh2

    intersect_mins = np.maximum(mins1, mins2)
    intersect_maxes = np.minimum(maxes1, maxes2)
    intersect_wh = np.maximum(0, intersect_maxes - intersect_mins)

    intersect_areas = np.prod(intersect_wh)

    areas1 = np.prod(wh1)
    areas2 = np.prod(wh2)

    union_areas = areas1 + areas2 - intersect_areas

    iou = intersect_areas / np.maximum(union_areas, 1e-6)

    return iou

def main():
    intrinsics_path = os.path.join(ROOT_DIR, 'models', 'intrinsics.yml')
    if not os.path.exists(intrinsics_path):
        raise RuntimeError('Can\'t find a intrinsics file!')

    extrinsics_path = os.path.join(ROOT_DIR, 'models', 'extrinsics.yml')
    if not os.path.exists(extrinsics_path):
        raise RuntimeError('Can\'t find a extrinsics file!')

    skeleton_model_path = os.path.join(ROOT_DIR, 'models', 'pose-unet-96x128.pb')
    if not os.path.exists(skeleton_model_path):
        raise RuntimeError('Can\'t find a skeleton detector model!')

    # facial_landmarks_model_path = os.path.join(ROOT_DIR, 'models', 'shape_predictor_68_face_landmarks.dat')
    # if not os.path.exists(facial_landmarks_model_path):
    #     raise RuntimeError('Can\'t find a facial landmark detection model!')

    # age_pca_path = os.path.join(ROOT_DIR, 'models', 'agedb-pca.pkl')
    # if not os.path.exists(age_pca_path):
    #     raise RuntimeError('Can\'t find an age pca model!')

    # gender_svm_path = os.path.join(ROOT_DIR, 'models', 'agedb-gender_svm.pkl')
    # if not os.path.exists(gender_svm_path):
    #     raise RuntimeError('Can\'t find a gender svm model!')

    # age_svm_path = os.path.join(ROOT_DIR, 'models', 'agedb-svm.pkl')
    # if not os.path.exists(age_svm_path):
    #     raise RuntimeError('Can\'t find an age svm model!')

    # age_svr_path = os.path.join(ROOT_DIR, 'models', 'agedb-svr.pkl')
    # if not os.path.exists(age_svr_path):
    #     raise RuntimeError('Can\'t find an age svr model!')

    # age_features_stat_path = os.path.join(ROOT_DIR, 'models', 'agedb-mean_std.npz')
    # if not os.path.exists(age_features_stat_path):
    #     raise RuntimeError('Can\'t find an age features statistics file!')

    skeleton_detector = SkeletonDetector(skeleton_model_path, (128, 96), 8)

    # age_gender_predictor = AgeGenderPredictor(
    #     facial_landmarks_model_path, age_pca_path, gender_svm_path, age_svm_path, age_svr_path, age_features_stat_path)

    stereo_params = StereoParams(intrinsics_path, extrinsics_path)

    if config.ENABLE_DISPARITY:
        disp_calc = DisparityCalculator(**config.SGBM_params)

    if isinstance(config.VIDEO_SOURCE, int):
        cap = StereoCapture(config.VIDEO_SOURCE, stereo_params)
    else:
        cap = StereoCapture(config.VIDEO_SOURCE)

    print('Capture: (%i, %i) %.2f' % (int(
        cap.get(cv.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)), cap.get(cv.CAP_PROP_FPS)), flush=True)

    if config.DEBUG_WRITE:
        writer = 0
        if config.ENABLE_DISPARITY:
            disp_writer = None
        left_writer = None
        right_writer = None

    try:
        print('Processing is started.', flush=True)

        people = []
        last_id = 0
        key = None

        start = time.time()
        while True:
            cap_start = time.time()
            ret, (left_frame, right_frame) = cap.read()
            cap_elapsed = time.time() - cap_start
            if not ret:
                break

            # disparity calculation

            if config.ENABLE_DISPARITY:
                disp_start = time.time()
                disparity_map = disp_calc(left_frame, right_frame)
                disp_elapsed = time.time() - disp_start
            else:
                disp_elapsed = 1e-3

            # skeleton detection

            skeleton_start = time.time()
            frame_people_skeletons, (joints_map, bones_map) = skeleton_detector(left_frame, ret_maps=True)
            skeleton_elapsed = time.time() - skeleton_start

            # tracking

            tracking_start = time.time()

            frame_people = []
            for person in frame_people_skeletons:
                head = person[0]
                neck = person[1]

                if head is not None and neck is not None:
                    dist = np.linalg.norm(np.array(head)-neck) * 1.25
                    bbox = [head[0]-dist, head[1]-dist, head[0]+dist, head[1]+dist]

                    frame_people.append((bbox, person))

            if not people:
                people = list(zip(range(last_id, len(frame_people)), frame_people, [None]*len(frame_people)))
                last_id = last_id + len(frame_people)
            else:
                new_people = []
                if frame_people:
                    for i in range(len(people)):
                        person = people[i]

                        best_j = None
                        best_iou = 0
                        for j in range(len(frame_people)):
                            iou = get_iou(person[1][0], frame_people[j][0])
                            if max(0, iou - 0.5) > 0 and best_iou < iou:
                                best_iou = iou
                                best_j = j

                        if best_j is not None:
                            new_people.append((person[0], frame_people[best_j], person[2]))
                            del frame_people[best_j]

                    if new_people:
                        last_id = max(new_people, key=lambda x: x[0])[0]+1
                    for person in frame_people:
                        new_people.append((last_id, person, None))
                        last_id += 1

                people = new_people

            tracking_elapsed = time.time() - tracking_start

            # age & gender detection

            if people and key is not None and key & 0xFF == ord('d'):
                # age_gender_start = time.time()

                # _, people_data, _ = zip(*people)
                # people_bboxes, _ = zip(*people_data)

                # predictions = age_gender_predictor(left_frame, people_bboxes)

                # for i, gender, age in predictions:
                #     people[i] = (people[i][0], people[i][1], (gender, age))

                age_gender_elapsed = 1e-3  # time.time() - age_gender_start
            else:
                age_gender_elapsed = 1e-3

            end = time.time()
            elapsed = end - start
            frame_elapsed = end - cap_start
            start = end

            print('\rFPS: %.2f; CAP: %.2f; DISP: %.2f; SKLTN: %.2f; TRACK: %.2f; AGE: %.2f; FRAME: %.2f, PPL_FOUND: %i' %
                  (1/elapsed, 1/cap_elapsed, 1/disp_elapsed, 1/skeleton_elapsed, 1/tracking_elapsed, 1/age_gender_elapsed, 1/frame_elapsed, len(people)), end='', flush=True)

            display_image = cv.cvtColor(left_frame, cv.COLOR_GRAY2BGR)

            display_image[..., 2] = np.uint8(
                255*np.clip((np.float32(display_image[..., 2]) / 255) + 0.9*min_max_norm(np.max(joints_map, axis=-1)), 0, 1))
            display_image[..., 1] = np.uint8(
                255*np.clip((np.float32(display_image[..., 1]) / 255) + 0.5*min_max_norm(np.max(np.linalg.norm(bones_map, axis=-1), axis=-1)), 0, 1))

            if people:
                _, people_data, _ = zip(*people)
                _, skeletons = zip(*people_data)

                display_image = draw_skeleton(display_image, skeletons)
                for id_, (bbox, _), extra in people:
                    cv.rectangle(display_image, (int(bbox[0]), int(bbox[1])),
                                 (int(bbox[2]), int(bbox[3])), (32, 32, 225))

                    person_line = ('#%i' % id_)
                    if extra is not None:
                        gender, age = extra

                        person_line = '%s a: %i' % (person_line, age)
                        person_line = '%s %s' % (person_line, 'M' if gender == 1 else 'F')

                    cv.putText(display_image, person_line, (int(bbox[0])+5, int(bbox[1])+16),
                               cv.FONT_HERSHEY_SIMPLEX, 0.5, (160, 32, 225), 2)

            cv.imshow('demo', display_image)
            if config.ENABLE_DISPARITY:
                cv.imshow('disparity', prepare_for_vis(disparity_map))

            if config.DEBUG_WRITE:
                if isinstance(writer, int):
                    writer += 1
                    if writer > 2:
                        writer = cv.VideoWriter('./demo.mp4', cv.VideoWriter_fourcc(*'avc1'), 1/elapsed, (int(
                            cap.get(cv.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))))
                        if config.ENABLE_DISPARITY:
                            disp_writer = cv.VideoWriter('./demo-disps.mp4', cv.VideoWriter_fourcc(*'avc1'), 1/elapsed, (int(
                                cap.get(cv.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))), False)
                        left_writer = cv.VideoWriter('./demo-lefts.mp4', cv.VideoWriter_fourcc(*'avc1'), 1/elapsed, (int(
                            cap.get(cv.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))), False)
                        right_writer = cv.VideoWriter('./demo-rights.mp4', cv.VideoWriter_fourcc(*'avc1'), 1/elapsed, (int(
                            cap.get(cv.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))), False)
                else:
                    writer.write(display_image)
                    if config.ENABLE_DISPARITY:
                        disp_writer.write(prepare_for_vis(disparity_map))
                    left_writer.write(left_frame)
                    right_writer.write(right_frame)

            key = cv.waitKey(1)
            if key == 27:
                break
            if key & 0xFF == ord('q'):
                break
    except:
        print('', flush=True)
        raise
    finally:
        cap.release()
        if config.DEBUG_WRITE:
            if not isinstance(writer, int):
                writer.release()
                if config.ENABLE_DISPARITY:
                    disp_writer.release()
                left_writer.release()
                right_writer.release()

if __name__ == "__main__":
    main()


import os
import numpy as np
import tvm
from tvm import autotvm
from tvm import relay
import tvm.relay.testing
from tvm.relay.testing.init import create_workload

from tvm.autotvm.tuner import XGBTuner, GATuner, RandomTuner, GridSearchTuner
from tvm.contrib.util import tempdir
import tvm.contrib.graph_runtime as runtime

from PIL import Image
from time import sleep
#from tvm.contrib import ndk

def conv_3x3(net, in_channles, out_channels, prefix, activation=True):
    weight = relay.var(prefix + "_weight")
    net = relay.nn.conv2d(net, weight, padding=(1,1), channels=out_channels, kernel_size=(3,3))

    # Bias add
    bias = relay.var(prefix + "_bias")
    net = relay.nn.bias_add(net, bias)

    # ReLU
    if activation:
        net = relay.nn.relu(net)

    return net

def get_net(batch_size, image_shape, num_classes, dtype):
    height = image_shape[1]
    width = image_shape[2]
    data_shape = (batch_size,) + image_shape
    net = relay.var("data", shape=data_shape, dtype=dtype)

    net = conv_3x3(net, 3, 64, "conv1_1")

    out = net
    args = relay.analysis.free_vars(out)

    return relay.Function(args, out)

def get_workload(batch_size=1,
                 num_classes=1000,
                 image_shape=(3, 224, 224),
                 dtype="float32",
                 **kwargs):
    net = get_net(batch_size=batch_size,
                  num_classes=num_classes,
                  image_shape=image_shape,
                  dtype=dtype,
                  **kwargs)
    return create_workload(net)

#target = tvm.target.cuda()
target_host = 'llvm -target=aarch64-linux-gnu'
target = tvm.target.cuda("-model=tx2")
#target_host = tvm.target.create('llvm -target=aarch64-linux-gnu')

network = 'sample'
log_file = 'gpu.log'
dtype = 'float32'

device_key = 'tx2'

tuning_option = {
    'log_filename': log_file,

    'tuner': 'xgb',
    'n_trial': 1000,
    'early_stopping': 600,

    'measure_option': autotvm.measure_option(
        builder=autotvm.LocalBuilder(timeout=10),
        #runner=autotvm.LocalRunner(number=20, repeat=3, timeout=4, min_repeat_ms=150),
        runner=autotvm.RPCRunner(
            device_key,  # change the device key to your key
            '0.0.0.0', 9190,
            number=5, repeat=3, timeout=100, min_repeat_ms=150)
    ),
}

def tune_tasks(tasks,
              measure_option,
              tuner='xgb',
              n_trial=1000,
              early_stopping=None,
              log_filename='tuning.log',
              use_transfer_learning=True,
              try_winograd=True):
    if try_winograd:
        for i in range(len(tasks)):
            try:    # try winograd template
                tsk = autotvm.task.create(tasks[i].name, tasks[i].args,
                                          tasks[i].target, tasks[i].target_host, 'winograd')
                input_channel = tsk.workload[1][1]
                if input_channel >= 64:
                    tasks[i] = tsk
            except Exception:
                pass

    # create tmp log file
    tmp_log_file = log_filename + ".tmp"
    if os.path.exists(tmp_log_file):
        os.remove(tmp_log_file)

    for i, tsk in enumerate(reversed(tasks)):
        prefix = "[Task %2d/%2d] " % (i+1, len(tasks))

        # create tuner
        if tuner == 'xgb' or tuner == 'xgb-rank':
            tuner_obj = XGBTuner(tsk, loss_type='rank')
        elif tuner == 'ga':
            tuner_obj = GATuner(tsk, pop_size=100)
        elif tuner == 'random':
            tunber_obj = RandomTuner(tsk)
        elif tuner == 'gridsearch':
            tuner_obj = GridSearchTuner(tsk)
        else:
            raise ValueError("Invalid tuner: " + tuner)

        if use_transfer_learning:
            if os.path.isfile(tmp_log_file):
                tuner_obj.load_history(autotvm.record.load_from_file(tmp_log_file))

        # do tuning
        n_trial = min(n_trial, len(tsk.config_space))
        tuner_obj.tune(n_trial=min(n_trial, len(tsk.config_space)),
                       early_stopping=early_stopping,
                       measure_option=measure_option,
                       callbacks=[
                           autotvm.callback.progress_bar(n_trial, prefix=prefix),
                           autotvm.callback.log_to_file(tmp_log_file)])

    # pick best records to a cache file
    autotvm.record.pick_best(tmp_log_file, log_filename)
    os.remove(tmp_log_file)

def main():
    # extract workloads from relay program
    input_shape = (1, 3, 224, 224)
    print("Extrack tasks...")
    mod, params= get_workload(image_shape = input_shape[1:], batch_size=input_shape[0])

    tasks = autotvm.task.extract_from_program(mod["main"], target=target, target_host=target_host, params=params, ops=(relay.op.nn.conv2d, relay.op.nn.dense,))

    # run tuning tasks
    print("Tuning...")

    tune_tasks(tasks, **tuning_option)

    with autotvm.apply_history_best(log_file):
        print("Compile...")
        with relay.build_config(opt_level=3):
            graph, lib, params = relay.build_module.build(mod, target=target, params=params)

            tmp = tempdir()
            filename = "net.tar"
            lib.export_library(tmp.relpath(filename))

            remote = autotvm.measure.request_remote(device_key, '0.0.0.0', 9190, timeout=10000)
            remote.upload(tmp.relpath(filename))
            rlib = remote.load_module(filename)

            ctx = remote.context(str(target), 0)
            module = runtime.create(graph, rlib, ctx)
            data_tvm = tvm.nd.array((np.random.uniform(size=input_shape)).astype(dtype))

            print("Run...")
            print("Set_input(\"data\")")
            module.set_input('data', data_tvm)
            print("Set_input(**param)")
            module.set_input(**params)

            #evaluate
            print("Evaluate inference time cost...")
            ftimer = module.module.time_evaluator("run", ctx, number=1, repeat=600)
            prof_res = np.array(ftimer().results) * 1000
            print("Mean inference time (std dev): %.2f ms (%.2f ms)" %(np.mean(prof_res), np.std(prof_res)))

if __name__ == "__main__":
    main()


from AidosMarket import AidosMarketAPI, AidosMarketProtectedAPI, AidosMarketKey

# 公開系API
aidos_market = AidosMarketAPI()

# order book
orders = aidos_market.order_book(market="ADK", currency="BTC")

# transactions
transactions = aidos_market.transactions(market="ADK", currency="BTC", limit=5)

auth = AidosMarketKey(
    secret="api secret",
    key="api key"
)

# 非公開系API
aidos_protected = AidosMarketProtectedAPI(keys=auth)

# my balances and info
balances_and_info = aidos_protected.balances_and_info()

open_orders = aidos_protected.open_orders(market="ADK", currency="BTC")