# -*- coding: utf-8 -*-
"""

 Stat 202A 2019 Fall - Homework 02
 Author:
 Date :

 INSTRUCTIONS: Please fill in the corresponding function. Do not change function names,
 function inputs or outputs. Do not write anything outside the function.

"""

import numpy as np

### Part 1 : Metropolis Algorithm

def sample_uniform(size=10000, low=0, high=1):

	"""
	Function 1A: Uniform (Same as HW1)
	Detail : 	This function return a np.ndarray with given size.
	"""
	import random
	X = np.zeros(size)

	m = 2**31 - 1
	a = 7**5
	c = 12345

	# Set the seed using the current system time in microseconds
	import time
	d = int(time.time() * random.random())
	for i in range (0,size):
		d = (a * d + c) % m
		X[i] = d / m * (high - low) + low

	return X

def sample_normal_chain(x0, c, chain_length=100, mean=0, var=1):
	import math
	"""
	Function 1B: Normal by Metropolis
	Detail : 	This function return multiple chains by Metropolis sampling from N(mean, var).
				For every train, the proposal distribution at x is y ~ Uniform[x-c, x+c].
				The input x0 is a 1 dimension np.ndarray.
				Return a np.ndarray X with size x0.shape[0] * chain_length. In X, each row X[i]
				should be a chain and t-th column X[:, t] corresponding to all different X_t.
	# """
	X = np.zeros((x0.shape[0], chain_length))
	eps = np.zeros((x0.shape[0], chain_length))
	probs = np.zeros((x0.shape[0], chain_length))
	X[:, 0] = x0
	for i in range (1, chain_length):
		eps[:,i] = sample_uniform(x0.shape[0], -c, c)
		probs[:,i] = sample_uniform(x0.shape[0], 0, 1)
		y = X[:,i-1] + eps[:,i]
		for j in range (0, x0.shape[0]):
			acceptance = min(1, math.exp(((X[j][i-1]-mean)**2-(y[j]-mean)**2)/(2*var)))
			if probs[j][i] < acceptance:
				X[j][i] = y[j]
			else:
				X[j][i] = X[j][i-1]
	return X

def metropolis_simulation(num_chain=1000, chain_length=100, mean=0, var=1):
	import matplotlib.pyplot as plt
	import matplotlib.animation as animation
	"""
	Function 1C: Simulate metropolis with different setting.
	Detail : 	Try different setting and output movie of histgrams.
	"""

	list_a = [0, -0.1, -1, -10] # Add other value as you want.
	list_b = [1, 0.1, 1, 10]
	list_c = [1, 2]

	for a, b, c in [(a, b, c) for a in list_a for b in list_b for c in list_c]:

		# Sample num_chain x0 from uniform[a,b]
		x0 = sample_uniform(num_chain, a, b)

		# Run Metropolis
		X_normal = sample_normal_chain(x0, c, chain_length, mean, var)

		# Define the graph for i-th frame
		def animate(i):
		    plt.cla()
		    plt.title("Metropolis Sampling (a=%s_b=%s_c=%s) Time: %d" % (a,b,c,i))
		    plt.hist(X_normal[:,i], bins=30)

		# Define the graph for first frame
		fig = plt.figure()
		hist = plt.hist(X_normal[:,0], bins=30)

		# Define animation class and save
		ani = animation.FuncAnimation(fig, animate, frames=chain_length, repeat_delay=3000, repeat=True)
		ani.save('Python_Metropolis_a=%s_b=%s_c=%s.gif' % (a, b, c), writer='imagemagick')

		# Plot movie and save
		# Here plot chain_length graphs, each of them is a histogram of num_chain point.
		# You may use matplotlib.animation and matplotlib.rc to save graphs into gif movies.

# ### Part 2 : Gibbs Sampling

def gibbs_sample(x0, y0, rho, num_chain=1000, chain_length=100, mean=0, var=1):
	"""
	Function 2A: Bivariate normal with correlation rho
	Detail : 	This function return multiple chains by Gibbs sampling
				The input x0, y0, rho, num_chain is a number. This time, we use same starting point.
				Return a np.ndarray X with size num_chain * chain_length * 2. In X, each row X[i]
				should be a chain and t-th column X[:, t] corresponding to all different pair (X_t, Y_t).
	"""
	X = np.zeros(((num_chain, chain_length, 2)))
	random_normal = np.random.normal(mean, var, (num_chain, chain_length, 2))
	X[:,0,0] = x0
	X[:,0,1] = y0
	for i in range (0, num_chain):
		for j in range (1, chain_length):
			X[i][j][0] = rho * X[i][j-1][1] + (1-rho**2)**(1/2)*random_normal[i][j][0]
			X[i][j][1] = rho * X[i][j][0] + (1-rho**2)**(1/2)*random_normal[i][j][1]
	return X

def gibbs_simulation():
	import matplotlib.pyplot as plt
	import matplotlib.animation as animation
	"""
	Function 2B: Simulate Gibbs with different rho and plot
	Detail : 	Try different setting and output movie of histgrams.
				Discard first 50 steps and output 50~100 steps only.
	"""
	list_rho = [0, -1, 1, -0.5, 0.5, -0.25, 0.25, -0.75, 0.75] # Add other value as you want.
	for rho in list_rho:
		X_gibbs = gibbs_sample(1, 1, rho)
		fig = plt.figure('Gibbs_Sample_Sccaterplot_rho=%s' % rho, figsize = (6,6)).add_subplot(111)
		fig.scatter(X_gibbs[1,50:,0],X_gibbs[1,50:,1])
		plt.savefig ("Python_Gibbs Single Chain Scatterplot(rho=%s).png" % (rho))

		def animate(i):
		    plt.cla()
		    plt.title("Gibbs 1000 Chain (rho=%s) Time: %d" % (rho,i))
		    plt.scatter(X_gibbs[:,i,0],X_gibbs[:,i,1])

		# Define the graph for first frame
		fig = plt.figure()
		hist = plt.scatter(X_gibbs[:,0,0],X_gibbs[:,0,1])

		# Define animation class and save
		ani = animation.FuncAnimation(fig, animate, frames=100, repeat_delay=3000, repeat=True)
		ani.save('Python_Gibbs_rho=%s.gif' % (rho), writer='imagemagick')
		# Run Gibbs Sampling

# def main():
# 	metropolis_simulation()
# 	gibbs_simulation()
# main()


from itertools import combinations
from queue import deque

AMOUNT_LINES = 18 # количесвто линий, подверженных КЗ

 # получение всевозможных позиций растановки КА в зависимочти от их количесвта
def combinationLines(amount_KA, amountLines = AMOUNT_LINES):
    positKA = [i for i in range(2, amountLines + 1)]  #
    combPosit = list(combinations(positKA, amount_KA))
    combinat_list = [list(item) for item in combPosit]
    return combinat_list

# -------------------- данные для схемы (линии электропередачи) ----------------
 # граф для линий (пока не нужен)
graph = {1: [2, 3],          2: [1],           3: [1, 4, 5],  4: [3],           5: [3, 6, 7],    6: [5],
         7: [5, 8, 11, 14],  8: [7, 9, 10],    9: [8],       10: [8],          11: [7, 12, 13], 12: [11],
        13: [11],            14: [7, 15, 16], 15: [14],      16: [14, 17, 18], 17: [16],        18: [16]}

 # матрица сопряженности линий (1 - если расположения КА на данных позициях
 #  будут влиять на функционирование друг друга
              # 1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18
matrixGraph = [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 1
               [0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 2
               [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 3
               [0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 4
               [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 5
               [0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # 6
               [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 7
               [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1],  # 8
               [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0],  # 9
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0],  # 10
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1],  # 11
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],  # 12
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],  # 13
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],  # 14
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],  # 15
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],  # 16
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],  # 17
               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]  # 18

 # для каждой позиции опаисано количество следующих за ней линий
dictProtectLines = {1: 18, 2: 16, 3: 1, 4: 14, 5: 1, 6: 12,
                    7: 1, 8: 5, 9: 3, 10: 3, 11: 3, 12: 1,
                    13: 1, 14: 1, 15: 1, 16: 1, 17: 1, 18: 1}

# ------------------------------------------------------------------------------

# количесвто защищаемых потребителей для каждой позиции КА
def functProtectConsumer(dictAmountConsumer):
    dictPositConsumer = {1: 0, 2: 0,
                         3: dictAmountConsumer[0], 4: 0,
                         5: dictAmountConsumer[2], 6: 0,
                         7: dictAmountConsumer[1], 8: 0,
                         9: 0, 10: 0,
                         11: 0, 12: dictAmountConsumer[7],
                         13: dictAmountConsumer[6], 14: dictAmountConsumer[5],
                         15: dictAmountConsumer[4], 16: dictAmountConsumer[3],
                         17: dictAmountConsumer[9], 18: dictAmountConsumer[8]
                         }

    arrayProtectConsumer = []
    for posKA1 in range(0, AMOUNT_LINES):
        protectConsumer = 0
        for posKA2 in range(0, AMOUNT_LINES):
            if matrixGraph[posKA1][posKA2] == 1:
                protectConsumer += dictPositConsumer[posKA2 + 1]
        arrayProtectConsumer.append(protectConsumer)
    return arrayProtectConsumer

# Функция нахождения зависимых линий. На выходе - зависмые линии для конкретного расположения
def lineDepend(arrayPos, matrixLine=matrixGraph):
    lineDependReturn = []
    for posJ in range(len(arrayPos)):
        sliceArray = arrayPos[posJ : len(arrayPos)]
        workArray = []
        if len(sliceArray) > 1:
            for i in range(1, len(sliceArray)):
                row = sliceArray[0] - 1
                col = sliceArray[i] - 1
                if matrixLine[row][col] == 1: # если линии зависимы, то оставляем
                    workArray.append(sliceArray[i])

            if len(workArray) > 1:
                delIt = 1
                array1 = workArray
                array3 = []
                while(delIt < len(array1)):
                    array2 = []
                    array3.append(array1[delIt-1])
                    for j in range(delIt, len(array1)):
                        row = array1[delIt-1]-1
                        col = array1[j] - 1
                        if matrixLine[row][col] == 0:  # если линии независимы, то оставляем
                            array2.append(array1[j])
                    array1 = array3+array2
                    delIt += 1
                workArray = array1
        lineDependReturn.append([sliceArray[0]]+workArray)

    return lineDependReturn

# функция подсчета защищаемых КА линий в зависимости от расположения
def lineProtect(dependLines, dictProtectLines):
    lineProtectReturn = []
    for arrPos in dependLines:
        protectLinesPosit = dictProtectLines[arrPos[0]]
        if len(arrPos) > 1:
            for i in range(1, len(arrPos)):
                protectLinesPosit -= dictProtectLines[arrPos[i]]
        lineProtectReturn.append(protectLinesPosit)

    if len(lineProtectReturn) != len(dependLines):
        print("lineProtect(): Количесвто значений в входном массиве не равно в выходном")
    else: return lineProtectReturn

 # рассчитывается количество линий, влияющих на функционирование КА на соответсвующей позиции
def lineInfluence(positions, protectLines, matrixLine=matrixGraph):
    #arrayInfl = []
    #arrayInfl.append(protectLines[0])
    #for i in range(1, len(positions)):
    #    for j in range(i):
    #        row = positions[i-j-1] - 1
    #        col = positions[i] - 1
    #        if matrixLine[row][col] == 1:  # если линии зависимы, то
    #            arrayInfl.append(protectLines[i] + arrayInfl[i-j-1])
    #            break
    #arrayInfl[0] = AMOUNT_LINES
    arrayInfl = [AMOUNT_LINES] * len(positions)
    return arrayInfl

 # рассчитывается количество защищаемых потребителей при установке КА на соответсвующей позиции
def consumerProtect(positions, arrayProtectConsumer):
    returnConsumerProtect = []
    for pos in positions:
        returnConsumerProtect.append(arrayProtectConsumer[pos-1])
    return returnConsumerProtect

 # рассчитывается математическое ожидание
def mathExpect(protectLines, influenceLines, consumers):
    me = 0
    for i in range(len(protectLines)):
        me += (protectLines[i] / influenceLines[i]) * consumers[i]
    return me

 # математическая модель для рассчета при установке N КА
def modelNka(nKA, arrayProtectConsumer, dictProtectLines=dictProtectLines):
     # формирование всевозможных позиций расположения КА (без повторений)
    combinatPositions = combinationLines(nKA)
    meMin = 1000000000
    optimalPositions = []
    for positions in combinatPositions:
        positions.append(1) # для расчета линий, незащищаемых ни одной КА
        positions.sort()    # сортировка по уровням графа линий
        #print(positions)
         # для каждой позиции находятся другие позиции КА, которые будут влиять на количесвто защищаемых ей линий
        dependLines = lineDepend(positions)
         # для каждой позиции рассчитываются соответсвующее количесвто защищаемых линий
        protectLines = lineProtect(dependLines, dictProtectLines)
        #print(protectLines)
         # для каждой позиции рассчитывается количесвто линий, влияющих на работу КА
        influenceLines = lineInfluence(positions, protectLines)
        #print(influenceLines)
         # для каждой позиции рассчитывается соответсвующее количесвто защищаемых потребителей
        consumers = consumerProtect(positions, arrayProtectConsumer)
        #print(consumers)
         # расчет мат ожидания
        me = mathExpect(protectLines, influenceLines, consumers)
        if me < meMin:
            optimalPositions.clear()
            meMin = me
            optimalPositions.append(positions[1:])
        elif me == meMin: optimalPositions.append(positions[1:]) # если оптимальных позиций окажется несколько

    return (meMin, nKA, optimalPositions)

 # рассчет для всех 1:N установленных КА
def calculationFrom1toNka(amountKA, numberCustomersTP):
    arrayCalculatData = []
    for i in range(1, amountKA+1):
        arrayCalculatData.append(modelNka(i, functProtectConsumer(numberCustomersTP)))
    return arrayCalculatData

#print(modelNka(17, functProtectConsumer([30, 20, 10, 5, 15, 40, 10, 30, 5, 10])))


import math
import random
import time
import struct
import os.path
from os import listdir
from os.path import isfile, isdir, join, split
import json
from pathlib import Path

import numpy as np
from scipy import spatial

import cv2
import hnswlib
import networkx as nx
import plyvel

from tensorflow.keras.applications import vgg16
from tensorflow.keras.applications.vgg16 import preprocess_input

class MemoryGraphWalker:
    def __init__(self, memory_graph, knn = 30, accurate_prediction_limit = 10, distance_threshold = 0.1,  adjacency_radius = 2, identical_distance=0.01):

        self.knn = knn
        self.accurate_prediction_limit = accurate_prediction_limit
        self.identical_distance = identical_distance
        self.distance_threshold = distance_threshold
        self.adjacency_radius = adjacency_radius

        self.memory_graph = memory_graph
        self.last_ids = dict()
        self.predictions = dict()

    def add_parrelell_observations(self, file, t, pos, adj, feats):
        return [self.add_observation(file, t, pos[i], adj[i], feats[i], i) for i in range(len(feats))]

    def add_observation(self, file, t, pos, adj, feats, walker_id):

        stats = {"adj":adj}

        observation_id = self.memory_graph.insert_observation({"file":file, "t":t, "y":pos[0], "x":pos[1]})

        l = d = None

        if self.memory_graph.index_count() >= self.knn:
            labels, distances = self.memory_graph.knn_query([feats], k = self.knn)
            l = labels[0]
            d = distances[0]

        if d is not None:
            #print("Nearest Neighbor", d[0])
            stats["nearest_neighbor"] = d[0]

        accurate_predictions = set()
        evaluated_ids = set()

        # find correct predictions and reinforce with adjacency
        if adj and walker_id in self.predictions:
            predictions = self.predictions[walker_id]

            for pred in predictions:
                a = pred["id_similar_to_prev"]
                b = pred["id_similar_to_curr"]

                if b in evaluated_ids:
                    continue

                f = pred['candidate_for_similar_to_curr']["f"]

                if self.memory_graph.distance(feats, f) <= self.distance_threshold:
                    # print("add_predicted_observations", b, observation_id)
                    self.memory_graph.add_predicted_observations([b], [observation_id])
                    accurate_predictions.add(a)

                if len(accurate_predictions) >= self.accurate_prediction_limit:
                    # print("Too many accurate_predictions")
                    break

                evaluated_ids.add(b)

            if len(predictions) > 0:
                stats["predictions"] = len(predictions)
                stats["accurate_predictions"] = len(accurate_predictions)
                # print("Predictions", len(accurate_predictions), "of", len(predictions))

        # print("frame", t, pos)

        if len(accurate_predictions) < self.accurate_prediction_limit:

            #if d is not None and sum([i < self.identical_distance for i in d]) > 3:
            if d is not None and (d[0] < self.identical_distance):
                node_id = l[0]
                stats["identical"] = True
                #print("Using Identical")
            else:
                node_id = self.memory_graph.insert_node({"f":feats})

            #print("NodeID", node_id)

            self.memory_graph.add_integrated_observations([node_id], [observation_id])

            if adj:
                if walker_id in self.last_ids and self.last_ids[walker_id] is not None :
                    self.memory_graph.insert_adjacency(self.last_ids[walker_id], node_id)

                for a in accurate_predictions:
                    self.memory_graph.insert_adjacency(a, node_id)

        else:
            node_id = None

        # make predictions
        self.predictions[walker_id] = []

        if self.memory_graph.index_count() >= self.knn and l is not None and d is not None:

            similar = 0

            for n in range(self.knn):
                label = l[n]
                distance = d[n]

                if distance <= self.distance_threshold:
                    # Found a previous similar observation

                    # find other observations that have been seen near this one
                    next_adjacencies = self.memory_graph.get_adjacencies(label, self.adjacency_radius)

                    for n in next_adjacencies:
                        props = self.memory_graph.get_node(n)
                        self.predictions[walker_id].append(dict(id_similar_to_prev=label, id_similar_to_curr=n, candidate_for_similar_to_curr=props))

                    similar += 1

        self.last_ids[walker_id] = node_id

        return node_id, observation_id, stats

class MemoryGraph:

    def __init__(self, path, space='cosine', dim=512, max_elements=1000000, ef=100, M=48):
        self.space = space
        self.dim = dim
        self.max_elements = max_elements
        self.ef = ef
        self.M = M
        self.path = path
        self.open()

    def close(self):
        self.db.close()
        self.graph = None
        self.index = None
        self.db = None

    def open(self):
        self.db = plyvel.DB(self.path, create_if_missing=True)

        self.graph = nx.Graph()

        self.index = hnswlib.Index(space=self.space, dim=self.dim)
        self.index.init_index(max_elements=self.max_elements, ef_construction=self.ef, M=self.M)
        self.index.set_ef(self.ef)

        print("MemoryGraph: loading nodes")
        nodes = self.load_all_nodes()
        for node in nodes:
            self.graph.add_node(node["id"], f=node["f"])
            self.index.add_items([node["f"]], [node["id"]])

        print("MemoryGraph: loading edges")
        edges = self.load_all_edges()
        for from_node_id, to_node_id in edges:
            self.graph.add_edge(from_node_id, to_node_id)

        print("MemoryGraph: loaded", len(nodes), "nodes,", len(edges), "edges")

    def increment_node_id(self, count):
        return self.increment_id(count, b'd:n')

    def increment_observation_id(self, count):
        return self.increment_id(count, b'd:o')

    def increment_id(self, count, key):
        b = self.db.get(key)
        if b is None:
            node_id = count
        else:
            node_id = struct.unpack('>I', b)[0]
            node_id += count
        b = struct.pack('>I', node_id)
        self.db.put(key, b)
        return list(range(node_id-count+1, node_id+1))

    ######################
    # NODES
    ######################

    @staticmethod
    def numpy_to_bytes(a):
        return a.tobytes()

    @staticmethod
    def numpy_from_bytes(b):
        return np.frombuffer(b, dtype=np.float32)

    # node:[node_id] -> [node_data]
    @staticmethod
    def encode_node(node):
        return MemoryGraph.numpy_to_bytes(node["f"])

    @staticmethod
    def decode_node(k, v):
        node = dict()
        node["f"] = MemoryGraph.numpy_from_bytes(v)
        node["id"] = struct.unpack_from('>I', k, offset=1)[0]
        return node

    @staticmethod
    def node_key(node_id):
        return b'n' + struct.pack('>I', node_id)

    def get_node(self, node_id):
        return self.graph.nodes[node_id]

    def insert_node(self, node):
        return self.insert_nodes([node])[0]

    def load_all_nodes(self):
        start = MemoryGraph.node_key(0)
        stop = MemoryGraph.node_key(4294967295)
        return [MemoryGraph.decode_node(key, value) for key, value in self.db.iterator(start=start, stop=stop)]

    def get_nodes(self, node_ids):
        return [self.get_node(node_id) for node_id in node_ids]

    def insert_nodes(self, nodes):
        node_ids = self.increment_node_id(len(nodes))
        wb = self.db.write_batch()
        for node_id, node in zip(node_ids, nodes):
            self.graph.add_node(node_id, f=node["f"])
            self.index.add_items([node["f"]], [node_id])
            wb.put(MemoryGraph.node_key(node_id), MemoryGraph.encode_node(node))
        wb.write()
        return node_ids

    ######################
    # EDGES
    ######################

    @staticmethod
    def edge_key(edge):
        return b'e' + struct.pack('>I', edge[0]) + struct.pack('>I', edge[1])

    def save_edges(self, edges):
        wb = self.db.write_batch()
        for from_node_id, to_node_id in edges:
            wb.put(MemoryGraph.edge_key((from_node_id, to_node_id)), b'')
        wb.write()

    def load_all_edges(self):
        start = MemoryGraph.edge_key((0,0))
        stop = MemoryGraph.edge_key((4294967295, 4294967295))
        return [(struct.unpack_from('>I', b, offset=1)[0], struct.unpack_from('>I', b, offset=5)[0]) for b in self.db.iterator(start=start, stop=stop, include_value=False)]

    ######################
    # OBSERVATIONS
    ######################

    # obs:[observation_id] -> [observation_data]
    @staticmethod
    def observation_key(observation_id):
        return b'o' + struct.pack('>I', observation_id)

    # get observation - observation is a dictionary
    def get_observation(self, observation_id):
        b = self.db.get(MemoryGraph.observation_key(observation_id))
        return json.loads(b.decode("utf-8") )

    def insert_observation(self, observation):
        return self.insert_observations([observation])[0]

    def get_observations(self, observation_ids):
        return [self.get_observation(observation_id) for observation_id in observation_ids]

    def insert_observations(self, observations):
        observation_ids = self.increment_observation_id(len(observations))
        wb = self.db.write_batch()
        for observation_id, observation in zip(observation_ids, observations):
            j = json.dumps(observation)
            b = j.encode("utf-8")
            wb.put(MemoryGraph.observation_key(observation_id), b)
        wb.write()
        return observation_ids

    # integrated_observation:[node_id]:[observation_id]
    @staticmethod
    def integrated_observations_key(node_id, observation_id):
        return b'i' + struct.pack('>I', node_id) + struct.pack('>I', observation_id)

    # observations that are integrated into node's features
    def get_integrated_observations(self, node_id):
        start = MemoryGraph.integrated_observations_key(node_id, 0)
        stop = MemoryGraph.integrated_observations_key(node_id, 4294967295)
        return [struct.unpack_from('>I', b, offset=5)[0] for b in self.db.iterator(start=start, stop=stop, include_value=False)]

    def add_integrated_observations(self, node_ids, observation_ids):
        wb = self.db.write_batch()
        for node_id, observation_id in zip(node_ids, observation_ids):
            wb.put(MemoryGraph.integrated_observations_key(node_id, observation_id), b'')
            wb.put(MemoryGraph.integrated_nodes_key(observation_id, node_id), b'')
        wb.write()

    # predicted_observation:[node_id]:[observation_id]
    @staticmethod
    def predicted_observations_key(node_id, observation_id):
        return b'p' + struct.pack('>I', node_id) + struct.pack('>I', observation_id)

    # observations that were predicted by node
    def get_predicted_observations(self, node_id):
        start = MemoryGraph.predicted_observations_key(node_id, 0)
        stop = MemoryGraph.predicted_observations_key(node_id, 4294967295)
        return [struct.unpack_from('>I', b, offset=5)[0] for b in self.db.iterator(start=start, stop=stop, include_value=False)]

    def add_predicted_observations(self, node_ids, observation_ids):
        wb = self.db.write_batch()
        for node_id, observation_id in zip(node_ids, observation_ids):
            wb.put(MemoryGraph.predicted_observations_key(node_id, observation_id), b'')
            wb.put(MemoryGraph.predicted_nodes_key(observation_id, node_id), b'')
        wb.write()

    # predicted_node:[observation_id]:[node_id]
    @staticmethod
    def predicted_nodes_key(observation_id, node_id):
        return b'q' + struct.pack('>I', observation_id) + struct.pack('>I', node_id)

    # nodes that predicted observation
    def get_predicted_nodes(self, observation_id):
        start = MemoryGraph.predicted_nodes_key(observation_id, 0)
        stop = MemoryGraph.predicted_nodes_key(observation_id, 4294967295)
        return [struct.unpack_from('>I', b, offset=5)[0] for b in self.db.iterator(start=start, stop=stop, include_value=False)]

    # integrated_node:[observation_id]:[node_id]
    @staticmethod
    def integrated_nodes_key(observation_id, node_id):
        return b'j' + struct.pack('>I', observation_id) + struct.pack('>I', node_id)

    # nodes that integrate observation
    def get_integrated_nodes(self, observation_id):
        start = MemoryGraph.integrated_nodes_key(observation_id, 0)
        stop = MemoryGraph.integrated_nodes_key(observation_id, 4294967295)
        return [struct.unpack_from('>I', b, offset=5)[0] for b in self.db.iterator(start=start, stop=stop, include_value=False)]

    def get_adjacencies(self, node_id, radius):
        return self._neighbors(node_id, radius)

    def _neighbors(self, v, radius, depth=0):
        result = set()
        for w in self.graph.neighbors(v):
            result.add(w)
            if depth + 1 < radius:
                result.update(self._neighbors(w, radius, depth+1))
        return result

    def insert_adjacency(self, from_id, to_id):
        self.save_edges([(from_id, to_id)])
        self.graph.add_edge(from_id, to_id)

    def knn_query(self, feats, k=1):
        return self.index.knn_query(feats, k)

    def dnn_query(self, feature, distance):
        k = 100
        n = self.index.get_current_count()

        while True:
            if k > n:
                k = n

            _labels, _distances = self.index.knn_query([feature], k)

            labels = _labels[0]
            distances = _distances[0]

            if distances[-1] > distance:
                idx = next(i for i,v in enumerate(distances) if v>distance)
                if idx == 0:
                    return [], []
                return labels[:(idx-1)], distances[:(idx-1)]

            if k == n:
                return labels, distances

            k = k * 2

            print("dnn expand", k)

    def index_count(self):
        return self.index.get_current_count()

    def random_walk(self, start, l, trials):
        visited = dict()

        for _ in range(trials):
            cur = start
            for _ in range(l):
                nei = list(self.graph.neighbors(cur))
                if len(nei) == 0:
                    break
                cur = random.choice(nei)
                if cur in visited:
                    visited[cur] += 1
                else:
                    visited[cur] = 1

        nodes = []
        count = []

        if not bool(visited):
            return [], []

        for key, value in visited.items():
            nodes.append(key)
            count.append(value)

        return zip(*sorted(zip(count, nodes), reverse=True))

    def get_community(self, node_id):
        counts, node_ids = self.random_walk(node_id, 10, 1000)

        n = 0
        for i in range(len(counts)):
            count = counts[i]
            if count < 200:
                break
            n += 1

        return node_ids[:n]

    # the goal here is to search through the set of all communities and find all the ones that have a
    # max_pool distance within a range of the max_pool distance of the query community
    # candidate communities are ones that contain any member that is near any member of the quey community
    def search_group(self, features, feature_dis, community_dis, k=30):

        results = set()
        lab, dis = self.knn_query(features, k=k)
        features_max = np.max(features, axis=0)

        visited_nodes = set()

        for j in range(len(features)):
            labels = lab[j]
            distances = dis[j]

            for i in range(k):
                if distances[i] > feature_dis:
                    # break because distance are sorted and only increase from here
                    break
                label = labels[i]

                if label in visited_nodes:
                    print("label in visited_nodes")
                    continue
                visited_nodes.add(label)

                community = self.get_community(label)
                print("len(community)", len(community))
                if len(community) == 0:
                    continue
                community_features = np.array([self.get_node(c)["f"] for c in community])
                community_features_max = np.max(community_features, axis=0)
                d = spatial.distance.cosine(community_features_max, features_max)
                print("distance", d)
                if d <= community_dis:
                    results.add(frozenset(community))

        return results

    def distance(self, a, b):
        if self.space == 'cosine':
            return spatial.distance.cosine(a, b)
        else:
            return np.linalg.norm(a-b)

def resize_frame(image):
    return image

def get_rad_grid(g_pos, rad, shape, stride):

    top_left = (g_pos[0]-rad, g_pos[1]-rad)
    g_width = math.floor((shape[0] - 32)/stride)
    g_height = math.floor((shape[1] - 32)/stride)

    res = []

    for i in range(2*rad+1):
        p = (top_left[0]+i, top_left[1])
        if p[0] >= 0 and p[1] >= 0 and p[0] < g_width and p[1] < g_height:
            res.append(p)

    for i in range(2*rad+1):
        p = (top_left[0]+i, top_left[1]+(2*rad+1))
        if p[0] >= 0 and p[1] >= 0 and p[0] < g_width and p[1] < g_height:
            res.append(p)

    for i in range(2*rad-1):
        p = (top_left[0], top_left[1]+(i+1))
        if p[0] >= 0 and p[1] >= 0 and p[0] < g_width and p[1] < g_height:
            res.append(p)

    for i in range(2*rad-1):
        p = (top_left[0]+(2*rad), top_left[1]+(i+1))
        if p[0] >= 0 and p[1] >= 0 and p[0] < g_width and p[1] < g_height:
            res.append(p)

    #print(rad, g_pos, res)
    return res

def first_pos(kp_grid):
    ## TODO: if there are no key points in frame
    loc = random.choice(list(kp_grid.keys()))
    return loc, random.choice(kp_grid[loc])

def next_pos_play(kp_grid, shape, g_pos, stride):
    rad_grid = get_rad_grid(g_pos, 1, shape, stride)
    print("rad_grid", rad_grid)
    candidates = []

    for loc in rad_grid:

        if loc in kp_grid:
            candidates.append(loc)

    if len(candidates) == 0:
        return None, None

    loc = random.choice(candidates)

    return loc, random.choice(kp_grid[loc])

def next_pos(kp_grid, shape, g_pos, walk_length, stride):

    if (g_pos is not None) and (random.random() > 1.0/walk_length):

        for rad in range(1, 3):
            rad_grid = get_rad_grid(g_pos, rad, shape, stride)

            if len(rad_grid) == 0:
                print("frame empty?")
                break

            random.shuffle(rad_grid)

            for loc in rad_grid:
                if loc in kp_grid:
                    return loc, random.choice(kp_grid[loc]), True

    loc, pos = first_pos(kp_grid)
    return loc, pos, False

def extract_windows(frame, pos, window_size, walker_count):
    windows = np.empty((walker_count, window_size, window_size, 3))

    for i in range(walker_count):
        windows[i] = extract_window(frame, pos[i], window_size)

    return windows

def extract_window(frame, pos, window_size):
    half_w = window_size/2.0
    bottom_left = [int(round(pos[0]-half_w)), int(round(pos[1]-half_w))]
    top_right = [bottom_left[0]+window_size, bottom_left[1]+window_size]

    if bottom_left[0] < 0:
        top_right[0] -= bottom_left[0]
        bottom_left[0] = 0

    if bottom_left[1] < 0:
        top_right[1] -= bottom_left[1]
        bottom_left[1] = 0

    if top_right[0] >= frame.shape[0]:
        bottom_left[0] -= (top_right[0]-frame.shape[0]+1)
        top_right[0] = frame.shape[0]-1

    if top_right[1] >= frame.shape[1]:
        bottom_left[1] -= (top_right[1]-frame.shape[1]+1)
        top_right[1] = frame.shape[1]-1

    return frame[bottom_left[0]:top_right[0], bottom_left[1]:top_right[1]]

def key_point_grid(orb, frame, stride):

    kp = orb.detect(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), None)

    grid = dict()

    grid_offset_x = ((frame.shape[0] - 32) % stride)/2.0 + 16
    grid_offset_y = ((frame.shape[1] - 32) % stride)/2.0 + 16

    for k in kp:
        p = (k.pt[1],k.pt[0])
        g = (int(math.floor((p[0]-grid_offset_x)/stride)), int(math.floor((p[1]-grid_offset_y)/stride)))
        if g in grid:
            grid[g].append(p)
        else:
            grid[g] = [p]

    return grid

def paint_windows(positions, windows, frame, window_size, rect=-1):
    for i in range(len(positions)):
        pos = positions[i]
        x1 = int(round(pos[1] - window_size/2.0))
        x2 = x1 + window_size
        y1 = int(round(pos[0] - window_size/2.0))
        y2 = y1 + window_size

        window = windows[i]

        if abs(y1-y2) != window_size and abs(x1-x2) != window_size:
            continue

        wx1 = 0
        wx2 = window_size
        wy1 = 0
        wy2 = window_size

        shape = frame.shape

        if y1 < 0:
            if y1 < -window_size:
                continue
            wy1 = -y1
            y1 = 0
        if y2 >= shape[0]:
            if y2 >= (shape[0] + window_size):
                continue
            wy2 = window_size - (y2 - shape[0] + 1)
            y2 = shape[0]-1
        if x1 < 0:
            if x1 < -window_size:
                continue
            wx1 = -x1
            x1 = 0
        if x2 >= shape[1]:
            if x2 >= (shape[1] + window_size):
                continue
            wx2 = window_size - (x2 - shape[1] + 1)
            x2 = shape[1]-1

        frame[y1:y2, x1:x2] = window[wy1:wy2, wx1:wx2]

        if rect > -1:
            x1 = int(round(pos[1] - window_size/2.0))
            x2 = int(round(pos[0] - window_size/2.0))
            y1 = x1 + window_size
            y2 = x2 + window_size

            cv2.rectangle(frame, (y1, y2), (x1,x2), colors[rect % len(colors)], 1)

def show_patches(path_windows, path_features, path_positions, frame_shape, memory_graph, window_size):
    print("show_patches")

    cv2.namedWindow("patches")

    frame = np.zeros((frame_shape[0], frame_shape[1], 3), np.uint8)

    paint_windows(path_positions, path_windows, frame, window_size, 0)

    # features, feature_dis, community_dis, k=30
    groups = list(memory_graph.search_group(path_features, .2, .2, 30))

    print("groups", groups)

    for i in range(len(groups)):
        group = list(groups[i])

        # node_ids = memory_graph.get_nodes(group)

        observation_ids = []
        for node_id in group:
            # print("node_id", node_id)
            integrated_observations = memory_graph.get_integrated_observations(node_id)
            observation_ids.extend(integrated_observations)
            predicted_observations = memory_graph.get_predicted_observations(node_id)
            observation_ids.extend(predicted_observations)

        windows = np.array([cv2.imread('./patches/patch'+str(observation_id)+'.png') for observation_id in observation_ids])

        observations = memory_graph.get_observations(observation_ids)

        positions = [(obs["y"], obs["x"]) for obs in observations]

        paint_windows(positions, windows, frame, window_size, i+1)

    cv2.imshow('patches', frame)

def play_video(db_path, playback_random_walk_length = 10, window_size = 32, stride = 16, max_elements=10000000):

    def on_click(event, x, y, flags, param):
        if event != cv2.EVENT_LBUTTONUP:
            return

        # kp = clostest_key_points(key_points, (x,y), 1)[0]

        res_frame = resize_frame(frame)
        kp_grid = key_point_grid(orb, res_frame, stride)
        print("len(kp_grid)", len(kp_grid))

        pos = (y, x)

        grid_offset_x = ((frame.shape[0] - 32) % stride)/2.0 + 16
        grid_offset_y = ((frame.shape[1] - 32) % stride)/2.0 + 16
        g_pos = (int(math.floor((pos[0]-grid_offset_x)/stride)), int(math.floor((pos[1]-grid_offset_y)/stride)))

        print("g_pos", g_pos)
        path = []

        for i in range(playback_random_walk_length):
            g_pos, pos = next_pos_play(kp_grid, res_frame.shape, g_pos, stride)
            print("g_pos, pos", g_pos, pos)
            if g_pos is None:
                break
            path.append(pos)

        path = list(set(path))

        windows = np.array([extract_window(res_frame, p, window_size) for p in path])

        preprocess_input(windows)
        features = model.predict(windows)
        features = features.reshape((windows.shape[0], 512))

        print("windows.shape, feats.shape", windows.shape, features.shape)
        show_patches(windows, features, path, frame.shape, memory_graph, window_size)

    orb = cv2.ORB_create(nfeatures=100000, fastThreshold=7)

    memory_graph = MemoryGraph(db_path, max_elements=max_elements, space='cosine', dim=512)

    model = vgg16.VGG16(weights="imagenet", include_top=False, input_shape=(32, 32, 3))

    cap = cv2.VideoCapture(video_file)

    # Check if camera opened successfully
    if (cap.isOpened() == False):
        print("Error opening video  file")

    cv2.namedWindow("preview")
    cv2.setMouseCallback("preview", on_click)

    # Read until video is completed
    while(cap.isOpened()):

        # Capture frame-by-frame
        ret, frame = cap.read()

        if ret == True:

            # Display the resulting frame
            cv2.imshow('preview', frame)

            # Press Q on keyboard to  exit
            key = cv2.waitKey(0)

            if key == 27: # exit on ESC
                break

        # Break the loop
        else:
            break

    cap.release()
    cv2.destroyAllWindows()

def build_graph(db_path, video_path, patch_dir, walk_length = 100, window_size = 32, stride = 16, runs = 1, max_frames=30*15, walker_count = 200, save_windows = True, max_elements=10000000):

    print("Starting...")

    if isdir(video_path):
        video_files = [f for f in listdir(video_path) if f.endswith('.mp4') and isfile(join(video_path, f))]
    else:
        sp = os.path.split(video_path)
        video_files = [sp[1]]
        video_path = sp[0]

    random.shuffle(video_files)

    orb = cv2.ORB_create(nfeatures=100000, fastThreshold=7)

    # initialize VGG16
    model = vgg16.VGG16(weights="imagenet", include_top=False, input_shape=(32, 32, 3))

    # memory graph
    memory_graph = MemoryGraph(db_path, space='cosine', dim=512, max_elements=max_elements)
    memory_graph_walker = MemoryGraphWalker(memory_graph, distance_threshold = 0.10, identical_distance=0.01)

    # for each run though the video
    for r in range(runs):

        print("Run", r)

        video_file_count = 0

        for video_file in video_files:

            video_file_count += 1

            # open video file for a run though
            cap = cv2.VideoCapture(join(video_path, video_file))

            if save_windows:
                Path(join(patch_dir, video_file)).mkdir(parents=True, exist_ok=True)

            # walkers
            g_pos = [None for _ in range(walker_count)]
            pos = [None for _ in range(walker_count)]
            adj = [False for _ in range(walker_count)]

            done = False

            # for each frame
            for t in range(max_frames):
                if done:
                    break

                ret, frame = cap.read()

                if ret == False:
                    done = True
                    break

                frame = resize_frame(frame)

                kp_grid = key_point_grid(orb, frame, stride)

                for i in range(walker_count):
                    g_pos[i], pos[i], adj[i] = next_pos(kp_grid, frame.shape, g_pos[i], walk_length, stride)

                windows = extract_windows(frame, pos, window_size, walker_count)

                # extract cnn features from windows
                preprocess_input(windows)
                feats = model.predict(windows)
                feats = feats.reshape((windows.shape[0], 512))

                ids = memory_graph_walker.add_parrelell_observations(video_file, t, pos, adj, feats)

                restart_count = 0
                near_neighbor_count = 0
                is_identical_count = 0
                has_predictions_count = 0
                has_accurate_predictions_count = 0
                has_too_many_accurate_predictions_count = 0

                for i in range(walker_count):
                    if ids[i][0] is None:
                        # restart walk because we are in a very predictable spot
                        g_pos[i] = None
                        pos[i] = None
                        adj[i] = False
                    if save_windows:
                        cv2.imwrite(join(patch_dir, video_file, 'patch_' + str(ids[i][1]) + '.png'), windows[i])

                    stats = ids[i][2]

                    if not stats["adj"]:
                        restart_count += 1
                    if "nearest_neighbor" in stats and stats["nearest_neighbor"] < .1:
                        near_neighbor_count += 1
                    if "predictions" in stats:
                        has_predictions_count += 1
                    if "accurate_predictions" in stats:
                        if stats["accurate_predictions"] > 0:
                            has_accurate_predictions_count += 1
                        if stats["accurate_predictions"] >= 10:
                            has_too_many_accurate_predictions_count += 1
                    if "identical" in stats and stats["identical"]:
                        is_identical_count += 1

                print(
                    "vid", video_file_count,
                    "frame", t+1,
                    "start", restart_count,
                    "near", near_neighbor_count,
                    "iden", is_identical_count,
                    "pred", has_predictions_count,
                    "accu", has_accurate_predictions_count,
                    "many", has_too_many_accurate_predictions_count,
                )

            cap.release()

    memory_graph.close()

    print("Done")

colors = [
    (1, 0, 103), (213, 255, 0), (255, 0, 86), (158, 0, 142), (14, 76, 161), (255, 229, 2), (0, 95, 57),(0, 255, 0),
    (149, 0, 58), (255, 147, 126), (164, 36, 0), (0, 21, 68), (145, 208, 203), (98, 14, 0),(107, 104, 130),
    (0, 0, 255), (0, 125, 181), (106, 130, 108), (0, 174, 126), (194, 140, 159), (190, 153, 112), (0, 143, 156),
    (95, 173, 78), (255, 0, 0), (255, 0, 246), (255, 2, 157), (104, 61, 59), (255, 116, 163), (150, 138, 232),
    (152, 255, 82), (167, 87, 64), (1, 255, 254), (255, 238, 232), (254, 137, 0), (189, 198, 255),(1, 208, 255),
    (187, 136, 0), (117, 68, 177), (165, 255, 210), (255, 166, 254), (119, 77, 0), (122, 71, 130), (38, 52, 0),
    (0, 71, 84), (67, 0, 44), (181, 0, 255), (255, 177, 103), (255, 219, 102), (144, 251, 146), (126, 45, 210),
    (189, 211, 147), (229, 111, 254), (222, 255, 116), (0, 255, 120), (0, 155, 255), (0, 100, 1), (0, 118, 255),
    (133, 169, 0), (0, 185, 23), (120, 130, 49), (0, 255, 198), (255, 110, 65), (232, 94, 190), (0, 0, 0)
]

# playback_random_walk_length = 10
# walk_length = 100
# window_size = 32
# stride = 16
# runs = 1
# max_frames=30*15
# walker_count = 200
# save_windows = True

# video_dir = './media/Tabletop Objects/videos/'
# db_path = "./data/tabletop_objects.db"


import numpy as np
import os

def get_arr_from_bin(fname):
    outdat = np.fromfile(fname)
    outdat = outdat.reshape(int(len(outdat)/7), 7)
    return(outdat[1:])

def get_ics_from_data(dcty):
    datagen = data_generator(dcty)
    ics = []
    for _, data in datagen:
        ics.append(data[-1, 1:-1])
    return(np.array(ics))

def data_generator(dcty, ending = 'bin'):
    for i in os.listdir(dcty):
        if i.endswith(ending):
            data = get_arr_from_bin(dcty+'/'+i)
            if len(data) > 10:
                yield (i, data)


import numpy as np
import matplotlib.pyplot as plt
import os
import dlipr
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping
from keras.utils.np_utils import to_categorical

# ----------------------------------------------
# Data
# ----------------------------------------------
data = dlipr.mnist.load_data()

# reshape the image matrices to vectors
X_train = data.train_images.reshape(-1, 28**2)
X_test = data.test_images.reshape(-1, 28**2)

# convert integer RGB values (0-255) to float values (0-1)
X_train = X_train.astype('float32') / 255
X_test = X_test.astype('float32') / 255

# convert class labels to one-hot encodings
Y_train = to_categorical(data.train_labels, 10)
Y_test = to_categorical(data.test_labels, 10)

# split off last 5000 training samples for validation
X_train, X_val = np.split(X_train, [55000])
Y_train, Y_val = np.split(Y_train, [55000])

# ----------------------------------------------
# Hyperparameter tuning: Grid search
# ----------------------------------------------

# make output directory
folder = 'results-gridsearch/'
if not os.path.exists(folder):
    os.makedirs(folder)

dropout = [0.0, 0.1, 0.2, 0.3, 0.5]
num_neurons = [32, 64, 128, 256, 512]

val_loss = np.ones((5, 5)) * 1E99
val_acc = np.zeros((5, 5))
best_fit = None

for i, neurons in enumerate(num_neurons):
    for j, drop in enumerate(dropout):
        print(i, j)

        model = Sequential([
            Dense(neurons, input_shape=(784,)),
            Activation('relu'),
            Dropout(drop),
            Dense(10),
            Activation('softmax')])

        model.compile(
            loss='categorical_crossentropy',
            optimizer=Adam(lr=1e-3),
            metrics=['accuracy'])

        fit = model.fit(
            X_train, Y_train,
            batch_size=128,
            epochs=50,
            verbose=0,
            validation_data=(X_val, Y_val),
            callbacks=[EarlyStopping(patience=2)])

        loss = fit.history['val_loss'][-1]
        acc = fit.history['val_acc'][-1]

        # save if new best model found
        if loss < np.amin(val_loss):
            best_fit = fit
            model.save(folder + 'best_model.h5')

        val_loss[i, j] = loss
        val_acc[i, j] = acc

        print('Neurons: %i, Dropout: %i, Loss = %.3f, Acc = %.3f' % (neurons, drop, loss, acc))

# save results
np.savez(
    folder + 'search',
    val_loss=val_loss,
    val_acc=val_acc,
    dropout=dropout,
    num_neurons=num_neurons)

# ----------------------------------------------
# Evaluation
# ----------------------------------------------

# 2D plot of the validation loss / accuracy for each hyperparameter point
fig, ax = plt.subplots(1)
im = ax.imshow(val_loss, interpolation='none')
cbar = plt.colorbar(im)
cbar.set_label('Validation loss')
plt.xticks(np.arange(5), num_neurons)
plt.yticks(np.arange(5), dropout)
plt.xlabel('neurons')
plt.ylabel('dropout')
plt.savefig(folder + 'loss.png', bbox_inches='tight')

# 2D plot of the validation accuracy for each hyperparameter point
fig, ax = plt.subplots(1)
im = ax.imshow(val_acc * 100, interpolation='none')
cbar = plt.colorbar(im)
cbar.set_label('Validation accuracy %')
plt.xticks(np.arange(5), num_neurons)
plt.yticks(np.arange(5), dropout)
plt.xlabel('neurons')
plt.ylabel('dropout')
plt.savefig(folder + 'accuracy.png', bbox_inches='tight')

# Training curve for the best set of hyperparameters
fig, ax = plt.subplots(1)
epoch = np.array(best_fit.epoch) + 1
ax.plot(epoch, best_fit.history['loss'], label='Training')
ax.plot(epoch, best_fit.history['val_loss'], label='Validation')
ax.legend(['Training', 'Validation'])
ax.grid()
ax.set_xlabel('Epoch')
ax.set_ylabel('Loss')
plt.savefig(folder + 'best_training.png', bbox_inches='tight')

# Test performance for the best model
model = keras.models.load_model(folder + 'best_model.h5')
loss, acc = model.evaluate(X_test, Y_test)
print('Best performance')
print('Test loss = %.3f' % loss)
print('Test accuracy = %.3f' % acc)

# Plot confusion matrix
Yp = model.predict(X_test)
yp = np.argmax(Yp, axis=1)
dlipr.utils.plot_confusion(yp, data.test_labels, data.classes,
                           fname=folder + 'confusion.png')


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 04 20:38:51 2017

@author: Pericle
"""

import binascii
import os
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import hashes, hmac
from cryptography.hazmat.backends import default_backend

# Cipher block size, expressed in bytes.
block = algorithms.AES.block_size/8

# Read the plaintext file.
filename = raw_input('Type the file to encrypt-then-MAC: ')
with open(filename, 'rb') as f:
    plaintext = f.read()

# Read the keys in hexadecimal digits from keyboard.
key_hex = raw_input('Type the encryption key in ' + str(2*block) + ' hexadecimal digits: ')
cryptkey = binascii.unhexlify(key_hex)
key_hex = raw_input('Type the authentication key in ' + str(2*block) + ' hexadecimal digits: ')
authkey = binascii.unhexlify(key_hex)

# Pad the plaintext to make it multiple of the block size.
ctx = padding.PKCS7(8*block).padder()
padded_plaintext = ctx.update(plaintext) + ctx.finalize()

# Encrypt the plaintext with a random IV.
iv = os.urandom(block)
cipher = Cipher(algorithms.AES(cryptkey), modes.CBC(iv), default_backend())
ctx = cipher.encryptor()
ciphertext = ctx.update(padded_plaintext) + ctx.finalize()

# Authenticate the ciphertext and the IV.
ctx = hmac.HMAC(authkey, hashes.SHA256(), default_backend())
ctx.update(iv)
ctx.update(ciphertext)
digest = ctx.finalize()

# Write the MAC, the IV, and the ciphertext in the output file.
with open(filename + '.enc', 'wb') as f:
    f.write(digest)
    f.write(iv)
    f.write(ciphertext)
print('Encrypt-then-MACed file: ' + filename + '.enc')


from seekr.fasta_reader import Reader
import numpy as np
import corefunctions
import argparse
import pickle
from tqdm import tqdm as tqdm
from scipy.special import logsumexp
from collections import defaultdict
import os

'''
Author: Daniel Sprague
Calabrese Lab, Department of Pharmacology UNC Chapel Hill
June 24, 2020

Purpose: Run the Baum-Welch MLE algorithm to identify putative transition parameters for mSEEKR given a set of training sequences.

Input:

-k : K-mer value to use
--db : Training sequences
--prior : Path to binary file output by train.py in the mSEEKR pipeline, essentially, initial guesses for the transition parameters,
    as well as trained values for the k-mer frequencies for query and null states

    e.g. markovModels/D_null/2/hmm.mkv

--its : How many iterations of the BW algorithm to run (default=100)

Output:

    A new .mkv python pickle file (binary) that contains the original k-mer frequencies and updated transition matrix

'''

# Initiate command line arguments

parser = argparse.ArgumentParser()
parser.add_argument("-k",type=int)
parser.add_argument('--db',type=str,help='Path to fasta file containing training sequences')
parser.add_argument('--prior',type=str,help='Path to binary .mkv file output from train.py (e.g. markovModels/D_null/2/hmm.mkv')
parser.add_argument('-cf','--createfile',action='store_true',help='Create new file rather than overwrite')
parser.add_argument('--its',type=int,help='Iterations to do, default=100',default=20)

args = parser.parse_args()

assert args.its > 0, 'Please provide an integer greater than or equal to 1'
assert args.k > 0, 'Please provide an integer greater than or equal to 1'

fa = Reader(args.db)
seqs = '$'.join(fa.get_seqs())
model = args.prior

k = args.k

# Identify the location of any ambiguous nucleotides (N)
O,oIdx,nBP = corefunctions.kmersWithAmbigIndex(seqs,k)
# Load in train.py output
hmm = pickle.load(open(args.prior,'rb'))

'''
A - transition matrix (dictionary)
E - emission matrix (dictionary)
pi - initial state probabilities (always 50/50)
states - list of states (query,null)
'''
A,E,pi,states = hmm['A'],hmm['E'],hmm['pi'],hmm['states']

data = defaultdict(list)
data['alpha'].append(A['+']['+'])
data['beta'].append(A['-']['-'])
# Perform --its number of baum-welch iterations
# BW converges to a LOCAL MINIMUM -- best to perform a grid search (run this script on a variety of different --priors)
for i in tqdm(range(args.its)):
    a = corefunctions.fwd(O,A,pi,states,E) # forward probabilities
    b = corefunctions.bkw(O,A,pi,states,E) # backwards probabilities
    A = corefunctions.update(a,b,O,states,A,E) # update parameter values

    # The above step yields probabilities that are not yet normalized (do not sum to 1)
    # That is, for prior state "i" (can be either query or null) transitioning to either query (+) or null (-): p(-|i) + p(+|j) != 1
    # Here, we calculate the marginal probability for transitioning from state i to either state i or j (normalizing constant)
    # Then, divide each probability p(-|i) and p(+|i) by the marginal probability
    # Now, p(-|i) + p(+|i) = 1
    for i in states:
        # marginal probability (normalizing constant)
        marg = logsumexp(list(A[i].values())) # cannot simply sum in log-space, logsumexp to sum log-space numbers without over/under-flow (see wiki page)
        A[i]['+']-=marg # log-space, so this represents the division by the normalizing constant
        A[i]['-']-=marg # as above
    data['alpha'].append(A['+']['+'])
    data['beta'].append(A['-']['-'])

# Data dictionary tracks the iterations of BW algorithm for manual inspection if necessary

arr = np.array(list(data.values()))
arr = 2**arr
arr = arr.T
np.savetxt(os.path.dirname(args.prior) +'/hmm_BWparameters.txt',arr,fmt='%.9f')
if args.createfile:
    bn = os.path.basename(args.prior)
    bn = bn.split('.')[0]
    bn+='_MLE'
    bn = os.path.dirname(args.prior) +'/'+ bn
    pickle.dump({'A':A,'E':E,'pi':pi,'states':states},open(f'{bn}.mkv','wb'))
elif not args.createfile:
    pickle.dump({'A':A,'E':E,'pi':pi,'states':states},open(f'{args.prior}','wb'))


from project.app import create_celery_app
from project.blueprints.page.models import Translate
from lib.util_request import action_request
from config.settings import UNBABEL_URL, HEADERS

from celery.schedules import crontab
from celery.task import periodic_task

celery = create_celery_app()

@celery.task(name='unbabel.add_database')
def add_database(uid, text, order_number, status):
    """
    Create the Translate object and save on database.

    :param uid: UID from the text
    :param text: Text to translate
    :param order_number: The order_number to translate
    :param status: The status from translation process
    :return: None
    """
    translate = Translate(
                    uid=uid,
                    original_text=text,
                    translated_text='Translating',
                    order_number=int(order_number),
                    status=status
                )
    translate.save()
    return None

@celery.task(name='unbabel.ask_translation')
def ask_translation(original_text):
    """
    Ask the Unbabel API to translate a text and return the response.

    :param original_text: The input_text to translate
    :return: None
    """
    send_data = {
        "text": original_text,
        "source_language": "en",
        "target_language": "es",
        "text_format": "text"
    }
    resp = action_request(method='POST', url=UNBABEL_URL, data=send_data, headers=HEADERS)
    if resp.status_code == 201:
        ret_data = resp.json()
        add_database.delay(uid=ret_data['uid'], text=ret_data['text'], order_number=ret_data['order_number'], status=ret_data['status'])
    return None

@celery.task(name='unbabel.update_database')
def update_database(uid, translated_text, status):
    translated = Translate.query.filter_by(uid=uid).first()
    if translated:
        translated.translated_text = translated_text
        translated.status = status
        translated.update()
    return None

#@periodic_task(run_every=crontab(minute='*/1'))
@celery.task(name='unbabel.get_translateds')
def get_translateds():
    """
    Get the translated texts yet not proccessed and check the translation proccess

    :return: None
    """
    translates = Translate.query.filter(Translate.status != 'completed').all()
    for t in translates:
        GET_UNBABEL_URL = '{0}{1}/'.format(UNBABEL_URL, t.uid)
        resp = action_request(method='GET', url=GET_UNBABEL_URL, headers=HEADERS)
        if resp.status_code == 200:
            ret_data = resp.json()
            if ret_data['status'] == 'completed':
                update_database.delay(uid=ret_data['uid'], translated_text=ret_data['translatedText'], status=ret_data['status'])
            else:
                update_database.delay(uid=ret_data['uid'], translated_text='Translating', status=ret_data['status'])
    return None


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
 This script evaluates all embeddings available in the package
 and saves .csv results

 Usage:

 ./evaluate_all_embeddings -o <output_dir>
"""

# external imports
# ---

from six import iteritems
from multiprocessing import Pool
from os import path
import logging
import optparse
import multiprocessing

# internal imports
# ---
from web.evaluate import evaluate_on_all
from web import embeddings

parser = optparse.OptionParser()

parser.add_option("-j", "--n_jobs", type="int", default=4)

parser.add_option("-o", "--output_dir", type="str", default="")

(opts, args) = parser.parse_args()

# Configure logging

logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%I:%M:%S')

logger = logging.getLogger(__name__)

job_selection = []

job_selection.append('conceptnet')
# job_selection.append('FastText')
# job_selection.append('PDC')
# job_selection.append('HDC')
# job_selection.append('GloVe')
# job_selection.append('SG_GoogleNews')
# job_selection.append('LexVec')

# WARNING
# NMT embeddings are no longer available through the provided url
# job_selection.append('NMT')

jobs = []

# ConceptNet Numberbatch
# ---

if 'conceptnet' in job_selection:

    jobs.append(["fetch_conceptnet_numberbatch", {}])

# FastText
# ---

if 'FastText' in job_selection:

    jobs.append(["fetch_FastText", {}])

# PDC and HDC
# ---

for dim in [50, 100, 300]:

    if 'PDC' in job_selection:

        jobs.append(["fetch_PDC", {"dim": dim}])

    if 'HDC' in job_selection:

        jobs.append(["fetch_HDC", {"dim": dim}])

# GloVe
# ---

if 'GloVe' in job_selection:

    for dim in [50, 100, 200, 300]:

        jobs.append(["fetch_GloVe", {"dim": dim, "corpus": "wiki-6B"}])

    for dim in [25, 50, 100, 200]:

        jobs.append(["fetch_GloVe", {"dim": dim, "corpus": "twitter-27B"}])

    for corpus in ["common-crawl-42B", "common-crawl-840B"]:

        jobs.append(["fetch_GloVe", {"dim": 300, "corpus": corpus}])

# SG
# ---
if 'SG_GoogleNews' in job_selection:

    jobs.append(["fetch_SG_GoogleNews", {}])

# LexVec
# ---

if 'LexVec' in job_selection:

    jobs.append(["fetch_LexVec", {}])

# NMT
# ---

if 'NMT' in job_selection:

    # WARNING
    # NMT embeddings are no longer available through the provided url

    jobs.append(["fetch_NMT", {"which": "FR"}])
    jobs.append(["fetch_NMT", {"which": "DE"}])

def run_job(job):

    fetch_function_name, kwargs = job

    embedding_name = fetch_function_name.split("_")[1]

    filename = embedding_name

    if kwargs:

        job_description = "_" + "_".join(str(k) + "=" + str(v)
                                         for k, v in iteritems(kwargs))

        filename += job_description

    filename += ".csv"

    outf = path.join(opts.output_dir, filename)

    logger.info("Processing " + outf)

    if not path.exists(outf):

        w = getattr(embeddings, fetch_function_name)(**kwargs)

        res = evaluate_on_all_fast(w)

        res.to_csv(outf)

if __name__ == "__main__":

    Pool(opts.n_jobs).map(run_job, jobs)


from django.conf.urls import patterns, url
from django.views.decorators.csrf import csrf_exempt
from . import views

urlpatterns = patterns('',
    url(r'^redirect/$', views.RedirectView.as_view(), name='sofort-redirect'),
    url(
        r'^return/$',
        views.SuccessResponseView.as_view(preview=True),
        name='sofort-success-response',
    ),
    url(
        r'^place-order/(?P<basket_id>\d+)/$',
        views.SuccessResponseView.as_view(),
        name='sofort-place-order',
    ),
    url(
        r'^cancel/(?P<basket_id>\d+)/$',
        views.CancelResponseView.as_view(),
        name='sofort-abort-response',
    ),
    url(
        r'^notify/$',
        csrf_exempt(views.NotificationView.as_view(preview=True)),
        name='sofort-notify'
    ),
)


from apps.messenger.components.buttons import URLButton

class Element(object):

    def __init__(self, title, item_url=None, image_url=None, subtitle=None, buttons=None, default_action=None):

        self._title = title
        self.item_url = item_url
        self.image_url = image_url
        self._subtitle = subtitle
        self._buttons = buttons
        self.default_action = default_action

    @property
    def title(self):
        if len(self._title) > 45:
            raise ValueError('title no puede tener mas de 45 caracteres')
        return self._title

    @property
    def subtitle(self):
        if self._subtitle:
            if len(self._subtitle) > 80:
                raise ValueError('El Subtitule no puede tener mas de 80 caracteres')
        return self._subtitle

    @property
    def buttons(self):
        return self._buttons

    def to_dict(self):

        data = dict()
        if self.title:
            data["title"] = self.title

        if self.item_url:
            data["item_url"] = self.item_url

        if self.image_url:
            data["image_url"] = self.image_url

        if self.subtitle:
            data["subtitle"] = self.subtitle

        if self.default_action:
            data["default_action"] = self.default_action.to_dict()

        if self.buttons:
            data['buttons'] = [
                button.to_dict() for button in self.buttons
            ]
        return data

class ListElement(Element):

    def __init__(self, title=None, item_url=None, image_url=None, subtitle=None, buttons=None, default_action=None):

        if item_url and default_action:
            raise ValueError('No puedes tener las propiedades item_url y default_action al mismo tiempo')

        if not item_url and default_action and not isinstance(default_action, URLButton):
            raise ValueError('default_action debe ser de tipo URLButton')

        self.default_action = default_action

        super(ListElement, self).__init__(title, item_url, image_url, subtitle, buttons)

    @property
    def buttons(self):
        if len(self._buttons) > 1:
            raise ValueError('El máximo de botones permitido es 1')
        return self._buttons

    def to_dict(self):
        serialised = super(ListElement, self).to_dict()
        if self.default_action and isinstance(self.default_action, URLButton):
            serialised["default_action"] = self.default_action.to_dict()
        return serialised

class ReceiptElement(object):
    def __init__(self, title=None, subtitle=None, quantity=None, price=None, currency=None, image_url=None):
        if not title:
            raise ValueError('title es requerido')
        if not price:
            raise ValueError('price es requerido')
        elif not price > 0:
            raise ValueError('price no puede ser 0 ni negativo')

        self.title = title
        self.subtitle = subtitle
        self.quantity = quantity
        self.price = price
        self.currency = currency
        self.image_url = image_url

    def to_dict(self):
        data = dict()
        data["title"] = self.title
        if self.subtitle:
            data["subtitle"] = self.subtitle
        if self.quantity:
            data["quantity"] = self.quantity
        data["price"] = self.price
        if self.currency:
            data["currency"] = self.currency
        if self.image_url:
            data["image_url"] = self.image_url
        return data

class ReceiptAddress(object):
    def __init__(self, street_1=None, street_2=None, city=None, postal_code=None, state=None, country=None):

        if not street_1:
            raise ValueError('street_1 es requerido')
        if not city:
            raise ValueError('city es requerido')
        if not postal_code:
            raise ValueError('postal_code es requerido')
        if not state:
            raise ValueError('state es requerido')
        if not country:
            raise ValueError('country es requerido')
        self.street_1 = street_1
        self.street_2 = street_2
        self.city = city
        self.postal_code = postal_code
        self.state = state
        self.country = country

    def to_dict(self):
        data = dict()
        data["street_1"] = self.street_1
        if self.street_2:
            data["street_2"] = self.street_2
        data["city"] = self.city
        data["postal_code"] = self.postal_code
        data["state"] = self.state
        data["country"] = self.country
        return data

class Summary(object):
    def __init__(self, subtotal=None, shipping_cost=None, total_tax=None, total_cost=None):

        if not total_cost:
            raise ValueError('total_cost es requerido')

        self.subtotal = subtotal
        self.shipping_cost = shipping_cost
        self.total_tax = total_tax
        self.total_cost = total_cost

    def to_dict(self):
        data = dict()
        if self.subtotal:
            data["subtotal"] = self.subtotal
        if self.shipping_cost:
            data["shipping_cost"] = self.shipping_cost
        if self.total_tax:
            data["total_tax"] = self.total_tax
        data["total_cost"] = self.total_cost

        return data

class Adjustment(object):

    def __init__(self, name=None, amount=None):
        self.name = name
        self.amount = amount

    def to_dict(self):
        data = dict()
        if self.name:
            data["name"] = self.name
        if self.amount:
            data["amount"] = self.amount
        return data


### ----------------------------------------------------------------------
###
### Example analayzer
###
###----------------------------------------------------------------------

LEPTON_SETUP = 2012
PD = ""
MCFILTER = ""

# Get absolute path
import os
PyFilePath = os.environ['CMSSW_BASE'] + "/src/ZZAnalysis/AnalysisStep/test/"

### ----------------------------------------------------------------------
### Standard sequence
### ----------------------------------------------------------------------

execfile(PyFilePath + "MasterPy/ZZ4lAnalysis.py")         # 2012 reference analysis

### ----------------------------------------------------------------------
  ### Replace parameters
### ----------------------------------------------------------------------
process.source.fileNames = cms.untracked.vstring(
#    '/store/cmst3/user/cmgtools/CMG/ZZTo4mu_8TeV-powheg-pythia6/Summer12-PU_S7_START52_V9-v1/AODSIM/V5/PAT_CMG_V5_4_0/cmgTuple_1.root'        # qqZZ
#    '/store/cmst3/user/cmgtools/CMG/GluGluToZZTo4L_8TeV-gg2zz-pythia6/Summer12-PU_S7_START52_V9-v1/AODSIM/V5/PAT_CMG_V5_4_0/cmgTuple_28.root' # ggZZ
#    'root://cmsphys05//data/b/botta/V5_4_0/patTuple_H126Summer12.root' #Summer12 H126 for FSR synch
       'root://cmsphys05//data/b/botta/V5_4_0/cmgTuple_H126Summer12.root' #Summer12 H126 for FSR synch
    )

process.maxEvents.input = -1

### ----------------------------------------------------------------------
### Output root file (monitoring histograms)
### ----------------------------------------------------------------------
process.TFileService=cms.Service('TFileService',
                                fileName=cms.string('ZZ4lAnalysis.root')
                                )

### ----------------------------------------------------------------------
### Additional Path
### ----------------------------------------------------------------------
process.CRPath = cms.Path(process.CR)
#process.CRPath = cms.Path(process.bareZCand + process.ZCand +  process.ZlCand)
#process.HF = cms.Path(process.heavyflavorfilter)

### ----------------------------------------------------------------------
### Analyzer for Plots
### ----------------------------------------------------------------------

PlotSetup = cms.EDAnalyzer("ZZ4lAnalyzer",
                           channel = cms.untracked.string('aChannel'),
                           candCollection = cms.untracked.string('aCand'),
                           isMC = cms.untracked.bool(IsMC),
                           sampleType = cms.int32(SAMPLE_TYPE),
                           setup = cms.int32(LEPTON_SETUP),
                           skimPaths = cms.vstring(SkimPaths),
                           PD = cms.string(PD),
                           MCFilterPath = cms.string(MCFILTER),
                           skipEmptyEvents = cms.bool(True),
                           )

process.Plots4mu = PlotSetup.clone()
process.Plots4mu.channel           = 'MMMM'
process.Plots4mu.candCollection    = 'MMMMCand'

process.Plots4e = PlotSetup.clone()
process.Plots4e.channel            = 'EEEE'
process.Plots4e.candCollection     = 'EEEECand'

process.Plots2e2mu = PlotSetup.clone()
process.Plots2e2mu.channel         = 'EEMM'
process.Plots2e2mu.candCollection  = 'EEMMCand'

# # All events together
# process.PlotsZZ    = cms.EDAnalyzer("ZZ4lAnalyzer",
#                                     channel = cms.untracked.string('ZZ'),
#                                     candCollection = cms.untracked.string('ZZCand'),
#                                     isMC = cms.untracked.bool(IsMC),
#                                     setup = cms.int32(LEPTON_SETUP),
#                                     skimPaths = cms.vstring(SkimPaths),
#                                     PD = cms.string(""),
#                                     MCFilterPath = cms.string(""),
#                                     )

### ----------------------------------------------------------------------
### Analyzer for Trees
### ----------------------------------------------------------------------

TreeSetup = cms.EDAnalyzer("HZZ4lNtupleMaker",
                                   channel = cms.untracked.string('aChannel'),
                                   CandCollection = cms.untracked.string('aCand'),
                                   fileName = cms.untracked.string('candTree'),
                                   isMC = cms.untracked.bool(IsMC),
                                   sampleType = cms.int32(SAMPLE_TYPE),
                                   setup = cms.int32(LEPTON_SETUP),
                                   skimPaths = cms.vstring(SkimPaths),
                                   PD = cms.string(PD),
                                   MCFilterPath = cms.string(MCFILTER),
                                   skipEmptyEvents = cms.bool(True),
                                   )

process.ZZ4muTree = TreeSetup.clone()
process.ZZ4muTree.channel = 'MMMM'
process.ZZ4muTree.CandCollection = 'MMMMCand'

process.ZZ4eTree = TreeSetup.clone()
process.ZZ4eTree.channel = 'EEEE'
process.ZZ4eTree.CandCollection = 'EEEECand'

process.ZZ2e2muTree = TreeSetup.clone()
process.ZZ2e2muTree.channel = 'EEMM'
process.ZZ2e2muTree.CandCollection = 'EEMMCand'

#process.source.eventsToProcess = cms.untracked.VEventRange("1:122490", "1:1343", "1:177684")

# Debug
process.dumpUserData =  cms.EDAnalyzer("dumpUserData",
     dumpTrigger = cms.untracked.bool(True),
     muonSrc = cms.InputTag("appendPhotons:muons"),
     electronSrc = cms.InputTag("appendPhotons:electrons"),
     candidateSrcs = cms.PSet(
        Zmm   = cms.InputTag("MMCand"),
        Zee   = cms.InputTag("EECand"),
        Z     = cms.InputTag("ZCand"),
        MMMM  = cms.InputTag("MMMMCand"),
        EEEE  = cms.InputTag("EEEECand"),
        EEMM  = cms.InputTag("EEMMCand"),
     )
)

#Print MC history
#process.p = cms.EndPath(process.printTree)

#Dump reconstructed variables
#process.dump = cms.Path(process.dumpUserData)

process.p = cms.EndPath( process.Plots4mu + process.Plots4e + process.Plots2e2mu )
process.trees = cms.EndPath(process.ZZ4muTree * process.ZZ4eTree * process.ZZ2e2muTree )


from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from tutorial.items import LiePinItem3
from scrapy_redis.spiders import RedisCrawlSpider

class MyCrawler(RedisCrawlSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    name = 'mycrawler_redis'
    redis_key = 'mycrawler:start_urls'

    rules = (
        Rule(LinkExtractor(allow=r'/zhaopin/\?init=-1.*?&curPage=\d+')),
        Rule(LinkExtractor(allow=r'https://www.liepin.com/job/\d+.shtml'), callback='parse_item'),
        Rule(LinkExtractor(allow=r'/a/\d+.shtml'), callback='parse_item2'),
        # 配置url提取的规律，注意不是数据的提取规律  要求所有的列表页以及所有的列表页上职位的详情页
        # 问题：它是怎么来实现你给了一个初始的url地址，怎么就找到了其它的列表页的url地址以及职位详情页的url地址
    )

    # 适合：网页的数据就在源代码里面，而且还要有下一页。
    # 怎么爬
    # 给一个起始的url地址
    # 再配置提取的url规则

    def parse_item(self, response):
        item = LiePinItem3()
        item["job_name"] = response.xpath("//div[@class='title-info']/h1/@title").extract()[0]
        item["com_name"] = response.xpath("//div[@class='title-info']/h3/a/@title").extract()[0]
        item["salary"] = response.xpath("//p[@class='job-item-title']/text()").extract()[0].replace("\r", "").replace(
            "\n", "")
        addr = response.xpath("//p[@class='basic-infor']/span/a/text()").extract()
        item["address"] = addr[0] if len(addr) > 0 else "没有公司地址，是一家黑公司"
        item["xueli"] = response.xpath("//div[@class='job-qualifications']/span[1]/text()").extract()[0]
        item["work_year"] = response.xpath("//div[@class='job-qualifications']/span[2]/text()").extract()[0]
        miaoshu = ""
        miaoshu_data = response.xpath("//div[@class='content content-word']/text()").extract()
        for ii in miaoshu_data:
            miaoshu += ii.replace("\r", "").replace("\n", "").strip()
        item["miaoshu"] = miaoshu

        return item

    def parse_item2(self, response):
        item = LiePinItem3()
        item["job_name"] = response.xpath("//div[@class='title-info ']/h1/text()").extract()[0]
        item["com_name"] = response.xpath("//div[@class='title-info ']/h3/text()").extract()[0]
        item["salary"] = response.xpath("//p[@class='job-main-title']/text()").extract()[0].replace("\r", "").replace(
            "\n", "")
        item["address"] = response.xpath("//p[@class='basic-infor']/span/text()").extract()[0]
        item["xueli"] = response.xpath("//div[@class='resume clearfix']/span[1]/text()").extract()[0]
        item["work_year"] = response.xpath("//div[@class='resume clearfix']/span[2]/text()").extract()[0]
        miaoshu = ""
        miaoshu_data = response.xpath("//div[@class='content content-word']/text()").extract()
        for ii in miaoshu_data:
            miaoshu += ii.replace("\r", "").replace("\n", "").strip()
        item["miaoshu"] = miaoshu

        return item


from ecm import Ecm
from assets import Rtu
from enumsets import FanSeq

class VafAutoHtg(Ecm.Ecm):
    def __init__(self, htg_fan_max_speed = 1.0,
                 htg_fan_min_speed = 0.333):
        self.htg_fan_min_speed = htg_fan_min_speed
        self.htg_fan_max_speed = htg_fan_max_speed
        self.htg_fan_cntrl_seq = FanSeq.FanSeq.VARIABLE_AIRFLOW

    def apply_ecm(self, asset):
        if (isinstance(asset, Rtu.Rtu)):
            asset.htg_fan_cntrl_seq = self.htg_fan_cntrl_seq
            asset.htg_fan_max_speed = self.htg_fan_max_speed
            asset.htg_fan_min_speed = self.htg_fan_min_speed
            asset.htg_fan_cntrl_seq = self.htg_fan_cntrl_seq

        else:
            raise NotImplementedError("ECM Type not supported for Asset Type")


# -*- coding: utf-8 -*-

import yaml
from Crypto.PublicKey import RSA
from boto import s3
from boto import iam
from boto import route53
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.key import Key
from boto.exception import S3ResponseError
import logging
from acme import client
from acme import messages
from acme import challenges
import io
import requests
import base64
import hashlib
from time import sleep
from OpenSSL import crypto
from datetime import datetime
from boto.ec2 import elb
from acme.jose.util import ComparableX509
import os

# this is the AWS region in which the function runs in
exec_region = os.environ['AWS_DEFAULT_REGION']

LOG = logging.getLogger("letslambda")
LOG.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
handler.setFormatter(formatter)
# add ch to logger
LOG.addHandler(handler)

def load_config(bucket):
    """
    Try to load the letlambda.yml out of the user bucket
    Will return None if the configuration file does not exist
    """
    conf = bucket.get_key("letslambda.yml")
    if conf == None:
        return None

    confString = conf.read()
    conf = yaml.load(confString)
    return conf

def loadAccountKey(bucket, conf):
    """
    Try to load the RSA account key from S3. If it doesn't
    succeed, it will create a new account key and try a registration
    with your provided information
    """
    LOG.info("Loading account key from s3")
    key = bucket.get_key("account.key.rsa")
    newAccountNeeded = False;
    if key == None:
        pem = createAndSaveKey(bucket, "account.key.rsa")
        newAccountNeeded = True
    else:
        pem = key.read()

    key = client.jose.JWKRSA.load(pem)
    if newAccountNeeded:
        registerNewAccount(conf, key)

    return key

def loadCSRKey(bucket, domain):
    LOG.info("Loading CSR key from S3")
    key = None
    if 'reuse_key' in domain.keys() and domain['reuse_key'] == True:
        LOG.info("Attempting to reuse old key for domain")
        name = domain['name'] + ".key.rsa"
        key = bucket.get_key(name)

    if key == None:
        pem = createAndSaveKey(bucket, name)
    else:
        pem = key.read()

    return crypto.load_privatekey(crypto.FILETYPE_PEM, pem)

def registerNewAccount(conf, key):
    """
    Attempt to create a new account on the ACME server
    with the key. No problem if it fails because this
    kye is already used.
    """
    LOG.info("Registering with ACME server with the new key")
    newReg = messages.NewRegistration(contact=tuple(conf['info']), key=key.public_key())
    acme_client = client.Client(conf['directory'], key)
    registration_resource = acme_client.register(newReg)
    LOG.info("Agreeing on the TOS on your behalf")
    acme_client.agree_to_tos(registration_resource)

def createAndSaveKey(bucket, name):
    """
    Generate an RSA 4096 key for general purpose (account or CSR)
    """
    LOG.info("Key not found! Generating new RSA key")
    key = RSA.generate(4096);
    saveToS3(bucket, key, name)
    return key.exportKey("PEM")

def saveToS3(bucket, rsakey, name):
    """
    Save the rsa key in PEM format to s3 .. for later use
    """
    LOG.info("Storing newly generated key")
    s3key = Key(bucket=bucket, name=name)
    fp = io.BytesIO(rsakey.exportKey("PEM"));
    s3key.set_contents_from_file(fp)

def get_authorization(client, domain):
    authorization_resource = client.request_domain_challenges(domain['name'])
    return authorization_resource;

def get_dns_challenge(authorization_resource):
    """
    Ask the ACME server to give us a list of challenges.
    Later, we will pick only the DNS one.
    """
    # Now let's look for a DNS challenge
    dns_challenges = filter(lambda x: isinstance(x.chall, challenges.DNS01), authorization_resource.body.challenges)
    return list(dns_challenges)[0]

def answer_dns_challenge(client, domain, challenge):
    """
    Compute the required answer and set it in the DNS record
    for the domain.
    """
    authorization = "{}.{}".format(
        base64.urlsafe_b64encode(challenge.get("token")).decode("ascii").replace("=", ""),
        base64.urlsafe_b64encode(client.key.thumbprint()).decode("ascii").replace("=", "")
        )

    dns_response = base64.urlsafe_b64encode(hashlib.sha256(authorization.encode()).digest()).decode("ascii").replace("=", "")

    # Let's update the DNS on our R53 account
    r53 = route53.connect_to_region(exec_region)
    zone = r53.get_zone(domain['r53_zone'])
    if zone == None:
        LOG.error("Cannot find R53 zone {}, are you controling it ?".format(domain['r53_zone']))
        exit(1)

    acme_domain = "_acme-challenge.{}".format(domain['name'])
    record = zone.find_records(name=acme_domain, type="TXT")
    if record:
        delete_status = zone.delete_record(record)

    add_status = zone.add_record("TXT", acme_domain, '"' + dns_response + '"')
    dns_updated = wait_until_sync(add_status)

    if dns_updated == False:
        LOG.error("We updated R53 but the servers didn't sync within 10 seconds. Bailing out.")
        exit(1)

    ## Now, let's tell the ACME server that we are ready
    challenge_response = challenges.DNS01Response(key_authorization=authorization)
    challenge_resource = client.answer_challenge(challenge, challenge_response)

    # We'd better poll to get the answer of this.

def generateCSR(bucket, conf):
    if 'reuse_key' in conf:
        key = loadCSRKey(bucket, conf)
    else:
        key = crypto.PKey()
        key.generate_key(crypto.TYPE_RSA, 4096)

    csr = crypto.X509Req()
    csr.get_subject().countryName = conf['countryName']
    csr.get_subject().CN = conf['name']
    csr.set_pubkey(key)
    csr.sign(key, "sha1")
    return (csr, key)

def requestCertificate(client, bucket, conf, auth_resource):
    (csr, key) = generateCSR(bucket, conf)
    # Gently wait for authorization to be valid
    # wait_until_valid(auth_resource)

    (certificate, ar) = client.poll_and_request_issuance(ComparableX509(csr), [auth_resource])
    return (certificate, key)

def wait_until_valid(auth):
    timeout = 30
    valid = False
    while valid and timeout > -1:
        status = requests.get(auth.uri)
        if status['status'] == "valid":
            valid = True

    return valid

def wait_until_sync(status):
    LOG.info("Waiting for DNS to synchronize with new TXT value")
    timeout = 30;
    while status.status != "INSYNC" and timeout > -1:
        sleep(1)
        timeout = timeout - 1;
        status.update()

    if timeout == -1:
        return False

    return True

def createIAMCertificate(domain, certificate, key):
    """
    Create a new IAM certificate from ACME and private key.
    It also fetched the chain certificate from ACME if provided
    """
    # First we fetch the chain certificate
    chain = requests.get(certificate.cert_chain_uri)
    chain_certificate = None
    if chain.status_code == 200:
        chain_certificate = crypto.load_certificate(crypto.FILETYPE_ASN1, chain.content)

    iam_connection = iam.connect_to_region(exec_region)
    res = iam_connection.upload_server_cert(
        domain['name'] + "-" + datetime.utcnow().strftime("%Y-%m-%dT%H-%M"),
        crypto.dump_certificate(crypto.FILETYPE_PEM, certificate.body.wrapped).decode("ascii"),
        crypto.dump_privatekey(crypto.FILETYPE_PEM, key).decode("ascii"),
        crypto.dump_certificate(crypto.FILETYPE_PEM, chain_certificate).decode("ascii"),
        "/")
    sleep(10)

    return res

def updateELB(conf, iam_certificate):
    LOG.info("Updating ELB with new certificate")
    elb_connection = elb.connect_to_region(conf['elb_region'])
    response = elb_connection.set_lb_listener_SSL_certificate(conf['elb'], 443, iam_certificate['upload_server_certificate_response']['upload_server_certificate_result']['server_certificate_metadata']['arn'])
    return response

def lambda_handler(event, context):
    bucket = event['bucket']
    region = event['region']

    LOG.info("Retrieving configuration file from bucket '{0}' in region '{1}' ".format(bucket, region))
    connection = s3.connect_to_region(region, calling_format=OrdinaryCallingFormat())
    try:
        bucket = connection.get_bucket(bucket);
    except S3ResponseError as e:
        print(e)
        LOG.error("Cannot fetch bucket : {}".format(bucket))
        exit(1)

    conf = load_config(bucket)
    if conf == None:
        LOG.error("Cannot find file 'letslambda.yml' in S3 bucket '{0}".format(bucket))
        exit(1)

    key = loadAccountKey(bucket, conf)
    acme_client = client.Client(conf['directory'], key)
    for domain in conf['domains']:
        authorization_resource = get_authorization(acme_client, domain)
        challenge = get_dns_challenge(authorization_resource)
        answer_dns_challenge(acme_client, domain, challenge)
        (certificate, key) = requestCertificate(acme_client, bucket, domain, authorization_resource)
        iam_cert = createIAMCertificate(domain, certificate, key)
        updateELB(domain, iam_cert)


# coding=utf-8
# 详细说明见书P132

# 使用任意数量的关键字实参,采用**

def build_profile(first, last, **user_info):
    """print all information"""
    profile = {}
    profile['first_name'] = first
    profile['last_name'] = last
    for key, value in user_info.items():
        profile[key] = value
    return profile

user_profile = build_profile('albert', 'einstein',
                            location = 'princeton',
                            field = 'physics',
                            country = 'china',
                            city = 'chengdu')
print(user_profile)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.core.exceptions import ImproperlyConfigured
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
from .decorators import verified_email_required
from .models import OrganizingHubFeature
import logging
logger = logging.getLogger(__name__)

@method_decorator(verified_email_required, name='dispatch')
class LocalGroupPermissionRequiredMixin(PermissionRequiredMixin):
    """
    Local Group based Permissions check with Email verification
    """
    local_group = None
    organizing_hub_feature = None
    skip_feature_check = False

    def get_local_group(self):
        """
        Override this method to override the local_group attribute.
        """
        if self.local_group is None:
            raise ImproperlyConfigured(
                '{0} is missing the local_group attribute. Define {0}.local_group, or override '
                '{0}.get_local_group().'.format(self.__class__.__name__)
            )
        return self.local_group

    def handle_no_permission(self):
        """
        Override default logic and redirect to Dashboard instead
        """
        messages.error(
            self.request,
            "You do not have permission to access this page."
        )
        return redirect(settings.ORGANIZING_HUB_DASHBOARD_URL)

    def has_feature_access(self):
        """Return True if we are skipping feature check"""
        if self.skip_feature_check:
            return True

        """Throw error if feature is missing"""
        if self.organizing_hub_feature is None:
            raise ImproperlyConfigured(
                '{0} is missing the organizing_hub_feature attribute. Define {0}.organizing_hub_feature, or override '
                '{0}.skip_feature_check.'.format(self.__class__.__name__)
            )

        """Check if Local Group has access to Feature"""
        local_group = self.get_local_group()
        if local_group is not None and hasattr(
            local_group,
            'organizinghubaccess',
        ):
            access = local_group.organizinghubaccess
            if access.has_feature_access(self.organizing_hub_feature):
                return True

        """Otherwise return False"""
        return False

    def has_permission(self):
        """
        Override this method to customize the way permissions are checked.
        """
        local_group = self.get_local_group()
        permissions = self.get_permission_required()

        """Return False for missing or non-approved local group"""
        if local_group is None or local_group.status != 'approved':
            return False

        """Return False for missing feature access"""
        if not self.has_feature_access():
            return False

        """Return True for empty permission list"""
        if not permissions:
            return True

        """Otherwise check permissions against Local Group Profile"""
        user = self.request.user
        if not hasattr(user, 'localgroupprofile'):
            return False
        else:
            profile = user.localgroupprofile
            has_permissions = profile.has_permissions_for_local_group(
                local_group,
                permissions
            )
            return has_permissions


n = int(input())
import random
ans = [int(10*random.random()) for i in xrange(n)]

f = open('output', 'w')
for x in ans:
    f.write(str(x) + ' ')

f.close()
f = open('input', 'w')
f.write(str(n)+'\n')
for i in range(n):
    for j in range(n):
        if i == j:
            f.write('0 ')
        else:
            f.write(str(ans[i]+ans[j]) + ' ')
    f.write('\n')


from _hll import _hll

class hll(_hll):
    def addh(self, item):
        ''' Hash an item and add it to the hll sketch. '''
        if item.__hash__:
            self.add(item.__hash__())
        else:
            self.addh_(int(item))

__doc__ = "HyperLogLog module"
__all__ = [hll]


from cv2 import cv2
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import os
import sys

'''
Pre-Processing Techniques
We do Pre-Processing images to improve the chances of successful in recognition.
1. De-skew — When the document is not aligned properly when scanned, it may need to be titled a few degrees clockwise or counterclockwise.In order to take it perfectly horizontal or vertical.
2. Despeckle — remove positive and negative spots, smoothing edges
3. Binarisation — Convert an image from color or greyscale to black-and-white (called a “binary image” because there are two colors). The task of binarisation is performed as a simple way of separating the text (or any other desired image component) from the background.
4. Line removal — Cleans up non-glyph boxes and lines
5. Layout analysis or “zoning” — Identifies columns, paragraphs, captions, etc. as distinct blocks.
6. Line and word detection — Establishes baseline for word and character shapes, separates words if necessary.
7. Character isolation or “segmentation” — For per-character OCR, multiple characters that are connected due to image artifacts must be separated; single characters that are broken into multiple pieces due to artifacts must be connected.
8. Normalize aspect ratio and scale.
'''

#some things to do in preprocessing
#Good Quality Original Source
#Binarize Image
#Scaling To The Right Size
#Increase Contrast
#Remove Noise and Scanning Artefacts
#Deskew
#Layout Analysis (or Zone Analysis)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        sys.stderr.write(
            "You must pass image name as argument !")
        sys.exit(1)
    path = os.path.dirname(os.path.abspath(__file__))
    file=os.path.join(path, sys.argv[1])
    print(file)

    matplotlib.interactive(True)
    image = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
    plt.figure(figsize=(8, 8))
    plt.imshow(image, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Gray", grayimg)
    #cv2.waitKey(0)

    '''
    image_enhanced = cv2.equalizeHist(image)
    plt.imshow(image_enhanced, cmap='gray')
    input('press return to continue')
    contrast_fixed = cv2.convertScaleAbs(image, alpha=2.5, beta=50)
    plt.figure(figsize=(8, 8))
    plt.imshow(contrast_fixed, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Contrast", contrast_fixed)
    #cv2.waitKey(0)
    '''

    #_, th = cv2.threshold(image, 200, 255, cv2.THRESH_BINARY_INV)
    th=cv2.adaptiveThreshold(image,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)
    cv2.imwrite(file+"-result.png", th)
    fig = plt.figure(figsize=(8, 8))
    fig.add_subplot(1, 2, 1)
    plt.imshow(image, cmap='gray')
    fig.add_subplot(1, 2, 2)
    plt.imshow(th, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Binarization", binary_thresh)
    #cv2.waitKey(0)

    lines = cv2.HoughLinesP(th, 1, np.pi/180, 100,
                            minLineLength= 600/2.0, maxLineGap=20)
    angle = 0
    for line in lines:
        x1, y1, x2, y2 = line[0]
        r = np.arctan2(y2 - y1, x2 -x1)
        angle += np.arctan2(y2 - y1, x2 - x1)

    avg_radian = angle / len(lines)
    avg_angle = avg_radian * 180 / np.pi
    print("Average angle is %f degrees" % avg_angle)


# 5430.py
# 2018.06.16

import collections

for T in range(int(input())):
	command = input()
	n = int(input())
	p = collections.deque(eval(input()))
	error, isreverse = [False]*2
	for c in command:
		if c == 'R':
			isreverse = False if isreverse else True
		elif not p:
			error = True
			break
		else:
			if isreverse:
				p.pop()
			else:
				p.popleft()
	if error:
		print('error')
	else:
		if isreverse:
			p.reverse()
		print('['+','.join(map(str, p))+']')

# 입력을 deque로 받고 입력받은 함수를 처리하면 된다.
# reverse를 마지막에만 수행하고 중간 부분에서는 pop의 위치를 바꿔주는 방식으로 간다.