import sys
import argparse
import numpy as np
import pandas as pds
import random
from matplotlib import pyplot as plt
import utilities
from os import listdir
from os.path import isfile, join

plt.switch_backend('agg')

'''
Describe: find the vertical seal
Return: return the list of vertical seals
'''
def get_vertical(matrix, window_size, pos_threshold, neg_threshold, exceed, exceed_rate = 0.25):
	# vertical_seal_list[i] is the "sum" of the ith col
	vertical_seal_list = []
	channel_num, sample_num = matrix.shape[0], matrix.shape[1]
	single_vertical_list = np.zeros(sample_num)
	num = channel_num*window_size

	for i in range(channel_num):
		# fixed a bug
		for j in range(sample_num):
			v = matrix[i][j]
			if v > pos_threshold:
				single_vertical_list[j] += v-pos_threshold
			elif v < neg_threshold:
				single_vertical_list[j] += neg_threshold-v
	for i in range(sample_num-window_size):
		for j in range(1, window_size):
			single_vertical_list[i] += single_vertical_list[i+j]
		if single_vertical_list[i]/num > exceed:
			exceed_cnt = 0
			for c in range(channel_num):
				v = matrix[c][i]
				if v > pos_threshold:
					exceed_cnt += 1
				elif v < neg_threshold:
					exceed_cnt += 1
			if exceed_cnt/(channel_num*1.0) > exceed_rate:
				vertical_seal_list.append((i, i+window_size))
				# print('rate: ', i, exceed_cnt/(channel_num*1.0))

			#vertical_seal_list.append((i, i+window_size))
	# print('vertical_seal_list: ', vertical_seal_list)
	return vertical_seal_list

''''
Describe: merge the nearby vertical seals
min_gap: if the distance between two vertical seals is less than min_gap, then they should be merged
'''
def merge_vertical_seals(matrix, vertical_seal_list, min_gap):
	merge_list = []
	l = len(vertical_seal_list)
	i = 0
	while i<l:
		start, end = vertical_seal_list[i][0], vertical_seal_list[i][1]
		i += 1
		while i<l:
			next_start, next_end = vertical_seal_list[i][0], vertical_seal_list[i][1]
			if next_start-end<=min_gap:
				end = next_end
				i += 1
			else:
				break
		merge_list.append((start, end))
	print('merge_list: ', merge_list)
	return merge_list

''''
Describe: partition the matrix into several small matrix, each small matrix is an independent steel tub
Return: the list of the small matrixes
'''
def partition(matrix, gap_list):
	submatrix_list = []

	# if len(gap_list)>0 and gap_list[0][0]>1:
	# 	start = 0
	# 	end = gap_list[0][0]-1
	# 	tmp = matrix[0:matrix.shape[0], start:end]
	# 	submatrix_list.append(tmp)
	for i in range(len(gap_list)-1):
		start, end = gap_list[i][1]+1, gap_list[i+1][0]-1
		print('start, end: ', start, end)
		tmp = matrix[0:matrix.shape[0], start:end]
		print('tmp.shape: ', tmp.shape)
		submatrix_list.append(tmp)
	# if len(gap_list)-1>=0 and gap_list[len(gap_list)-1][1]<matrix.shape[1]:
	# 	start = gap_list[len(gap_list)-1][1]
	# 	end = matrix.shape[1]
	# 	tmp = matrix[0:matrix.shape[0], start:end]
	# 	submatrix_list.append(tmp)
	return submatrix_list

'''
Replace the vertical heave spot and the lean heave spot by the average value of the matrix
value k can be viewed as slop
'''
def clean(matrix, channel_num, sample_num, average_val, k, window_size,
			pos_threshold, neg_threshold, exceed):
	print('clean')

	for i in range(int(sample_num-k*channel_num-window_size)):
		cur_val = 0
		for j in range(channel_num):
			for ws in range(window_size):
				v = matrix[j][int(i+j*k+ws)]
				if v > pos_threshold:
					cur_val += v-pos_threshold
				elif v < neg_threshold:
					cur_val += neg_threshold-v
		if cur_val/(channel_num*window_size) > exceed:
			print('remove, position: %d,   val:%.4f'%(i,cur_val/(channel_num*window_size)))
			for j in range(channel_num):
				for ws in range(window_size):
					matrix[j][int(i+j*k+ws)] = average_val
	move = 2000
	xcoord = [i for i in range(sample_num)]
	for i in range(matrix.shape[0]):
		plt.plot(xcoord, matrix[i]+move*i)
	plt.show()

'''Attention: must set appropriate start_step and max_step MANUALLY'''
'''
TODO: this version have bugs, (max_step)
submatrix[j][(j-i)*k+l]  might out of boundary
'''
def each_converge(submatrix, window_size, pos,
				   pos_threshold, neg_threshold, exceed,
					move, partition_flag=False, start_step = 8):
	sub_std = np.std(submatrix)
	pos_threshold *= sub_std
	neg_threshold *= -sub_std
	exceed *= sub_std
	print(pos_threshold, neg_threshold, exceed)
	channel_num, sample_num = submatrix.shape[0], submatrix.shape[1]
	tmp_list = []
	max_val = 0
	converge_point = -1 # point to recoord channel num
	converge_step = -1

	all_max_val = 0
	all_converge_point = -1
	all_converge_step = -1
	for i in range(channel_num):
		# max_step = (sample_num-window_size)//(channel_num)
		max_step = 28
		# -----------------------------------------
		for k in range(start_step, max_step):
			cur_sum = 0
			cur_cnt = 0
			for j in range(i, channel_num+i):
				for l in range(window_size):
					cur_cnt += 1
					sample_index = (j-i)*k+l
					if partition_flag:
						group = 8
						if j-i > group:
							n = ((j//group) - (i//group))*group*k
							sample_index += n
					if sample_index >= submatrix.shape[1]:
						break
					v = submatrix[j%channel_num][sample_index]
					if v > pos_threshold:
						cur_sum += v-pos_threshold
					elif v < neg_threshold:
						cur_sum += neg_threshold-v
			cur_average = cur_sum/cur_cnt
			if cur_average > all_max_val:
				all_max_val = cur_average
				all_converge_point = i
				all_converge_step = k
			if cur_average > exceed:
				print('converge coordinate: (%d, %d)    step: %d, val: %.4f'%(pos, i, k, cur_average))
				tmp_list.append(i)
				if cur_average > max_val:
					converge_point = i
					max_val = cur_average
					converge_step = k
		# -----------------------------------------
		# cur_sum = 0
		# cur_cnt = 0
		# k = 18
		# for j in range(i, channel_num+i):
		# 	for l in range(window_size):
		# 		cur_cnt += 1
		# 		sample_index = (j-i)*k+l
		# 		if partition_flag:
		# 			group = 8
		# 			if j-i > group:
		# 				n = ((j//group) - (i//group))*group*k
		# 				sample_index += n
		# 		if sample_index >= submatrix.shape[1]:
		# 			break
		# 		v = submatrix[j%channel_num][sample_index]
		# 		if v > pos_threshold:
		# 			cur_sum += v-pos_threshold
		# 		elif v < neg_threshold:
		# 			cur_sum += neg_threshold-v
		# cur_average = cur_sum/cur_cnt
		# if cur_average > all_max_val:
		# 	all_max_val = cur_average
		# 	all_converge_point = i
		# 	all_converge_step = k
		# if cur_average > exceed:
		# 	print('converge coordinate: (%d, %d)    step: %d, val: %.4f'%(pos, i, k, cur_average))
		# 	tmp_list.append(i)
		# 	if cur_average > max_val:
		# 		converge_point = i
		# 		max_val = cur_average
		# 		converge_step = k
	if converge_step != -1:
		print(' ')
		print('*'*20)
		print('converge coordinate: (%d, %d)    step: %d, val: %.4f'%(pos, converge_point, converge_step, max_val))
		print('*'*20)
	else:
		print('found no converge point, all max val: %.4f, pos: %d, step: %.4f'%(all_max_val, all_converge_point, all_converge_step))

	# [which channel, start position, end position]
	# if converge_point is -1, it means that there is no converge point.
	return converge_point, max_val, converge_step

def func_4_generated_data():
	data_filename = './data/data_generated.csv'
	matrix = pds.read_csv(data_filename, header=None)
	matrix = matrix.values
	print('std is: %.4f'%np.std(matrix))
	move = 8
	show_size = 1024
	window_size = 20

	########################
	pos_threshold = 0.8
	neg_threshold = -0.8
	exceed = 0.4

	_std = np.std(matrix)
	e0 = 1.0
	e1 = 0.5
	########################

	xcoord = [i for i in range(show_size)]
	for i in range(matrix.shape[0]):
		plt.plot(xcoord, matrix[i][:show_size]+move*i)
	plt.show()

	vertical_seal_list = get_vertical(matrix=matrix, window_size=window_size,
									pos_threshold=_std, neg_threshold=-_std, exceed=e0*_std)
	gap_list = merge_vertical_seals(matrix=matrix, vertical_seal_list=vertical_seal_list, min_gap=100)
	submatrix_list = partition(matrix=matrix, gap_list=gap_list)
	cnt = 0
	for submatrix in submatrix_list:
		pos = gap_list[cnt][1]+1
		cnt += 1
		each_converge(submatrix=submatrix,window_size=window_size, pos=pos,
					  pos_threshold=_std, neg_threshold=-_std, exceed=e1*_std, move=move)

def func_4_real_data(data_filename, DIR):

	matrix = pds.read_csv(DIR+ data_filename, header=None)
	matrix = matrix.values
	matrix_converge_real_data(data_filename=data_filename, DIR=DIR, matrix=matrix)

def func_4_real_data_partition(data_filename, DIR):
	move = 2000
	print(data_filename)
	matrix = 0
	if data_filename.split('.')[-1] == 'npy':
		matrix = np.load(DIR + data_filename)
	elif data_filename.split('.')[-1] == 'csv':
		matrix = pds.read_csv(DIR+ data_filename, header=None)
		matrix = matrix.values
	# matrix = matrix[:, :70000]

	result_path = utilities.PIC_DIR + '%s/'%(data_filename.split('.')[0])
	utilities.make_sure_path_exists(result_path)

	matrix_a, matrix_b = matrix_partition(matrix = matrix)
	gap_list_A, converge_list_A = matrix_converge_real_data(data_filename=data_filename, DIR=DIR, matrix=matrix_a)
	gap_list_B, converge_list_B = matrix_converge_real_data(data_filename=data_filename, DIR=DIR, matrix=matrix_b)
	cnt = 0
	for converge in converge_list_A:
		front_converge_point_A, front_max_val_A, front_step_A, pos_A, back_converge_point_A, back_max_val_A, back_step_A, end_pos_A = converge[0], converge[1], converge[2], converge[3], converge[4], converge[5], converge[6], converge[7]

		front_converge_point_B, front_max_val_B, front_step_B, pos_B, back_converge_point_B, back_max_val_B, back_step_B, end_pos_B = front_converge_point_A, front_max_val_A, front_step_A, pos_A, back_converge_point_A, back_max_val_A, back_step_A, end_pos_A
		front_min, back_min = sys.maxint, sys.maxint
		for converge_B in converge_list_B:
			cur_front_converge_point_B, cur_front_max_val_B, cur_front_step_B, cur_pos_B, cur_back_converge_point_B, cur_back_max_val_B, cur_back_step_B, cur_end_pos_B = converge_B[0], converge_B[1], converge_B[2], converge_B[3], converge_B[4], converge_B[5], converge_B[6], converge_B[7]
			if abs(cur_pos_B - pos_A) < front_min or abs(cur_end_pos_B - end_pos_A) < back_min:
				front_converge_point_B, front_max_val_B, front_step_B, pos_B, back_converge_point_B, back_max_val_B, back_step_B, end_pos_B = cur_front_converge_point_B, cur_front_max_val_B, cur_front_step_B, cur_pos_B, cur_back_converge_point_B, cur_back_max_val_B, cur_back_step_B, cur_end_pos_B
				front_min = abs(cur_pos_B - pos_A)
				back_min = abs(cur_end_pos_B - end_pos_A)

		bias_len = pos_A - pos_B

		extra_len = 1000
		pos = pos_A
		end_pos = end_pos_A
		for i in range(matrix.shape[0]):
			plt.xlabel('bias_len: %d'%bias_len, fontsize=18)
			# additional_move = move*(i//8)*8
			additional_move = 0
			if (i//8)%2 == 0:
				length = min(end_pos+extra_len, matrix.shape[1]) - max(pos-extra_len, 0)
				xcoord = [pos+x-extra_len for x in range(length)]
				plt.plot(xcoord, matrix[i, max(pos-extra_len, 0):min(end_pos+extra_len, matrix.shape[1])]+move*i+additional_move)
			else:
				length = min(end_pos+extra_len-bias_len, matrix.shape[1]) - max(pos-extra_len-bias_len, 0)
				xcoord = [pos+x-extra_len for x in range(length)]
				plt.plot(xcoord, matrix[i, max(pos-extra_len-bias_len, 0):min(end_pos+extra_len-bias_len, matrix.shape[1])]+move*i+additional_move)
			if front_converge_point_A!=-1:
				# additional_move = move*(front_converge_point//8)*8
				plt.plot([pos], front_converge_point_A*move+additional_move,
						marker='o', markersize=10, color="red")
			if back_converge_point_A!=-1:
				# additional_move = move*(back_converge_point//8)*8
				plt.plot([end_pos], back_converge_point_A*move+additional_move,
						marker='o', markersize=10, color="red")

			if front_converge_point_B!=-1:
				# additional_move = move*(front_converge_point//8)*8
				plt.plot([pos_B + bias_len], front_converge_point_B*move+additional_move,
						marker='o', markersize=10, color="blue")
			if back_converge_point_B!=-1:
				# additional_move = move*(back_converge_point//8)*8
				plt.plot([end_pos_B + bias_len], back_converge_point_B*move+additional_move,
						marker='o', markersize=10, color="blue")
		plt.savefig(result_path + '%s%dth_%d_%d.jpg'%('', cnt, pos, end_pos), dpi=100)
		# plt.show()
		plt.close()
		cnt += 1

def matrix_partition(matrix):
	matrix_a = []
	matrix_b = []
	channel_num, sample_num = matrix.shape[0], matrix.shape[1]
	tmp = np.full((sample_num,), matrix.mean(), dtype=float)
	print(tmp.mean())
	for i in range(channel_num):
		if (i//8)%2 == 0:
			matrix_a.append(matrix[i])
			matrix_b.append(tmp)
		else:
			matrix_a.append(tmp)
			matrix_b.append(matrix[i])
	return np.asarray(matrix_a), np.asarray(matrix_b)

def matrix_converge_real_data(data_filename, DIR, matrix, prefix=''):
	if len(data_filename.split('.'))!=2 or (data_filename.split('.')[1]!='csv' and data_filename.split('.')[1]!='npy'):
		print('file incorrect: %s'%data_filename)
		return
	result_path = utilities.PIC_DIR + '%s/'%(data_filename.split('.')[0])
	utilities.make_sure_path_exists(result_path)
	# result_txt = prefix + data_filename.split('.')[0] + '.txt'
	# result_file = open(result_path+result_txt, 'w+')
	# vertical_txt = prefix + data_filename.split('.')[0] + 'vertical' + '.txt'
	# vertical_file = open(result_path+vertical_txt, 'w+')

	move = 2000
	############################
	# pos_threshold = 700
	# neg_threshold = -700
	# exceed = 80
	# _std = np.std(matrix)
	# print('   ', _std)
	# e0 = 2.0
	# e1 = 0.4
	# window_size = 40
	_std = np.std(matrix)
	e0 = 2.0
	e1 = 0.15
	window_size = 10
	############################

	vertical_seal_list = get_vertical(matrix=matrix, window_size=window_size,
			pos_threshold=1.0*_std, neg_threshold=-1.0*_std, exceed=1.0*_std)
	gap_list = merge_vertical_seals(matrix=matrix, vertical_seal_list=vertical_seal_list, min_gap=20)
	submatrix_list = partition(matrix=matrix, gap_list=gap_list)
	cnt = 0

	converge_list = []

	for submatrix in submatrix_list:
		pos = gap_list[cnt][1]+1
		end_pos = pos + submatrix.shape[1]
		front_converge_point, front_max_val, front_step = each_converge(submatrix=submatrix,window_size=window_size, pos=pos,
					  pos_threshold=e0, neg_threshold=e0, exceed=e1, move=move)
		submatrix_flipped = np.copy(submatrix)
		if np.may_share_memory(submatrix_flipped, submatrix) == True:
			print('share memory')
		else:
			print('not share memory')
		submatrix_flipped = submatrix_flipped[::-1]
		for i in range(submatrix_flipped.shape[0]):
			submatrix_flipped[i] = submatrix_flipped[i][::-1]
		back_converge_point, back_max_val, back_step = each_converge(submatrix=submatrix_flipped,window_size=window_size, pos=pos,
					  pos_threshold=e0, neg_threshold=e0, exceed=e1, move=move)
		# if back_converge_point is -1, it means that there is no converge point in the end of the submatrix
		if back_converge_point != -1:
			back_converge_point = submatrix.shape[0]-back_converge_point-1

		converge_list.append((front_converge_point, front_max_val, front_step, pos, back_converge_point, back_max_val, back_step, end_pos))

		# extra_len = 200
		# length = min(end_pos+extra_len, matrix.shape[1]) - max(pos-extra_len, 0)
		# xcoord = [pos+i-extra_len for i in range(length)]
		# for i in range(submatrix.shape[0]):
		# 	# additional_move = move*(i//8)*8
		# 	additional_move = 0
		# 	plt.plot(xcoord, matrix[i, max(pos-extra_len, 0):min(end_pos+extra_len, matrix.shape[1])]+move*i+additional_move)
		# 	if front_converge_point!=-1:
		# 		# additional_move = move*(front_converge_point//8)*8
		# 		plt.plot([pos], submatrix[front_converge_point][0]+front_converge_point*move+additional_move,
		# 				marker='o', markersize=10, color="red")
		# 	if back_converge_point!=-1:
		# 		# additional_move = move*(back_converge_point//8)*8
		# 		plt.plot([end_pos], submatrix[back_converge_point][-1]+back_converge_point*move+additional_move,
		# 				marker='o', markersize=10, color="red")
		# plt.savefig(result_path + '%s%dth_%d_%d.jpg'%(prefix, cnt, pos, end_pos), dpi=100)
		# plt.close()

		# result_file.write('%d, %d, 0\n'%(pos, front_converge_point))
		# result_file.write('%d, %d, 1\n'%(end_pos, back_converge_point))
		# print('save file to %dth_%d_%d.jpg'%(cnt, pos, end_pos))
		# print(' ')
		cnt += 1
	# cnt = 0
	# result_file.close()
	# for gap in gap_list:
	# 	vertical_file.write('%d, %d\n'%(cnt, (gap[0]+gap[1])//2))
	# 	cnt += 1
	# vertical_file.close()
	return gap_list, converge_list

def main():
	# func_4_generated_data()
	# DIR = utilities.CSV_DIR
	# DIR = './data/data0525_csv/'
	parser = argparse.ArgumentParser(description='Please enter the input directory')
	parser.add_argument("-d", dest="DIR", help="the input directory")
	args = parser.parse_args()
	DIR = args.DIR
	if DIR == '' or DIR==None:
		parser.print_help()
		return

	allfilenames = [f for f in listdir(DIR) if isfile(join(DIR, f))]
	print(allfilenames)
	for filename in allfilenames:
		if filename.split('.')[-1] == 'npy':
			func_4_real_data_partition(data_filename=filename, DIR=DIR)
		elif filename.split('.')[-1] == 'csv':
			func_4_real_data_partition(data_filename=filename, DIR=DIR)

if __name__ == '__main__':
	main()


import sys, traceback
import json
import os
import shutil
from pathlib import Path
import zipfile
import pprint

from os import path as pat
from shutil import copy

extension_mp3 = '.mp3'
extension_mp4 = '.mp4'
extension_jpg = '.jpg'

def only_one_element(a_list):
    element_num = len(a_list)
    if element_num > 1:
        try:
            exception = Exception("Folder have 2 or more same type src")
            print(exception)
            raise exception
        except:
            traceback.print_exc()
            sys.exit(1)

    return element_num != 0

def get_file_extension(path):
    return pat.splitext(path)[-1]
class node:
    nodes = {}
    counter = 0
    def get(path):
        if node.nodes.get(path,None):
            return node.nodes[path]
        return None

    def __init__(self,path,node_name):
        self.id = node.counter
        self.child = [None,None]
        self.src = []
        self.path = path
        self.node_name = node_name
        node.nodes[pat.join(path,node_name)] = self
        node.counter += 1

    def set_childL(self,child):
        self.child[0] = child
    def set_childR(self,child):
        self.child[1] = child

    def add_src(self,src):
        self.src.append(src)

    def get_dict(self):
        default_dict = {
                "nodeid":self.id,
                "video":"0x00",
                "audio":"0x00",
                "pic":"0x00",
                "childNodeL":'',
                "childNodeR":'',
                "name":self.node_name,
                }

        video_candiate = list(filter(lambda x: x.find(extension_mp4) != -1, self.src))
        audio_candiate = list(filter(lambda x: x.find(extension_mp3) != -1, self.src))
        picture_candiate = list(filter(lambda x: x.find(extension_jpg) != -1, self.src))

        if only_one_element(video_candiate): default_dict['video'] = str(node.id).zfill(5) + extension_mp4
        if only_one_element(audio_candiate): default_dict['audio'] = str(node.id).zfill(5) + extension_mp3
        if only_one_element(picture_candiate): default_dict['pic'] = str(node.id).zfill(5) + extension_jpg

        if self.child[0]: default_dict['childNodeL'] = self.child[0].id
        if self.child[1]: default_dict['childNodeR'] = self.child[1].id
        # print(json.dumps(default_dict))
        return default_dict

def unzip_file(file_name,output):
    def prev_dir(path):
        return os.path.relpath(os.path.join(path,'..'))

    with zipfile.ZipFile(file_name, 'r') as f:
        zipInfo = f.infolist()
        for member in zipInfo:
            member.filename = member.filename.encode('cp437').decode('utf-8')
            if member.filename.find('_MACOSX') != -1:
                continue
            f.extract(member,output)

root_path: str = './unzipped'
# file_to_unzip = '/Users/homemac/Desktop/今天我們要去哪探險呢？-2.zip' #sys.argv[1]
# unzip_file(file_to_unzip,root_path)
output_folder = './output'
# if not os.path.exists(output_folder):
#     os.makedirs(output_folder)

for _path, dirs, files in os.walk(root_path,topdown=True):
    path = os.path.abspath(_path)
    if _path == root_path:
        continue
    if not node.get(path,):
        _ = node(*pat.split(path))

    visiting_node = node.nodes[path]
    for dir in dirs:
        if not visiting_node.child[0]:
            visiting_node.set_childL( node(path,dir) )
        elif not visiting_node.child[1]:
            visiting_node.set_childR( node(path,dir) )
        else:
            try:
                exception = Exception("Cannot set child node, for parent node has full node")
                print(exception)
                raise exception

            except:
                traceback.print_exc()
                sys.exit(2)
    for file in files:
        if file.find('.') == 0 or file.find('_MACOSX') != -1:
            continue
        node.nodes[path].add_src(pat.join(path,file))

if True:
    output_path = pat.realpath(pat.join(root_path,'..',output_folder))
    json_info = {"Product":"StoryBook",
                 "DataStructure":"BinaryTree",
                 "node_nums":len(node.nodes),
                 "root_nodeid":0,
                 "node":[],
                 "action":[
                     {"btn_Name": "btn_left", "action": "do_left"},
                     {"btn_Name": "btn_right", "action": "do_right"},
                     {"btn_Name": "btn_confirm", "action": "do_confirm"},
                     {"btn_Name": "btn_home", "action": "do_home"},
                 ],

                 }
    for path,node in node.nodes.items():
        for src in node.src:
            src_extension = get_file_extension(src)
            src_path_output = pat.join( output_path \
                                        ,str(node.id).zfill(5)+src_extension)
            copy(src, src_path_output)
        json_info['node'].append(node.get_dict())
    #print(json.dumps(json_info))

    tree_info = {"nodes":{},
                 'struct':None}
    for node in json_info['node']:
        tree_info['nodes'][node['nodeid']] = node
    # print(tree_info)
    for node in tree_info['nodes'].values():
        children = []
        if node['childNodeL'] != '':
            lookup_node_id = node['childNodeL']
            children.append(tree_info["nodes"][lookup_node_id])
        if node['childNodeR'] != '':
            lookup_node_id = node['childNodeR']
            children.append(tree_info["nodes"][lookup_node_id])
        if len(children) != 0:
            node['children'] = children

        delete_attr = ['childNodeL','childNodeR','audio','video','pic','nodeid']
        for attr in delete_attr:
            # print(node)
            if node.get(attr,None) or node.get(attr,None) == '':
                del node[attr]
    print(json.dumps(tree_info['nodes'][0]))
    # with open('./tree.json','w') as file:
    #     file.write(json.dumps(tree_info['nodes'][0]))


import functools
import json
import os
import click
import dotenv
from .api import JenkinsApi

dotenv.load_dotenv(dotenv_path=os.path.join(os.getenv('HOME', '.'), '.jenkins-env'))

HOSTNAME = os.getenv('JENKINS_HOST')
VERIFY_SSL = not bool(os.getenv('JENKINS_NOVERIFY'))
USERNAME = os.getenv('JENKINS_USERNAME')
TOKEN = os.getenv('JENKINS_TOKEN')

def api_command(group):
    def decorator(func):
        @group.command()
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            api = JenkinsApi(HOSTNAME, auth=(USERNAME, TOKEN), verify=VERIFY_SSL)
            response = func(api, *args, **kwargs)
            result = response if isinstance(response, str) else json.dumps(response, indent=2)
            print(result)
        return wrapper
    return decorator

@click.group()
def main():
    pass

@api_command(main)
def jobs(api):
    return api.jobs()

@api_command(main)
@click.option('-j', '--job', required=True)
def job(api, job):
    return api.job(job)

@api_command(main)
@click.option('-j', '--job', required=True)
@click.option('-r', '--run', required=True)
@click.option('-g', '--git', 'job_action', flag_value='git')
@click.option('-s', '--stdout', 'job_action', flag_value='stdout')
@click.option('--default', 'job_action', flag_value='run', default=True)
def run(api, job, run, job_action):
    action = getattr(api, job_action)
    return action(job, run)

@api_command(main)
def computer(api):
    return api.computer()

@api_command(main)
def people(api):
    return api.people()

@api_command(main)
def builds(api):
    return api.view_all()


import sys
import os
import csv
import mgmnet.bio_nets as bn
import mgmnet.union_nets as un
import mgmnet.syn_nets as sn
import mgmnet.ranRxn_nets as rn
import mgmnet.bioRanRxn_nets as brn
import mgmnet.bioUniRxn_nets as bui
import mgmnet.kegg_nets as kn
import mgmnet.topo_measure as tm
import shutil
import pandas as pd

#---------- results to be connected ==> topo_ave ----------#
topo = tm.topoMeasure()
header = topo.header

#---------- class of networks ----------#
module_name = sys.argv[1]
module_dict = {'bio': bn.bio(), \
               'union': un.union(), \
               'syn': sn.syn(), \
               'ranRxn': rn.ranRxn(), \
               'bioRanRxn':brn.bioRanRxn(), \
               'bioUniRxn':bui.bioUniRxn()}

class_name = module_dict[module_name]

#---------- directories ----------#
dr_collection = ''
for ds in ('../results_cluster', '/topo_ave', '/%s'%(module_name)):
    dr_collection = dr_collection + ds
    if not os.path.exists(dr_collection):
        os.makedirs(dr_collection)

dr_missing = ''
for ds in ('../results_cluster', '/topo_ave', '/%s'%(module_name)):
    dr_missing = dr_missing + ds
    if not os.path.exists(dr_missing):
        os.makedirs(dr_missing)

dr_missing_batch = ''
for ds in ('../cluster', '/missing', '/topo_ave'):
    dr_missing_batch = dr_missing_batch + ds

#---------- collecting results and identifying missing results ----------#
df_list = []
for system_name in class_name.number_of_species.iterkeys():
    dr_results = ''
    for ds in ('../results_cluster', '/topo_ave', '/%s'%(module_name), '/%s'%(system_name)):
        dr_results = dr_results + ds
        if not os.path.exists(dr_results):
            os.makedirs(dr_results)

    collectedFileName = dr_collection + '/topo_ave_%s.csv'%(system_name)
    with open(collectedFileName, "w") as of:
        cof = csv.writer(of)
        cof.writerow(header)

    missingFileName = dr_missing + '/missing_topo_ave_%s'%(system_name)
    with open(missingFileName, "w") as mf:
       cmf = csv.writer(mf)

    for species in range(1, class_name.number_of_species[system_name] + 1):
        result_file_string = {'bio': '%s-%d.csv'%(system_name, species), \
                       'union': '%s-upto-%d.csv'%(system_name, species), \
                       'syn': '%s-%d.csv'%(system_name, species), \
                       'ranRxn': '%s-%d.csv'%(system_name, species),\
                       'bioRanRxn': '%s-%d.csv'%(system_name, species), \
                       'bioUniRxn':'%s-%d.csv'%(system_name, species)} #==>change the string after generate syn results

        resultFileName = dr_results + '/' + result_file_string[module_name]
        if not os.path.isfile(resultFileName):
            sample = [species]
            with open(missingFileName, "a") as mf:
                cmf = csv.writer(mf)
                cmf.writerow(sample)
            continue

        with open(resultFileName, "r") as rf:
            crf = csv.reader(rf, delimiter = ",")
            data = list(crf)
            row_count = len(data)

        if row_count == class_name.lines_in_topo_ave[system_name]:
            l = class_name.lines_in_topo_ave[system_name] - 1
            outcome = data[l]
        else:
            sample = [species]
            with open(missingFileName, "a") as mf:
                cmf = csv.writer(mf)
                cmf.writerow(sample)
            continue

        with open(collectedFileName, "a") as of:
            cof = csv.writer(of)
            cof.writerow(outcome)

    df_list.append(pd.read_csv(collectedFileName))

    missingBatchName = dr_missing_batch + '/missing_topo_ave_%s'%(system_name)
    if os.path.isfile(missingBatchName):
        os.remove(missingBatchName)

    #------copy missingFile to cluster folder -------#
    if os.stat(missingFileName).st_size > 0:
        if not os.path.exists(dr_missing_batch):
            os.makedirs(dr_missing_batch)
        shutil.copyfile(missingFileName, missingBatchName)
    else:
        os.remove(missingFileName)

#-------- Merge all files into one-----------#
finalFileName = dr_collection + '/topo_ave_%s.csv'%(module_name)
full_df = pd.concat(df_list)
full_df.to_csv(finalFileName)


# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import boto3
import botocore

from datetime import datetime, timedelta
import time

INSTANCE_NAME_TAG = 'Name'
SNAPSHOT_AGE_IN_HOURS = 24
SLEEP_IN_SECONDS = 5

def instances_and_volumes(region_name):
    while True:
        try:
            ec2 =  boto3.client('ec2', region_name=region_name)
            instances_paginator = ec2.get_paginator('describe_instances')
            #volumes_paginator = ec2.get_paginator('describe_volumes')

            for instances_page in instances_paginator.paginate():
                for instance in instances_page['Reservations']:
                    for i in instance['Instances']:
                        print('\n%s,%s' %(region_name, i['InstanceId']), end='')
                        for bdm in i['BlockDeviceMappings']:
                            print(', %s ' %bdm['Ebs']['VolumeId'], end='')

        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'RequestLimitExceeded':
                print("^^^^^^^^^^   REQUEST LIMIT EXCEEDED ^^^^^^^^^^^^^: %s" % e)
                time.sleep(SLEEP_IN_SECONDS)  # boto's built-in exponential backoff expired so just sleep
            else:
                print("%%%%%%%%%%%%%%%%%%%% Unexpected error: %s" % e)
            continue
        break

'''
Per instance, display volumes with no snapshots or
volumes that have not been snapshotted recently.
'''
def report_by_volumes(region_name):
    while True:  # in case of an exception, redo volume or snapshot check.  See http://stackoverflow.com/questions/2083987/how-to-retry-after-exception-in-python
        try:
            ec2 = boto3.client('ec2', region_name=region_name)
            volumes_paginator = ec2.get_paginator('describe_volumes')
            snapshots_paginator = ec2.get_paginator('describe_snapshots')

            for page in volumes_paginator.paginate():
                for volume in page['Volumes']:
                    time_since_latest_snapshot = timedelta.max
                    instance_id = volume['Attachments'][0]['InstanceId'] if volume['Attachments'] else 'Unattached'
                    volume_id, az = volume['VolumeId'], volume['AvailabilityZone']
                    print('%s attached to %s in %s\t\t\t' % (volume_id, instance_id, az), end='')
                    snapshots_iterator = snapshots_paginator.paginate(OwnerIds=['self'], Filters=[{'Name': 'volume-id', 'Values': [volume_id]}])
                    snapshot_taken = False

                    for snapshots in snapshots_iterator:
                        for snapshot in snapshots['Snapshots']:
                            snapshot_taken = True
                            snapshot_date = snapshot['StartTime']
                            time_since_snapshot = datetime.utcnow().replace(microsecond=0) - snapshot_date.replace(
                                tzinfo=None).replace(microsecond=0)
                            if time_since_snapshot < time_since_latest_snapshot:
                                time_since_latest_snapshot = time_since_snapshot
                                latest_snapshot = snapshot

                            snapshot_age_msg = str(time_since_latest_snapshot) + " elapsed since last snapshot %s." % (snapshot['SnapshotId'])

                    if not snapshot_taken:
                        print("\t*** WARNING: No snapshots taken for this volume.")
                    else:
                        snapshot_out_of_date = latest_snapshot['StartTime'].replace(
                            tzinfo=None) < datetime.utcnow() - timedelta(
                            hours=SNAPSHOT_AGE_IN_HOURS)
                        if snapshot_out_of_date:
                            print("\t### WARNING: It's been %s" % snapshot_age_msg)
                        else:
                            print(
                                "\t@@@ %s is the latest snapshot for this volume and it's current. " % latest_snapshot[
                                    'SnapshotId'],
                                snapshot_age_msg)

        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'RequestLimitExceeded':
                print("^^^^^^^^^^   REQUEST LIMIT EXCEEDED ^^^^^^^^^^^^^: %s" % e)
                time.sleep(SLEEP_IN_SECONDS)  # boto's built-in exponential backoff expired so just sleep
            else:
                print("%%%%%%%%%%%%%%%%%%%% Unexpected error: %s" % e)
            continue
        break

def check_volumes():
    ec2 = boto3.resource('ec2')

    instances = ec2.instances.all()  # .filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
    for instance in instances:
        instance_name = ''
        for tag in instance.tags:
            instance_name = tag['Value'] if tag['Key'] == INSTANCE_NAME_TAG else None

        print("\n\nInstance name: '%s' \t Instance ID: %s" % (instance_name, instance.id))

        while True:  # in case of an exception, redo volume or snapshot check.  See http://stackoverflow.com/questions/2083987/how-to-retry-after-exception-in-python
            try:
                for volume in instance.volumes.all():
                    print("Volume ID: %s\t" % (volume.id), end = '')
                    snapshot_taken = False
                    time_since_latest_snapshot = timedelta.max
                    latest_snapshot = None

                    for snapshot in volume.snapshots.all():
                        snapshot_taken = True
                        snapshot_date = snapshot.start_time

                        time_since_snapshot = datetime.utcnow().replace(microsecond=0) - snapshot_date.replace(tzinfo=None).replace(
                            microsecond=0)
                        if time_since_snapshot < time_since_latest_snapshot:
                            time_since_latest_snapshot = time_since_snapshot
                            latest_snapshot = snapshot

                        snapshot_age_msg = str(time_since_latest_snapshot) + " elapsed since last snapshot."

                    if not snapshot_taken:
                        print("\t*** WARNING: No snapshots taken for this volume.")
                    else:
                        snapshot_out_of_date = latest_snapshot.start_time.replace(tzinfo=None) < datetime.utcnow() - timedelta(
                            hours=SNAPSHOT_AGE_IN_HOURS)
                        if snapshot_out_of_date:
                            print("\t### WARNING: It's been %s" % snapshot_age_msg)
                        else:
                            print("\t@@@ %s is the latest snapshot for this volume and it's current. " %  latest_snapshot.id, snapshot_age_msg)

            except botocore.exceptions.ClientError as e:
                if e.response['Error']['Code'] == 'RequestLimitExceeded':
                    #print("^^^^^^^^^^   REQUEST LIMIT EXCEEDED ^^^^^^^^^^^^^: %s" % e)
                    time.sleep(SLEEP_IN_SECONDS) # boto's built-in exponential backoff wasn't good enough so just sleep
                else:
                    print("%%%%%%%%%%%%%%%%%%%% Unexpected error: %s" % e)
                continue
            break

def main():
    ec2 = boto3.client('ec2')

    regions = ec2.describe_regions()
    for region in regions['Regions']:
        #instances_and_volumes(region['RegionName'])
        report_by_volumes(region['RegionName'])
        #check_volumes()

if __name__ == "__main__":
    main()


import unittest
import numpy as np
from os import path
from assimpcy import (aiImportFile, aiImportFileFromMemoryWithProperties,
                      aiPropertyStore, aiPostProcessSteps as pp,
                      aiComponent as co, AI_CONFIG_PP_RVC_FLAGS)

VERTEX_COUNT = 318
ANIMATION_COUNT = 2
EXAMPLE_FILE_PATH = path.join(path.dirname(__file__), path.pardir, 'examples', 'models', 'cil', 'cil.x')

class SimpleTest(unittest.TestCase):
    def setUp(self):
        flags = pp.aiProcess_JoinIdenticalVertices | pp.aiProcess_Triangulate | pp.aiProcess_CalcTangentSpace | \
                pp.aiProcess_OptimizeGraph | pp.aiProcess_OptimizeMeshes | pp.aiProcess_FixInfacingNormals | \
                pp.aiProcess_GenUVCoords | pp.aiProcess_LimitBoneWeights | pp.aiProcess_SortByPType | \
                pp.aiProcess_RemoveRedundantMaterials

        self.scene = aiImportFile(EXAMPLE_FILE_PATH, flags)

    def test_vertexCount(self):
        self.assertEqual(VERTEX_COUNT, self.scene.mMeshes[0].mNumVertices)

    def test_vertex_value(self):
        scene = self.scene
        v = int(scene.mMeshes[0].mNumVertices / 2)
        value = scene.mMeshes[0].mVertices[v]
        self.assertTrue(np.all(value == np.array([0.707107, 0.707107, 0.188462], np.float32)))

    def test_animationCount(self):
        self.assertEqual(ANIMATION_COUNT, self.scene.mNumAnimations)

class ImportPropertyTest(unittest.TestCase):
    def setUp(self):
        props = aiPropertyStore()
        props.SetImportPropertyInteger(AI_CONFIG_PP_RVC_FLAGS, co.aiComponent_ANIMATIONS)
        flags = pp.aiProcess_JoinIdenticalVertices | pp.aiProcess_RemoveComponent
        with open(EXAMPLE_FILE_PATH, "rb") as f:
            data = f.read()

        self.scene = aiImportFileFromMemoryWithProperties(data, len(data),
                                                          flags, "", props)

    def test_animationCount(self):
        self.assertEqual(0, self.scene.mNumAnimations)

if __name__ == '__main__':
    unittest.main()


import os
from glob import glob
from subprocess import Popen, PIPE
import json
from nipype.interfaces import fsl
from nipype.interfaces.fsl import RobustFOV, Reorient2Std
from nipype.interfaces.c3 import C3dAffineTool
import argparse
import shutil
from nipype.interfaces.fsl import BinaryMaths, UnaryMaths, ErodeImage, ImageStats, Threshold, DilateImage
import numpy as np
import math
from nipype.interfaces.base import CommandLine, CommandLineInputSpec, SEMLikeCommandLine, TraitedSpec, File, Directory, traits, isdefined, InputMultiPath, OutputMultiPath
import pbr
from pbr.base import _get_output

"takes the difference between the two flair scans at different timepoints, multiplies the difference by the wm mask (created from sienax), create zscore image, multiplies that images by the lst lesion mask, and thresholds that image to get a new lesion mask"

PBR_long = '/data/henry10/PBR_long/subjects/'
no_wm = "/data/henry6/PBR/surfaces/MNI152_T1_1mm/mri/no_wm_MNI_new.nii.gz"

class imageData():
    def __init__(self, lesion_MNI, wm_MNI, flair_file, t1_file, gm_MNI):
        self.lesion_MNI = lesion_MNI
        self.wm_MNI = wm_MNI
        self.gm_MNI = gm_MNI
        self.flair_file = flair_file
        self.t1_file = t1_file

def find_lesion_MNI(mseid):

    if not os.path.exists(_get_output(mseid)+"/"+mseid+"/alignment/status.json"):
        run_pbr_align(mseid)

    with open(_get_output(mseid)+"/"+mseid+"/alignment/status.json") as data_file:
        data = json.load(data_file)
        if len(data["t1_files"]) == 0:
            #print("no {0} t1 files".format(tp))
            run_pbr_align(mseid)
            t1_file = "none"
        else:
            t1_file = data["t1_files"][-1]

        if len(data["flair_files"]) == 0:
            flair_file = "none"
        else:
            flair_file = data["flair_files"][-1]
            flair_file = format_to_baseline_mni(flair_file, "_T1mni.nii.gz")
            print(flair_file)

    lesion_MNI = _get_output(mseid)+ mseid + "/alignment/lesion_MNI.nii.gz"
    mni_long = PBR_long + msid + "/MNI/"
    wm_MNI = _get_output(mseid)+ mseid + "/alignment/WM_MNI.nii.gz"
    gm_MNI = _get_output(mseid)+ mseid + "/alignment/GM_MNI.nii.gz"

    if os.path.exists(wm_MNI):
        print(wm_MNI)
        if not os.path.exists(mni_long):
            os.mkdir(mni_long)

        print(wm_MNI,mni_long + "/wm_"+mseid + ".nii.gz")

    if os.path.exists(gm_MNI):
        if not os.path.exists(mni_long):
            os.mkdir(mni_long)
        print(gm_MNI)
        cmd = ["fslmaths", gm_MNI, "-bin", gm_MNI]
        proc = Popen(cmd)
        proc.wait()

        shutil.copyfile(gm_MNI,mni_long + "/gm_"+mseid + ".nii.gz")
        print(gm_MNI,mni_long + "/gm_"+mseid + ".nii.gz")

    if os.path.exists(lesion_MNI):
        print(lesion_MNI)

        if not os.path.exists(mni_long):
             os.mkdir(mni_long)
             print(mni_long)
        if not os.path.exists(mni_long+ "/lesion_"+mseid + ".nii.gz"):
            shutil.copyfile(lesion_MNI,mni_long + "/lesion_"+mseid + ".nii.gz")
            print(lesion_MNI,mni_long + "/lesion_"+mseid + ".nii.gz")
        if flair_file.endswith(".nii.gz"):

            flair_path = os.path.split(lesion_MNI)[0] +'/'+ os.path.split(flair_file)[-1]

            shutil.copyfile(flair_path, mni_long +'/'+ os.path.split(flair_path)[-1])
            print(flair_path, mni_long +'/'+ os.path.split(flair_path)[-1])

    return imageData(lesion_MNI, wm_MNI, flair_file, t1_file, gm_MNI)
    print("FINISHED COPYING OVER MNI FILES....")

def run_pbr_align(mseid):

    alignment_folder = _get_output(mseid)+  '/' + mseid + "/alignment/"
    if os.path.exists(alignment_folder):
        cmd_rm = ['rm','-r', alignment_folder]
        print (cmd_rm)
        proc = Popen(cmd_rm)
        proc.wait()

    cmd = ['pbr', mseid, '-w', 'align', '-R']
    print (cmd)
    proc = Popen(cmd)
    proc.wait()

def format_to_baseline_mni(in_file,extension,message="show"):
    out_path = in_file.split("alignment")[0] + "alignment/baseline_mni"
    file_name = in_file.split('/')[-1].split('.')[0] + extension
    if not os.path.exists(out_path):
        os.makedirs(out_path)
    out_file = os.path.join(out_path, file_name)
    return out_file

def format_to_lesion_mni(in_file,extension,message="show"):
    out_path = in_file.split("alignment")[0] + "lesion_mni"
    file_name = in_file.split('/')[-1].split('.')[0] + extension
    if not os.path.exists(out_path):
        os.makedirs(out_path)
    out_file = os.path.join(out_path, file_name)
    return out_file

def create_flair_lesions(mseid, msid):
    if not os.path.exists(_get_output(mseid) + "/" + mseid + "/sienax_flair/"):
        find_lesion_MNI(mseid)
        print(mseid, msid)
        mni_long = PBR_long + msid + "/MNI/"
        if not os.path.exists(mni_long):
            print("This subject does not have any FLAIR images to register the lesions to")
        else:
            with open(_get_output(mseid)+"/"+mseid+"/alignment/status.json") as data_file:
                data = json.load(data_file)
                if len(data["t1_files"]) == 0:
                    #print("no {0} t1 files".format(tp))
                    run_pbr_align(mseid)
                    t1_file = "none"
                else:
                    t1_file = data["t1_files"][-1]

                if len(data["flair_files"]) == 0:
                    flair_file = "none"
                else:
                    flair_file = data["flair_files"][-1]
                    flair_file = format_to_baseline_mni(flair_file, "_T1mni.nii.gz")
                    print(flair_file)

            if os.path.exists(flair_file):
                if not os.path.exists(_get_output(mseid) +"/"+mseid + "/lesion_mni"):
                    os.mkdir(_get_output(mseid) +"/"+mseid + "/lesion_mni")

                flair_MNI = str(glob(mni_long + "ms*.nii.gz")[0])
                wm_MNI = str(glob(mni_long + "wm*.nii.gz")[0])
                gm_MNI = str(glob(mni_long + "gm*.nii.gz")[0])
                lesion_MNI = str(glob(mni_long + "lesion_mse*.nii.gz")[0])

                cmd = ["fslmaths", lesion_MNI, "-bin", lesion_MNI.replace("lesion_", "lesion_bin_")]
                proc = Popen(cmd)
                proc.wait()

                lesion_bin_MNI = str(glob(mni_long + "lesion_bin_mse*.nii.gz")[0])

                base_dir =  os.path.split(format_to_lesion_mni(flair_file, "_WM.nii.gz"))[0]
                wm_eroded = base_dir + "/wm_eroded"
                wm_flair = base_dir + "/wm_flair"

                cmd = ["N4BiasFieldCorrection", "-d", "3", "-i", flair_file,"-w", wm_MNI, "-o", flair_file.replace(".nii.gz", "_n4corr.nii.gz")]
                proc = Popen(cmd)
                proc.wait()

                flair_file = flair_file.replace(".nii.gz", "_n4corr.nii.gz")

                # calculating estimated median and standard deviation for nawm

                cmd = ["fslmaths", wm_MNI, "-ero","-sub", no_wm, "-thr", ".1", "-mul",flair_file, wm_eroded]
                proc = Popen(cmd)
                proc.wait()

                wm_with_les = base_dir+ "/wm_withles.nii.gz"
                lesion_dil = base_dir + "/lesion_dil.nii.gz"

                cmd = ["bet",flair_file, base_dir + "/skull.nii.gz", "-s"]
                proc = Popen(cmd)
                proc.wait()

                cmd = ["fslstats", wm_eroded, "-P", "50"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                output = output[0]
                median_nawm = float(output[0])
                print(median_nawm, "THIS IS THE MEDIAN of the NAWM")

                cmd = ["fslmaths", base_dir + "/skull.nii.gz", "-uthr",str(median_nawm),"-bin",base_dir+ "/thr.nii.gz"]
                proc = Popen(cmd)
                proc.wait()

                #dilating the lesion mask and subtracting the gray matter
                cmd = ["fslmaths",lesion_bin_MNI , "-dilM",  "-sub", gm_MNI,"-thr", ".1","-bin", lesion_dil]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]

                cmd = ["fslmaths", lesion_dil, "-add", wm_MNI, "-bin", wm_with_les]
                proc = Popen(cmd)
                proc.wait()

                cmd = ["fslmaths", wm_with_les, "-mul",flair_file, wm_flair]
                proc = Popen(cmd)
                proc.wait()

                new_median_nawm = median_nawm - .000001
                print("NEW MEDIAN", new_median_nawm)

                cmd = ["fslmaths",  wm_eroded, "-uthr", str(new_median_nawm), base_dir+"/ero_WM_Lhalf.nii.gz"]
                proc = Popen(cmd)
                proc.wait()

                cmd = ["fslstats", base_dir+ "/ero_WM_Lhalf.nii.gz", "-S"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                output = output[0]
                std_nawm = float(output[0])
                print(std_nawm, "THIS IS THE STANDARD DEVIATION of NAWM")

                est_std = float(std_nawm) * 1.608

                cmd = ["fslstats", wm_eroded, "-V"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                output = output[0]
                vol_nawm = float(output[0])
                print(vol_nawm, "THIS IS THE VOLUME OF THE NAWM")

                #calculate wm histogram
                std_times2 = std_nawm*std_nawm*2
                print(std_times2, "this is the standard deviation squared times 2")
                part1 = vol_nawm/(math.sqrt(std_times2*(math.pi)))
                print(part1, "this is part1 times 2")
                cmd = ["fslmaths", wm_flair, "-sub",str(median_nawm),"-sqr", "-div", str(std_times2), "-mul", "-1", "-exp", "-mul", str(part1), base_dir+"/wm_hist.nii.gz"]
                print(cmd, "COMMAND")
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                print(cmd)

                # calculating estimated median and standard deviation for lesions
                lesion_mul_flair = base_dir + "/lesion.nii.gz"

                cmd = ["fslmaths", lesion_bin_MNI,"-sub",base_dir + "/thr.nii.gz","-thr", ".1", "-bin", "-mul",flair_file,  lesion_mul_flair]
                proc = Popen(cmd)
                proc.wait()

                cmd = ["fslstats", lesion_mul_flair, "-P", "50"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                output = output[0]
                median_lesion = float(output[0])
                print(median_lesion, "THIS IS THE LESION MEDIAN")

                new_median_lesion = median_lesion + .000001
                print("NEW MEDIAN", new_median_lesion)

                cmd = ["fslmaths",  lesion_mul_flair, "-thr", str(new_median_lesion), base_dir+ "/lesion_Uhalf.nii.gz"]
                proc = Popen(cmd)
                proc.wait()

                cmd = ["fslstats",base_dir +"/lesion_Uhalf.nii.gz" , "-S"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                std_lesion = output[0]
                std_lesion = float(std_lesion[0])
                print(std_lesion, "THIS IS THE STANDARD DEVIATION for the lesion upper half")

                est_std = std_lesion * 1.608
                #echo 20.077990*1.608 | bc

                cmd = ["fslstats", lesion_mul_flair, "-V"]
                proc = Popen(cmd, stdout=PIPE)
                output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                output = output[0]
                vol_lesion = float(output[0])
                print(vol_lesion, "THIS IS THE VOLUME OF THE lesion")

                if not vol_lesion == 0.0:
                # lesion histogram
                    lesion_times2 = std_lesion*std_lesion*2
                    print(lesion_times2, "thisis the standard deviation times 2")
                    part2 = vol_lesion/(math.sqrt(lesion_times2*(math.pi)))
                    print(part2, "this is part2 times 2")
                    cmd = ["fslmaths", wm_flair, "-sub",str(median_lesion),"-sqr", "-div", str(lesion_times2), "-mul", "-1", "-exp", "-mul", str(part1),base_dir+"/lesion_hist.nii.gz"]
                    print(cmd, "COMMAND")
                    proc = Popen(cmd, stdout=PIPE)
                    output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]
                    print(cmd)

                    prob_map = base_dir +"/prob_map_new.nii.gz"
                    prob_map_nobs = base_dir +"/prob_map_nowmbs.nii.gz"

                    final_lesion = base_dir + "/lesion_final_new.nii.gz"
                    wm_no_bs = base_dir+ "/wm_no_bs.nii.gz"

                    cmd = ["fslmaths", gm_MNI, "-dilM","-dilM", base_dir + "/gm_dil.nii.gz"]
                    proc = Popen(cmd)
                    proc.wait()

                    # eroding the wm mask and adding in the dilated lesion mask
                    cmd = ["fslmaths",wm_MNI, "-sub", no_wm, "-thr", ".1","-ero","-add",lesion_dil,"-bin", wm_no_bs] # can try adding back later "-sub",gm_MNI,
                    proc = Popen(cmd)
                    proc.wait()

                    # making the probability map
                    cmd = ["fslmaths", base_dir+"/lesion_hist.nii.gz", "-add", base_dir+ "/wm_hist.nii.gz",base_dir+"/add_his.nii.gz"]
                    proc = Popen(cmd)
                    proc.wait()

                    cmd = ["fslmaths",base_dir +"/lesion_hist.nii.gz", "-div",base_dir+ "/add_his.nii.gz","-mul", wm_with_les, prob_map]
                    proc = Popen(cmd)
                    proc.wait()

                    #or just look in the dialated lesion minus the gray matter mask
                    cmd = ["fslmaths",lesion_dil,"-mul",prob_map, "-thr", ".99","-bin","-mul",wm_with_les,"-bin", "-sub",base_dir+ "/thr.nii.gz","-sub", gm_MNI, "-thr", ".1", "-bin", final_lesion]
                    proc = Popen(cmd)
                    proc.wait()

                    cmd = ["fslmaths",wm_no_bs, "-mul", prob_map, "-ero","-ero", prob_map_nobs]
                    proc = Popen(cmd)
                    proc.wait()

                    cmd = ["fslmaths", prob_map_nobs,"-thr", ".99","-sub",base_dir+ "/thr.nii.gz","-thr", ".1", "-bin", "-add", final_lesion, "-bin", base_dir+ "/lesion_prob_map.nii.gz"]
                    proc = Popen(cmd)
                    proc.wait()

                    cmd = ["fslmaths", lesion_MNI, "-dilM", "-mul", base_dir+ "/lesion_prob_map.nii.gz", base_dir+ "/lesion_labeled.nii.gz" ]
                    proc = Popen(cmd)
                    proc.wait()

                    """cmd = ["fslview",wm_eroded, wm_with_les,prob_map, base_dir+ "/lesion_labeled.nii.gz",final_lesion, lesion_bin_MNI,flair_file,base_dir+ "/lesion_prob_map.nii.gz"]
                    proc = Popen(cmd, stdout=PIPE)
                    output = [l.decode("utf-8").split() for l in proc.stdout.readlines()[:]]"""

                    #if not os.path.exists(PBR_base_dir + "/" + mseid + "/sienax/"):
                    print("sienax_optibet", format_to_baseline_mni(t1_file, "_T1mni.nii.gz"), "-lm", final_lesion.replace("_reorient",""), "-r", "-d", "-o", _get_output(mseid)+ "/" + mseid + "/sienax_flair/")
                    cmd = ["sienax_optibet", format_to_baseline_mni(t1_file, "_T1mni.nii.gz"), "-lm", final_lesion.replace("_reorient",""), "-r", "-d", "-o", _get_output(mseid)+ "/" + mseid + "/sienax_flair/"]
                    proc = Popen(cmd)
                    proc.wait()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('ms', nargs="+")
    args = parser.parse_args()
    ms = args.ms
    print("msid is:", ms)

    for msid in ms:
        text_file = '/data/henry6/mindcontrol_ucsf_env/watchlists/long/VEO/msid/{0}.txt'.format(msid)
        if os.path.exists(text_file):
            with open(text_file,'r') as f:
                timepoints = f.readlines()
                timepoints = timepoints[::-1]
                for timepoint in timepoints:
                    mseid = timepoint.replace("\n","")
                    print("these are the mseID's", mseid)
                    create_flair_lesions(mseid, msid)


from __future__ import (absolute_import, unicode_literals, division,
                        print_function)

__all__ = ["ema_parallel", "model_ensemble", "parameters"
           "model", "outcomes", "samplers", "uncertainties",
           'RealUncertainty', "IntegerUncertainty", "CategoricalUncertainty",
           "Model", 'FileModel', "ModelEnsemble",
           "Outcome", "ScalarOutcome", "TimeSeriesOutcome",
           "RealParameter", "IntegerParameter", "CategoricalParameter",
           "Scenario", "Policy", "Experiment", "Constant", "create_parameters",
           "parameters_to_csv", "Category"
           ]

from .outcomes import ScalarOutcome, TimeSeriesOutcome, Outcome
from .uncertainties import (ParameterUncertainty, CategoricalUncertainty)
from .model import Model, FileModel
from .ensemble import (ModelEnsemble)
from .parameters import (RealParameter, IntegerParameter, CategoricalParameter,
                     Scenario, Policy, Constant, Experiment, create_parameters,
                     parameters_to_csv, Category)
from .samplers import (MonteCarloSampler, FullFactorialSampler, LHSSampler,
                       PartialFactorialSampler, LHS, MC, FF, PFF, SAMPLERS)

def perform_experiments(models, cases, policies=[Policy('None')],
                        sampling=LHS, parallel=False, reporting_interval=None,
                        uncertainty_union=False, outcome_union=False):
    '''convenience function for running experiments

    Parameters
    ----------
    models : list or model instance
    cases : int or recarray
    policies : list, optional
    sampling : {LHS, MC, FF, PFF}
    parallel : bool, optional
    reporting_interval : int, optional
                         defaults to 1/10 of the number of cases
    uncertainty_union : bool, optional
    outcome_union : bool, optional

    Returns
    -------
    tuple

    '''

    import numbers

    ensemble = ModelEnsemble(sampler=SAMPLERS[sampling])
    ensemble.parallel = parallel
    ensemble.model_structures = models
    ensemble.policies = policies

    return ensemble.perform_experiments(cases,
                                        reporting_interval=reporting_interval,
                                        uncertainty_union=uncertainty_union,
                                        outcome_union=outcome_union)


from django import template
import markdown
from django.db import models
from promotions.models import Affectation_Etudiant, Bulletin
from django.utils.safestring import mark_safe
from django.db.models import Avg
import os
from django.http import QueryDict

register = template.Library()

@register.simple_tag
def query_transform(request, page):
    dict_ = request.GET.copy()
    dict_["page"] = page
    return dict_.urlencode()

@register.simple_tag
def test_url_active(request, menu,value):
    test_string="{}/{}/".format(menu,value)
    if test_string in request.path:
        output="active"
    else:
        output=""
    return output

def find_note(bulletin, module_pk):

    note=Note.objects.filter(bulletin=bulletin,module__pk=module_pk)

    if note.exists():
        output=note[0].moyenne
    else:
        output="Indéterminé"

    return output

def find_nb_etu(parcours, promotion_pk):

    affectations=Affectation_Etudiant.objects.filter(parcours=parcours,promotion__pk=promotion_pk)
    nb_etu=affectations.count()
    return nb_etu

def find_moy(parcours, promotion_pk):

    bulletin=Bulletin.objects.filter(affectation__parcours=parcours,affectation__promotion__pk=promotion_pk)
    result=bulletin.aggregate(Avg('moyenne'))
    moyenne=result["moyenne__avg"]
    if isinstance(moyenne,float):
        moyenne=round(moyenne,2)

    return moyenne

def as_p(object):

    html=""

    for field in object._meta.get_fields():

        if not (isinstance(field,models.ManyToManyRel)) and not (isinstance(field,models.OneToOneRel)) and not (isinstance(field,models.ManyToOneRel)):

            if field.name is not "id":

                value=getattr(object,field.name)
                html=html+"
"+str(field.verbose_name)+""
                html=html+"
"+markdown.markdown(str(value))+"

"

    return mark_safe(html)

def as_description(object,list_display):

    html="
"

    for fieldname in list_display:
        fieldvalue=getattr(object,fieldname)
        html=html+"
"+str(fieldname)+"
"
        html=html+"
"+str(fieldvalue)+"
"

    html=html+"
"

    return mark_safe(html)

def as_markdown_description(object,list_display):

    html="
"

    for fieldname in list_display:
        fieldvalue=getattr(object,fieldname)
        html=html+"
"+str(fieldname)+"
"
        html=html+"
"+markdown.markdown(str(fieldvalue))+"
"

    html=html+"
"

    return mark_safe(html)

def as_markdown(object):
    if object is None:
        html=""
    else:
        html=markdown.markdown(object)
    return mark_safe(html)

def file_extension(object):
    name, extension = os.path.splitext(object.url)
    return extension

def as_row(object,list_display):

    #first column
    fieldvalue=getattr(object,list_display[0])
    if callable(fieldvalue):
        fieldvalue=fieldvalue()

    html="
"+str(fieldvalue)+"	"

    # other columns
    for fieldname in list_display[1:]:

        fieldvalue=getattr(object,fieldname)

        if callable(fieldvalue):
            fieldvalue=str(fieldvalue())
        else:
            field=object._meta.get_field(fieldname)

            if isinstance(field,models.EmailField):
                fieldvalue=""

            if isinstance(field,models.DateTimeField):
                fieldvalue=fieldvalue.strftime("%Y-%m-%d %H:%M:%S")

            if isinstance(field,models.BooleanField):
                if fieldvalue==True:
                    fieldvalue=""
                else:
                    fieldvalue=""

            if isinstance(field,models.CharField):
                fieldvalue=str(fieldvalue)
                fieldvalue=(fieldvalue[:40] + '...') if len(fieldvalue) > 40 else fieldvalue

            if isinstance(field,models.FloatField):
                fieldvalue=str(fieldvalue)

            if isinstance(field,models.IntegerField):
                fieldvalue="%d" % fieldvalue

        html=html+""+str(fieldvalue)+"	"

    return mark_safe(html)

def as_progress_bar(object):
    try:
        value=int(object)
        class_name=["danger","warning","info","info","success"]
        value_100=(value+1)*20

        html="
"
        html=html+"

" %(class_name[value],value_100,value_100)
        html=html+"
"
    except:
        html="None"
    return mark_safe(html)

def as_table(object_list,list_display):

    action=list_display[-1]

    html=""

    for object in object_list:

        #first column
        fieldvalue=getattr(object,list_display[0])
        if callable(fieldvalue):
            fieldvalue=fieldvalue()

        html=html+"
"+str(fieldvalue)+"	"

        # other columns
        for fieldname in list_display[1:-1]:

            fieldvalue=getattr(object,fieldname)

            if callable(fieldvalue):
                fieldvalue=fieldvalue()
            else:
                field=object._meta.get_field(fieldname)

                if isinstance(field,models.EmailField):
                    fieldvalue=""

                if isinstance(field,models.DateTimeField):
                    fieldvalue=fieldvalue.strftime("%Y-%m-%d %H:%M:%S")

                if isinstance(field,models.BooleanField):
                    if fieldvalue==True:
                        fieldvalue=""
                    else:
                        fieldvalue=""

                if isinstance(field,models.CharField):
                    fieldvalue=str(fieldvalue)
                    fieldvalue=(fieldvalue[:40] + '...') if len(fieldvalue) > 40 else fieldvalue

                if isinstance(field,models.FloatField) or isinstance(field,models.IntegerField):
                    fieldvalue=str(fieldvalue)

            html=html+""+fieldvalue+"	"

        if action == "D":
            html=html+"	"

        html=html+"
"

    return mark_safe(html)

register.filter('as_markdown_description', as_markdown_description)
register.filter('as_description', as_description)
register.filter('as_table', as_table)
register.filter('as_row', as_row)
register.filter('as_markdown', as_markdown)
register.filter('as_progress_bar', as_progress_bar)
register.filter('file_extension', file_extension)
register.filter('find_note', find_note)
register.filter('find_nb_etu', find_nb_etu)
register.filter('find_moy', find_moy)


def solve(solution, r_idx, c_idx):
    # fill options of numbers ranging 1-9
    for fill in range(1, 10):
        found = False
        # check the column
        for row in range(9):
            if solution [row][c_idx] == fill:
                found = True
                break
        # check the row
        for col in range(9):
            if solution[r_idx][col] == fill:
                found = True
                break
        # check the box
        box_row_spot = r_idx % 3
        box_col_spot = c_idx % 3
        for row in range(r_idx - box_row_spot, r_idx - box_row_spot + 3):
            for col in range(c_idx - box_col_spot, c_idx - box_col_spot + 3):
                if solution[row][col] == fill:
                    found = True
                    break
        # not found -> fill box and move on
        if found == False:
            solution [r_idx][col_idx] = fill
            next_r = r_idx
            next_c = c_idx
            while solution [next_r][next_c] != 0:
                 next_c = (next_c + 1) % 9
                 if next_c == 0:
                     next_r += 1
                     if next_r == 9:
                         return True
            ret = solve(solution, next_r, next_c)
            # we did it!
            if ret == True:
                return ret
            # else reset box to 0 and move on to try next fill options
            else:
                solution[r_idx][c_idx] = 0
    # none of the fill options worked -> messed up somewhere further back -> go to last box filled and retry
    if solution[r_idx][c_idx] == 0:
        return False


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import TimeoutException
from urllib.parse import quote

broswer=webdriver.Chrome()
wait=WebDriverWait(broswer,20)
KEYWORD="ipad"

def index_page(page):
    print("正在爬取第",page,"页")
    try:
        url="https://s.taobao.com/search?q="+quote(KEYWORD)
        broswer.get(url)
        if page>1:
            input=wait.until(
                EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager div.form>input')))
            submit=wait.until(
                EC.element_to_be_clickable((By.CSS_SELECTOR,'#mainsrp-pager div.form>span.btn.J_Submit')))
            input.clear()
            input.send_keys(page)
            submit.click()
        wait.until(
            EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager li.item.active>span'),str(page)))
        wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR,'.m-itemlist.item.item')))
        get_products()
    except TimeoutException:
        index_page(page)
from pyquery import PyQuery as pq
def get_products():
    html=broswer.page_source
    doc=pq(html)
    items=doc('#mainsrp-itemlist.items.item.').items()
    for item in items:
        product={
            'image':item.find('.pic.img').attr('date-src'),
            'price':item.find('.price').text(),
            'deal':item.find('.deal-cnt').text(),
            'title':item.find('.title').text(),
            'shop':item.find('.shop').text(),
            'location':item.find('.location').text()
        }
        print(product)
        save_to_mogo(product)
import pymongo
MONGO_URL='localhost'
MONGO_DB='taobao'
MONGO_COLLECTION='products'
client=pymongo.MongoClient(MONGO_URL)
db=client[MONGO_DB]
def save_to_mogo(result):
    try:
        if db[MONGO_COLLECTION].insert(result):
            print("储存到MongoDB成功")
    except Exception:
        print("储存到MongoDB失败")

MAX_PAGE=100
if __name__ == '__main__':
    for i in range(1,MAX_PAGE+1):
        index_page(i)

脚本爬虫/其他脚本爬虫/selenium淘宝爬取-失败.py


'''
Created on Nov 4, 2018

@author: taivanbat

Script to run queries
'''
import os
import argparse
import logging

import torch.autograd
import torch.cuda
import torch.nn as nn
import torch.optim
from torch.utils.data import DataLoader
from skimage import io as img_io
import numpy as np
import json
from tqdm import tqdm
from PIL import Image

from src.cnn_ws.evaluation.retrieval import run_query
from src.cnn_ws.utils.save_load import my_torch_load, my_torch_save
from src.cnn_ws.models.myphocnet import PHOCNet
from src.cnn_ws.utils import utils

parser = argparse.ArgumentParser()
# eval arguments
parser.add_argument('--gpu_id', '-gpu', action='store',
                    type=lambda str_list: [int(elem) for elem in str_list.split(',')],
                    default='0',
                    help='The ID of the GPU to use. If not specified, training is run in CPU mode.')
parser.add_argument('--model_dir', default='models/base_model', help="Directory containing params.json")
parser.add_argument('--debug', action='store_true', help='Set to true if we want to debug')
parser.add_argument('--save_im_results', default=True, help='Set to true if we want to save top num_save_im results of search')
parser.add_argument('--num_save_im', default=10, help='how many images to save')

# set if we're loading words from the wiener files
parser.add_argument('--wiener_from_json', action='store_true')

def eval(params):
    # if candidates aren't known, make them
    if 'candidates.npy' not in os.listdir(params.model_dir) or 'candidates_labels.json' not in os.listdir(params.model_dir):
        candidates, candidates_labels = make_candidates(params)
    else:
        # load candidates
        candidates = np.load(os.path.join(params.model_dir,'candidates.npy'))

        # load candidates_labels
        with open(os.path.join(params.model_dir,'candidates_labels.json'), 'r') as filehandle:
            candidates_labels = json.load(filehandle)

    # TODO enable handling situations where character is not in the character set
    # queries = ['und', 'ich', 'bin', 'eigen', 'besonders']
    with open('word_search.txt') as f:
        queries = f.read().split()

    queries = [query.lower() for query in queries]

    results = run_query(params.num_save_im, candidates, candidates_labels, queries, params)

    # save the results. This is a num_queries x num_candidates sized matrix where the indices
    # of the most likely result is stored in column 1, and the second most likely is stored
    # in column 2 and so on
    np.save(os.path.join(params.model_dir, 'results.npy'), results)

def make_candidates(params):
    # load model, create candidates
    # TODO path to candidates should be an argument
    # TODO specify whether we're loading wiener from json files
    if params.wiener_from_json:
        # this is the base path
        # within it are wiener_images_cropped (containing original images with original size)
        # and also wiener_images_segmented (containing the .json files with bounding box information)
        path_to_cands = '/specific/disk1/home/adiel/wiener'
    else:
        path_to_cands = 'data/wiener/candidates/word_images'

    # load model
    # 675 is what we have by default
    cnn = PHOCNet(n_out=690,
                  input_channels=1,
                  gpp_type=params.gpp_type,
                  pooling_levels=params.pooling_levels)

    cnn.init_weights()

    my_torch_load(cnn, os.path.join(params.model_dir, 'PHOCNET_best.pt'))

    if params.gpu_id is not None:
        cnn.cuda(params.gpu_id[0])

    # set to eval mode
    cnn.eval()

    candidates = []
    candidates_labels = []

    # TODO!!!
    # enable loading from json files
    # want to store:
    # PHOC representation of each word
    # coordinates in original image of each word
    # as well as the name of the image to which it belongs

    if params.wiener_from_json:
        cropped_img_folder = os.path.join(path_to_cands, 'wiener_images_cropped')
        segmented_img_folder = os.path.join(path_to_cands, 'wiener_images_segmented')

        word_phocs = []
        candidates_labels = []

        # get list of images
        collections = [collection for collection in os.listdir(cropped_img_folder) if collection not in ['.', '..']]

        for coll_idx, collection in enumerate(collections):
            # get names of all images
            page_names = os.listdir(os.path.join(cropped_img_folder, collection))
            page_names = sorted([page_name for page_name in page_names if page_name.endswith('.jpg')])

            logging.info('going over collection %s with %s images', collection, str(len(page_names)))

            # go over up to a 100 images

            num_pages = min(len(page_names),100)

            with tqdm(total=num_pages) as t:
                # iterate over pages
                for page_idx, page_name in enumerate(page_names[:num_pages]):
                    # numpy --> page_img.shape is in row x col
                    page_img = img_io.imread(os.path.join(cropped_img_folder, collection, page_name))

                    ratio_yx = float(page_img.shape[0])/page_img.shape[1]

                    if ratio_yx < float(1200)/900:
                        r = page_img.shape[1]/float(900)
                    else:
                        r = page_img.shape[0]/float(1200)

                    with open(os.path.join(segmented_img_folder, collection, page_name[:-4] + '.json')) as json_f:
                        json_info = json.load(json_f)

                    word_imgs = []
                    bboxes = []
                    bboxes_idx = []
                    ratios = []

                    for box_idx in range(json_info['predictions']):
                        # get coordinates in original image that's why r*coord
                        bbox = [int(r*coord) for coord in json_info['box_' + str(box_idx)]['pred']]

                        # make sure we're not going over the image limits
                        bbox[1] = max(0, bbox[1])
                        bbox[3] = min(page_img.shape[0], bbox[3])
                        bbox[0] = max(0,bbox[0])
                        bbox[2] = min(page_img.shape[1], bbox[2])

                        area = (bbox[3]-bbox[1])*(bbox[2]-bbox[0])
                        if area < 100:
                            continue

                        word_img = page_img[bbox[1]:bbox[3], bbox[0]:bbox[2]]

                        bboxes += [bbox]
                        bboxes_idx += ['box_' + str(box_idx)]

                        # save ratio of x/y of image
                        ratios += [(bbox[2]-bbox[0])/float(bbox[3] - bbox[1])]

                        word_imgs += [word_img]
                        # if params.debug:
                            # word = Image.fromarray(word_img)
                            # word.show()

                    # save information about where each image is from, and its bounding box info
                    candidates_labels += [{'page': os.path.join(collection, page_name),
                                           'bbox': bbox,
                                           'bbox_idx': bboxes_idx[idx],
                                           'xy_ratio': ratios[idx]} for idx, bbox in enumerate(bboxes)]

                    # get phoc representation of words
                    word_imgs = [torch.autograd.Variable(torch.from_numpy(word_img)).float() for word_img in word_imgs]

                    if params.gpu_id is not None:
                        word_imgs = [word_img.cuda(params.gpu_id[0]) for word_img in word_imgs]

                    word_imgs = [word_img.unsqueeze(0) for word_img in word_imgs]
                    word_imgs = [word_img.unsqueeze(0) for word_img in word_imgs]

                    # get phoc representation of words
                    word_phocs = []
                    for word_img in word_imgs:
                        word_phocs += [torch.sigmoid(cnn(word_img))]

                    word_phocs = [word_phoc.data.cpu().numpy().flatten() for word_phoc in word_phocs]
                    word_phocs = np.vstack(word_phocs)
                    # concatenate along the row dimension as a numpy array
                    # if very first item
                    if coll_idx == 0 and page_idx == 0:
                        candidates = np.vstack(word_phocs)
                    else:
                        candidates = np.vstack((candidates, word_phocs))

                    t.update(1)
    else:
        folders = os.listdir(path_to_cands)
        folders = [folder for folder in folders if folder != '.DS_Store']

        if params.debug:
            folders = folders[:2]

            for i, folder in enumerate(folders):
                word_img_names = sorted(
                    [elem for elem in os.listdir(os.path.join(path_to_cands, folder)) if elem.endswith('.jpg')])

                candidates_labels += ['_'.join([folder, word_img_name[:-4]]) for word_img_name in word_img_names]
                word_imgs = [img_io.imread(os.path.join(path_to_cands, folder, word_img_name))
                             for word_img_name in word_img_names]

                word_imgs = [torch.autograd.Variable(torch.from_numpy(word_img)).float() for word_img in word_imgs]

                if params.gpu_id is not None:
                    word_imgs = [word_img.cuda(params.gpu_id[0]) for word_img in word_imgs]

                word_imgs = [word_img.unsqueeze(0) for word_img in word_imgs]
                word_imgs = [word_img.unsqueeze(0) for word_img in word_imgs]

                # get phoc representation of words
                word_phocs = [torch.sigmoid(cnn(word_img)) for word_img in word_imgs]
                word_phocs = [word_phoc.data.cpu().numpy().flatten() for word_phoc in word_phocs]
                word_phocs = np.vstack(word_phocs)
                # concatenate along the row dimension as a numpy array
                # if first in list of folders, candidates are just phocs from this folder
                if i == 0:
                    candidates = np.vstack(word_phocs)
                else:
                    candidates = np.vstack((candidates, word_phocs))

                t.update(1)

    # save candidates as npy file
    np.save(os.path.join(params.model_dir,'candidates.npy'), candidates)

    # save candidates_labels as json file
    with open(os.path.join(params.model_dir, 'candidates_labels.json'), 'w') as filehandle:
        json.dump(candidates_labels, filehandle)

    return candidates, candidates_labels

if __name__ == '__main__':
    args = parser.parse_args()
    args.num_save_im = int(args.num_save_im)

    # if GPU is not available, use CPU
    if not torch.cuda.is_available():
        args.gpu_id = None

    utils.set_logger(os.path.join(args.model_dir, 'eval.log'))

    json_path = os.path.join(args.model_dir, 'params.json')
    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
    params = utils.Params(json_path)
    # merge params and args into one object
    params.args_to_params(args)
    eval(params)


# python code to solve the linear advection equation using CTCS

import numpy as np # External library for numerical analysis
import matplotlib.pyplot as plt #plotting library

def initialBell (x):
    return np.where(x%1. < 0.5, np.power(np.sin(2*x*np.pi),2),0)

def main():
    # setting the intial pi profile and courant number
    nx = 40 # number of points plotting in space
    c = 0.2 # the courant number

    # spacial variable going from zero to one inclusive
    x = np.linspace(0.0, 1.0, nx+1)

    # three time levels of the dependent variable, phi
    phi = initialBell(x)
    phiNew = phi.copy()
    phiOld = phi.copy()

    #FTCS for the first timestep becuase we don't have a time -1
    for j in range (1,nx):
        phi[j] = phiOld[j] - 0.5*c*(phi[j+1] - phi[j-1])

    #apply periodic boundary conditions -> phi 0 and nx
    phi[0] = phiOld[0] - 0.5*c*(phiOld[1] - phiOld[nx-1])
    phi[nx] = phi[0]

    nt = 40
    for n in range (1, nt): #loop over time
        #loop over space
        for j in range (1,nx):
            phiNew[j] = phiOld[j] - c*(phi[j+1] - phi[j-1])
        phiNew[0] = phiOld[0] - c*(phi[1] - phi[nx-1])
        phiNew[nx] = phiNew[0]
        #update phi for the next step
        phiOld = phi.copy()
        phi = phiNew.copy()
        #plt.plot(x,phi)

    #derive quantities
    u = 1.
    dx = 1./nx
    dt = c*dx/u
    t = nt*dt

    plt.plot(x, initialBell(x-u*t), 'k', label = 'analytic')
    minimum = ('min' , min(initialBell(x-u*t)-phi))  #a method for numerically analyzing how well its working
    maximum = ('max' , max(initialBell(x-u*t)-phi))
    plt.plot(x, phi, 'b', label ='CTS n = 40')
    plt.legend(loc='best')
    plt.ylabel('$\phi$')
    plt.axhline(0,linestyle=':', color='black')

    plt.show()

main()
print('y')


def prime(n):
    if n>1:
        for i in range(2,n):
            if n %i==0:
                print("Not Prime")
                break
            else:
                print("Prime Number")
    else:
        print("Not Prime")

if __name__ == '__main__':
    prime(int(input()))


import pandas as pd
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
import pylab

#Problem 1
data = pd.read_csv('DJIA.csv', index_col = ['DATE'], na_values=["."])
data.VALUE = pd.Series(data.VALUE)
data.index = pd.to_datetime(data.index)
data["VALUE"] = data["VALUE"].fillna(np.nan).astype(float)

#Problem 2
paychecks = pd.read_csv('paychecks.csv', header = None)
fri1 = pd.date_range(start = '03/13/2008', periods = 93, freq = 'WOM-1FRI')
fri3 = pd.date_range(start = '03/13/2008', periods = 93, freq = 'WOM-3FRI')
fri1 = fri1.union(fri3)[:93]
paychecks.index = fri1

#Problem 4
finances = pd.read_csv('finances.csv')
finances.index = pd.period_range("1978-09", "1999-06", freq ="Q-DEC")

#Problem 5
traffic = pd.read_csv('website_traffic.csv')
a = traffic["ENTER"] = pd.to_datetime(traffic["ENTER"])
b= traffic["LEAVE"] = pd.to_datetime(traffic["LEAVE"])
c = b - a
traffic["DURATION"] = c
print(np.mean(c))

traffic["HOURS"] = 1
traffic.index = pd.to_datetime(traffic["ENTER"])
traffic = traffic.drop("IP", 1)
traffic = traffic.drop("ENTER", 1)
traffic = traffic.drop("LEAVE", 1)
traffic = traffic.drop("DURATION", 1)
print(traffic.resample('H').sum())

#Problem 6
data["DIFFERENCE"] = data - data.shift(1)
maxdaygain = np.max(data["DIFFERENCE"])
mindaygain = np.min(data["DIFFERENCE"])

maxdayindex = np.where(data["DIFFERENCE"]==maxdaygain)
mindayindex = np.where(data["DIFFERENCE"]==mindaygain)
print("Day with largest gain: ", str(data.index.values[maxdayindex])[2:12])
print("Day with largest loss: ", str(data.index.values[mindayindex])[2:12])

data = data.resample('M').sum()

maxmonthgain = np.max(data["DIFFERENCE"])
minmonthgain = np.min(data["DIFFERENCE"])
maxmonthindex = np.where(data["DIFFERENCE"]==maxmonthgain)
minmonthindex = np.where(data["DIFFERENCE"]==minmonthgain)
print("Month with largest gain: ", str(data.index.values[maxmonthindex])[2:9])
print("Month with largest loss: ", str(data.index.values[minmonthindex])[2:9])

#Problem 7
data = pd.read_csv('DJIA.csv', index_col = ['DATE'], na_values=["."])
data.VALUE = pd.Series(data.VALUE)
data.index = pd.to_datetime(data.index)
data["VALUE"] = data["VALUE"].fillna(np.nan).astype(float)
plt.plot(data, label = "Original")
plt.title("Dow Jones Industrial Average")

plt.plot(data.rolling(window = 30, min_periods = 20).mean(), label = "Rolling")
plt.plot(data.rolling(window = 365, min_periods = 20).mean())
plt.plot(data.ewm(span = 30, min_periods = 20).mean(), label = "EWMA")
plt.plot(data.ewm(span = 365, min_periods = 20).mean())
plt.legend(loc = 'lower right')
plt.show()


#!/usr/bin/env python3
# vim: set encoding=utf-8 tabstop=4 softtabstop=4 shiftwidth=4 expandtab
#########################################################################
#  Copyright 2014-2015 Thomas Ernst                       offline@gmx.net
#########################################################################
#  This file is part of SmartHome.py.
#
#  SmartHome.py is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  SmartHome.py is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with SmartHome.py. If not, see <http://www.gnu.org/licenses/>.
#########################################################################
from . import AutoBlindConditionSet
from . import AutoBlindLogger

# Class representing a list of condition sets
class AbConditionSets:
    # Initialize the list of condition sets
    # smarthome: Instance of smarthome.py-class
    def __init__(self, smarthome):
        self.__sh = smarthome
        self.__name = None
        self.__condition_sets = {}

    # Return number of condition sets in list
    def count(self):
        return len(self.__condition_sets)

    # Fill a condition set
    # conditionset_name: Name of condition set
    # item: item containing settings for condition set
    # grandparent_item: grandparent item of item (containing the definition if items and evals)
    # logger: Instance of AbLogger to write log messages to
    def fill(self, conditionset_name, item, grandparent_item, logger: AutoBlindLogger.AbLogger):
        # Add condition set if not yet existing
        if conditionset_name not in self.__condition_sets:
            self.__condition_sets[conditionset_name] = AutoBlindConditionSet.AbConditionSet(self.__sh,
                                                                                conditionset_name)
        # Update this condition set
        self.__condition_sets[conditionset_name].update(item, grandparent_item, logger)

    # Check the condition sets, optimize and complete them
    # item_state: item to read from
    # abitem_object: Related AbItem instance for later determination of current age and current delay
    # logger: Instance of AbLogger to write log messages to
    def complete(self, item_state, abitem_object, logger):
        for conditionset_name in self.__condition_sets:
            self.__condition_sets[conditionset_name].complete(item_state, abitem_object, logger)

    # Write all condition sets to logger
    # logger: Instance of AbLogger to write log messages to
    def write_to_logger(self, logger: AutoBlindLogger.AbLogger):
        for conditionset_name in self.__condition_sets:
            logger.info("Condition Set '{0}':", conditionset_name)
            logger.increase_indent()
            self.__condition_sets[conditionset_name].write_to_logger(logger)
            logger.decrease_indent()

    # check if one of the conditions sets in the list is matching.
    # logger: Instance of AbLogger to write log messages to
    # returns: True = one condition set is matching or no condition sets are defined, False: no condition set matching
    def one_conditionset_matching(self, logger: AutoBlindLogger.AbLogger):
        if self.count() == 0:
            logger.debug("No condition sets defined -> matching")
            return True
        for conditionset_name in self.__condition_sets:
            if self.__condition_sets[conditionset_name].all_conditions_matching(logger):
                return True
        return False


import csv
import os
import multiprocessing
from zipfile import ZipFile
import pandas

from histdata.api import download_hist_data, Platform

def mkdir_p(path):
    import errno
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def extract_zip_to_folder(path):
    from pathlib import Path
    folder_name = ".".join(Path(path).name.split('.')[:-1])
    parent_folder = Path(path).parent.absolute()

    extract_folder = Path.joinpath(parent_folder, folder_name)
    os.makedirs(extract_folder.absolute())

    with ZipFile(path, 'r') as zip:
        zip.extractall(path=extract_folder)
        return extract_folder

def handle_download(row):
    currency_pair_name, pair, history_first_trading_month = row
    year = int(history_first_trading_month[0:4])
    print('[START] for currency', currency_pair_name)
    output_folder = os.path.join('output', pair)

    if os.path.exists(output_folder):
        import shutil
        shutil.rmtree(output_folder)

    mkdir_p(output_folder)

    try:
        while True:
            could_download_full_year = False
            path = None
            try:
                path = download_hist_data(year=year,
                                                pair=pair,
                                                output_directory=output_folder,
                                                platform=Platform.META_TRADER,
                                                verbose=False)
                print('-', path)
                extract_zip_to_folder(path)
                could_download_full_year = True
            except AssertionError:
                pass  # lets download it month by month.
            month = 1
            while not could_download_full_year and month <= 12:
                path = download_hist_data(year=str(year),
                                                month=str(month),
                                                pair=pair,
                                                output_directory=output_folder,
                                                platform=Platform.META_TRADER,
                                                verbose=False)
                print('-', path)
                extract_zip_to_folder(path)
                month += 1
            year += 1
            os.remove(path)
    except Exception as e:
        import glob
        # combines all csvs
        extension = 'csv'
        all_filenames = [i for i in glob.glob('{}/**/*.{}'.format(output_folder, extension))]

        file_name = "{}.csv".format(output_folder)
        if os.path.exists(file_name):
            os.remove(file_name)

        combined_csv = open(file_name, 'a')

        for file in all_filenames:
            f = open(file)
            for line in f:
                combined_csv.write(line)

        import shutil
        shutil.rmtree(output_folder)

        print('[DONE] for currency', currency_pair_name)

def download_all():
    with open('pairs.csv', 'r') as f:
        reader = csv.reader(f, delimiter=',')
        next(reader, None)  # skip the headers

        nr_parallel_jobs = os.cpu_count()

        pool = multiprocessing.Pool(nr_parallel_jobs)
        rows = [row for row in reader]

        pool.map(handle_download, rows)

if __name__ == '__main__':
    import time
    start_time = time.time()

    download_all()

    print("Took %s seconds to download, extract and join" % (time.time() - start_time))


from random import randint
from threading import Thread
from time import sleep ,time
from os import getpid

def  download_task(string):
    print('进程：%s' % getpid())
    print('开始下载%s' % string)
    sleep_time = randint(3,7)
    sleep(sleep_time)
    print('下载完成%s,耗时%d' % (string, sleep_time))

def main():
    start = time()
    t1 = Thread(target = download_task, args = ('111',))
    t1.start()
    t2 = Thread(target = download_task, args = ('222',))
    t2.start()

    t1.join()
    t2.join()
    end = time()
    print('总耗时%s'% (end-start))

if __name__ == "__main__":
    main()