import argparse
import os

from datasets.data_split import divide_data

import numpy as np
import torch.nn.functional as F
from torchvision import transforms

from skimage import io
import os
import openslide as ops
from PIL import Image
from metrics.evaluation import *

from models.u_net import U_Net, AttU_Net, XXU_Net
from models.unet import UNet, UNet_V1, UNet_V2, UNet_V3, UNet_V4 # new version

import torch
import csv

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def wsi_prediction(s, model, foreground, level, n_class=2, patch_size = 384, step = 200, foreground_thres = 0.25, t=transforms.ToTensor()):
    step = patch_size//2

    whole_size_i = s.level_dimensions[level][1] # y
    whole_size_j = s.level_dimensions[level][0] # x

    patch_size_i = patch_size
    patch_size_j = patch_size

    result = np.zeros((n_class,whole_size_i,whole_size_j))
    for i in range(0, whole_size_i, step):
        for j in range(0, whole_size_j, step):
            if i+patch_size_i>whole_size_i:
                i = whole_size_i-patch_size_i
            if j+patch_size_j>whole_size_j:
                j = whole_size_j-patch_size_j

            sub_foreground=foreground[i:i+patch_size, j:j+patch_size]
            if sub_foreground.sum()/(patch_size*patch_size)<=foreground_thres:
                continue

            img = s.read_region((int(j*s.level_downsamples[level]),int(i*s.level_downsamples[level])),level,(patch_size,patch_size))
            img = t(img)[0:3,:,:].unsqueeze(0)
            SR, CR = model(img)
            SR = SR.squeeze().detach().numpy()

            if i==0 and j==0:
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            elif i==0 and j>0:
                overlay1 = result[:,i:i+patch_size_i,j:j+patch_size_j-step]
                overlay2 = SR[:,:,0:patch_size_j-step]
                SR[:,:,0:patch_size_j-step] = (overlay1+overlay2)/2
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            elif j==0 and i>0:
                overlay1 = result[:,i:i+patch_size_i-step,j:j+patch_size_j]
                overlay2 = SR[:,0:patch_size_j-step,:]
                SR[:,0:patch_size_j-step,:] = (overlay1+overlay2)/2
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            else:
                overlay1 = result[:,i:i+patch_size_i-step,j:j+patch_size_j]
                overlay2 = SR[:,0:patch_size_j-step,:]
                SR[:,0:patch_size_j-step,:] = (overlay1+overlay2)/2

                overlay3 = result[:,i+patch_size_i-step:i+patch_size_i,j:j+patch_size_j-step]
                overlay4 = SR[:,patch_size_i-step:patch_size_i,0:patch_size_j-step]
                SR[:,patch_size_i-step:patch_size_i,0:patch_size_j-step] = (overlay3+overlay4)/2

                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
    return result

def test_wsi(model_type, unet, svsf, p1, level, patch_size = 384, category = 1, t=transforms.ToTensor()):
    '''
    svsf: (list) wsi to validate
    '''
    Image.MAX_IMAGE_PIXELS = 933120000000

    score = 0
    jss = []
    for i, f in enumerate(svsf):
        print('%d/%d'%(i+1,len(svsf)))

        s=ops.open_slide(p1+f)
        foreground=io.imread(p1+f.split('.')[0]+'_level%s.tif'%level)
        result = wsi_prediction(s, unet, foreground, level, patch_size = patch_size, foreground_thres = 0.25)

        # save numpy as binary map
        heatmap = result[category,:,:]
        segmap = result.argmax(0).astype('uint8')

        heatmap_pil = Image.fromarray((heatmap*255).astype('uint8'), 'L')
        heatmapf = model_type + '_'+f.split('.')[0]+'_heatmap.png'
        heatmap_pil.save(os.path.join(p1, heatmapf))

        segmap_pil = Image.fromarray((segmap*255), 'L')
        segmapf = model_type + '_'+f.split('.')[0]+'_segmap.png'
        segmap_pil.save(os.path.join(p1, segmapf))

    return 0

def test_patch(model_type, unet, patches, root, category = 1, t=transforms.ToTensor()):
    '''

    '''
    Image.MAX_IMAGE_PIXELS = 933120000000

    for i, f in enumerate(patches):
        print('%d/%d'%(i+1,len(patches)))

        img = Image.open(os.path.join(root, f)).convert("RGB")
        img = t(img)[0:3,:,:].unsqueeze(0)
        SR, CR = unet(img)
        result = SR.squeeze().detach().numpy()

        # save numpy as binary map
        heatmap = result[category,:,:]
        segmap = result.argmax(0).astype('uint8')

        heatmap_pil = Image.fromarray((heatmap*255).astype('uint8'), 'L')
        heatmapf = model_type + '_'+f.split('.')[0]+'_heatmap.png'
        heatmap_pil.save(os.path.join(root, heatmapf))

        segmap_pil = Image.fromarray((segmap*255), 'L')
        segmapf = model_type + '_'+f.split('.')[0]+'_segmap.png'
        segmap_pil.save(os.path.join(root, segmapf))

    return 0

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    # model
    parser.add_argument('--depth', type=int, default=5)
    parser.add_argument('--width', type=int, default=32)
    parser.add_argument('--image_size', type=int, default=384)
    parser.add_argument('--n_classes', type=int, default=2)
    parser.add_argument('--category', type=int, default=1, help='category for evaluation label')
    parser.add_argument('--t', type=int, default=2, help='t for Recurrent step of R2U_Net or R2AttU_Net')
    parser.add_argument('--reduction_ratio', type=int, default=None, help='reduction ratio for attention layer')
    parser.add_argument('--n_skip', type=int, default=4, help='number of skip-connection layers, <= depth-1')
    parser.add_argument('--n_head', type=int, default=1, help='number of heads for prediction, 1 <= depth-1')
    parser.add_argument('--att_mode', type=str, default='cbam', help='cbam/bam/se')
    parser.add_argument('--conv_type', type=str, default='basic', help='basic/sk')
    parser.add_argument('--is_shortcut', type=str2bool, default=False)

    # training hyper-parameters
    parser.add_argument('--img_ch', type=int, default=3)
    parser.add_argument('--num_epochs', type=int, default=100)
    parser.add_argument('--num_epochs_decay', type=int, default=30)
    parser.add_argument('--batch_size', type=int, default=4)
    parser.add_argument('--num_workers', type=int, default=8)
    parser.add_argument('--lr', type=float, default=0.0001)
    parser.add_argument('--beta1', type=float, default=0.5)        # momentum1 in Adam
    parser.add_argument('--beta2', type=float, default=0.999)      # momentum2 in Adam
    parser.add_argument('--augmentation_prob', type=float, default=0.5)

    # log
    parser.add_argument('--log_step', type=int, default=1)
    parser.add_argument('--val_step', type=int, default=2)

    # loss
    parser.add_argument('--loss_type', type=str, default='nll', help='l1/nll/focal/iou/dice/multitask/multiloss')
    parser.add_argument('--alpha', type=float, default=1)        # alpha for l1 loss
    parser.add_argument('--gamma', type=float, default=1)        # gamma for l1/Focal loss
    parser.add_argument('--balance', type=list, default=None)   # balance factor

    # misc
    parser.add_argument('--mode', type=str, default='train', help='train/test')
    parser.add_argument('--model_type', type=str, default='XXU_Net', help='XXU_Net/U_Net/R2U_Net/AttU_Net/R2AttU_Net')
    parser.add_argument('--model_path', type=str, default='/mnt/DATA_OTHER/paip2020/results/checkpoint/')
    parser.add_argument('--root_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/')
    parser.add_argument('--train_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/train/')
    parser.add_argument('--train_anno_path', type=str, default=None)
    parser.add_argument('--valid_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/validation/')
    parser.add_argument('--wsi_path', type=str, default='/mnt/DATA_OTHER/paip2020/original/images/')
    parser.add_argument('--valid_anno_path', type=str, default=None)
    parser.add_argument('--test_path', type=str, default='./fundus_images/test/')
    parser.add_argument('--result_path', type=str, default='./results/')
    parser.add_argument('--fold', type=int, default=3, help='5-fold cross validation')
    parser.add_argument('--level', type=int, default=3, help='1/2/3')

    # other
    parser.add_argument('--cuda_idx', type=int, default=1)

    config = parser.parse_args()
    config.root_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/'%(config.level, config.image_size)
    config.train_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/train/'%(config.level, config.image_size)
    config.valid_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/validation/'%(config.level, config.image_size)
    if config.n_skip>config.depth-1:
        config.n_skip = config.depth-1

    # divide data for k-fold cross validation
#     dd = divide_data(config.root_path)
#     dd.reset() # move data to all/
#     dd.divide(config.fold)

    """Build model"""
    unet = None
    if config.model_type =='U_Net':
        unet = U_Net(img_ch=3, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type =='R2U_Net':
        unet = R2U_Net(img_ch=3,t=config.t, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type =='AttU_Net':
        unet = AttU_Net(img_ch=3, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'R2AttU_Net':
        unet = R2AttU_Net(img_ch=3,t=config.t, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['UNet', 'SEU_Net', 'CBAMU_Net', 'BAMU_Net']:
        unet = UNet(img_ch=3, n_classes=config.n_classes, init_features=config.width, network_depth=config.depth, reduction_ratio=config.reduction_ratio, att_mode = config.att_mode, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['SKU_Net', 'SK-SC-U_Net', 'SK-SE-U_Net']:
        unet = UNet_V1(reduction_ratio=config.reduction_ratio, att_mode = config.att_mode, is_shortcut = config.is_shortcut, conv_type = config.conv_type, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'MHU_Net':
        unet = UNet_V2(img_ch=3, n_classes=config.n_classes, n_head = config.n_head, is_head_selective = False, is_shortcut = False, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'SCU_Net':
        unet = UNet_V3(img_ch=3, n_classes=config.n_classes, n_head = config.n_head, is_scale_selective = False, is_shortcut = True, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['SSU_Net', 'SK-SSU_Net', 'SE-SSU_Net', 'SC-SSU_Net' ]:
        unet = UNet_V4(reduction_ratio=config.reduction_ratio, n_head = config.n_head, att_mode = config.att_mode, is_scale_selective = True, is_shortcut = config.is_shortcut, conv_type = config.conv_type, activation = torch.nn.Softmax(dim=1))
    else:
        raise NotImplementedError(config.model_type+" is not implemented")

    unet_path = os.path.join(config.model_path, '%s-%s-level%s-size%s-depth%s-width%s-n_classes%s-alpha%s-gamma%s-nhead%s-fold%s.pkl'%(config.model_type, config.loss_type, config.level, config.image_size, config.depth, config.width, config.n_classes, config.alpha, config.gamma, config.n_head, config.fold))
    print('try to load weights from: %s'%unet_path)
    unet.load_state_dict(torch.load(unet_path))
    unet.train(False)
    unet.eval()

    # validation slide, fold1
    valf=['13', '24', '28', '32', '38', '39', '4', '44', '47']
    svsf=[]
    for f in os.listdir(config.wsi_path):
        if f.endswith(('.svs','.SVS')) and f.split('.')[0] in valf:
            svsf.append(f)
    print(svsf)

    # wsi prediction
    s = test_wsi(config.model_type+config.loss_type, unet, svsf, p1 = config.wsi_path, level = config.level, patch_size = config.image_size, category = config.category, t=transforms.ToTensor())
    print(s)

#     patchesf = []
#     proot = config.test_path
#     for f in os.listdir(proot):
#         if f.endswith(('.png','.PNG')):
#             patchesf.append(f)
#     print(patchesf)

#     # patches prediction
#     stage = test_patch(config.model_type, unet, patchesf, proot, category = 1, t=transforms.ToTensor())
#     print(stage)


import os
import sys
import pickle

import argparse
import numpy as np
from numpy.lib.format import open_memmap

from ntu_read_skeleton import read_xyz

#代表这些人的动作用于训练，其他人的动作用于验证，即Cross-Subject划分准则
training_subjects = [
    1, 2, 4, 5, 8, 9, 13, 14, 15, 16, 17, 18, 19, 25, 27, 28, 31, 34, 35, 38
]
#代表只用2号、3号相机产生的数据进行训练，1号相机采集的数据用于验证，即Cross-View划分准则
training_cameras = [2, 3]
#最多两个人
max_body = 2
#总共关节数为25
num_joint = 25
#最多300帧
max_frame = 300

'''
输出过程信息
'''
toolbar_width = 30

def print_toolbar(rate, annotation=''):
    # setup toolbar
    sys.stdout.write("{}[".format(annotation))
    for i in range(toolbar_width):
        if i * 1.0 / toolbar_width > rate:
            sys.stdout.write(' ')
        else:
            sys.stdout.write('-')
        sys.stdout.flush()
    sys.stdout.write(']\r')

def end_toolbar():
    sys.stdout.write("\n")

'''
data_path:数据存放路径

out_path:输出路径

ignored_sample_path:用于存放忽略文件名的地方，如果为空则代表不忽略文件

benchmark:('xview','xsub')
    若为xview则代表按照不同相机来划分训练集和验证集（即相机1采集的数据作为验证集，相机2、3采集的数据作为训练集）
    若为xsub则代表按照不同的人来划分训练集和验证集（即在training_subjects中的人对应的数据为训练集，其他的人对应的数据为验证集）

part:('eval','train')，为eval代表验证集，为train代表训练集
'''
def gendata(data_path,
            out_path,
            ignored_sample_path=None,
            benchmark='xview',
            part='eval'):

    # step1.看是否有需要忽略的样本
    # ignored_sample_path中出现的文件名即代表忽略对应样本
    if ignored_sample_path != None:
        with open(ignored_sample_path, 'r') as f:
            ignored_samples = [
                line.strip() + '.skeleton' for line in f.readlines()
            ]
    else:
        ignored_samples = []

    # step2.依次遍历数据集目录下的文件名，通过是否需要忽略样本以及样本是否符合当前状态（是在训练还是在验证）以将需要读取的样本以及标签加入sample_name和sample_label中
    sample_name = []
    sample_label = []
    for filename in os.listdir(data_path):
        # 如果该文件名字在忽略样本中，则忽略该文件
        if filename in ignored_samples:
            continue
        # 文件名A开始的部分代表动作的分类
        action_class = int(
            filename[filename.find('A') + 1:filename.find('A') + 4])
        # 文件名P开始的部分代表人物的id
        subject_id = int(
            filename[filename.find('P') + 1:filename.find('P') + 4])
        # 文件名C开始的部分代表相机id
        camera_id = int(
            filename[filename.find('C') + 1:filename.find('C') + 4])

        # 看基准是xview还是xsub，若为xview则代表按相机来划分训练集和验证集，则看当前文件的文件名中对应的相机是否为训练相机
        # 若为xsub则代表以人为基准，则看文件名中对应的人是否在训练集对应的人中
        # istraining用于代表是否为训练部分
        if benchmark == 'xview':
            istraining = (camera_id in training_cameras)
        elif benchmark == 'xsub':
            istraining = (subject_id in training_subjects)
        else:
            raise ValueError()

        # 看传入的模式是在训练还是在验证，若在训练则取istraining的值以看当前文件是否为训练样本
        # 若在验证则对istraining的值取反以看当前文件是否为验证样本
        # issample代表当前文件是否需要读取（采样）
        if part == 'train':
            issample = istraining
        elif part == 'val':
            issample = not (istraining)
        else:
            raise ValueError()

        # 如果当前文件需要读取，则加入到列表中
        # action_class为由文件名给出的ground_truth动作类别号
        if issample:
            sample_name.append(filename)
            sample_label.append(action_class - 1)

    # 将以上获得的需要读取的样本文件名以及ground_truth动作类别号存入out_path的train_label.pkl（训练集）或eval_label.pkl（验证集）中
    with open('{}/{}_label.pkl'.format(out_path, part), 'wb') as f:
        # 将待取样的sample_name和sample_label以ascii形式存入f指向的文件中
        pickle.dump((sample_name, list(sample_label)), f)
    # np.save('{}/{}_label.npy'.format(out_path, part), sample_label)

    # 再打开一个numpy类型的存数据文件流
    fp = open_memmap(
        '{}/{}_data.npy'.format(out_path, part),
        dtype='float32',
        mode='w+',
        shape=(len(sample_label), 3, max_frame, num_joint, max_body))

    # 依次读取样本文件并写入
    # 数据的第0维是序号即代表是哪一个样本，第一维存的是关节的x,y,z坐标，第二维用于标识是哪一帧、第三维用于标识是哪个关节、第四维用于表示是哪个身体
    for i, s in enumerate(sample_name):
        print_toolbar(i * 1.0 / len(sample_label),
                      '({:>5}/{:<5}) Processing {:>5}-{:<5} data: '.format(
                          i + 1, len(sample_name), benchmark, part))
        data = read_xyz(
            os.path.join(data_path, s), max_body=max_body, num_joint=num_joint)
        # data:第一维存的是关节的x,y,z坐标，第二维用于标识是哪一帧、第三维用于标识是哪个关节、第四维用于表示是哪个身体
        fp[i, :, 0:data.shape[1], :, :] = data
    end_toolbar()

# 只有当运行该文件时才执行，其中的参数需要改为自己数据集的位置
if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='NTU-RGB-D Data Converter.')
    parser.add_argument(
        '--data_path', default='D:/ntu-rgb+d/raw_txt')
    parser.add_argument(
        '--ignored_sample_path',
        default='D:/ntu-rgb+d/missing_list/samples_with_missing_skeletons.txt')
    parser.add_argument('--out_folder', default='D:/ntu-rgb+d/cooked_dat')

    benchmark = ['xsub', 'xview']
    part = ['train', 'val']
    arg = parser.parse_args()

    for b in benchmark:
        for p in part:
            out_path = os.path.join(arg.out_folder, b)
            if not os.path.exists(out_path):
                os.makedirs(out_path)
            gendata(
                arg.data_path,
                out_path,
                arg.ignored_sample_path,
                benchmark=b,
                part=p)


import tensorflow as tf

x = tf.Variable([[1,2,3,4],[2,3,4,5],[4,3,2,1]])

init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
#Scope begin
# Find armax acros axis = 1. Result : [3 3 0]
print(sess.run(tf.argmax(x,1)))

# Find armax acros axis = 0. Result : [2,1,1,1]
print(sess.run(tf.argmax(x,0)))

#Scope end
sess.close()


#from django.test import LiveServerTestCase
from .base import FunctionalTest

#from django.contrib.staticfiles.testing import StaticLiveServerTestCase
#from selenium import webdriver
#from selenium.webdriver.common.keys import Keys
#import sys
#from unittest import skip

class LayoutAndStylingTest(FunctionalTest):
	def test_layout_and_styling(self):
		self.browser.get(self.server_url)
		self.browser.set_window_size(1024,768)

		inputbox = self.get_item_input_box()
		self.assertAlmostEqual(inputbox.location['x'] + inputbox.size['width'] / 2, 512, delta=5)

		inputbox.send_keys('testing\n')
		inputbox = self.get_item_input_box()
		self.assertAlmostEqual(inputbox.location['x'] + inputbox.size['width'] / 2, 512, delta=5)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<i@binux.me>
#         http://binux.me
# Created on 2014-02-22 23:20:39

import socket
import time
import datetime
from six import iteritems, itervalues
from flask import render_template, request, json

try:
    import flask_login as login
except ImportError:
    from flask.ext import login

from .app import app

index_fields = ['name', 'group', 'status', 'comments', 'rate', 'burst', 'updatetime']

@app.route('/')
def index():
    projectdb = app.config['projectdb']
    projects = sorted(projectdb.get_all(fields=index_fields),
                      key=lambda k: (0 if k['group'] else 1, k['group'] or '', k['name']))
    return render_template("index.html", projects=projects)

@app.route('/queues')
def get_queues():
    def try_get_qsize(queue):
        if queue is None:
            return 'None'
        try:
            return queue.qsize()
        except Exception as e:
            return "%r" % e

    result = {}
    queues = app.config.get('queues', {})
    for key in queues:
        result[key] = try_get_qsize(queues[key])
    return json.dumps(result), 200, {'Content-Type': 'application/json'}

@app.route('/update', methods=['POST', ])
def project_update():
    projectdb = app.config['projectdb']
    project = request.form['pk']
    name = request.form['name']
    value = request.form['value']

    project_info = projectdb.get(project, fields=('name', 'group'))
    if not project_info:
        return "no such project.", 404
    if 'lock' in projectdb.split_group(project_info.get('group')) \
            and not login.current_user.is_active():
        return app.login_response

    if name not in ('group', 'status', 'rate'):
        return 'unknown field: %s' % name, 400
    if name == 'rate':
        value = value.split('/')
        if len(value) != 2:
            return 'format error: rate/burst', 400
        rate = float(value[0])
        burst = float(value[1])
        update = {
            'rate': min(rate, app.config.get('max_rate', rate)),
            'burst': min(burst, app.config.get('max_burst', burst)),
        }
    else:
        update = {
            name: value
        }

    ret = projectdb.update(project, update)
    if ret:
        rpc = app.config['scheduler_rpc']
        if rpc is not None:
            try:
                rpc.update_project()
            except socket.error as e:
                app.logger.warning('connect to scheduler rpc error: %r', e)
                return 'rpc error', 200
        return 'ok', 200
    else:
        return 'update error', 500

@app.route('/counter')
def counter():
    rpc = app.config['scheduler_rpc']
    if rpc is None:
        print(11111111111111111)
        return json.dumps({})

    result = {}
    try:
        data = rpc.webui_update()
        for type, counters in iteritems(data['counter']):
            for project, counter in iteritems(counters):
                result.setdefault(project, {})[type] = counter
        for project, paused in iteritems(data['pause_status']):
            result.setdefault(project, {})['paused'] = paused
    except socket.error as e:
        app.logger.warning('connect to scheduler rpc error: %r', e)
        return json.dumps({}), 200, {'Content-Type': 'application/json'}

    return json.dumps(result), 200, {'Content-Type': 'application/json'}

@app.route('/run', methods=['POST', ])
def runtask():
    rpc = app.config['scheduler_rpc']
    if rpc is None:
        return json.dumps({})

    projectdb = app.config['projectdb']
    project = request.form['project']
    project_info = projectdb.get(project, fields=('name', 'group'))
    if not project_info:
        return "no such project.", 404
    if 'lock' in projectdb.split_group(project_info.get('group')) \
            and not login.current_user.is_active():
        return app.login_response

    newtask = {
        "project": project,
        "taskid": "on_start",
        "url": "data:,on_start",
        "process": {
            "callback": "on_start",
        },
        "schedule": {
            "age": 0,
            "priority": 9,
            "force_update": True,
        },
    }

    try:
        ret = rpc.newtask(newtask)
    except socket.error as e:
        app.logger.warning('connect to scheduler rpc error: %r', e)
        return json.dumps({"result": False}), 200, {'Content-Type': 'application/json'}
    return json.dumps({"result": ret}), 200, {'Content-Type': 'application/json'}

@app.route('/robots.txt')
def robots():
    return """User-agent: *
Disallow: /
Allow: /$
Allow: /debug
Disallow: /debug/*?taskid=*
""", 200, {'Content-Type': 'text/plain'}

@app.route('/health')
def health():
    """
    health check, service:
    webui
    scheduler
    fetcher
    processor
    result_worker
    :return:
    """
    # all service return dict
    good = {
        "status": "OK",
        "type": "HARD",
        "checked_at": gettime(),
        "spent_time": "1ms",
        "info": "ok"
    }
    bad = {
        "status": "ERROR",
        "type": "HARD",
        "checked_at": gettime(),
        "spent_time": "1ms",
        "info": "ERROR"
    }

    _health = {
        "status": "OK",
        "dependencies": {
            "webui": good.copy()
        }
    }

    # scheduler fetcher procsssor result
    mapping = {
        'fetcher2processor': 'processor',
        'newtask_queue': 'scheduler',
        'processor2result': 'result_worker',
        'scheduler2fetcher': 'fetcher'
    }

    result = get_queues()
    queue_info = dict(json.loads(result[0]))
    # redis down
    try:
        alert_info = {k: int(v) for k, v in queue_info.items()}
        _health['dependencies']['redis'] = good.copy()
    except:
        _health['dependencies']['redis'] = bad.copy()
        _health['dependencies']['redis']['info'] = 'redis down'
        return json.dumps(_health)

    # check num
    for k, v in alert_info.items():
        service = mapping.get(k)
        if service:
            if v >= 100:
                _health['dependencies'][service] = bad.copy()
                _health['dependencies'][service]['info'] = '{} down'.format(service)
            else:
                _health['dependencies'][service] = good.copy()

    # check error table
    resultdb = app.config['resultdb']
    project = 'error'
    offset = int(request.args.get('offset', 0))
    limit = int(request.args.get('limit', 20))
    try:
        results = list(resultdb.select(project, offset=offset, limit=limit))
        _health['dependencies']['db'] = good.copy()
    except:
        # db down
        _health['dependencies']['db'] = bad.copy()
        _health['dependencies']['db']['info'] = 'db down'

    # scheduler xmlrpc detect
    try:
        app.config['scheduler_rpc_client'].ping()
    except:
        _health['dependencies']['scheduler']['status'] = 'ERROR'
        _health['dependencies']['scheduler']['info'] = 'scheduler server down'

    try:
        app.config['fetcher_rpc_client'].ping()
    except:
        _health['dependencies']['fetcher']['status'] = 'ERROR'
        _health['dependencies']['fetcher']['info'] = 'fetcher server down'

    return json.dumps(_health)

@app.route('/ping')
def ping():
    return 'pong'

def gettime():
    return datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + str(time.time() % 1)[1:5] + 'Z'


import dash
from dash import html, dcc
import dash_bootstrap_components as dbc
from dash import html, Input, Output, State, callback_context, dash_table,callback,ctx
import json, operator

dash.register_page(__name__, path='/lastPage')

layout = html.Div(children=[
    html.H1(children='Please enter your user testing ID.'),

    html.Div(
        [
            html.P("Please enter your user testing ID."),
            dbc.Input(id="userTestingID", type="text"),
        ],
        id="estimateInputDiv"),
    html.Div([
        html.A(dbc.Button('Submit', id='submitUserId', n_clicks=0),id="thankYouPageLink",href="/thankYouPage")
    ],style={'display':'flex','justifyContent':'center'}),
    # html.Div(children='''
    #     Thank you for taking this survey! Please paste the following code into Prolific to receive compensation: 'uniqueCode'
    # '''),
],style={'margin-left':'150px','margin-right':'150px','margin-top':'50px'})

def write_json(new_data, filename='assets/files/studyFiles/userDataInteractions.json'):
    print("inside write json")
    with open(filename, 'r+') as file:
        # First we load existing data into a dict.
        file_data = json.load(file)
        # Join new_data with file_data inside emp_details
        file_data["interaction_details"].append(new_data)
        # Sets file's current position at offset.
        file.seek(0)
        # convert back to json.
        json.dump(file_data, file, indent=4)

@callback(
    Output('thankYouPageLink', 'href'),
    [Input('submitUserId', 'n_clicks_timestamp'),
     Input("userIDStore", "data"),
     Input("userTestingID", "value"),
     ])
def saveUserTestingID(n, userIDStore,userTestingID):

    elementTriggered = ctx.triggered_id

    if elementTriggered == "submitUserId":
        matchingIDRecord= {"dashUserID":userIDStore['userID'],
                           "userTestingID":userTestingID
        }
        #open file and find record with same user ID
        write_json(matchingIDRecord)

    return "/thankYouPage"


from ..models import EMETariffs, Retailers, Distributors
from django.core import files
from io import BytesIO
import requests

from BillWise_Site.settings.base import BASE_DIR

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import os, re, datetime, sys

# import django
# from django.conf import settings
# from BillComparison.models import *

def get_tariffs_by_postcode(postcode=None, customerType="S"):
    """
    Performs generic search by postcode and customerType.
    returns pandas DataFrame with all tariff listings. Columns: 'tariff_id','tariff_name','url'
    """

    columns = ['tariff_id','tariff_name','url']
    df = pd.DataFrame(columns=columns)
    url = "https://www.energymadeeasy.gov.au/results.html?postcode={0}&fuelType=E&customerType={1}".format(postcode,customerType)

    browser.get(url)
    all_plans_button = browser.find_elements_by_xpath("//button[@id='sort-all']")[0]
    all_plans_button.click()

    while True:
        show_more_plans_button = browser.find_elements_by_xpath("//button[text()='Show more plans']")

        if len(show_more_plans_button) > 0:
            show_more_plans_button[0].click()
        else:
            False
            break

    html = browser.page_source
    soup = BeautifulSoup(html, 'html.parser')

    all_results = soup.find_all("div", {"class": "result__row"})

    for result_row in all_results[2:]:
        if re.search(r'Offer available if',result_row.text, re.IGNORECASE) == None:

            href = result_row.find('a')['href']
            href = 'https://www.energymadeeasy.gov.au' + href
            tariff_name = result_row.find('a').text
            tariff_id = re.findall(r'offer/[0-9]{6}',href)[0]
            tariff_id = tariff_id.split('/')[1]

            newrow = pd.DataFrame([[tariff_id,tariff_name,href]] ,  columns=columns)
            df = df.append(newrow)
        else:
            pass

    return(df)

class EnergyMadeEasy_AllTariffListings:
    def __init__(self, postcode_list=None):
        if postcode_list is None:
            self.postcode_list = []
        else:
            self.postcode_list = postcode_list

        self.tariff_list = []
        print(self.postcode_list)

    def listTariffsForPostcode(self):
        """
        Batch performs get_tariffs_by_postcode() for a list of postcodes. Removes duplicate entries.
        Thus, returns pandas DataFrame with all tariff listings. Columns: 'tariff_id','tariff_name','url'
        """

        assert isinstance(self.postcode_list, list), "'postcode_list' must be a list"

        for postcode in self.postcode_list:
            try:
                newlist = get_tariffs_by_postcode(postcode=postcode)
            except IndexError:
                pass

            try:
                self.tariff_list = self.tariff_list.append(newlist, ignore_index=True)
            except TypeError:
                self.tariff_list = newlist
            except NameError:
                self.tariff_list = newlist

        # Specify writer
        writer = pd.ExcelWriter('Tariff_Listings.xlsx', options={'encoding':'utf-8'})
        # Write the frame to excel
        #self.tariff_list.to_excel(writer, sheet_name='FullList', index=False)
        self.tariff_list.drop_duplicates('tariff_id').to_excel(writer, sheet_name='NoDuplicates', index=False)
        # Save the file
        writer.save()

class EnergyMadeEasy_Tariff:
    def __init__(self, url=None):

        self.warning = False
        self.skip = False
        self.url = url

        browser.get(self.url)
        html = browser.page_source
        soup = BeautifulSoup(html, 'html.parser')

        # Check if this BPIP is a 3rd party referral (e.g. FastConnect, or iSelect, etc.)
        if soup.find("div", {"class": "panel__item--warning"}) is not None:
            if re.search(r'Contact',soup.find("div", {"class": "panel__item--warning"}).text, re.IGNORECASE) is not None:
                self.skip = True

        # Populate mandatory fields
        self.tariff_name = soup.findAll("div", {"class": "v2-title"})[0].text
        self.plan_id = soup.findAll("div", {"class": "plan-id"})[0].text.split(": ")[1]

        retailer_name = soup.findAll("div", {"class": "bp__title"})[0].text
        self.retailer = Retailers.objects.filter(retailer__contains=retailer_name)
        self.retailer = self.retailer.first() if self.retailer.exists() else Retailers.objects.create(retailer=retailer_name)

        self.retailer_logo = soup.find("div", {"class": "bp__summary-icon"}).find('img')['src']

        if not self.skip:
            imagepath = os.path.join(BASE_DIR,'../../static_files/BillComparison/images/Retailer_Logos/',(str(self.retailer)+'.png'))
            imagepathexists = os.path.isfile(imagepath)
            if imagepathexists:
                pass
            else:
                url = self.retailer_logo
                r = requests.get(url, allow_redirects=True)
                open(imagepath, 'wb').write(r.content)

        distributor_name = soup.find(text="Distributor").parent.parent.findNext('p').text
        self.distributor = Distributors.objects.filter(distributor__contains=distributor_name)
        self.distributor = self.distributor.first() if self.distributor.exists() else Distributors.objects.create(distributor=distributor_name)

        self.effective_from = soup.find(text="Effective from").parent.parent.findNext('p').text
        self.effective_from = datetime.datetime.strptime(self.effective_from, '%d %b %Y').date()

        if re.search('smallBusiness', self.url, re.IGNORECASE):
            self.premises_type = 'sm_busn'
        else:
            self.premises_type = 'resi'

        ############# Contract details
        try:
            self.valid_until = soup.find(text="Valid until").parent.parent.findNext('p').text
            self.valid_until = datetime.datetime.strptime(self.valid_until, '%d %b %Y').date()
        except AttributeError:
            self.valid_until = None
        try:
            self.contract_length = soup.find(text=re.compile(r'Contract Length')).parent.parent.findNext('p').text.strip()
        except AttributeError:
            self.contract_length = None
        try:
            self.contract_expiry = soup.find(text=re.compile(r'Contract expiry')).parent.parent.findNext('p').text.strip()
        except AttributeError:
            self.contract_expiry = None
        try:
            self.meter_restrictions = soup.find(text=re.compile(r'Meter restrictions')).parent.parent.findNext('p').text.strip()
        except AttributeError:
            self.meter_restrictions = None
        try:
            self.billing_period = soup.find(text=re.compile(r'Billing period')).parent.parent.findNext('p').text.strip()
        except AttributeError:
            self.billing_period = None
        try:
            self.payment_options = soup.find(text=re.compile(r'Payment options')).parent.parent.findNext('ul').text.strip()
        except AttributeError:
            self.payment_options = None

        ############# Tariff rates
        # Supply Charge
        try:
            self.usage_primary_rate_SUPPLYCHARGE = float(soup.find(text=re.compile(r'Daily supply charge', flags=re.IGNORECASE)).parent.findNext('strong').text.split(" ")[0])/100
        except AttributeError:
            self.warning += "Error parsing 'Daily supply charge'"
            pass

        # Demand Charge
        if soup.find(text=re.compile(r'demand charge', flags=re.IGNORECASE)) is not None:
            self.warning = True # Flag warning to indicate manual change needed
            self.warning_notes += 'Demand charge (manual entry required) \n'

        # Flat Tariffs
        try:
            self.usage_primary_rate_FLAT = float(soup.find(text=re.compile(r'Usage at all times')).parent.findNext('strong').text.split(" ")[0])/100
        except AttributeError:
            self.usage_primary_rate_FLAT = None
        try:
            self.export_primary_rate_FLAT = float(soup.find(text=re.compile(r'Solar feed')).parent.parent.findNext('strong').text)/100
        except AttributeError:
            self.export_primary_rate_FLAT = None

        # Block Tariffs
        try: # Try to find the first block rate
            self.usage_primary_rate_BLOCK1 = float(soup.find(text=re.compile(r'First .* kWh')).parent.findNext('strong').text.split(" ")[0])/100
            self.usage_primary_ceiling_BLOCK1 = float(soup.find(text=re.compile(r'First .* kWh')).split(" ")[1])
            self.usage_primary_timescale = soup.find("div", {"class": "panel__subnote panel__subnote--right"}).find('strong').text

            try:  # Try to find the 'next' and 'remaining' block rate
                self.usage_primary_rate_BLOCK2 = float(soup.find(text=re.compile(r'Next .* kWh')).parent.findNext('strong').text.split(" ")[0])/100
                self.usage_primary_rate_BLOCK3 = float(soup.find(text=re.compile(r'Remaining')).parent.findNext('strong').text.split(" ")[0])/100
                self.usage_primary_ceiling_BLOCK2 = self.usage_primary_ceiling_BLOCK1 + float(soup.find(text=re.compile(r'Next .* kWh')).split(" ")[1])
                self.usage_primary_ceiling_BLOCK3 = 9999999

            except AttributeError:   # If this fails, it means there is only a 'remaining' block rate (i.e. 2 tier block rate)
                self.usage_primary_rate_BLOCK2 = float(soup.find(text=re.compile(r'Remaining')).parent.findNext('strong').text.split(" ")[0])/100
                self.usage_primary_rate_BLOCK3 = None
                self.usage_primary_ceiling_BLOCK2 = 9999999

        except AttributeError: # No 'first' block. Assumes this is not a block tariff
            self.usage_primary_rate_BLOCK1 = None
            self.usage_primary_rate_BLOCK2 = None
            self.usage_primary_rate_BLOCK3 = None
            self.usage_primary_ceiling_BLOCK1 = None
            self.usage_primary_ceiling_BLOCK2 = None
            self.usage_primary_timescale = None

        # Time of Use Tariffs
        weekday_button = browser.find_elements_by_xpath("//button[text()='Mon-Fri']")
        weekend_button = browser.find_elements_by_xpath("//button[text()='Weekends']")
        self.usage_primary_rate_PEAK = None
        self.usage_primary_rate_SHOULDER = None
        self.usage_primary_rate_OFFPEAK = None

        self.Peak_weekday_1_start = None
        self.Peak_weekday_1_end = None
        self.Peak_weekday_2_start = None
        self.Peak_weekday_2_end = None
        self.Peak_weekday_3_start = None
        self.Peak_weekday_3_end = None
        self.Shoulder_weekday_1_start = None
        self.Shoulder_weekday_1_end = None
        self.Shoulder_weekday_2_start = None
        self.Shoulder_weekday_2_end = None
        self.Shoulder_weekday_3_start = None
        self.Shoulder_weekday_3_end = None
        self.OffPeak_weekday_1_start = None
        self.OffPeak_weekday_1_end = None
        self.OffPeak_weekday_2_start = None
        self.OffPeak_weekday_2_end = None
        self.OffPeak_weekday_3_start = None
        self.OffPeak_weekday_3_end = None
        self.Peak_weekend_1_start = None
        self.Peak_weekend_1_end = None
        self.Peak_weekend_2_start = None
        self.Peak_weekend_2_end = None
        self.Peak_weekend_3_start = None
        self.Peak_weekend_3_end = None
        self.OffPeak_weekend_1_start = None
        self.OffPeak_weekend_1_end = None
        self.OffPeak_weekend_2_start = None
        self.OffPeak_weekend_2_end = None
        self.OffPeak_weekend_3_start = None
        self.OffPeak_weekend_3_end = None
        self.Shoulder_weekend_1_start = None
        self.Shoulder_weekend_1_end = None
        self.Shoulder_weekend_2_start = None
        self.Shoulder_weekend_2_end = None
        self.Shoulder_weekend_3_start = None
        self.Shoulder_weekend_3_end = None

        try: # First make 'Mon-Fri' is already selected
            weekday_button[0].click()
            html = browser.page_source
            soup = BeautifulSoup(html, 'html.parser')
        except IndexError:
            pass
        TOU_Panel = soup.find("div", {"id": "bp-pricing-tou"})
        if TOU_Panel is not None:
            try:
                TOU_RatesTable = TOU_Panel.find("table", {"class": "panel__table"})
                TOU_RatesTable = pd.read_html(str(TOU_RatesTable))[0]
                self.usage_primary_rate_OFFPEAK = float(TOU_RatesTable['Off-Peak'][0].split('c/kWh')[0])/100
                self.usage_primary_rate_SHOULDER = float(TOU_RatesTable['Shoulder'][0].split('c/kWh')[0])/100
                self.usage_primary_rate_PEAK = float(TOU_RatesTable['Peak'][0].split('c/kWh')[0])/100
            except KeyError:
                pass
            except ValueError:
                pass

            try:
                string = re.sub(r"\s*\|*\s*","",soup.find("div", {"class": 'pie-chart__accessible-tooltips'}).text)

                OffPeak_Periods = re.findall(r"Off-Peak[0-9]{4}-[0-9]{4}",string)
                try:
                    self.OffPeak_weekday_1_start, self.OffPeak_weekday_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[0])]
                    string = re.sub(OffPeak_Periods[0],"",string)
                    self.OffPeak_weekday_2_start, self.OffPeak_weekday_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[1])]
                    string = re.sub(OffPeak_Periods[1],"",string)
                    self.OffPeak_weekday_3_start, self.OffPeak_weekday_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[2])]
                    string = re.sub(OffPeak_Periods[2],"",string)
                except IndexError:
                    pass

                Shoulder_Periods = re.findall(r"Shoulder[0-9]{4}-[0-9]{4}",string)
                try:
                    self.Shoulder_weekday_1_start, self.Shoulder_weekday_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[0])]
                    string = re.sub(Shoulder_Periods[0],"",string)
                    self.Shoulder_weekday_2_start, self.Shoulder_weekday_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[1])]
                    string = re.sub(Shoulder_Periods[1],"",string)
                    self.Shoulder_weekday_3_start, self.Shoulder_weekday_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[2])]
                    string = re.sub(Shoulder_Periods[2],"",string)
                except IndexError:
                    pass

                Peak_Periods = re.findall(r"Peak[0-9]{4}-[0-9]{4}",string)
                try:
                    self.Peak_weekday_1_start, self.Peak_weekday_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[0])]
                    string = re.sub(Peak_Periods[0],"",string)
                    self.Peak_weekday_2_start, self.Peak_weekday_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[1])]
                    string = re.sub(Peak_Periods[1],"",string)
                    self.Peak_weekday_3_start, self.Peak_weekday_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[2])]
                    string = re.sub(Peak_Periods[2],"",string)
                except IndexError:
                    pass

            except AttributeError:
                pass

        try: # Next switch to 'Weekend'
            weekend_button[0].click()
            html = browser.page_source
            soup = BeautifulSoup(html, 'html.parser')
        except IndexError:
            pass
        TOU_Panel = soup.find("div", {"id": "bp-pricing-tou"})
        if TOU_Panel is not None:
            try:
                TOU_RatesTable = TOU_Panel.find("table", {"class": "panel__table"})
                TOU_RatesTable = pd.read_html(str(TOU_RatesTable))[0]
                self.usage_primary_rate_OFFPEAK = float(TOU_RatesTable['Off-Peak'][0].split('c/kWh')[0])/100
                self.usage_primary_rate_SHOULDER = float(TOU_RatesTable['Shoulder'][0].split('c/kWh')[0])/100
                self.usage_primary_rate_PEAK = float(TOU_RatesTable['Peak'][0].split('c/kWh')[0])/100
            except KeyError:
                pass
            except ValueError:
                pass

            try:
                string = re.sub(r"\s*\|*\s*","",soup.find("div", {"class": 'pie-chart__accessible-tooltips'}).text)

                OffPeak_Periods = re.findall(r"Off-Peak[0-9]{4}-[0-9]{4}",string)
                try:
                    self.OffPeak_weekend_1_start, self.OffPeak_weekend_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[0])]
                    string = re.sub(OffPeak_Periods[0],"",string)
                    self.OffPeak_weekend_2_start, self.OffPeak_weekend_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[1])]
                    string = re.sub(OffPeak_Periods[1],"",string)
                    self.OffPeak_weekend_3_start, self.OffPeak_weekend_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',OffPeak_Periods[2])]
                    string = re.sub(OffPeak_Periods[2],"",string)
                except IndexError:
                    pass

                Shoulder_Periods = re.findall(r"Shoulder[0-9]{4}-[0-9]{4}",string)
                try:
                    self.Shoulder_weekend_1_start, self.Shoulder_weekend_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[0])]
                    string = re.sub(Shoulder_Periods[0],"",string)
                    self.Shoulder_weekend_2_start, self.Shoulder_weekend_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[1])]
                    string = re.sub(Shoulder_Periods[1],"",string)
                    self.Shoulder_weekend_3_start, self.Shoulder_weekend_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Shoulder_Periods[2])]
                    string = re.sub(Shoulder_Periods[2],"",string)
                except IndexError:
                    pass

                Peak_Periods = re.findall(r"Peak[0-9]{4}-[0-9]{4}",string)
                try:
                    self.Peak_weekend_1_start, self.Peak_weekend_1_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[0])]
                    string = re.sub(Peak_Periods[0],"",string)
                    self.Peak_weekend_2_start, self.Peak_weekend_2_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[1])]
                    string = re.sub(Peak_Periods[1],"",string)
                    self.Peak_weekend_3_start, self.Peak_weekend_3_end = [datetime.datetime.strptime(i, '%H%M').time() for i in re.findall(r'[0-9]{4}',Peak_Periods[2])]
                    string = re.sub(Peak_Periods[2],"",string)
                except IndexError:
                    pass

            except AttributeError:
                pass

        ############# Fees / Incentives / Discounts
        # Fees
        FeesChargesDiscountsIncentives = soup.find("div", {"class": "bp__fees"})

        try:
            self.connection_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'^Connection fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.connection_fee_NOMINAL = None
        try:
            self.disconnection_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'^Disconnection fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.disconnection_fee_NOMINAL = None
        try:
            self.annual_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'annual fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.annual_fee_NOMINAL = None
        try:
            self.exit_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'exit fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.exit_fee_NOMINAL = None
        try:
            self.latePayment_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'Late payment', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.latePayment_fee_NOMINAL = None
        try:
            self.paymentDishonour_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'dishonour', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.paymentDishonour_fee_NOMINAL = None
        try:
            self.specialMeterRead_fee_NOMINAL = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'meter read', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('$')[1])
        except AttributeError:
            self.specialMeterRead_fee_NOMINAL = None
        try:
            self.creditCard_fee_PERCENTAGE = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'^Credit card payment fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('%')[0])/100
        except AttributeError:
            self.creditCard_fee_PERCENTAGE = None
        try:
            paymentProcessing_fee = FeesChargesDiscountsIncentives.find(text=re.compile(r'^Payment processing fee', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text
            if re.compile(paymentProcessing_fee).search('$'):
                self.paymentProcessing_fee_NOMINAL = float(paymentProcessing_fee.split('$')[1])
            if re.compile(paymentProcessing_fee).search('%'):
                self.paymentProcessing_fee_PERCENTAGE = float(paymentProcessing_fee.split('%')[0])
        except AttributeError:
            self.paymentProcessing_fee_NOMINAL= None

        # Discounts
        try:
            self.payOnTime_discount_PERCENTAGE = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'Pay on time', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('%')[0])/100
        except AttributeError:
            self.payOnTime_discount_PERCENTAGE = None
        try:
            self.bundling_discount_PERCENTAGE = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'Bundling discount', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('%')[0])/100
        except AttributeError:
            self.bundling_discount_PERCENTAGE = None
        try:
            self.guaranteed_usage_discount_PERCENTAGE = float(FeesChargesDiscountsIncentives.find(text=re.compile(r'Guaranteed discount', flags=re.IGNORECASE)).findNext("div", {"class": "panel__list-value"}).text.split('%')[0])/100
        except AttributeError:
            self.guaranteed_usage_discount_PERCENTAGE = None
        except ValueError:
            self.guaranteed_usage_discount_PERCENTAGE = None

        try:
            incentives_list = FeesChargesDiscountsIncentives.find("span", text="Incentives").findNext('section').find_all("div", {"class": "panel__list-item"})
            for item in incentives_list:
                self.incentives += item.text + "\n"
        except AttributeError:
            self.incentives = None

    def Add_to_DB(self):

        if not self.skip:
            NewTariff, created = EMETariffs.objects.update_or_create(

                # Basic tariff details
                tariff_name = self.tariff_name,
                plan_id = self.plan_id,

                defaults = {
                    # Flags
                    'warning' : self.warning,

                    # 'premises_type' : self.premises_type,
                    'premises_type' : 'Residential',
                    'EME_url' : self.url,
                    'retailer' : self.retailer,
                    #'retailer_logo' : self.retailer_logo,
                    'distributor' : self.distributor,

                    # Contract details
                    'last_updated' : datetime.date.today(),
                    'effective_from' : self.effective_from,
                    'valid_until' : self.valid_until,
                    'contract_length' : self.contract_length,
                    'contract_expiry' : self.contract_expiry,
                    'meter_restrictions' : self.meter_restrictions,
                    'billing_period' : self.billing_period,
                    'payment_options' : self.payment_options,

                    # Tariff rates
                    'usage_primary_rate_SUPPLYCHARGE' : self.usage_primary_rate_SUPPLYCHARGE,

                    'usage_primary_rate_FLAT' : self.usage_primary_rate_FLAT,
                    'export_primary_rate_FLAT' : self.export_primary_rate_FLAT,

                    'usage_primary_rate_BLOCK1' : self.usage_primary_rate_BLOCK1,
                    'usage_primary_rate_BLOCK2' : self.usage_primary_rate_BLOCK2,
                    'usage_primary_rate_BLOCK3' : self.usage_primary_rate_BLOCK3,
                    'usage_primary_ceiling_BLOCK1' : self.usage_primary_ceiling_BLOCK1,
                    'usage_primary_ceiling_BLOCK2' :self.usage_primary_ceiling_BLOCK2,
                    'usage_primary_timescale' : self.usage_primary_timescale,

                    'usage_primary_rate_PEAK' : self.usage_primary_rate_PEAK,
                    'usage_primary_rate_SHOULDER' : self.usage_primary_rate_SHOULDER,
                    'usage_primary_rate_OFFPEAK' : self.usage_primary_rate_OFFPEAK,

                    'usage_primary_peak_weekday_1_start' : self.Peak_weekday_1_start,
                    'usage_primary_peak_weekday_1_end' : self.Peak_weekday_1_end,
                    'usage_primary_peak_weekday_2_start' : self.Peak_weekday_2_start,
                    'usage_primary_peak_weekday_2_end' : self.Peak_weekday_2_end,
                    'usage_primary_peak_weekday_3_start' : self.Peak_weekday_3_start,
                    'usage_primary_peak_weekday_3_end' : self.Peak_weekday_3_end,
                    'usage_primary_shoulder_weekday_1_start' : self.Shoulder_weekday_1_start,
                    'usage_primary_shoulder_weekday_1_end' : self.Shoulder_weekday_1_end,
                    'usage_primary_shoulder_weekday_2_start' : self.Shoulder_weekday_2_start,
                    'usage_primary_shoulder_weekday_2_end' : self.Shoulder_weekday_2_end,
                    'usage_primary_shoulder_weekday_3_start' : self.Shoulder_weekday_3_start,
                    'usage_primary_shoulder_weekday_3_end' : self.Shoulder_weekday_3_end,
                    'usage_primary_offpeak_weekday_1_start' : self.OffPeak_weekday_1_start,
                    'usage_primary_offpeak_weekday_1_end' : self.OffPeak_weekday_1_end,
                    'usage_primary_offpeak_weekday_2_start' : self.OffPeak_weekday_2_start,
                    'usage_primary_offpeak_weekday_2_end' : self.OffPeak_weekday_2_end,
                    'usage_primary_offpeak_weekday_3_start' : self.OffPeak_weekday_3_start,
                    'usage_primary_offpeak_weekday_3_end' : self.OffPeak_weekday_3_end,

                    'usage_primary_peak_weekend_1_start' : self.Peak_weekend_1_start,
                    'usage_primary_peak_weekend_1_end' : self.Peak_weekend_1_end,
                    'usage_primary_peak_weekend_2_start' : self.Peak_weekend_2_start,
                    'usage_primary_peak_weekend_2_end' : self.Peak_weekend_2_end,
                    'usage_primary_peak_weekend_3_start' : self.Peak_weekend_3_start,
                    'usage_primary_peak_weekend_3_end' : self.Peak_weekend_3_end,
                    'usage_primary_shoulder_weekend_1_start' : self.Shoulder_weekend_1_start,
                    'usage_primary_shoulder_weekend_1_end' : self.Shoulder_weekend_1_end,
                    'usage_primary_shoulder_weekend_2_start' : self.Shoulder_weekend_2_start,
                    'usage_primary_shoulder_weekend_2_end' : self.Shoulder_weekend_2_end,
                    'usage_primary_shoulder_weekend_3_start' : self.Shoulder_weekend_3_start,
                    'usage_primary_shoulder_weekend_3_end' : self.Shoulder_weekend_3_end,
                    'usage_primary_offpeak_weekend_1_start' : self.OffPeak_weekend_1_start,
                    'usage_primary_offpeak_weekend_1_end' : self.OffPeak_weekend_1_end,
                    'usage_primary_offpeak_weekend_2_start' : self.OffPeak_weekend_2_start,
                    'usage_primary_offpeak_weekend_2_end' : self.OffPeak_weekend_2_end,
                    'usage_primary_offpeak_weekend_3_start' : self.OffPeak_weekend_3_start,
                    'usage_primary_offpeak_weekend_3_end' : self.OffPeak_weekend_3_end,

                    # ############# Discounts & Fees
                    'connection_fee_NOMINAL' : self.connection_fee_NOMINAL,
                    'disconnection_fee_NOMINAL' : self.disconnection_fee_NOMINAL,
                    'annual_fee_NOMINAL' : self.annual_fee_NOMINAL,
                    'exit_fee_NOMINAL' : self.exit_fee_NOMINAL,
                    'latePayment_fee_NOMINAL' : self.latePayment_fee_NOMINAL,
                    'paymentDishonour_fee_NOMINAL' : self.paymentDishonour_fee_NOMINAL,
                    'specialMeterRead_fee_NOMINAL' : self.specialMeterRead_fee_NOMINAL,
                    'creditCard_fee_PERCENTAGE' : self.creditCard_fee_PERCENTAGE,

                    'payOnTime_discount_PERCENTAGE' : self.payOnTime_discount_PERCENTAGE,
                    'bundling_discount_PERCENTAGE' : self.bundling_discount_PERCENTAGE,
                    'guaranteed_usage_discount_PERCENTAGE' : self.guaranteed_usage_discount_PERCENTAGE,

                    'incentives' : self.incentives,
                    }
                )

        print("Finished {0}".format(self.tariff_name))

browser = webdriver.Chrome()

# A = EnergyMadeEasy_AllTariffListings(postcode_list=[2600,2900,2617,2914,2125,2535,2767,2850,2611,2423,2722,2588,2800,3707,2540,2000,2026,2073,2114,2198,2211,2337,2899])
# A.listTariffsForPostcode()

df = pd.read_excel('Tariff_Listings.xlsx', sheet_name='NoDuplicates')

for url in df.url[250:].values:             # In production this will for-loop through all urls in the df
    try:
        A = EnergyMadeEasy_Tariff(url)
        A.Add_to_DB()
    except IndexError:
        pass
#
# # for i in range(20,100):
# #     A = EnergyMadeEasy_Tariff(df.url.values[i])
# #     A.Add_to_DB()
#
# browser.close()


# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from bs4 import BeautifulSoup
import pandas as pd
import time
from camera_name_dict import make_dict

portal_sites = ['kakaku.com',
'www.yodobashi.com',
'biccamera.com',
'amazon.co.jp',
'mapcamera.com',
'rakuten.co.jp',
'shopping.yahoo.co.jp',
'sony.jp',
'shop.kitamura.jp',
'yamada-denkiweb.com',
'nojima.co.jp',
'joshinweb.jp',
'kojima.net',
'edion.com'
]

name_check_list = make_dict()

df_target_cameras = pd.read_csv('datasets/target_cameras.csv')
cam_names_list = df_target_cameras['name']

cam_list_del_id = 0

driver = webdriver.Chrome('./chromedriver')

driver_reset = 1

for model in cam_names_list:

  if driver_reset > 3:
    driver.quit()
    driver = webdriver.Chrome('./chromedriver')
    driver_reset = 1

  results =[]
  driver.implicitly_wait(5)
  driver.get('https://www.google.co.jp/search?q=' + model + 'レビュー')
  driver.implicitly_wait(5)

  try:
    while True:
      html = driver.page_source
      soup = BeautifulSoup(html, "html.parser")

      all_search = soup.find_all(class_='rc')

      for data in all_search:
        url = data.find('a').get('href')
        for pt_site in portal_sites:
          exit_flag = False
          if pt_site in url:
            exit_flag = True
          else:
            title = data.find('h3').string
            dis_element = data.find(class_='st')
            if dis_element is None:
              dis = ''
            else:
              dis = dis_element.string
            result = [title, dis, url]
            results.append(result)
            exit_flag = True
          if exit_flag is True:
            break

      driver.find_element_by_id('pnnext').click()

  except NoSuchElementException:
      print('最後のページまできたよ')

  index_of_results = 0
  del_list = []

  for item in results:
    dele = True
    stop_loop = False
    for check_words in name_check_list:
      if stop_loop is True:
        stop_loop = False
        break
      for check_word in check_words:
        if check_word in item[0]:
          dele = False
          stop_loop = True
          break
        else:
          pass
    if dele is True:
      results.pop(index_of_results)
    index_of_results += 1

  print(model)
  print(str(len(results)) + '件のデータが取れたよ')

  if len(results) ==0:
    print('スクレイピング失敗！処理を終えます！')
    driver.quit()
    break

  df_review = pd.read_csv('datasets/reviews.csv', index_col=0 )
  df_inter_table = pd.read_csv('datasets/camera_reviews.csv', index_col=0)

  if len(df_inter_table) == 0:
    last_cam_id = 0
  else:
    camera_ids = df_inter_table['camera_id']
    table_len = len(camera_ids)
    last_cam_id = int(camera_ids[table_len - 1])

  if len(df_review) == 0:
    review_index = 1
  else:
    review_index = len(df_review) + 1

  for result in results:
    df_result = pd.DataFrame()
    df_result['title'] = [result[0]]
    df_result['body'] = [result[1]]
    df_result['url'] = [result[2]]
    df_review = pd.concat([df_review, df_result])

    df_cam_rev = pd.DataFrame()
    df_cam_rev['camera_id'] = [last_cam_id + 1]
    df_cam_rev['review_id'] = [review_index]
    df_inter_table = pd.concat([df_inter_table, df_cam_rev])
    review_index += 1

  df_review = df_review.reset_index(drop=True)
  df_inter_table = df_inter_table.reset_index(drop=True)
  df_review.to_csv('datasets/reviews.csv')
  df_inter_table.to_csv('datasets/camera_reviews.csv')

  cam_names_list = cam_names_list.drop(cam_list_del_id)
  df_update_cameras = pd.DataFrame()
  df_update_cameras['name'] = cam_names_list
  df_update_cameras.to_csv('datasets/target_cameras.csv')
  cam_list_del_id += 1

  driver_reset += 1


from silo import log, config
import io

def test_logging():
    stream = io.BytesIO()
    config(file=stream, handler=lambda x: x['message'].encode('utf-8'))

    def reset():
        val = stream.getvalue()
        stream.truncate(0)
        stream.seek(0)
        return val

    log('')
    assert reset() == b''

    log('hello')
    assert reset() == b'hello'

    log('hello {name}', name='world')
    assert reset() == b'hello world'

    log('{missing}')
    assert reset() == b'{missing}'


# -*- coding: utf-8 -*-
"""
Created on Sun Mar 31 12:23:59 2019
手写识别系统
@author: Kylin
"""

import numpy as np
from os import listdir
from knntest import classify

#编写函数img2vector()，将图像格式转化为分类器使用的向量格式

def img2vector(filename):
    #将图像转换为向量
    '''必须将图像格式化处理为一个向量，我们把32×32的二进制图像矩阵转换为1×1024的向量，
    这样就可以使用classifier分类器了。
    '''
    returnVect = np.zeros((1,1024))
    fr = open(filename)
    for i in range(32):
        lineStr = fr.readline()
        for j in range(32):
            returnVect[0,32*i+j] = int(lineStr[j])
    return returnVect

#4. 测试算法
def handwritingClassTest():
    #1. 收集数据:提供文本文件
    #trainingDigits中包含了2000个例子，testDigits中包含了大学900个测试数据
    hwLabels = []
    trainingFileList = listdir('trainingDigits') #加载训练集
    m = len(trainingFileList)
    trainingMat = np.zeros((m,1024))
    for i in range(m):
        fileNameStr = trainingFileList[i] #获得第i个文件名
        fileStr = fileNameStr.split('.')[0] #获得不含后缀的文件名
        #classNumStr为类标
        classNumStr = int(fileStr.split('_')[0])
        hwLabels.append(classNumStr)
        #2. 将数据转化为分类器所需要的格式
        trainingMat[i,:] = img2vector('trainingDigits/%s' % fileNameStr)

    #3. 加载测试集
    testFileList = listdir('testDigits')
    errorCount = 0.0
    mTest = len(testFileList)

    for i in range(mTest):
        fileNameStr = testFileList[i]
        fileStr = fileNameStr.split('.')[0] #take off .txt
        classNumStr = int(fileStr.split('_')[0])
        vectorUnderTest = img2vector('testDigits/%s' % fileNameStr)
        #4. 训练数据
        classifierResult = classify(vectorUnderTest, trainingMat, hwLabels, 3)
        print("the classifier came back with: %d, the real answer is: %d"%(classifierResult, classNumStr))
        if (classifierResult != classNumStr):
            errorCount += 1.0
    #5. 输出测试结果，及总体的错误率
    print("\n在测试集中的错误次数为: %d" % errorCount)
    print("\n总体错误率是: %f" % (errorCount/float(mTest)))

if __name__ == '__main__':
    handwritingClassTest()


# -*- coding:utf-8 -*-

# All DNA is composed of a series of nucleotides abbreviated as A, C, G, and T, for example: "ACGAATTCCG". When studying DNA, it is sometimes useful to identify repeated sequences within the DNA.
#
# Write a function to find all the 10-letter-long sequences (substrings) that occur more than once in a DNA molecule.
#
# Example:
#
#
# Input: s = "AAAAACCCCCAAAAACCCCCCAAAAAGGGTTT"
#
# Output: ["AAAAACCCCC", "CCCCCAAAAA"]
#
#

class Solution(object):
    def findRepeatedDnaSequences(self, s):
        """
        :type s: str
        :rtype: List[str]
        """
        l = len(s)
        ht = set()
        res = set()
        for i in range(l-9):
            ss = s[i:i+10]
            if ss in ht:
                res.add(ss)
            else:
                ht.add(ss)
        return list(res)


import os.path
from string import ascii_lowercase

f = open(os.path.dirname(__file__) + '/../inputs/input02.txt')
s = f.read().split('\n')
c2 = c3 = 0
for x in s:
    twos = threes = False
    for c in ascii_lowercase:
        twos = twos or x.count(c) == 2
        threes = threes or x.count(c) == 3
    c2 += twos
    c3 += threes
print("Task 1: ", c2*c3)

ret = ""
for x in s:
    for y in s:
        diff = ""
        for i in range(0, len(y)):
            diff += x[i] if x[i] == y[i] else ""
        if len(diff)+1 == len(y):
            print("Task 2: ", diff)
            exit(0)


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

df = pd.read_csv("tp3.csv",sep=r'\s*,\s*',header=0, engine='python')
Y = df['Tiempo promedio'].tolist()
X = df['RC(FΩ)'].tolist()
yerr=df['error t'].tolist()
xerr=df['error RC'].tolist()

print(xerr)
z, residuals, rank, singular_values, rcond = np.polyfit(X, Y, 1, full=True)
print("residuals RC")
print(residuals)
p = np.poly1d(z)
plt.plot(X,p(X),"r--",color='red')
plt.errorbar(X, Y,xerr=xerr,yerr=yerr)
#plt.set_title('Resistores A y B en Serie', fontsize=14)
plt.xlabel('RC (FΩ = s)',fontsize=18)
plt.ylabel('Tiempo promedio (s)',fontsize=18)
plt.title('Tiempo característico', fontsize=20)
plt.show()


from styx_msgs.msg import TrafficLight

import os
import sys
import time
from PIL import Image

import numpy as np
import tensorflow as tf

import rospy

DETECTION_THRESHOLD = 0.8

class TLClassifier(object):
    def __init__(self, model_path):
        # Import tensorflow graph
        self.detection_graph = tf.Graph()

        with self.detection_graph.as_default():
            od_graph_def = tf.GraphDef()

            with tf.gfile.GFile(model_path, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

            # get all necessary tensors
            self.image_tensor = self.detection_graph.get_tensor_by_name('image_tensor:0')
            self.d_boxes = self.detection_graph.get_tensor_by_name('detection_boxes:0')
            self.d_scores = self.detection_graph.get_tensor_by_name('detection_scores:0')
            self.d_classes = self.detection_graph.get_tensor_by_name('detection_classes:0')

        self.sess = tf.Session(graph=self.detection_graph)

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        # Bounding Box Detection.
        tic = time.time()
        with self.detection_graph.as_default():
            # BGR to RGB conversion
            image = image[:, :, ::-1]

            # Expand dimension since the model expects image to have shape [1, None, None, 3].
            img_expanded = np.expand_dims(image, axis=0)

            # run classifier
            (scores, classes) = self.sess.run(
                [self.d_scores, self.d_classes],
                feed_dict={self.image_tensor: img_expanded})

            # find the top score for a given image frame
            idx = np.argmax(np.squeeze(scores))
            top_score = np.squeeze(scores)[idx]

            elapsed_time = time.time() - tic
            rospy.loginfo("Time spent on classification=%.2f sec" % (elapsed_time))

            # figure out traffic light class based on the top score
            if top_score > DETECTION_THRESHOLD:
                tl_state = int(np.squeeze(classes)[idx])

                if tl_state == 1:
                    return TrafficLight.RED
                elif tl_state == 2:
                    return TrafficLight.YELLOW
                else:
                    return TrafficLight.GREEN
            else:
                return TrafficLight.UNKNOWN


# -*- coding: utf-8-*-
import random
import re
import sys
sys.path.append('/home/pi/Desktop/autoh/Lights')

from serial_led import serialControl

WORDS = ["TURN", "ON", "ALL", "LIGHTS"]

def handle(text, mic, profile):
    """
        Responds to user-input, typically speech text, by relaying the
        meaning of life.

        Arguments:
        text -- user-input, typically transcribed speech
        mic -- used to interact with the user (for both input and output)
        profile -- contains information related to the user (e.g., phone
                   number)"""

    messages = ["TURNING LIGHTS ON...",
                "HOLD ON."]

    message = random.choice(messages)

    mic.say(message)

    serialControl("2000")
    serialControl("3000")
    serialControl("4000")
    serialControl("5000")

def isValid(text):
	return bool(re.search(r'\bturn all lights on\b', text, re.IGNORECASE))


import os
from mp3_tagger import MP3File, VERSION_1, VERSION_2, VERSION_BOTH

# 1 Recupera os arquivos MP3
# Lista os arquivos MP3 em uma determinada pasta e subpastas
def encontraArquivosEmPastaRecursivamente(pasta, extensao):
    print(pasta)
    print(extensao)
    arquivosMP3 = []
    caminhoAbsoluto = os.path.abspath(pasta)
    for pastaAtual, subPastas, arquivos  in os.walk(caminhoAbsoluto):
        arquivosMP3.extend([os.path.join(pastaAtual,arquivo) for arquivo in arquivos if arquivo.endswith(extensao)])
    return arquivosMP3

# 2 Monta lista e testa atualização
def atualizaTagArquivos(diretorio,arquivos,delimitador_arquivo,atualizar):
    total_de_arquivos = 0

    for arq in arquivos:
        nome_mp3 = (arq.split('\\')[2])
        delimitador = nome_mp3.find(delimitador_arquivo)

        if ( delimitador > 0 ):
            total_de_arquivos += 1
            musica = nome_mp3.split(delimitador_arquivo)[1]
            # Retira o ".mp3" do nome da música
            musica = musica.replace('.mp3','')
            # Retira espaços do nome da música
            musica = musica.strip(' ')
            artista = nome_mp3.split(delimitador_arquivo)[0]
            # Retira espaços do nome do artista
            artista = artista.strip(' ')
            print('arquivo original: [{}]  \n   musica: {}  \n  artista: {}'.format(nome_mp3, musica, artista))

            # Atualiza tags
            if ( atualizar == 'S' ):
                print('Vai atualizar a música: {} '.format(musica))
                atualizarArquivos(diretorio, nome_mp3, artista, musica)

    print('--------------------------------------------------')
    print('Total de Arquivos MP3 lidos: {}. '.format(total_de_arquivos))

# 3 Atualiza
def atualizarArquivos(diretorio,nome_arquivo, artista, musica):
    barra = '\\'
    nome_completo_arquivo = diretorio + str(barra) + str(nome_arquivo)
    print(nome_completo_arquivo)
    mp3 = MP3File(nome_completo_arquivo)
    tags = mp3.get_tags()
    print(' Tags Antes:\n {}'.format(tags))
    # Seta o novo artista
    mp3.artist = artista
    mp3.song = musica
    mp3.save()
    tags = mp3.get_tags()
    print(' Tags Depois:\n {}'.format(tags) )

if __name__ == '__main__':
    arquivos = []
    diretorio = 'E:\Teste'
    extensao = 'mp3'

    #Recebe os arquivos MP3
    arquivos = encontraArquivosEmPastaRecursivamente(diretorio,extensao)
    delimitador = '-'

    #Lista os arquivos MP3
    print('-------------------Arquivos MP3--------------------')
    atualizaTagArquivos(diretorio,arquivos,delimitador,'S')
    print('--------------------------------------------------')


import Pyro4

#from StringUtil import StringUtil

#util = StringUtil()

util = Pyro4.Proxy("PYRONAME:StringUtil")    # use name server object lookup uri shortcut

s = util.toUpperCase("hello world")
print(s)

s = util.reverse(s)
print(s)


import logging
import os

from datetime import datetime

import monasca_agent.collector.checks as checks

log = logging.getLogger(__name__)

class Crash(checks.AgentCheck):

    def __init__(self, name, init_config, agent_config):
        super(Crash, self).__init__(name, init_config, agent_config)
        self.crash_dir = self.init_config.get('crash_dir', '/var/crash')
        log.debug('crash dir: %s', self.crash_dir)

    def check(self, instance):
        """Capture crash dump statistics
        """
        dimensions = self._set_dimensions(None, instance)
        dump_count = 0
        value_meta = {'latest': ''}

        # Parse the crash directory
        if os.path.isdir(self.crash_dir):
            for entry in sorted(os.listdir(self.crash_dir), reverse=True):
                if os.path.isdir(os.path.join(self.crash_dir, entry)):
                    try:
                        dt = datetime.strptime(entry, '%Y%m%d%H%M')
                    except ValueError:
                        continue

                    # Found a valid crash dump directory
                    log.debug('found crash dump dir: %s',
                              os.path.join(self.crash_dir, entry))
                    dump_count += 1

                    # Return the date-/timestamp of the most recent crash
                    if dump_count == 1:
                        value_meta = {'latest': unicode(dt)}

        log.debug('dump_count: %s', dump_count)
        self.gauge('crash.dump_count', dump_count, dimensions=dimensions,
                   value_meta=value_meta)


import pandas as pd
import numpy as np

# Read in data
data = pd.read_csv('AnalyticsChallenge2020Data.csv')

# on_off_df = pd.DataFrame(columns = ['position', 'epa on', 'epa off', 'diff', 'count_on', 'count_off])

# expand position types to include side of ball
data['TechniqueName'] = data['SideOfBall'].astype(str) + ' ' + data['TechniqueName']
data['PlayID'] = data['GameID'].astype(str) + ' ' + data['EventID'].astype(str)
# eliminates either passing or rushing plays
data = data[~data['EventType'].str.contains('pass')]

# create a set of positions and list of plays associated with each position
position_set = set()
play_dict = {}
for index, row in data.iterrows():
    position_set.add(row['TechniqueName'])
    if row['TechniqueName'] in play_dict:
        play_dict[row['TechniqueName']].append(row['PlayID'])
    else:
        play_dict[row['TechniqueName']] = [row['PlayID']]

# create a blank spot in dict for each position
on_off_dict = {}
for x in position_set:
    on_off_dict[x] = [[],[],0,0,0]

# make list of epa on for each position
for index, row in data.iterrows():
    tech = row['TechniqueName']
    on_off_dict[tech][0].append(row['EPA'])
    on_off_dict[tech][3] += 1

# fill list of epa off
current_play = 0
count = 0
for index, row in data.iterrows():
    if row['PlayID'] != current_play:
        count += 1
        current_play = row['PlayID']
        for position in play_dict:
            if current_play not in play_dict[position]:
                on_off_dict[position][1].append(row['EPA'])
                on_off_dict[position][4] += 1

# change lists to average
for x in on_off_dict:
    avg = np.nanmean(on_off_dict[x][0])
    on_off_dict[x][0] = avg
    avg2 = np.nanmean(on_off_dict[x][1])
    on_off_dict[x][1] = avg2
    on_off_dict[x][2] = avg - avg2

# convert dict to df
on_off_df = pd.DataFrame.from_dict(on_off_dict, orient = 'index', columns =['epa on', 'epa off', 'diff', 'count_on', 'count_off'])


import requests
import json
import re
import time
import random
import redis
redis=redis.Redis(host="127.0.0.1",port=6379,db=2)
url="http://fundgz.1234567.com.cn/js/{}.js?rt=1589463125600"
r = requests.get('http://fund.eastmoney.com/js/fundcode_search.js')
print(r)
cont = re.findall('var r = (.*])', r.text)[0]  # 提取list
ls = json.loads(cont)  # 将字符串个事的list转化为list格式
print(ls)
daihao=[i[0] for i in ls]
num=0
redis.delete("errorFundCode")
for i in daihao:
    try:
        r=requests.get(url.format(i),timeout=3)
        text = re.findall('\((.*?)\)', r.text)[0]
        dic=json.loads(text)
        print("[{}] ".format(num),text)
    except Exception as e:
        print("wrong daihao: ",i)
        redis.lpush("errorFundCode",i)
        continue
    latest=redis.lindex(i,-1)
    if latest==None:
        redis.rpush(i,text)
        num+=1
        print("[{}] new add: ".format(num),i)
    else:
        latest=json.loads(latest)
        if latest["gztime"]!=dic["gztime"]:
            redis.rpush(i,text)
            num += 1
            print("[{}] add: ".format(num), i)
    time.sleep(random.random())


import tensorflow as tf
import os

# TFrecord reading
filename_queue = tf.train.string_input_producer(tf.train.match_filenames_once("./output2/training_images/*.tfrecords"))
#filename_queue = tf.train.string_input_producer(tf.train.match_filenames_once("./output/training_images/*.tfrecords"))

reader = tf.TFRecordReader()
_, serialized = reader.read(filename_queue)
features = tf.parse_single_example(serialized,
    features={
    'label': tf.FixedLenFeature([], tf.string),
    'image': tf.FixedLenFeature([], tf.string),
    })
record_image = tf.decode_raw(features['image'], tf.uint8)
# Changing the image into this shape helps train and visualize the output by
# converting it to be organized like an image.
image = tf.reshape(record_image, [250, 250, 1])
label = tf.cast(features['label'], tf.string)
min_after_dequeue = 10
batch_size = 100
capacity = min_after_dequeue + 3 * batch_size
image_batch, label_batch = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue)

# Converting the images to a float of [0,1) to match the expected input to convolution2d
float_image_batch = tf.image.convert_image_dtype(image_batch, tf.float32)

conv2d_layer_one = tf.contrib.layers.convolution2d(
    float_image_batch,
    num_outputs=32,     # The number of filters to generate
    kernel_size=(2,2),          # It's only the filter height and width.
    activation_fn=tf.nn.relu,
    #weights_initializer=tf.random_normal,
    stride=(2, 2),
    trainable=True)
pool_layer_one = tf.nn.max_pool(conv2d_layer_one,
    ksize=[1, 2, 2, 1],
    strides=[1, 2, 2, 1],
    padding='SAME')

conv2d_layer_two = tf.contrib.layers.convolution2d(
    pool_layer_one,
    num_outputs=64,        # More output channels means an increase in the number of filters
    kernel_size=(2,2),
    activation_fn=tf.nn.relu,
    #weights_initializer=tf.random_normal,
    stride=(1, 1),
    trainable=True)
pool_layer_two = tf.nn.max_pool(conv2d_layer_two,
    ksize=[1, 2, 2, 1],
    strides=[1, 2, 2, 1],
    padding='SAME')

flattened_layer_two = tf.reshape(
    pool_layer_two,
    [
        batch_size,  # Each image in the image_batch
        -1           # Every other dimension of the input
    ])

# The weight_init parameter can also accept a callable, a lambda is used here  returning a truncated normal
# with a stddev specified.
hidden_layer_three = tf.contrib.layers.fully_connected(
    flattened_layer_two,
    512,
    #weights_initializer=lambda i, dtype: tf.truncated_normal([38912, 512], stddev=0.1),
    activation_fn=tf.nn.relu
)

# Dropout some of the neurons, reducing their importance in the model
hidden_layer_three = tf.nn.dropout(hidden_layer_three, 0.5)

# The output of this are all the connections between the previous layers and the 120 different dog breeds
# available to train on.
final_fully_connected = tf.contrib.layers.fully_connected(
    hidden_layer_three,
    6,  # Number of dog breeds in the ImageNet Dogs dataset
    #weights_initializer=lambda i, dtype: tf.truncated_normal([512, 3], stddev=0.1)
)

import glob

# Find every directory name in the imagenet-dogs directory (n02085620-Chihuahua, ...)
labels = list(map(lambda c: c.split("\\")[-1], glob.glob("./input/*")))

# Match every label from label_batch and return the index where they exist in the list of classes
train_labels = tf.map_fn(lambda l: tf.where(tf.equal(labels, l))[0,0:1][0], label_batch, dtype=tf.int64)

loss = tf.reduce_mean(
    tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=final_fully_connected, labels=train_labels))

batch = tf.Variable(0)
increment_batch=batch.assign_add(1)
learning_rate = tf.train.exponential_decay(
    0.05,
    batch * 3,
    120,
    0.95,
    staircase=True)

optimizer = tf.train.AdamOptimizer(
    learning_rate, 0.9).minimize(
    loss, global_step=batch)

train_prediction = tf.nn.softmax(final_fully_connected)
num_correct=tf.cast(tf.equal(tf.argmax(train_prediction,1),train_labels),"float")
accuracy=tf.reduce_mean(num_correct)

saver=tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)

ckpt = tf.train.get_checkpoint_state(os.path.dirname('./my-model'))
if ckpt and ckpt.model_checkpoint_path:
  saver.restore(sess, ckpt.model_checkpoint_path)

training_steps = 1
for step in range(training_steps):
  p, n, a = sess.run([train_prediction, num_correct, accuracy])
  print(p)
  print(n)
  print(a)

coord.request_stop()
coord.join(threads)
sess.close()


import subprocess

def url_xpath(url, path):
    import scrapelib
    import lxml.html
    doc = lxml.html.fromstring(scrapelib.urlopen(url))
    return doc.xpath(path)

def convert_pdf(filename, type='xml'):
    commands = {'text': ['pdftotext', '-layout', filename, '-'],
                'text-nolayout': ['pdftotext', filename, '-'],
                'xml':  ['pdftohtml', '-xml', '-stdout', filename],
                'html': ['pdftohtml', '-stdout', filename]}
    try:
        pipe = subprocess.Popen(commands[type], stdout=subprocess.PIPE,
                                close_fds=True).stdout
    except OSError:
        raise EnvironmentError("error running %s, missing executable?" %
                               ' '.join(commands[type]))
    data = pipe.read()
    pipe.close()
    return data

def pdf_to_lxml(filename, type='html'):
    import lxml.html
    text = convert_pdf(filename, type)
    return lxml.html.fromstring(text)


import os, sys

usage_string = 'Usage: txt_to_csv.py input_file output_file'

if len(sys.argv) != 3:
	print(usage_string)
	exit(1)

source_path, dest_path = sys.argv[1:]

# You may modify this line with the features of your dataset
header = 'Id,Feature 1'
# You may modify this line depending on the symbols that separate your data in the .txt file
separator = '\t'

with open(dest_path, 'w') as f:
	f.write(header + '\r\n')
	for line in open(source_path, 'r'):
		line = line.replace(separator, ',')
		f.write(line.replace('\n', '\r\n'))


from .my_upload import sheet_info_list # noqa
from .my_class_kit import Kit # noqa

class BaseDownload():

    def __init__(self, token, sheets_info, context):

        import uuid
        import os

        self.context = context
        self.token = token
        self.temp_dir = '/tmp/{}'.format(uuid.uuid4())
        os.mkdir(self.temp_dir)

        self.sheets = sheets_info  # 存储sheets 的表名信息
        self.sheets_list = []  # 存储写入excel 在的 row

    def update_sheets_list(self, sheets_list):
        """
          将清洗的db data 赋予给sheets_list
        """
        self.sheets_list = sheets_list

    def open_excel(self, path):
        import xlsxwriter
        self.workbook = xlsxwriter.Workbook(path)

    def write_sheet_row(self, workbook, sheet, row_value_list, row_index, is_bold, context):
        i = 0
        cell_format = workbook.add_format({'bold': is_bold, 'font_color': 'black'})
        for s_value in row_value_list:
            str_value = str(s_value)
            sheet.write(row_index, i, str_value, cell_format)
            i = i + 1

    def write_sheet_title(self, sheet_info):

        sheet_name = sheet_info.get('label')
        self.sheet_obj = self.workbook.add_worksheet(sheet_name)
        head_list = sheet_info.get('d_title')
        self.write_sheet_row(self.workbook, self.sheet_obj, head_list, 0, True, self.context)

class MixinHandle():

    def handle_row(self, data, sheet_info):
        db_name = sheet_info.get('related_db')
        if db_name == 'table4':
            return self.handle_row_table4(data, sheet_info)
        if db_name == 'table3':
            return self.handle_row_table3(data, sheet_info)

    def handle_row_table4(self, data_obj, sheet_info):
        data = data_obj.data
        new_arr = []
        new_arr.append(data.get('express_id'))

        new_arr.append(data.get('send_name'))
        new_arr.append(data.get('send_tel'))
        start_place = data.get('start_place')
        start_list = Kit.area_str_to_list(start_place)
        new_arr.append(start_list[0])
        new_arr.append(start_list[1])

        new_arr.append(data.get('get_name'))
        new_arr.append(data.get('get_tel'))
        end_place = data.get('end_place')
        end_list = Kit.area_str_to_list(end_place)
        new_arr.append(end_list[0])
        new_arr.append(end_list[1])

        new_arr.append(data.get('goods_type'))
        new_arr.append(data.get('goods_weight'))

        start_date_time = data.get('start_date_time')
        self.context.log('start_date_time==>', start_date_time)
        datetime_list = Kit.isostr_to_list(start_date_time)
        self.context.log('datetime_list==>', datetime_list)

        new_arr.append(datetime_list[0])
        new_arr.append(datetime_list[1])

        return new_arr

class DownloadUtil(MixinHandle, BaseDownload, Kit):
    def __init__(self, token, sheets_info, context):
        import time
        super().__init__(token, sheets_info, context)
        self.timestamp = int(time.time())
        self.temp_path = self.temp_dir+'/%s.xls' % (self.timestamp)

    def start_running(self):
        new_arr = []

        self.open_excel(self.temp_path)

        for sheet_info in self.sheets:  # 遍历每一个sheet, 从数据库中获取raw data, 清洗raw data 同时将数据写入excel中
            db_name = sheet_info.get('related_db')
            qs = self.context.get_rows(db_name)
            self.write_sheet_title(sheet_info)
            new_dict = {
              'sheet_name': sheet_info.get('label'),
              'rows': [],
            }
            rows = []
            for index, obj in enumerate(qs):
                new_row = self.handle_row(obj, sheet_info)
                self.write_sheet_row(self.workbook, self.sheet_obj, new_row, index+1, False, self.context)
                rows.append(new_row)
            new_dict.update(rows=rows)
            new_arr.append(new_dict)

        self.workbook.close()
        self.update_sheets_list(new_arr)

def download(event, context):
    """ 将管理员上传到七牛服务器的excel存储到数据库"""

    """
      1. 读取数据库中的相关数据
      2. 清洗读出的数据
      3. 写入excel
      4. 上传至七牛
    """
    # import requests
    import qiniu
    import shutil

    token = event.get('token')

    downloadutil = DownloadUtil(token, sheet_info_list, context)
    downloadutil.start_running()

    # 将excel上传至七牛服务器, 待服务器返回文件key  application/vnd.ms-excel
    ret, info = qiniu.put_file(token, '%s.xls' % downloadutil.timestamp, downloadutil.temp_path, mime_type='application/vnd.ms-excel')

    context.log(ret, 'ret<====')
    context.log('------------------------------------', type(ret), type(info))
    context.log(info, 'info<====')
    shutil.rmtree(downloadutil.temp_dir)
    if ret:
        return {
          'status': 'success',
          'data': {
            'file_url': ret.get('file_path')
          }
        }
    else:
        return {
          'status"': 'error',
          'msg': '下载失败'
        }


from sklearn.externals import joblib
import json
import pickle
import pandas as pd
import thulac
import numpy as np
import json
# # 生成词向量
# thu1 = thulac.thulac(seg_only=True)  #默认模式
# text = thu1.cut("工商注册地、税务征管关系及统计关系在广州市南沙区范围内", text=True)  #进行一句话分词
# print(text)

def get_pretrained_vec():
    vocabu2id=joblib.load('../open_data/vocabulary2id.pkl')
    # df=pd.read_csv('../data/train_people_relation.txt',delimiter='\t')
    # print(df['rel'].value_counts())

    # for each in df['sentence'].head():
    #     print(each)

    # with open('../data/sgns.baidubaike.bigram-char',encoding='utf8') as f:
    #     all_lines=f.read()
    #     all_lines_split=all_lines.split('\n')
    #     print(len(all_lines_split))

    word_vec_new=[]
    with open('../data/sgns.baidubaike.bigram-char',encoding='utf8') as f:

        for index ,each0 in enumerate(f):
            word_dict = {}
            if index==0:
                print('index=0,each',each0)
                continue

            # if index>30:
            #     break
            #     print(each0)
            each=each0.strip().split(' ')

            # print('each:',len(each),each[0],len(each[1:]))
            # print(len(each),each[0],each[1:])
            if len(each[1:])!=300:
                print(index,len(each[1:]))
                print(each0)
                continue
            else:
                if each[0] in vocabu2id:
                    word_dict['word']=each[0]
                    word_dict['vec']=each[1:]
                    word_vec_new.append(word_dict)
                if index%50000==0:
                    print('index={},word_dict={}'.format(index,word_dict))
    print('len word_vec_new',len(word_vec_new))
    print('word_vec_new\n',word_vec_new[0:10])
    print('开始保存文件')

    with open('../data/word_vec_people.json','w',encoding='utf8') as f:
        #两种效果一样
        ## f.write(json.dumps(all_rel_data,indent=4))
        json.dump(word_vec_new,f,ensure_ascii=False)

    # with open('../data/word_vec_people.pkl','wb') as f:
    #     #两种效果一样
    #     ## f.write(json.dumps(all_rel_data,indent=4))
    #     # json.dump(word_vec_new,f,ensure_ascii=False)
    #     pickle.dump(word_vec_new,f)

    # with open('../data/word_vec_people.pkl', 'rb') as f:
    #      data = pickle.load(f)
    #      print(data)

def convert_people_train():
    path = '../open_data/sent_dev.txt'
    df=pd.read_csv(path,delimiter='\t',header=-1,names=['id','e1','e2','sentence'])
    print(df.head())

    df_rel=pd.read_csv('../open_data/sent_relation_dev.txt',delimiter='\t',header=-1,names=['rel_id','class'])
    print('df_rel.head',df_rel.head())
    with open('../data/rel2id_people.json',encoding='utf8') as f:
      rel2id=json.load(f)
      id2rel={}
      for k,v in rel2id.items():
          # print(type(k),type(v))
          id2rel[v]=k
    print('id2rel',id2rel)
    # df_rel['class'].apply(lambda row: print(type(row),row))
    def id2rel_process(class_input):
        # print('type',type(class_input),class_input)
        return  id2rel[int(class_input)]

    df_rel['rel']=df_rel['class'].apply(lambda row:id2rel_process(row))
    print('df_rel.tail\n',df_rel.tail())
    rel_class_id={}
    for index ,row in df_rel.iterrows():
        rel_class_id[row['rel_id']]=row['rel']

    # with open('../data/rel_class_id.json', 'w', encoding='utf8') as f:
    # # #两种效果一样
    #     ## f.write(json.dumps(all_rel_data,indent=4))
    #     json.dump(rel_class_id,f,ensure_ascii=False,indent=4)

    print('rel_class_id\n',rel_class_id)
    all_train=[]
    for index,row in df.iterrows():
        print('row',row['id'],rel_class_id[row['id']])
        each_train={}
        each_train['head']={'id':row['id'],'word':row['e1']}
        each_train['relation']=rel_class_id[row['id']]
        each_train['sentence']=row['sentence']
        each_train['tail']={'id':row['id'],'word':row['e2']}
        # print('each_train',each_train)
        all_train.append(each_train)

    with open('../data/dev_people.json','w',encoding='utf8') as f:
        #两种效果一样
        ## f.write(json.dumps(all_rel_data,indent=4))
        json.dump(all_train,f,ensure_ascii=False,indent=4)
        #显示比较少时前面的关系确实都为NA，转换程序并没有错

    pass
def convert_people_rel2id():
    rel2id_people={}
    path = '../open_data/relation2id.txt'
    df = pd.read_csv(path, delimiter='\t',header=-1,names=['relation','class'])
    print('df',df)
    for index,each in df.iterrows():
        rel2id_people[each['relation']]=each['class']
    with open('../data/rel2id_people.json','w',encoding='utf8') as f:
        #两种效果一样
        ## f.write(json.dumps(all_rel_data,indent=4))
        json.dump(rel2id_people,f,ensure_ascii=False)

    pass
if __name__=='__main__':
    # convert_people_rel2id()
    convert_people_train()

    # with open('../data/train_people.json',encoding='utf8') as f:
    #     # 两种效果一样
    #     ## f.write(json.dumps(all_rel_data,indent=4))
    #     list_data=json.load(f)
    #     print(list_data[-10:-1])
    pass