'''handles messages in a loop'''
from helpers.queuehelper import QueueName
from when_discovered import dodiscovered
from when_alert import doalert
from when_provision import doprovision
from when_monitorminer import domonitorminer
from when_runrules import dorules
from when_offline import dooffline
from when_online import doonline
from when_restart import dorestart

def dispatchmessages(mainapp, entries):
    '''process the messages'''
    if entries is None:
        return
    for entry in entries.entries:

        if entry.queuename == QueueName.Q_DISCOVERED:
            miner = mainapp.messagedecodeminer(entry.message)
            more = dodiscovered(miner)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_ALERT:
            doalert(entry.message)

        elif entry.queuename == QueueName.Q_PROVISION:
            miner = mainapp.messagedecodeminer(entry.message)
            more = doprovision(miner)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_MONITORMINER:
            miner = mainapp.messagedecodeminer(entry.message)
            more = domonitorminer(miner)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_STATISTICSUPDATED:
            msg = mainapp.messagedecodeminerstats(entry.message)
            more = dorules(msg.miner, msg.minerstats, msg.minerpool)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_OFFLINE:
            miner = mainapp.messagedecodeminer(entry.message)
            more = dooffline(miner)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_ONLINE:
            miner = mainapp.messagedecodeminer(entry.message)
            more = doonline(miner)
            dispatchmessages(mainapp, more)

        elif entry.queuename == QueueName.Q_RESTART:
            minermsg = mainapp.messagedecodeminercommand(entry.message)
            dorestart(minermsg.miner, minermsg.command)
        else:
            print('{0} is not handled'.format(entry.queuename))


import requests
from lxml import etree
import datetime
import time, json
from datetime import timedelta

now_table = str(datetime.datetime.now()).split(' ')[0]

from with_hdfs import HdfsClient
hdfs = HdfsClient(url='http://192.168.1.209:14000', user='dpp-executor')
sjc = str(time.time()).split('.')[0]
daily_date = str(datetime.datetime.now()- timedelta(hours=2)).split(' ')[0].replace('-', '')

ss_name = str(datetime.datetime.now()).split('.')[0]
ss_name = ss_name.replace(':', '-').split('-')
del ss_name[-1]
ss_names = "-".join(ss_name)
import sys
canss = sys.argv[1]

st_1 = str(datetime.datetime.now()).split(' ')[1].split(':')[0]
st_2 = str(datetime.datetime.now() - timedelta(hours=2)).split(' ')[1].split(':')[0]
st_3 = st_2 + '_' + st_1

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
    'Host': 'v.qq.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',
    'If-Modified-Since': 'Thu, 06 Sep 2018 02:10:38 GMT',
    'Upgrade-Insecure-Requests': '1'
}

def main(url,word):
    for i in range(20):
        time.sleep(1)
        urls = url+'&stag=3&cur='+str(i+1)+'&cxt=tabid%3D0%26sort%3D0%26pubfilter%3D2%26duration%3D0'
        print(urls)
        response = requests.get(urls, timeout=10, headers=headers)
        html = etree.HTML(response.text)
        href = html.xpath('//*[@class="result_title"]/a/@href')
        times = html.xpath('//*[@class="result_info"]/div[1]/div[1]/span[2]/text()')
        title = html.xpath('//*[@class="result_title"]')

        wirte_json(title, href, times,word)
        if href == []:
            break
def wirte_json(title,href,time,word):

    num = 0
    for j in href:
        # print(j)
        ti = title[num].xpath('string(.)')
        response1 = requests.get(j, timeout=10, headers=headers)
        response1.encoding = 'utf-8'
        html = etree.HTML(response1.text)

        try:
            des = html.xpath('//*[@class="summary _video_summary"]/text()')[0]
        except:
            des = ''
        try:
            autor = html.xpath('//*[@class="user_name"]/text()')[0]
        except:
            autor = ''
        try:
            autor_id = html.xpath('//*[@class="user_name"]/../@href')[0].split('/')[-1]
        except:
            autor_id = ''
        try:
            fans = html.xpath('//*[@class="video_user _video_user"]/a[2]/span/text()')[1]
        except:
            fans = 0

        try:
            view = html.xpath('//*[@id="mod_cover_playnum"]/text()')[0]
        except:
            view = 0
        try:
            timeaa = html.xpath('//*[@class="date _date"]/text()')[0]
            if '年' in timeaa:
                date = timeaa.replace('年','-').replace('月','-').replace('日','').replace('发布','')
            else:
                date = ''
        except:
            date = ''
        topic_id = j.split('.html')[0].split('/')[-1]
        if autor == 'undefined':
            autor = ''
        car = {"platform": "腾讯视频", "date": date, "time": '',"title": ti,"description":des,
               "source_author": autor,"followers_count":fans,"clicks": 0, "play": view,"categroy":'',
                   'comments_count': 0, "likes": 0, "keyword": word, "url": j,"topic_id":topic_id,"author_id":autor_id,"content_id":''
               ,"file_code":'4',"reposts_count":'',"audiourl": '', "imageurl": ""}
        data = json.dumps(car, ensure_ascii=False) + '\n'
        # TODO
        hdfs.new_write('/user/cspider_daily/nike_2h/video/%s/%s/4_%s_%s_tecent.json' % (
            daily_date,st_3, st_3, sjc), data,
                       encoding='utf-8')
        num+=1
def clean_date(x):
    now = datetime.datetime.now()
    if str(x).find('昨天')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(days=-1), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('前天')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(days=-2), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('天前')!=-1:
        x = datetime.datetime.strftime(now + datetime.timedelta(days=-int(str(x).replace('天前',''))), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('小时前')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(hours=-int(str(x).replace('小时前',''))), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('分钟前')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(minutes=-int(str(x).replace('分钟前',''))), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('今天')!=-1:
        x = str(x).replace('今天',now.strftime('%Y-%m-%d'))
    elif str(x).find('刚刚')!=-1:
        x = now.strftime('%Y-%m-%d %H:%M:%S')
    elif str(x).find('月前')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(weeks=-4*int(str(x).replace('月前',''))), '%Y-%m-%d %H:%M:%S')
    elif str(x).find('周前')!=-1:
        x = datetime.datetime.strftime(now+datetime.timedelta(weeks=-int(str(x).replace('周前',''))), '%Y-%m-%d %H:%M:%S')
    return x

def clean_deily():
    global st_3
    if st_3 == '22_00':
        st_3 = '22_24'

if __name__ == '__main__':

    clean_deily()

    hdfs.makedirs('/user/cspider_daily/nike_2h/video/{}/{}'.format(daily_date,st_3)) # 创建每日文件夹
    h1 = time.time()
    with open("word.txt", "r", encoding="utf-8") as f:
        lines = f.readlines()

    for i in lines:
        h2 = time.time()
        h3 = h2 - h1
        print(h3)
        if int(h3) > int(canss) * 3600:
            break
        url = 'http://v.qq.com/x/search/?q=' +i.strip()+ '&filter=sort%3D1%26pubfilter%3D1%26duration%3D0%26tabid%3D0%26resolution%3D0'
        try:
            main(url,i.strip())
        except:
            try:
                url = 'http://v.qq.com/x/search/?q=' + i.strip() + '&filter=sort%3D1%26pubfilter%3D1%26duration%3D0%26tabid%3D0%26resolution%3D0'

                main(url, i.strip())
            except:
                pass


import sys
import os
import time
import copy
import math

import numpy as np
import matplotlib.pyplot as plt
from sklearn import cluster, datasets, mixture
import seaborn as sns
from sklearn.neighbors import KernelDensity
from scipy.stats import wasserstein_distance

import torch.nn as nn
import torch
from torch import distributions
import torch.nn.utils.spectral_norm as spectral_norm

from utils.filehandler import *
from utils.printer import *
from gu.argparser import ArgParser
from datagenerator.gausunifmixture import GausUniffMixture

curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)

result_path = os.path.join(rootPath, 'result', 'gu_1', 'realnvp')
if not os.path.exists(result_path):
    os.makedirs(result_path)

args = ArgParser().get_parser()
opt = args.parse_args()
device = torch.device(f"cuda:{opt.cuda}" if torch.cuda.is_available() else "cpu")

# LOWER, UPPER = 10, 10.1
# r = 1
r = opt.r
bsz = 64
k = 5
# iteration = 30000
iteration = opt.i

lr = .0001
b1 = 0.5
b2 = 0.9
lambda_ = opt.ln

test_size = 100000
kde_num = 200

prior_dim = 5
depth = opt.d
fix_g = opt.fg

gu_std = opt.width / 10
ur = opt.ur

scale_g = opt.sc_g
scale_d = opt.sc_d

up = 'sgd'
up = 'adam'

class RealNVP(nn.Module):
    def __init__(self, nett, nets, prior, masks):
        super().__init__()
        self.prior = prior
        self.masks = masks
        depth = len(self.masks)
        self.t = nn.ModuleList([nett() for _ in range(depth)])
        self.s = nn.ModuleList([nets() for _ in range(depth)])

    def f(self, x):
        log_det_J, z = x.new_zeros(x.size(0), device=device), x
        for i in range(len(self.t)):
            bz = self.masks[i] * z
            s = self.s[i](bz)
            t = self.t[i](bz)
            z = bz + (1 - self.masks[i]) * (z * torch.exp(s) + t)
            log_det_J += torch.sum(s * (1 - self.masks[i]), dim=1)
            # log_det_J += torch.sum(s, dim=1)

        return z, log_det_J

    def g(self, z):
        x = z
        for i in reversed(range(len(self.t))):
            bx = self.masks[i] * x
            s = self.s[i](bx)
            t = self.t[i](bx)
            x = bx + (1 - self.masks[i]) * (x - t) * torch.exp(-s)

        return x

    def log_prob(self, x):
        z, log_p = self.f(x)
        return self.prior.log_prob(z) + log_p

    def sample(self, bsz):
        z = self.prior.sample((bsz,)).to(device)
        return self.g(z)

nets = lambda: nn.Sequential(
    nn.Linear(2, 256),
    nn.LeakyReLU(inplace=True),
    # nn.Linear(256, 256),
    # nn.LeakyReLU(inplace=True),
    nn.Linear(256, 2),
    nn.Tanh()
)

# nets_sn = lambda: nn.Sequential(
#     spectral_norm(nn.Linear(2, 256)),
#     nn.LeakyReLU(inplace=True),
#     spectral_norm(nn.Linear(256, 256)),
#     nn.LeakyReLU(inplace=True),
#     spectral_norm(nn.Linear(256, 2))
# )

nett = lambda: nn.Sequential(
    nn.Linear(2, 256),
    nn.LeakyReLU(inplace=True),
    # nn.Linear(256, 256),
    # nn.LeakyReLU(inplace=True),
    nn.Linear(256, 2)
)

def train(nett, nets):
    masks = torch.from_numpy(np.array([[0, 1], [1, 0]] * 3).astype(np.float32)).to(device)
    prior = distributions.MultivariateNormal(torch.zeros(2).to(device), torch.eye(2).to(device))
    netF = RealNVP(nett=nett, nets=nets, masks=masks, prior=prior).to(device)
    # netF = RealNVP(nets, nett, masks, prior)

    optimizer_F = torch.optim.Adam([p for p in netF.parameters() if p.requires_grad], lr=.0001, betas=(b1, b2))

    dataloader = GausUniffMixture(n_mixture=1, mean_dist=10., sigma=gu_std,
                                  unif_intsect=5, unif_ratio=ur,
                                  centralized=False,
                                  device=device, seed=2019)

    # real = dataloader.get_sample(test_size)
    # extra_real = torch.rand(size=real.size(), device=device)
    # # extra_real = real
    # real = torch.cat((real, extra_real), 1)
    # z_1 = netF.f(real)[0]
    # z_1 = z_1.cpu().data.numpy().squeeze()
    # z_2 = np.random.multivariate_normal(np.zeros(2), np.eye(2), test_size)
    # plt.scatter(z_1[:, 0], z_1[:, 1], color='orange', label='fake gaussian', alpha=0.1)

    checkpoint = {}
    for e in range(1, 11):

        running_loss_f = 0.
        for i in range(iteration):

            # ---------------------------
            # Train RealNVP

            optimizer_F.zero_grad()
            real = dataloader.get_sample(bsz)
            extra_real = torch.rand(size=real.size(), device=device)
            # extra_real = real
            real = torch.cat((real, extra_real), 1)
            log_p_loss = - netF.log_prob(real).mean()
            log_p_loss.backward(retain_graph=True)
            optimizer_F.step()
            running_loss_f += log_p_loss.item()

        # ------------------
        # Save Trained Discriminator and Generator

        tmpF = RealNVP(nett=nett, nets=nets, masks=masks, prior=prior).to(device)
        tmpF.load_state_dict(copy.deepcopy(netF.state_dict()))
        checkpoint[e] = tmpF

        print(f'Epoch: {e} / {10} ============> '
              f'Loss: {running_loss_f / iteration}')

    plt.cla()
    fig = plt.figure(figsize=(75, 30))

    for n in range(1, 6):
        netF = checkpoint[2 * n - 1]

        ax = fig.add_subplot(2, 5, n)
        real = dataloader.get_sample(test_size)
        extra_real = torch.rand(size=real.size(), device=device)
        real = torch.cat((real, extra_real), 1)
        real = real.cpu().data.numpy().squeeze()

        netF = checkpoint[n]
        fake = netF.sample(test_size)
        fake = fake.cpu().data.numpy().squeeze()

        ax.scatter(fake[:, 0], fake[:, 1], color='orange', label='fake gaussian', alpha=0.3)
        ax.scatter(real[:, 0], real[:, 1], color='green', label='true gaussian', alpha=0.3)
        ax.legend(loc=2, fontsize=24)
        ax.tick_params(labelsize=24)
        ax.set_title(f"epoch: [{2 * n - 1}]")

        ax = fig.add_subplot(2, 5, n + 5)
        real = dataloader.get_sample(test_size)
        real = real.cpu().data.numpy().squeeze()

        fake = netF.sample(test_size)
        fake = fake.cpu().data.numpy().squeeze()
        fake = fake[:, 0]

        sns.distplot(real, kde=True, color='green', label='real', norm_hist=True)
        sns.distplot(fake, kde=True, color='orange', label='fake', norm_hist=True)
        w_dist = wasserstein_distance(real, fake)
        ax.legend(loc=2, fontsize=24)
        ax.tick_params(labelsize=24)
        ax.set_title(f"epoch: [{2 * n - 1}], w distance: [{np.round(w_dist, 5)}]")

    # for n in range(1, 6):
    #     ax = fig.add_subplot(2, 5, n)
    #     # real = dataloader.get_sample(test_size)
    #     # real = real.cpu().data.numpy().squeeze()
    #     #
    #     # netF = checkpoint[n]
    #     # fake = netF.sample(test_size)
    #     # fake = fake.cpu().data.numpy().squeeze()
    #     # fake = fake[:, 0]
    #     #
    #     # sns.distplot(real, kde=True, color='green', label='real', norm_hist=True)
    #     # sns.distplot(fake, kde=True, color='orange', label='fake', norm_hist=True)
    #
    #     real = dataloader.get_sample(test_size)
    #     extra_real = torch.rand(size=real.size(), device=device)
    #     real = torch.cat((real, extra_real), 1)
    #     real = real.cpu().data.numpy().squeeze()
    #
    #     netF = checkpoint[n]
    #     fake = netF.sample(test_size)
    #     fake = fake.cpu().data.numpy().squeeze()
    #
    #     ax.scatter(fake[:, 0], fake[:, 1], color='orange', label='fake gaussian', alpha=0.3)
    #     ax.scatter(real[:, 0], real[:, 1], color='green', label='true gaussian', alpha=0.3)
    #
    #     # _t = np.concatenate([real, fake])
    #     # x_min, x_max = min(_t), max(_t)
    #     # x_d = np.linspace(x_min, x_max, 1000)
    #     #
    #     # range_width = x_max - x_min
    #     # kde_width = kde_num * range_width / test_size
    #     # kde = KernelDensity(bandwidth=kde_width, kernel='gaussian')

    plt_name = f'Generated Data: [Total iteration: {10 * iteration}]-[Test: {test_size}]'
    # plt.title(plt_name)
    plt.legend()
    plt_path = os.path.join(result_path, plt_name.replace('.', ',') + '.png')
    sep()
    print(f"Saving plot to: [{plt_path}]")
    plt.savefig(plt_path)

    # plt.cla()
    # fig = plt.figure(figsize=(75, 30))
    # for n in range(1, 11):
    #     ax = fig.add_subplot(2, 5, n)
    #
    #     netF = checkpoint[n]
    #     real = dataloader.get_sample(test_size)
    #     extra_real = torch.rand(size=real.size(), device=device)
    #     # extra_real = real
    #     real = torch.cat((real, extra_real), 1)
    #     z_1 = netF.f(real)[0]
    #     z_1 = z_1.cpu().data.numpy().squeeze()
    #     z_2 = np.random.multivariate_normal(np.zeros(2), np.eye(2), test_size)
    #     ax.scatter(z_1[:, 0], z_1[:, 1], color='orange', label='fake gaussian', alpha=0.1)
    #     ax.scatter(z_2[:, 0], z_2[:, 1], color='green', label='true gaussian', alpha=0.1)
    #
    #     ax.legend(loc=2, fontsize=24)
    #     ax.tick_params(labelsize=23)
    #
    # plt_name = f'Transformed Data: [Total iteration: {10 * iteration}]-[Test: {test_size}]'
    # plt.title(plt_name)
    # plt.legend()
    # plt_path = os.path.join(result_path, plt_name.replace('.', ',') + '.png')
    # sep()
    # print(f"Saving plot to: [{plt_path}]")
    # plt.savefig(plt_path)

if __name__ == '__main__':
    train(nett=nett, nets=nets)
    # masks = torch.from_numpy(np.array([[0, 1], [1, 0]] * 3).astype(np.float32))
    # prior = distributions.MultivariateNormal(torch.zeros(2), torch.eye(2))
    # flow = RealNVP(t=nett, s=nets, masks=masks, prior=prior)
    # optimizer = torch.optim.Adam([p for p in flow.parameters() if p.requires_grad], lr=.0001, betas=(b1, b2))
    #
    # for t in range(8001):
    #     noisy_moons = datasets.make_moons(n_samples=100, noise=.05)[0].astype(np.float32)
    #     loss = -flow.log_prob(torch.from_numpy(noisy_moons)).mean()
    #
    #     optimizer.zero_grad()
    #     loss.backward(retain_graph=True)
    #     optimizer.step()
    #
    #     if t % 500 == 0:
    #         print('iter %s:' % t, 'loss = %.3f' % loss)
    #
    # noisy_moons = datasets.make_moons(n_samples=1000, noise=.05)[0].astype(np.float32)
    # z = flow.f(torch.from_numpy(noisy_moons))[0].detach().numpy()
    # plt.subplot(221)
    # plt.scatter(z[:, 0], z[:, 1])
    # plt.title(r'$z = f(X)$')
    #
    # z = np.random.multivariate_normal(np.zeros(2), np.eye(2), 1000)
    # plt.subplot(222)
    # plt.scatter(z[:, 0], z[:, 1])
    # plt.title(r'$z \sim p(z)$')
    #
    # plt.subplot(223)
    # x = datasets.make_moons(n_samples=1000, noise=.05)[0].astype(np.float32)
    # plt.scatter(x[:, 0], x[:, 1], c='r')
    # plt.title(r'$X \sim p(X)$')
    #
    # plt.subplot(224)
    # x = flow.sample(1000).detach().numpy()
    # plt.scatter(x[:, 0, 0], x[:, 0, 1], c='r')
    # plt.title(r'$X = g(z)$')
    #
    # plt.savefig('test_realnvp.png')


# uncompyle6 version 3.7.4
# Python bytecode 2.4 (62061)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/uncompyle6/semantics/customize3.py
# Compiled at: 2020-04-20 22:50:15
"""Isolate Python 3 version-specific semantic actions here.
"""
from uncompyle6.semantics.consts import TABLE_DIRECT
from xdis.util import co_flags_is_async
from xdis import iscode
from uncompyle6.scanner import Code
from uncompyle6.semantics.helper import find_code_node, gen_function_parens_adjust
from uncompyle6.semantics.make_function3 import make_function3_annotate
from uncompyle6.semantics.customize35 import customize_for_version35
from uncompyle6.semantics.customize36 import customize_for_version36
from uncompyle6.semantics.customize37 import customize_for_version37
from uncompyle6.semantics.customize38 import customize_for_version38

def customize_for_version3(self, version):
    TABLE_DIRECT.update({'comp_for': (' for %c in %c', (2, 'store'), (0, 'expr')), 'if_exp_not': ('%c if not %c else %c', (2, 'expr'), (0, 'expr'), (4, 'expr')), 'except_cond2': ('%|except %c as %c:\n', (1, 'expr'), (5, 'store')), 'function_def_annotate': ('\n\n%|def %c%c\n', -1, 0), 'call_generator': ('%c%P', 0, (1, -1, ', ', 100)), 'importmultiple': ('%|import %c%c\n', 2, 3), 'import_cont': (', %c', 2), 'kwarg': ('%[0]{attr}=%c', 1), 'raise_stmt2': ('%|raise %c from %c\n', 0, 1), 'tf_tryelsestmtl3': ('%c%-%c%|else:\n%+%c', 1, 3, 5), 'store_locals': ('%|# inspect.currentframe().f_locals = __locals__\n',), 'with': ('%|with %c:\n%+%c%-', 0, 3), 'withasstmt': ('%|with %c as (%c):\n%+%c%-', 0, 2, 3)})
    assert version >= 3.0

    def tryfinallystmt(node):
        suite_stmts = node[1][0]
        if len(suite_stmts) == 1 and suite_stmts[0] == 'stmt':
            stmt = suite_stmts[0]
            try_something = stmt[0]
            if try_something == 'try_except':
                try_something.kind = 'tf_try_except'
            if try_something.kind.startswith('tryelsestmt'):
                if try_something == 'tryelsestmtl3':
                    try_something.kind = 'tf_tryelsestmtl3'
                else:
                    try_something.kind = 'tf_tryelsestmt'
        self.default(node)

    self.n_tryfinallystmt = tryfinallystmt

    def listcomp_closure3(node):
        """List comprehensions in Python 3 when handled as a closure.
        See if we can combine code.
        """
        p = self.prec
        self.prec = 27
        code_obj = node[1].attr
        assert iscode(code_obj)
        code = Code(code_obj, self.scanner, self.currentclass)
        ast = self.build_ast(code._tokens, code._customize)
        self.customize(code._customize)
        while len(ast) == 1 or ast in ('sstmt', 'return') and ast[(-1)] in ('RETURN_LAST',
                                                                            'RETURN_VALUE'):
            self.prec = 100
            ast = ast[0]

        n = ast[1]
        collections = [
         node[(-3)]]
        list_ifs = []
        if self.version == 3.0 and n != 'list_iter':
            stores = [
             ast[3]]
            assert ast[4] == 'comp_iter'
            n = ast[4]
            while n == 'comp_iter':
                if n[0] == 'comp_for':
                    n = n[0]
                    stores.append(n[2])
                    n = n[3]
                elif n[0] in ('comp_if', 'comp_if_not'):
                    n = n[0]
                    if n[0].kind == 'expr':
                        list_ifs.append(n)
                    else:
                        list_ifs.append([1])
                    n = n[2]
                else:
                    break

            self.preorder(n[1])
        else:
            assert n == 'list_iter'
            stores = []
            while n == 'list_iter':
                n = n[0]
                if n == 'list_for':
                    stores.append(n[2])
                    n = n[3]
                    if n[0] == 'list_for':
                        c = n[0][0]
                        if c == 'expr':
                            c = c[0]
                        if c == 'attribute':
                            c = c[0]
                        collections.append(c)
                elif n in ('list_if', 'list_if_not'):
                    if n[0].kind == 'expr':
                        list_ifs.append(n)
                    else:
                        list_ifs.append([1])
                    n = n[2]
                elif n == 'list_if37':
                    list_ifs.append(n)
                    n = n[(-1)]
                elif n == 'list_afor':
                    collections.append(n[0][0])
                    n = n[1]
                    stores.append(n[1][0])
                    n = n[3]

            assert n == 'lc_body', ast
            self.preorder(n[0])
        n_colls = len(collections)
        for (i, store) in enumerate(stores):
            if i >= n_colls:
                break
            if collections[i] == 'LOAD_DEREF' and co_flags_is_async(code_obj.co_flags):
                self.write(' async')
            self.write(' for ')
            self.preorder(store)
            self.write(' in ')
            self.preorder(collections[i])
            if i < len(list_ifs):
                self.preorder(list_ifs[i])

        self.prec = p

    self.listcomp_closure3 = listcomp_closure3

    def n_classdef3(node):
        """Handle "classdef" nonterminal for 3.0 >= version 3.0 <= 3.5
        """
        assert 3.0 <= self.version <= 3.5
        cclass = self.currentclass
        subclass_info = None
        if node == 'classdefdeco2':
            if self.version <= 3.3:
                class_name = node[2][0].attr
            else:
                class_name = node[1][2].attr
            build_class = node
        else:
            build_class = node[0]
            class_name = node[1][0].attr
            build_class = node[0]
        assert 'mkfunc' == build_class[1]
        mkfunc = build_class[1]
        if mkfunc[0] in ('kwargs', 'no_kwargs'):
            if 3.0 <= self.version <= 3.2:
                for n in mkfunc:
                    if hasattr(n, 'attr') and iscode(n.attr):
                        subclass_code = n.attr
                        break
                    elif n == 'expr':
                        subclass_code = n[0].attr

            for n in mkfunc:
                if hasattr(n, 'attr') and iscode(n.attr):
                    subclass_code = n.attr
                    break

            if node == 'classdefdeco2':
                subclass_info = node
            else:
                subclass_info = node[0]
        elif build_class[1][0] == 'load_closure':
            load_closure = build_class[1]
            if hasattr(load_closure[(-3)], 'attr'):
                subclass_code = find_code_node(load_closure, -3).attr
            elif hasattr(load_closure[(-2)], 'attr'):
                subclass_code = find_code_node(load_closure, -2).attr
            else:
                raise 'Internal Error n_classdef: cannot find class body'
            if hasattr(build_class[3], '__len__'):
                if not subclass_info:
                    subclass_info = build_class[3]
            elif hasattr(build_class[2], '__len__'):
                subclass_info = build_class[2]
            else:
                raise 'Internal Error n_classdef: cannot superclass name'
        elif not subclass_info:
            if mkfunc[0] in ('no_kwargs', 'kwargs'):
                subclass_code = mkfunc[1].attr
            else:
                subclass_code = mkfunc[0].attr
            if node == 'classdefdeco2':
                subclass_info = node
            else:
                subclass_info = node[0]
        if node == 'classdefdeco2':
            self.write('\n')
        else:
            self.write('\n\n')
        self.currentclass = str(class_name)
        self.write(self.indent, 'class ', self.currentclass)
        self.print_super_classes3(subclass_info)
        self.println(':')
        self.indent_more()
        self.build_class(subclass_code)
        self.indent_less()
        self.currentclass = cclass
        if len(self.param_stack) > 1:
            self.write('\n\n')
        else:
            self.write('\n\n\n')
        self.prune()
        return

    self.n_classdef3 = n_classdef3
    if version == 3.0:
        TABLE_DIRECT.update({'ifstmt30': ('%|if %c:\n%+%c%-', (0, 'testfalse_then'), (1, '_ifstmts_jump30')), 'ifnotstmt30': ('%|if not %c:\n%+%c%-', (0, 'testtrue_then'), (1, '_ifstmts_jump30')), 'try_except30': ('%|try:\n%+%c%-%c\n\n', (1, 'suite_stmts_opt'), (4, 'except_handler'))})

        def n_comp_iter(node):
            if node[0] == 'expr':
                n = node[0][0]
                if n == 'LOAD_FAST' and n.pattr[0:2] == '_[':
                    self.prune()
            self.default(node)

        self.n_comp_iter = n_comp_iter
    elif version == 3.3:

        def n_yield_from(node):
            assert node[0] == 'expr'
            if node[0][0] == 'get_iter':
                template = (
                 'yield from %c', (0, 'expr'))
                self.template_engine(template, node[0][0])
            else:
                template = (
                 'yield from %c', (0, 'attribute'))
                self.template_engine(template, node[0][0][0])
            self.prune()

        self.n_yield_from = n_yield_from
    if 3.2 <= version <= 3.4:

        def n_call(node):
            mapping = self._get_mapping(node)
            key = node
            for i in mapping[1:]:
                key = key[i]

            if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
                pass
            elif key.kind.startswith('CALL_FUNCTION_VAR'):
                argc = node[(-1)].attr
                nargs = argc & 255
                kwargs = argc >> 8 & 255
                if kwargs != 0:
                    if nargs == 0:
                        template = (
                         '%c(*%c, %C)', 0, -2, (1, kwargs + 1, ', '))
                    else:
                        template = (
                         '%c(%C, *%c, %C)', 0, (1, nargs + 1, ', '), -2, (-2 - kwargs, -2, ', '))
                    self.template_engine(template, node)
                    self.prune()
            else:
                gen_function_parens_adjust(key, node)
            self.default(node)

        self.n_call = n_call
    elif version < 3.2:

        def n_call(node):
            mapping = self._get_mapping(node)
            key = node
            for i in mapping[1:]:
                key = key[i]

            gen_function_parens_adjust(key, node)
            self.default(node)

        self.n_call = n_call

    def n_mkfunc_annotate(node):
        if node[(-2)] == 'EXTENDED_ARG':
            i = -1
        else:
            i = 0
        if self.version <= 3.2:
            code = node[(-2 + i)]
        elif self.version >= 3.3 or node[(-2)] == 'kwargs':
            code = node[(-3 + i)]
        elif node[(-3)] == 'expr':
            code = node[(-3)][0]
        else:
            code = node[(-3)]
        self.indent_more()
        for annotate_last in range(len(node) - 1, -1, -1):
            if node[annotate_last] == 'annotate_tuple':
                break

        if self.f.getvalue()[-4:] == 'def ':
            self.write(code.attr.co_name)
        make_function3_annotate(self, node, is_lambda=False, code_node=code, annotate_last=annotate_last)
        if len(self.param_stack) > 1:
            self.write('\n\n')
        else:
            self.write('\n\n\n')
        self.indent_less()
        self.prune()

    self.n_mkfunc_annotate = n_mkfunc_annotate
    TABLE_DIRECT.update({'tryelsestmtl3': ('%|try:\n%+%c%-%c%|else:\n%+%c%-', (1, 'suite_stmts_opt'), 3, (5, 'else_suitel')), 'LOAD_CLASSDEREF': ('%{pattr}',)})
    if version >= 3.4:
        TABLE_DIRECT.update({'LOAD_CLASSDEREF': ('%{pattr}',), 'yield_from': ('yield from %c', (0, 'expr'))})
        if version >= 3.5:
            customize_for_version35(self, version)
            if version >= 3.6:
                customize_for_version36(self, version)
                if version >= 3.7:
                    customize_for_version37(self, version)
                    if version >= 3.8:
                        customize_for_version38(self, version)
    return


import argparse

import numpy as np
import sklearn
from parallelm.mlops import mlops as mlops
# use below import if user wants to user RegressionMetrics predefined metrics names.
from parallelm.mlops.stats.bar_graph import BarGraph
from sklearn.datasets import make_regression
from sklearn.svm import SVR

def parse_args():
    """
    Parse Arguments from component
    :return:
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("--num_samples", help="# samples")
    parser.add_argument("--num_features", help="# features")

    parser.add_argument("--kernel", help="Kernel")
    parser.add_argument("--degree", help="Degree")
    parser.add_argument("--gamma", help="Gamma")
    parser.add_argument("--tol", help="Tol")
    parser.add_argument("--max_iter", dest="max_iter", type=int, required=False, default=100,
                        help='Maximum number of iterations')
    parser.add_argument("--output-model", help="Data file to save model")
    options = parser.parse_args()
    return options

def main():
    pm_options = parse_args()

    print("PM: Configuration:")
    print("PM: # Sample:                    [{}]".format(pm_options.num_samples))
    print("PM: # Features:                  [{}]".format(pm_options.num_features))

    print("PM: Kernel:                      [{}]".format(pm_options.kernel))
    print("PM: Degree:                      [{}]".format(pm_options.degree))
    print("PM: Gamma:                       [{}]".format(pm_options.gamma))
    print("PM: Tolerance:                   [{}]".format(pm_options.tol))
    print("PM: Maximum iterations:          [{}]".format(pm_options.max_iter))

    print("PM: Output model:                [{}]".format(pm_options.output_model))

    # Initialize MLOps Library
    mlops.init()

    num_samples = int(pm_options.num_samples)
    num_features = int(pm_options.num_features)

    # Create synthetic data using scikit learn
    X, y = make_regression(n_samples=num_samples,
                           n_features=num_features,
                           n_informative=2,
                           random_state=42)

    # for making labels all positive
    y = y + -1 * np.min(y)

    # Separate into features and labels
    features = X
    labels = y

    # Add noise to the data
    noisy_features = np.random.uniform(0, 10) * \
                     np.random.normal(0, 1,
                                      (num_samples, num_features))
    features = features + noisy_features

    # Output Health Statistics to MCenter
    # MLOps API to report the distribution statistics of each feature in the data
    mlops.set_data_distribution_stat(features)

    hist, bin_edges = np.histogram(labels)

    # Output label distribution as a BarGraph using MCenter
    bar = BarGraph().name("User Defined: Label Distribution") \
        .cols((bin_edges).astype(str).tolist()) \
        .data((hist).tolist()) \
        .as_continuous()

    mlops.set_stat(bar)

    # Create a model that should be deployed into production
    final_model = SVR(kernel=pm_options.kernel,
                      degree=int(pm_options.degree),
                      gamma=str(pm_options.gamma),
                      tol=float(pm_options.tol),
                      max_iter=int(pm_options.max_iter))

    final_model.fit(features, labels)

    labels_pred = final_model.predict(features)
    hist_pred, bin_edges_pred = np.histogram(labels_pred)

    # Output prediction label distribution as a BarGraph using MCenter
    pred_label_bar = BarGraph().name("User Defined: Prediction Label Distribution") \
        .cols((bin_edges_pred).astype(str).tolist()) \
        .data((hist_pred).tolist()) \
        .as_continuous()

    mlops.set_stat(pred_label_bar)

    y_true = labels
    y_pred = labels_pred

    # Regression Metrics
    ##########################################################################
    #################### Start: Output Explained Variance ####################
    ##########################################################################

    evs = sklearn.metrics.explained_variance_score(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: Explained Variance ", evs)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.EXPLAINED_VARIANCE_SCORE, evs)

    # OR

    # Third Way
    # mlops.metrics.explained_variance_score(y_true=labels, y_pred=labels_pred)
    #################### DONE NEW WAY ####################

    ########################################################################
    #################### End: Output Explained Variance ####################
    ########################################################################

    ######################################################################
    #################### Start: Output Mean Abs Error ####################
    ######################################################################

    mae = sklearn.metrics.mean_absolute_error(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: Mean Abs Error", mae)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.MEAN_ABSOLUTE_ERROR, mae)

    # OR

    # Third Way
    # mlops.metrics.mean_absolute_error(y_true=labels, y_pred=labels_pred)
    #################### DONE NEW WAY ####################

    ####################################################################
    #################### End: Output Mean Abs Error ####################
    ####################################################################

    ##########################################################################
    #################### Start: Output Mean Squared Error ####################
    ##########################################################################

    mse = sklearn.metrics.mean_squared_error(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: Mean Squared Error", mse)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.MEAN_SQUARED_ERROR, mse)

    # OR

    # Third Way
    # mlops.metrics.mean_squared_error(y_true=labels, y_pred=labels_pred)
    #################### DONE NEW WAY ####################

    ########################################################################
    #################### End: Output Mean Squared Error ####################
    ########################################################################

    ##############################################################################
    #################### Start: Output Mean Squared Log Error ####################
    ##############################################################################

    msle = sklearn.metrics.mean_squared_log_error(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: Mean Squared Log Error", msle)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.MEAN_SQUARED_LOG_ERROR, msle)

    # OR

    # Third Way
    # mlops.metrics.mean_squared_log_error(y_true=labels, y_pred=labels_pred)

    #################### DONE NEW WAY ####################

    ############################################################################
    #################### End: Output Mean Squared Log Error ####################
    ############################################################################

    ########################################################################
    #################### Start: Output Median Abs Error ####################
    ########################################################################

    median_ae = sklearn.metrics.median_absolute_error(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: Median Abs Error", median_ae)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.MEDIAN_ABSOLUTE_ERROR, median_ae)

    # OR

    # Third Way
    # mlops.metrics.median_absolute_error(y_true=labels, y_pred=labels_pred)
    #################### DONE NEW WAY ####################

    ######################################################################
    #################### End: Output Median Abs Error ####################
    ######################################################################

    ################################################################
    #################### Start: Output R2 Score ####################
    ################################################################

    r2_s = sklearn.metrics.r2_score(y_true, y_pred)

    #################### OLD WAY ####################
    # First Way
    mlops.set_stat("User Defined: R2 Score", r2_s)
    #################### DONE OLD WAY ####################

    #################### NEW WAY ####################
    # Second Way
    # mlops.set_stat(RegressionMetrics.R2_SCORE, r2_s)

    # OR

    # Third Way
    # mlops.metrics.r2_score(y_true=labels, y_pred=labels_pred)
    #################### DONE NEW WAY ####################

    ##############################################################
    #################### End: Output R2 Score ####################
    ##############################################################

    # Save the model
    import pickle
    model_file = open(pm_options.output_model, 'wb')
    pickle.dump(final_model, model_file)
    model_file.close()
    # Terminate MLOPs
    mlops.done()

if __name__ == "__main__":
    main()


from __future__ import print_function, division

__author__ = 'Amrit, Zhe, Jerry, Jack'

from Tkinter import *
import os
from func_GUI import *
import re
import random
from email import message_from_string

# BASE_DIR = os.path.dirname(os.path.abspath(__file__))
Mails_directory = "../Mails/initial2.txt"

"convert the original email to string list"

def email_parser(mailText):
    content = message_from_string(mailText)  # extract email content totally
    subject = content['subject']  # parse for email subject
    body = []
    if content.is_multipart():  # parse for email body
        for payload in content.get_payload():
            body.append(payload.get_payload())
    else:
        body.append(content.get_payload())
    body_segments = re.sub(r"\n|(\\(.*?){)|}|[!$%^&*#()_+|~\-={}\[\]:\";'<>?,.\/\\]|[0-9]|[@]", '',
                           ''.join(body))  # filter characters for email body
    body_keywords = re.sub('\s+', ' ', body_segments)
    subject_segments = re.sub(r"\n|(\\(.*?){)|}|[!$%^&*#()_+|~\-={}\[\]:\";'<>?,.\/\\]|[0-9]|[@]", '',
                              ''.join(subject))  # filter characters for email subject
    subject_keywords = re.sub('\s+', ' ', subject_segments)
    mail_subject = subject_keywords.lower()
    mail_body = body_keywords.lower()
    return subject, body, mail_subject, mail_body  # return original subject, body, and processed subject and body

"read the initial setup from initial.txt"

def readfile(filename):
    global my_folder, pool, list_mails
    with open(filename, 'r') as f:
        for doc in f.readlines():
            try:
                mailText=''
                with open("../"+doc.strip(), 'r') as fd:
                    for docc in fd.readlines():
                        mailText+=docc
                subject, body, mail_subject, mail_body = email_parser(mailText)
                true_label = get_true_label(mailText).lower()
                mail_body = mail_subject + " "+mail_body
                newmail = Email(mail_body)
                newmail.set_read()
                newmail.raw_email = body[0]
                newmail.subject=subject
                newmail.trueLabel = true_label
                newmail.set_folder(true_label)
                newmail.set_label(true_label)
                newmail.credit=1
                if not newmail.label in my_folder.names:
                    my_folder.addfolder(newmail.label)
                pool.append(newmail)
            except:
                pass
    my_folder.train(pool)
    list_mails = pool

def readnew():
    with open("../Mails/newmails.txt", 'r') as f:
        for doc in f.readlines():
            new_mails.append(doc.strip())

def get_true_label(raw_email):
    for line in raw_email.split('\r\n'):
        if line.startswith('X-Folder:'):
            folder = line.split('X-Folder:')[1]
            true_lable = folder.split('\\')[-1]
            print ('folder: ' + true_lable)
            return true_lable

"when new email comes in"
def new_email(mailText):
    global list_mails, my_folder
    subject, body, mail_subject, mail_body = email_parser(mailText)
    true_label = get_true_label(mailText).lower()
    mail_body = mail_subject + " "+mail_body
    newmail = Email(mail_body)
    newmail.raw_email = body[0]
    newmail.trueLabel = true_label
    newmail.subject=subject
    list_mails.append(newmail)
    my_folder.predict(newmail)

class Application(Frame):

    def refresh(self):
        global list_mails, currentfolder, my_folder
        self.unread.delete(first=0, last=self.unread.size())
        self.read.delete(first=0, last=self.read.size())
        folder_unreadNum = {foldername: 0 for foldername in my_folder.names}
        for i, mail in enumerate(list_mails):
            # Count the unread emails for each folder.
            for f in mail.folder:
                if f not in folder_unreadNum:
                    folder_unreadNum[f] = 0
                if not mail.read:
                    folder_unreadNum[f] += 1
            ## update Read and Unread Box Widget
            if currentfolder in mail.folder:
                if mail.read:
                    self.read.insert('end', "%0.4d : %s" % (i, mail.subject))
                else:
                    self.unread.insert('end', "%0.4d : %s" % (i, mail.subject))
        for folder, button in self.folderName_button.iteritems():
            # if folder in ['uncertain', 'trash']:
            #     continue
            unreadNum = folder_unreadNum[folder]
            if unreadNum == 0:
                button.config(text=folder)
                button.config(bg='White')
            else:
                button.config(text=folder + ' (' + str(unreadNum) + ')' )

    def button_command(self, a):
        global currentfolder
        currentfolder = a
        self.refresh()

    # def del_un(self):
    #     x= int(self.unread.get(ANCHOR).split(' : ')[0])
    #     self.unread.delete(ANCHOR)
    #     for i,k in enumerate(list_labels_mails[x]):
    #         if k == "trash":
    #             list_labels_mails[x][i] = ""
    #         if k == "inbox":
    #             list_labels_mails[x][i] = "trash"
    #         if k == "uncertain":
    #             list_labels_mails[x][i] = "trash"
    #
    # def del_re(self):
    #     x= int(self.read.get(ANCHOR).split(' : ')[0])
    #     self.read.delete(ANCHOR)
    #     for i,k in enumerate(list_labels_mails[x]):
    #         if k == "trash":
    #             list_labels_mails[x][i] = ""
    #         if k == "inbox":
    #             list_labels_mails[x][i] = "trash"
    #         if k == "uncertain":
    #             list_labels_mails[x][i] = "trash"

    def read_user(self, event):
        global list_mails, currentfolder
        w = event.widget
        email_id = w.get(w.curselection()).split(' : ')[0]
        email = list_mails[int(email_id)]
        true_label = email.get_trueLabel()
        if email.get_raw() == '':
            pop_message = email.get_body()
        else:
            pop_message = '*** '+ true_label + ' *** \n' + email.get_raw()[:500]
        self.popup1(pop_message)
        if not (currentfolder == 'uncertain' or currentfolder == 'trash'):
            activity_yes(list_mails[int(w.get(w.curselection()).split(' : ')[0])], currentfolder)
        self.refresh()

    def showscore(self):
        global my_folder
        message1="Precision: %f" %my_folder.P
        message2="Recall: %f" %my_folder.R
        message3="F1: %f" %my_folder.F1
        print("Precision: %f" %my_folder.P)
        print("Recall: %f" %my_folder.R)
        print("F1: %f" %my_folder.F1)
        self.popup1(message1+'\n'+message2+'\n'+message3)

    def incoming(self):
        try:
            x=random.randint(0,len(new_mails)-1)
            l=''
            with open("../"+new_mails.pop(x), 'r') as f:
                for doc in f.readlines():
                    l+=doc
            new_email(l)

            self.refresh()
        except:
            self.showscore()

    def unread_user(self, event):
        # w = event.widget
        # self.popup1(w.get(w.curselection()).split(' : ')[1])
        # global list_mails, currentfolder
        global list_mails, currentfolder
        w = event.widget
        email_id = w.get(w.curselection()).split(' : ')[0]
        email = list_mails[int(email_id)]
        true_label = email.get_trueLabel()
        if email.get_raw() == '':
            pop_message = email.get_body()
        else:
            pop_message = '*** '+ true_label + ' *** \n' + email.get_raw()[:500]
        self.popup1(pop_message)

        list_mails[int(w.get(w.curselection()).split(' : ')[0])].set_read()
        if not (currentfolder == 'uncertain' or currentfolder == 'trash'):
            activity_yes(list_mails[int(w.get(w.curselection()).split(' : ')[0])], currentfolder)
        self.refresh()

    def mov_command(self, targetfolder):
        global list_mails, currentfolder
        list_mails[int(self.waste)].folder = [targetfolder]
        if not (targetfolder == 'uncertain' or targetfolder == 'trash'):
            activity_no(list_mails[int(self.waste)], targetfolder)
        self.refresh()

    def c_labels_command(self):
        self.popup()
        if (self.entryValue() != ''):
            a = self.entryValue()
            global my_folder
            my_folder.addfolder(a)
            self.create_folder(a)

    def create_folder(self, name):
        self.buttons.append(name)
        x1 = Button(self.m, text=name, fg="red", command=lambda name=name: self.button_command(name), state=ACTIVE)
        self.m.add(x1)
        self.folderName_button[name] = x1
        self.count += 1

    def change_checkbox(self, bv, id):
        now = bv.get()
        self.checkbox[id] = now
        # print (now)
        features[id] = now
        print(features)

    def create_checkbox(self, id, default=True):
        bv = BooleanVar()
        bv.set(default)
        bv.trace("w", lambda name, index, mode, bv=bv: self.change_checkbox(bv, id))
        self.checkbox[id] = bv
        c = Checkbutton(self.m3, text=feature_names[id], variable=bv)
        self.m3.add(c)

    def createWidgets(self):
        self.m = PanedWindow(self, orient=VERTICAL)
        self.m.pack(side=LEFT, expand=1)
        for name in my_folder.names:
            self.create_folder(name)
        # self.USER.pack(side=BOTTOM, anchor=W, fill=X)
        self.m2 = PanedWindow(self, orient=VERTICAL)
        self.m2.pack(fill=BOTH, side=LEFT, expand=1)
        # self.m.add(self.m2)
        # New incoming mail

        self.top_right = Label(self.m2, text="Unread")
        self.m2.add(self.top_right)

        self.unread = Listbox(self.m2, width=120, height=20)
        self.unread.grid(column=0, row=0, sticky='nwes')
        self.s = Scrollbar(command=self.unread.yview, orient='vertical')
        # self.s.grid(column=0, row=0, sticky="ns")
        self.unread.config(yscrollcommand=self.s.set)
        self.unread.insert('end', "Unread Mails")
        self.unread.bind("<Double-Button-1>", self.unread_user)
        self.unread.bind("<Button-2>", self.popup_menu)
        self.unread.pack()
        self.m2.add(self.unread)

        self.bottom_right = Label(self.m2, text="Read")
        self.m2.add(self.bottom_right)
        self.read = Listbox(self.m2, width=120, height=20)
        self.read.grid(column=0, row=0, sticky='nwes')
        self.s = Scrollbar(command=self.read.yview, orient='vertical')
        # self.s.grid(column=0, row=0, sticky="ns")
        self.read.config(yscrollcommand=self.s.set)
        self.read.insert('end', "Read Mails")
        self.read.bind("<Double-Button-1>", self.read_user)
        self.read.bind("<Button-2>", self.popup_menu)

        self.read.pack()
        self.m2.add(self.read)

        self.m3 = PanedWindow(self, orient=VERTICAL)
        self.m3.pack(side=RIGHT, expand=1)
        # self.delete_un = Button(self, text="DELETE UNREAD", command=self.del_un)
        # self.delete_re = Button(self, text="DELETE READ", command=self.del_re)
        self.CREATE_Labels = Button(self, text="CREATE_LABELS", fg="red", command=self.c_labels_command, state=ACTIVE)
        self.newm = Button(self, text="New Mail", fg="red", command=self.incoming, state=ACTIVE)
        self.show = Button(self, text="Show Score", fg="red", command=self.showscore, state=ACTIVE)
        self.m3.add(self.newm)
        self.m3.add(self.CREATE_Labels)
        self.m3.add(self.show)

        for feature_id in range(len(features)):
            self.create_checkbox(feature_id)

        self.button_command('uncertain')

    def popup_menu(self, event):
        w = event.widget
        self.aMenu = Menu(self, tearoff=0)
        # print(labels_gui)
        for i in my_folder.names:
            if i != 'uncertain' and i != 'trash':
                self.aMenu.add_command(label=i, command=lambda i=i: self.mov_command(i.lower()))
        self.waste = w.get(w.curselection()).split(' : ')[0]
        self.aMenu.post(event.x_root, event.y_root)

    def popup(self):
        self.w = popupWindow(self.Frame)
        self.Frame.wait_window(self.w.top)

    def popup1(self, data):
        self.w = popupWindow1(self.Frame, data)
        self.Frame.wait_window(self.w.top)

    def entryValue(self):
        return self.w.value

    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.Frame = master
        self.count = 0
        self.buttons = []
        self.folderName_button = {}
        self.checkbox = {}
        self.pack()
        self.waste = ''
        self.createWidgets()
        # print(Application.__getitem__(self, Button)

class popupWindow1(object):
    def __init__(self, master, data):
        self.data = data
        top = self.top = Toplevel(master)
        self.l = Label(top, text=self.data)
        self.l.pack()
        self.b = Button(top, text='Ok', command=self.cleanup)
        self.b.pack()

    def cleanup(self):
        self.top.destroy()

class popupWindow(object):
    def __init__(self, master):
        top = self.top = Toplevel(master)
        self.l = Label(top, text="Label Creation")
        self.l.pack()
        self.e = Entry(top)
        self.e.pack()
        self.b = Button(top, text='Ok', command=self.cleanup)
        self.b.pack()

    def cleanup(self):
        self.value = self.e.get()
        self.top.destroy()

"format email, str to csr"

def format(email):
    global hashemail
    x = vsm([email])
    x = hashemail(x)
    x = l2normalize(x)
    return x

class Folder(object):
    def __init__(self, thres=0.35):
        self.names = ['uncertain', 'trash']
        self.classifier = []
        self.thres = thres
        self.num = 0
        self.T={}
        self.F={}
        self.P=0
        self.R=0
        self.F1=0
        self.FR={}

    def addfolder(self, name):
        self.names.append(name)
        self.num = self.num + 1
        self.T[name]=0
        self.F[name]=0
        self.FR[name]=0

    def update(self):
        folders=self.names[2:]
        self.P=0
        self.R=0
        self.F1=0
        if folders:
            for fold in folders:
                if self.T[fold]==0:
                    self.P=self.P+0
                    self.R=self.R+0
                else:
                    self.P=self.P+self.T[fold]/(self.T[fold]+self.F[fold])
                    self.R=self.R+self.T[fold]/(self.T[fold]+self.FR[fold])
            self.P=self.P/len(folders)
            self.R=self.R/len(folders)
            if self.P and self.R:
                self.F1=self.P*self.R*2/(self.P+self.R)
            else:
                self.F1=0

    "call this to train the classifier"

    def train(self, training_set):
        labels = []
        bodies = []
        for mail in training_set:
            labels.append(mail.label)
            tmp = mail.mat
            if not bodies == []:
                bodies = csr_vstack(bodies, tmp)
            else:
                bodies = tmp
        self.classifier = do_SVM(labels, bodies)

    "return the list of folder names the email would belong to, also will change mail.folder"

    def predict(self, mail):
        global features
        proba = self.classifier.predict_proba(mail.mat)[0]
        mail.folder = []
        for ind, label in enumerate(self.classifier.classes_):
            mail.proba[label] = proba[ind]
            if proba[ind] > self.thres:
                mail.folder.append(label)
        if not mail.folder:
            mail.folder = ['uncertain']
        if not features[2]:
            if len(mail.folder) > 1:
                mail.folder = [self.classifier.classes_[np.argmax(proba)]]

        for fold in mail.folder:
            if fold == mail.trueLabel:
                try:
                    self.T[fold]=self.T[fold]+1
                except:
                    self.T[fold]=1
            else:
                try:
                    self.F[fold]=self.F[fold]+1
                except:
                    self.F[fold]=1
        if mail.trueLabel not in mail.folder:
            try:
                self.FR[mail.trueLabel]=self.FR[mail.trueLabel]+1
            except:
                self.FR[mail.trueLabel]=1
        self.update()

        return mail.folder

class Email(object):
    def __init__(self, body=""):
        self.body = body
        if body:
            self.mat = format(body)
        else:
            self.mat = []
        self.folder = []
        self.label = ''
        self.trueLabel = ''
        self.credit = 0
        self.proba = {}
        self.read = False
        self.raw_email = ''
        self.subject=""

    def set_read(self):
        self.read = True

    def set_label(self, label):
        self.label = label

    def set_body(self, body):
        self.body = body
        self.mat = format(body)

    def set_credit(self, credit):
        self.credit = credit

    def set_folder(self, foldername):
        self.folder = [foldername]

    def get_body(self):
        return self.body

    def get_raw(self):
        return self.raw_email

    def get_probab(self):
        return self.proba

    def get_label(self):
        return self.label

    def get_trueLabel(self):
        return self.trueLabel

    def get_credit(self):
        return self.credit

"If user read, forward, reply this email. Will not trigger if current_folder is uncertain or trash"

def activity_yes(email, current_folder):
    global pool, features
    if features[0]:
        email.set_label(current_folder)
        try:
            email.set_credit(1 - email.proba[current_folder])
        except:
            pass
        if email not in pool:
            pool.append(email)
    check_credit()

"If user move this email into a target_folder. Will not trigger if target_folder is uncertain or trash."

def activity_no(email, target_folder):
    global pool, features
    if features[0] or features[1]:
        email.set_label(target_folder)
        try:
            email.set_credit(1 - email.proba[target_folder])
        except:
            email.set_credit(1)
        if email not in pool:
            pool.append(email)
    check_credit()

"Runs every mid-night? Retrain the classifier or not"

def check_credit():
    global pool, my_folder, saturation, lastcan
    can = []
    for label in my_folder.names:
        if label=='uncertain' or label=='trash': continue
        collect = [x for x in pool if x.label == label]
        credits = [x.credit for x in collect]
        many=int(len(pool)/(len(my_folder.names)-2))
        can.extend(list(np.array(collect)[np.argsort(credits)[-many:]]))
    if can!=lastcan:
        my_folder.train(can)
        lastcan=can
    if len(pool) > saturation:
        pool=can
    return True

if __name__ == '__main__':
    global feature_number, hashemail, my_folder, pool, saturation, currentfolder, features, feature_names, new_mails, list_mails, lastcan

    "Global variables, need to be initialized"
    feature_number = 4000
    hashemail = hasher(feature_number)
    my_folder = Folder()
    pool = []
    list_mails=[]
    saturation = 1000
    currentfolder = "uncertain"
    features = [True, True, True]
    feature_names = ['implicit feedback', 'explicit feedback', 'multi-folder']
    "feature[0]: implicit user feedback; feature[1]: explicit user feedback; feature[2]: multi-folder"
    new_mails=[]
    lastcan=[]

    #new mail reading
    readnew()

    # pool of mails with intial labels.
    readfile(Mails_directory)
    # check_credit()
    # GUI
    root = Tk()
    root.title("Mailbox")
    root.minsize(width=1000, height=600)
    app = Application(master=root)
    app.mainloop()
    root.destroy()


from xai.brain.wordbase.nouns._numismatist import _NUMISMATIST

#calss header
class _NUMISMATISTS(_NUMISMATIST, ):
	def __init__(self,):
		_NUMISMATIST.__init__(self)
		self.name = "NUMISMATISTS"
		self.specie = 'nouns'
		self.basic = "numismatist"
		self.jsondata = {}


"""
This file contains the main code for bonsai command line, a script
that can be run to interact with braind in place of Mastermind.

The `main` function in this file will be an entry point for execution
as specified by setup.py.
"""
import os
import time
import logging

import click
from tabulate import tabulate

from bonsai_config import BonsaiConfig
from bonsai_cli.api import BonsaiAPI, BrainServerError
from bonsai_cli import __version__
from bonsai_cli.dotbrains import DotBrains
from bonsai_cli.projfile import ProjectDefault
from bonsai_cli.projfile import ProjectFile, ProjectFileInvalidError

def _verify_required_configuration(bonsai_config):
    """This function verifies that the user's configuration contains
    the information required for interacting with the Bonsai BRAIN api.
    If required configuration is missing, an appropriate error is
    raised as a ClickException.
    """
    messages = []
    missing_config = False

    if not bonsai_config.access_key():
        messages.append("Your access key is not configured.")
        missing_config = True

    if not bonsai_config.username():
        messages.append("Your username is not confgured.")
        missing_config = True

    if missing_config:
        messages.append(
            "Run 'bonsai configure' to update required configuration.")
        raise click.ClickException("\n".join(messages))

def _raise_as_click_exception(*args):
    """This function raises a ClickException with a message that contains
    the specified message and the details of the specified exception.
    This is useful for all of our commands to raise errors to the
    user in a consistent way.

    This function expects to be handed a BrainServerError, an Exception (or
    one of its subclasses), or a message string followed by an Exception.
    """
    if args and len(args) == 1:
        if isinstance(args[0], BrainServerError):
            raise click.ClickException(str(args[0]))
        else:
            raise click.ClickException('An error occurred\n'
                                       'Details: {}'.format(str(args[0])))
    elif args and len(args) > 1:
        raise click.ClickException("{}\nDetails: {}".format(args[0], args[1]))
    else:
        raise click.ClickException("An error occurred")

def _api():
    """
    Convenience function for creating and returning an API object.
    :return: An API object.
    """
    bonsai_config = BonsaiConfig()
    _verify_required_configuration(bonsai_config)
    return BonsaiAPI(access_key=bonsai_config.access_key(),
                     user_name=bonsai_config.username(),
                     api_url=bonsai_config.brain_api_url())

def _default_brain():
    """
    Look up the currently selected brain.
    :return: The default brain from the .brains file
    """
    dotbrains = DotBrains()
    brain = dotbrains.get_default()
    if brain is None:
        raise click.ClickException(
            "Missing brain name. Specify a name with `--brain NAME`.")
    return brain.name

def _brain_fallback(brain, project):
    """
    Implements the fallback options for brain name.
    If a brain is given directly, use it.
    If a project is specified, check that for a brain.
    If neither is given, use .brains locally.
    """
    if brain:
        return brain
    if project:
        pf = ProjectFile.from_file_or_dir(project)
        db = DotBrains(pf.directory())
        b = db.get_default()
        if b:
            return b.name
        else:
            raise click.ClickException(
                "No Brains found with the given project")
    return _default_brain()

def _add_or_default_brain(directory, brain_name):
    """
    Verifies that a .brains file exists for given brain_name.
    Will create .brains file if it doesn't exist
    :param directory: Path to check/create .brains at
    :param brain_name: BRAIN name to set as default
    """
    db = DotBrains(directory)
    brain = db.find(brain_name)
    if brain is None:
        click.echo("Adding {} to '.brains', ".format(brain_name), nl=False)
        db.add(brain_name)
        click.echo("added.")
    else:
        db.set_default(brain)
        click.echo("Brain {} is in '.brains'.".format(brain_name))

def _show_version():
    click.echo("bonsai_cli %s" % __version__)

@click.group()
@click.option('--debug/--no-debug', default=False,
              help='Enable/disable verbose debugging output.')
@click.version_option(version=__version__)
def cli(debug):
    """Command line interface for the Bonsai Artificial Intelligence Engine.
    """
    log_level = logging.DEBUG if debug else logging.ERROR
    logging.basicConfig(level=log_level)

@click.group()
def brain():
    """Create, delete BRAINs."""
    pass

@click.command()
@click.option('--key', help='Provide an access key.')
def configure(key):
    """Authenticate with the BRAIN Server."""
    bonsai_config = BonsaiConfig()

    if key:
        access_key = key
    else:
        access_key_message = ("You can get this access key from "
                              "https://beta.bons.ai/accounts/key.")
        click.echo(access_key_message)

        access_key = click.prompt(
            "Access Key (typing will be hidden)", hide_input=True)

    click.echo("Validating access key...")
    api = BonsaiAPI(access_key=access_key, user_name=None,
                    api_url=bonsai_config.brain_api_url())
    content = None

    try:
        content = api.validate()
    except BrainServerError as e:
        _raise_as_click_exception(e)

    if 'username' not in content:
        raise click.ClickException("Server did not return a username for "
                                   "access key {}".format(access_key))
    username = content['username']
    bonsai_config.update_access_key_and_username(access_key, username)

    click.echo("Success! Your username is {}".format(username))

@click.command()
@click.argument("profile")
@click.option("--url", default=None, help="Set the brain api url.")
def switch(profile, url):
    """Change the active configuration section."""
    bonsai_config = BonsaiConfig()
    bonsai_config.update(Profile=profile)
    if url:
        bonsai_config.update(Url=url)

    url = bonsai_config.brain_api_url()
    click.echo("Success! Switched to {}. "
               "Commands will target: {}".format(profile, url))

@click.group()
def sims():
    """Retrieve information about simulators."""
    pass

@click.command("list", short_help="Lists BRAINs owned by current user.")
def brain_list():
    """Lists BRAINs owned by current user or by the user under a given
    URL.
    """
    try:
        content = _api().list_brains()
        rows = []
        if content and 'brains' in content and len(content['brains']) > 0:
            for item in content['brains']:
                try:
                    name = item['name']
                    state = item['state']
                    rows.append([name, state])
                except KeyError:
                    pass  # If it's missing a field, ignore it.
        if rows:
            table = tabulate(rows, headers=['BRAIN', 'State'],
                             tablefmt='simple')
            click.echo(table)
        else:
            click.echo('The current user has not created any brains.')
    except BrainServerError as e:
        _raise_as_click_exception(e)

# This command is currently disabled,
# brain_create_local is used instead.
@click.command("create")
@click.argument("brain_name")
def brain_create(brain_name):
    """ Creates a BRAIN on the server."""
    brain_create_server(brain_name)

def brain_create_server(brain_name, project_file=None, project_type=None):
    try:
        brain_list = _api().list_brains()
        brains = brain_list['brains']
    except BrainServerError as e:
        click.echo("Error:")
        _raise_as_click_exception(e)

    names = [b['name'] for b in brains]
    if brain_name in names:
        click.echo("Brain {} exists.".format(brain_name))
    else:
        click.echo("Creating {}, ".format(brain_name), nl=False)
        try:
            if project_type:
                _api().create_brain(brain_name, project_type=project_type)
            elif project_file:
                _api().create_brain(brain_name, project_file=project_file)
            else:
                _api().create_brain(brain_name)
        except BrainServerError as e:
            click.echo("error:")
            _raise_as_click_exception(e)
        else:
            click.echo("created.")

def _is_empty_dir(dir):
    for file_or_dir in os.listdir(dir):
        if file_or_dir.startswith("."):
            # Omit .brains, .gitignore, etc.
            pass
        else:
            return False
    return True

@click.command("create",
               short_help="Create a BRAIN and set the default BRAIN.")
@click.argument("brain_name")
@click.option("--project",
              help='Override to target another project directory.')
@click.option("--project-type",
              help='Specify to download and use demo/starter project files '
                   '(e.g. "demos/cartpole").')
def brain_create_local(brain_name, project, project_type):
    """Creates a BRAIN and sets the default BRAIN for future commands."""
    if project_type:
        # Create brain using project_type.

        # Make sure clean directory before continuing.
        cur_dir = os.getcwd()
        if not _is_empty_dir(cur_dir):
            message = ("Refusing to create and download project files using "
                       "project-type in non-empty directory.  Please run in an"
                       " empty directory.")
            raise click.ClickException(message)

        brain_create_server(brain_name, project_type=project_type)
        _brain_download(brain_name, cur_dir)
        bproj = ProjectFile.from_file_or_dir(cur_dir)
    else:
        # Create brain using project file.
        if project:
            bproj = ProjectFile.from_file_or_dir(project)
        else:
            bproj = ProjectFile()
        _validate_project_file(bproj)

        brain_create_server(brain_name, project_file=bproj)

    _add_or_default_brain(bproj.directory(), brain_name)

    ProjectDefault.apply(bproj)
    bproj.save()
    click.echo("Saving project file, saved.")
    click.echo("")
    click.echo("Run 'bonsai push' to push inkling and "
               "training sources into {}.".format(brain_name))

@click.command("push")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help="Override to target another project directory")
def brain_push(brain, project):
    """Uploads project file(s) to a BRAIN."""
    brain = _brain_fallback(brain, project)
    directory = project if project else os.getcwd()

    # Load project file.
    path = ProjectFile.find(directory)
    logging.debug("Reading project file (%s)", path)
    if not path:
        message = ("Unable to locate project file (.bproj) in "
                   "directory={}".format(directory))
        raise click.ClickException(message)
    bproj = ProjectFile(path=path)
    _validate_project_file(bproj)

    # Upload file(s) based on project file
    files = list(bproj._list_paths()) + [bproj.project_path]
    click.echo("Uploading {} file(s) to {}... ".format(len(files), brain))
    logging.debug("Uploading files=%s", files)
    try:
        response = _api().edit_brain(brain, bproj)
    except BrainServerError as e:
        click.echo("error:")
        _raise_as_click_exception(e)
    else:
        num_files = len(response["files"])
        click.echo("Push succeeded.  {} updated with {} files".format(
            brain, num_files))

def _validate_project_file(project_file):
    """ Sends error message to user if project file invalid. """
    try:
        project_file.validate_content()
    except ProjectFileInvalidError as e:
        _raise_as_click_exception(e)

@click.command("load")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help='Override to target another project directory.')
def brain_load(brain, project):
    """Loads an inkling file into the given BRAIN."""

    brain = _brain_fallback(brain, project)
    bproj = ProjectFile.from_file_or_dir(project) if project else ProjectFile()
    inkling_content = None
    try:
        inkling_path = bproj.inkling_file
        inkling_path = os.path.join(bproj.directory(), inkling_path)
    except ProjectFileInvalidError as e:
        _raise_as_click_exception("Inkling Path Lookup", e)

    try:
        # we could check if the file path ends in .ink here
        with open(inkling_path, 'r') as inkling_stream:
            inkling_content = inkling_stream.read()
    except (FileNotFoundError, IsADirectoryError, PermissionError) as e:
        _raise_as_click_exception(
            "Could not open '{}'".format(inkling_path), e)
    except UnicodeDecodeError as e:
        _raise_as_click_exception(
            "Could not read '{}', was it a .ink file?".format(inkling_path), e)

    try:
        content = _api().load_inkling_into_brain(brain_name=brain,
                                                 inkling_code=inkling_content)
    except BrainServerError as e:
        _raise_as_click_exception(e)

    click.echo("Load request succeeded; a new brain version was created.")
    click.echo("Note: 'bonsai load' has been deprecated.  Use `bonsai push` "
               "instead.")
    try:
        click.echo("Connect simulators to {}{} for training".format(
            BonsaiConfig().brain_websocket_url(),
            content["simulator_connect_url"]))
    except KeyError:
        click.echo("But missing the simulator connection path in "
                   "the response.")

@click.command("download")
@click.argument("brain_name")
def brain_download(brain_name):
    """Downloads all the files related to a BRAIN."""
    _brain_download(brain_name, brain_name)

    _add_or_default_brain(brain_name, brain_name)

    click.echo(("Download request succeeded. "
                "Files saved to directory '{}'".format(brain_name)))

def _brain_download(brain_name, dest_dir):
    if os.path.exists(dest_dir) and not _is_empty_dir(dest_dir):
        err_msg = ("Directory '{}' already exists and "
                   "is not an empty directory".format(dest_dir))
        _raise_as_click_exception(err_msg)

    try:
        click.echo("Downloading files...")
        files = _api().get_brain_files(brain_name=brain_name)
        with click.progressbar(files,
                               bar_template='%(label)s %(info)s',
                               label="Saving files...",
                               item_show_func=lambda x: x,
                               show_eta=False,
                               show_pos=True) as files_wrapper:
            for filename in files_wrapper:
                # respect directories
                file_path = os.path.join(dest_dir, filename)
                dirname = os.path.dirname(file_path)
                if dirname and not os.path.exists(dirname):
                    os.makedirs(dirname)

                with open(file_path, "w") as outfile:
                    outfile.write(files[filename])
    except BrainServerError as e:
        _raise_as_click_exception(e)

@click.group("train", short_help="Start and stop training on a BRAIN.")
def brain_train():
    """Start and stop training on a BRAIN, as well as get training
    status information.
    """
    pass

@click.command("list")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help='Override to target another project directory.')
def sims_list(brain, project):
    """List the simulators connected to the BRAIN server."""

    brain = _brain_fallback(brain, project)
    try:
        content = _api().list_simulators(brain)
        rows = []
        for sim_name, sim_details in content.items():
            rows.append([sim_name, 1, 'connected'])

        table = tabulate(rows,
                         headers=['NAME', 'INSTANCES', 'STATUS'],
                         tablefmt='simple')
        click.echo(table)
    except BrainServerError as e:
        _raise_as_click_exception(e)

@click.command("start")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help='Override to target another project directory.')
@click.option("--remote", 'sim_local', flag_value=False, default=True,
              help='Run a simulator remotely on Bonsai\'s servers.')
def brain_train_start(brain, project, sim_local):
    """Trains the specified BRAIN."""

    brain = _brain_fallback(brain, project)
    try:
        content = _api().start_training_brain(brain, sim_local)
        click.echo(
            "When training completes, connect simulators to {}{} "
            "for predictions".format(
                BonsaiConfig().brain_websocket_url(),
                content["simulator_predictions_url"]))
    except KeyError:
        pass
    except BrainServerError as e:
        _raise_as_click_exception(e)

@click.command("status")
@click.option("--brain", help="Override to target another BRAIN.")
@click.option('--json', default=False, is_flag=True,
              help='Output status as json.')
@click.option("--project",
              help='Override to target another project directory.')
def brain_train_status(brain, json, project):
    """Gets training status on the specified BRAIN."""

    brain = _brain_fallback(brain, project)
    status = None
    try:
        status = _api().get_brain_status(brain)
    except BrainServerError as e:
        _raise_as_click_exception(e)

    if json:
        click.echo(status)
    else:
        keys = list(status.keys())
        keys.sort()
        rows = ((k, status[k]) for k in keys)
        table = tabulate(rows,
                         headers=['KEY', 'VALUE'],
                         tablefmt='simple')
        click.echo(table)

@click.command("stop")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help='Override to target another project directory.')
def brain_train_stop(brain, project):
    """Stops training on the specified BRAIN."""

    brain = _brain_fallback(brain, project)
    try:
        _api().stop_training_brain(brain)
        click.echo("Stopped.")
    except BrainServerError as e:
        _raise_as_click_exception(e)

@click.command("log",
               short_help="Display logs from remote training.")
@click.option("--brain",
              help="Override to target another BRAIN.")
@click.option("--project",
              help='Override to target another project directory.')
def brain_log(brain, project):
    """Displays last 1000 lines of the running simulator's output."""
    brain = _brain_fallback(brain, project)

    # TODO: extend for multiple simulators
    sims = ["1"]

    try:
        log_lines = []
        for sim in sims:
            result = _api().get_simulator_logs(brain, "latest", sim)
            log_lines.extend(result)
    except BrainServerError as e:
        _raise_as_click_exception(e)

    click.echo("\n".join(log_lines))

# Compose the commands defined above.
# The top level commands: configure, sims and switch
cli.add_command(configure)
cli.add_command(sims)
cli.add_command(switch)
# T1666 - break out the actions of brain_create_local
# cli.add_command(brain)

# The brain commands: create, list, download, load, and train
cli.add_command(brain_create_local)
cli.add_command(brain_push)
cli.add_command(brain_list)
cli.add_command(brain_download)
cli.add_command(brain_load)
cli.add_command(brain_train)
cli.add_command(brain_log)

# The brain sub commands: create
# This is disabled in favor of brain_create_local.
brain.add_command(brain_create)

# This sims command has one sub command: list
sims.add_command(sims_list)

# The brain train command has three sub commands: start, status, and stop
brain_train.add_command(brain_train_start)
brain_train.add_command(brain_train_status)
brain_train.add_command(brain_train_stop)

def main():
    if os.environ.get('STAGE') == 'dev':
        # Pause while brain gets ready... not necessary in other environments
        time.sleep(3)

    cli()

if __name__ == '__main__':
    raise RuntimeError("run ../bonsai.py instead.")


#Determine if a 9 x 9 Sudoku board is valid. Only the filled cells need to be validated according to the following rules:

#Each row must contain the digits 1-9 without repetition.
#Each column must contain the digits 1-9 without repetition.
#Each of the nine 3 x 3 sub-boxes of the grid must contain the digits 1-9 without repetition.
#Note:

#A Sudoku board (partially filled) could be valid but is not necessarily solvable.
#Only the filled cells need to be validated according to the mentioned rules.

#Example 1:

#Input: board =
#[["5","3",".",".","7",".",".",".","."]
#,["6",".",".","1","9","5",".",".","."]
#,[".","9","8",".",".",".",".","6","."]
#,["8",".",".",".","6",".",".",".","3"]
#,["4",".",".","8",".","3",".",".","1"]
#,["7",".",".",".","2",".",".",".","6"]
#,[".","6",".",".",".",".","2","8","."]
#,[".",".",".","4","1","9",".",".","5"]
#,[".",".",".",".","8",".",".","7","9"]]
#Output: true
#Example 2:

#Input: board =
#[["8","3",".",".","7",".",".",".","."]
#,["6",".",".","1","9","5",".",".","."]
#,[".","9","8",".",".",".",".","6","."]
#,["8",".",".",".","6",".",".",".","3"]
#,["4",".",".","8",".","3",".",".","1"]
#,["7",".",".",".","2",".",".",".","6"]
#,[".","6",".",".",".",".","2","8","."]
#Output: false
#Explanation: Same as Example 1, except with the 5 in the top left corner being modified to 8. Since there are two 8's in the top left 3x3 sub-box, it is invalid.

#Constraints:

#board.length == 9
#board[i].length == 9
#board[i][j] is a digit or '.'.

from typing import List
class Solution:
    def isValidSudoku(self, board: List[List[str]]) -> bool:
        row = [{} for i in range(9)]
        col = [{} for i in range(9)]
        box = [{} for i in range(9)]

        for i in range(9):
            for j in range(9):
                num = board[i][j]
                if num != '.':
                    num = int(num)
                    box_index = (i // 3) * 3 + j // 3

                    row[i][num] = row[i].get(num, 0) + 1
                    col[j][num] = col[j].get(num, 0) + 1
                    box[box_index][num] = box[box_index].get(num, 0) + 1

                    if row[i][num] > 1 or col[i][num] > 1 or box[box_index][num] > 1:
                        return False

        return True


#!/usr/bin/env python
# -*- coding: utf-8 -*-
########################################################################
#
# Copyright (c) 2016 All Rights Reserved
#
########################################################################

"""
File: prepare_train.py
Author: gong(li.gong@outlook.com)
"""
import sys
import argparse
import codecs
from random import shuffle

if __name__ == '__main__':
    readme = """
    """
    parser = argparse.ArgumentParser(description=readme)
    parser.add_argument('-d', "--data",  required=True, help = "data")
    parser.add_argument('-p', "--split", type = float, default = 0.7, help = "corss validation")
    parser.add_argument('-s', "--dsr",   type = float, default = 1.0, help = "down sampling rate")
    args = parser.parse_args()

    pos_inst = []
    neg_inst = []
    for line in open(args.data, 'r'):
        line = ' '.join(line.split())
        label, info = line.split(' ', 1)
        if label == '1':
            pos_inst.append(line.strip())
        else:
            neg_inst.append(line.strip())
    shuffle(pos_inst)
    shuffle(neg_inst)

    if args.split >= 1.0 or args.split <= 0.0:
        if args.dsr >= 1.0 or args.dsr <= 0.0:
            pos_inst.extend(neg_inst)
        else:
            dsr_idx = int(args.dsr * len(neg_inst) + 1)
            pos_inst.extend(neg_inst[:dsr_idx])
        train_outf = open('train.inst', 'w')
        for line in pos_inst:
            train_outf.write('%s\n' % line)
        train_outf.close()
    else:
        pos_idx = int(len(pos_inst) * args.split + 1)
        neg_idx = int(len(neg_inst) * args.split + 1)

        pos_train = pos_inst[:pos_idx]
        neg_train = neg_inst[:neg_idx]
        pos_test  = pos_inst[pos_idx:]
        neg_test  = neg_inst[neg_idx:]

        pos_test.extend(neg_test)
        test_outf = open('test.inst', 'w')
        for line in pos_test:
            test_outf.write('%s\n' % line)
        test_outf.close()

        if args.dsr >= 1.0 or args.dsr <= 0.0:
            pos_train.extend(neg_train)
        else:
            dsr_idx = int(args.dsr * len(neg_train) + 1)
            pos_train.extend(neg_train[:dsr_idx])
        train_outf = open('train.inst', 'w')
        for line in pos_train:
            train_outf.write('%s\n' % line)
        train_outf.close()


import sys, json, requests, datetime
try:
	import sh
	shell=True
except:
	shell=False

try:
	HGR_ID = sys.argv[1]
except IndexError:
	HGR_ID = input("HGR ID: ")

try:
	GEO_ID = int(sys.argv[2])
except IndexError:
	GEO_ID = int(input("Geonames ID: "))

dataset_file = "output/enhanced_dataset.json"
with open(dataset_file,'r',encoding='utf-8') as fp:
	dataset = json.load(fp)

dataset[HGR_ID]['geo'] = {}

R = requests.get("http://api.geonames.org/getJSON?geonameId={}&username=jaguillette".format(GEO_ID))
geoinfo = R.json()

dataset[HGR_ID]['geo']['geonameId'] = "http://www.geonames.org/{}".format(GEO_ID)
dataset[HGR_ID]['geo']['pres_country'] = geoinfo['countryName']
dataset[HGR_ID]['geo']['x_coord'] = geoinfo['lng']
dataset[HGR_ID]['geo']['y_coord'] = geoinfo['lat']
dataset[HGR_ID]['geo']['coded'] = "manually"
dataset[HGR_ID]['geo']['country_code'] = geoinfo['countryCode']
dataset[HGR_ID]['geo']['pres_loc'] = "{}, {}".format(geoinfo['name'],geoinfo['adminName1'])
dataset[HGR_ID]['query_datetime'] = datetime.datetime.strftime(datetime.datetime.now(),"%Y-%m-%dT%H:%M:%S")

with open(dataset_file,'w',encoding='utf-8') as fp:
	json.dump(dataset,fp,sort_keys=True,indent=2)

if shell:
	git = sh.git.bake(_cwd='/home/jeremy/Documents/pro/fung_library/gazetteer/fungHGR/')
	print(git.add(dataset_file))
	print(git.commit(m="Added {} to dataset after confirming it corresponds to GeoNames ID {}".format(dataset[HGR_ID]['name_old_orth'],GEO_ID)))

#print("Georeferencing successfully added for {}.\nLocation currently corresponds to modern location {}, {}.".format(dataset[HGR_ID]['ru_old_orth'],dataset[HGR_ID]['geo']['pres_loc'],dataset[HGR_ID]['geo']['pres_country']))


#    ######       /=================================================\
#   #######       | CLASS EXAMPLE FOR "COMPUTER SCIENCES" (07JCJ**) |
#   ####   \      | https://github.com/squillero/computer-sciences  |
#    ###G  c\     |                                                 |
#    ##     _\    | © 2021 Giovanni Squillero <squillero@polito.it> |
#    |    _/      | Free for personal or classroom use.             |
#    |   _/       \=================================================/

extra_variable = True

sum_ = 0
while extra_variable:
    input_ = input("Oh no: ")

    # if len(input_) == 0:
    # if input_ == '':
    if not input_:  # ie. if bool(input_) == False
        extra_variable = False
    else:
        num = int(input_)
        sum_ += num

    # Alternative solution
    #if input_.isdigit():
    #    num = int(input_)
    #    sum_ += num
    #else:
    #    extra_variable = False

print(f"Sum: {sum_:,}")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May  1 01:18:11 2019

@author: xie
"""

n = int(input())
ans = 0
while n // 5 > 1:
    ans += n // 5
    n = n // 5
else:
    if n >= 5:
        ans += 1

print(ans)

五一思维训练/美丽的阶乘.py


from odoo import api, fields, models

class ProjectTeam(models.Model):

    _name = 'project.team'
    _description = 'Project Team'

    name = fields.Char(required=True)

    lead_id = fields.Many2one(comodel_name='res.users',
                              string='Team Lead',
                              required=True)

    member_ids = fields.Many2many(comodel_name='res.users',
                                  string='Team Members')

    @api.model
    def create(self, values):
        if not values['member_ids']:
            values['member_ids'] = [(4, values['lead_id'])]
        else:
            values['member_ids'][0][2].append(values['lead_id'])

        return super(ProjectTeam, self).create(values)

    @api.multi
    def write(self, values):

        if 'lead_id' not in values and self.lead_id:
            values['lead_id'] = self.lead_id.id

        if 'lead_id' in values:
            if not values.get('member_ids', None):
                values['member_ids'] = [(4, values['lead_id'])]
            else:
                values['member_ids'][0][2].append(values['lead_id'])

        return super(ProjectTeam, self).write(values)


"""This app. will collect the data about Mowbray Court Hotel  from Booking.com
Prepared by Vytautas Bielinskas at Ekistics Property Advisors LLP"""

from selenium import webdriver
options = webdriver.chrome.options.Options()
options.add_argument("--disable-extensions")

chrome_path = r"C:\Users\user\Desktop\Python\JSscrapping\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)

# Import basic libraries
import pandas as pd
import os

# Get the list of reservation days for the next year
def getPeriod():
    from datetime import date, timedelta

    d1 = date(2017,12,15)
    d2 = date(2018,1,10)

    dd = [d1 + timedelta(days = x) for x in range((d2 - d1).days + 1)]

    dateStr = []
    for x in range(0, len(dd), 1):
        dateStr.append(str(dd[x]))
        print(dateStr)

    return dateStr

days = getPeriod()

# Prepare URLs list
urls = []
url_full = 'https://www.booking.com/hotel/gb/the-mowbray-court.en-gb.html?label=gen173nr-1FCAEoggJCAlhYSDNiBW5vcmVmaIgBiAEBmAEuwgEKd2luZG93cyAxMMgBDNgBAegBAfgBC5ICAXmoAgM;sid=b936752e86136715926a341ad92c5f93;all_sr_blocks=183707102_91687460_2_41_0;checkin=2018-01-10;checkout=2018-01-11;dest_id=-2601889;dest_type=city;dist=0;group_adults=2;hapos=1;highlighted_blocks=183707102_91687460_2_41_0;hpos=1;room1=A%2CA;sb_price_type=total;srepoch=1507188269;srfid=432e588605f8385de45b06ab319925e007d99be3X1;srpvid=db6134168cf40284;type=total;ucfs=1&#hotelTmpl'

def getURLs(urls, endday, url_full):
    urls = []
    for i in range(0, endday, 1):
        url_divided = url_full.split("checkin=")
        url_first_part = url_divided[0] + "checkin=" + days[i]
        url_second_part = ";checkout=" + days[i+1] + ((url_divided[1].split("checkout="))[1])[10:]
        url_full = url_first_part + url_second_part
        print(url_full)
        urls.append(url_full)
    return urls

url_list = getURLs(urls, len(days)-1, url_full)

# Crawling the Booking.com pages
for i in range(0, len(days)-1, 1):
    allowing = True;
    driver.get(url_list[i])

    #Preparing free space for new data
    d = {}
    l = []

    #Get reservation date
    reservation_date = driver.find_element_by_class_name("sb-date-field__display").text
    d["(Reservation date)"] = reservation_date

    #Scanners
    rooms = driver.find_elements_by_class_name("js-track-hp-rt-room-name")
    prices = driver.find_elements_by_class_name("rooms-table-room-price")

    #Scanning
    def scanner(rooms, prices, df, allowing):
        if len(rooms) * 2 == len(prices):
            i_price = 0
            for item in range(0, len(rooms), 1):
                #print(rooms[item].text)
                for pricer in range(0, 2, 1):
                    if i_price % 2 == 0:
                        d[rooms[item].text + " 1"] = float(prices[i_price].text.partition(' ')[2])
                        i_price = i_price + 1
                        continue
                    elif i_price % 2 == 1:
                        d[rooms[item].text + " 2"] = float(prices[i_price].text.partition(' ')[2])
                        i_price = i_price + 1
                        continue
        else:
            print("Need change criterions inside the code!")
            allowing = False

        l.append(d)
        return d, allowing

    scanner(rooms, prices, d, allowing)

    #It is time to take Panda to the job and appen the existed database
    if allowing == True and l != []:
        df = pd.DataFrame(l)

    def writeToFile(df):
        #--> If file does not exist write header
        if not os.path.isfile('Hotel_Price_Data.csv'):
            df.to_csv('Hotel_Price_Data.csv')
        else: # --> Otherwise append data to existing database file
            df.to_csv('Hotel_Price_Data.csv', mode = 'a', header = False)
        return(True)

    if allowing == True and l != []:
        writeToFile(df)

print("Finished")


"""
Complements of Scott Robinson
https://stackabuse.com/download-files-with-python/
"""

import os.path
from os import path

if not path.exists('pima-indians-diabetes.csv'):
    import wget
    url = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv'
    wget.download(url, 'pima-indians-diabetes.csv')
    print('')

# Visualize training history
from keras.models import Sequential
from keras.layers import Dense
import matplotlib.pyplot as plt
import numpy
# load pima indians dataset
dataset = numpy.loadtxt("pima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]
# create model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Fit the model
history = model.fit(X, Y, validation_split=0.33, epochs=150, batch_size=10, verbose=0)
# list all data in history
print(history.history.keys())
# summarize history for accuracy
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.savefig('accuracy.png')
plt.close()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.savefig('loss.png')
plt.close()