import numpy as np
import pandas as pd

import gc
import random
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

import seaborn as sns
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.nn.utils.rnn as rnn_utils
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from datetime import datetime as dt
import os
import glob
import pickle
import json
from feature_engineering.feature_factory_for_transformer import FeatureFactoryForTransformer
from experiment.common import get_logger

torch.manual_seed(0)
np.random.seed(0)
is_debug = True
is_make_feature_factory = True
epochs = 25
device = torch.device("cuda")

output_dir = f"../output/{os.path.basename(__file__).replace('.py', '')}/{dt.now().strftime('%Y%m%d%H%M%S')}/"
os.makedirs(output_dir, exist_ok=True)

class SAKTDataset(Dataset):
    def __init__(self, group, n_skill, n_part=8, max_seq=100, is_test=False, predict_mode=False):
        super(SAKTDataset, self).__init__()
        self.max_seq = max_seq
        self.n_skill = n_skill
        self.samples = group
        self.is_test = is_test
        self.n_part = n_part
        self.predict_mode = predict_mode

        self.user_ids = []
        for user_id in group.keys():
            q = group[user_id][("content_id", "content_type_id")]
            if not is_test:
                self.user_ids.append([user_id, -1])
            else:
                is_val = group[user_id]["is_val"]
                for i in range(len(q)):
                    if is_val[i]:
                        self.user_ids.append([user_id, i + 1])

    def __len__(self):
        return len(self.user_ids)

    def __getitem__(self, index):
        user_id = self.user_ids[index][0]
        end = self.user_ids[index][1]
        q_ = self.samples[user_id][("content_id", "content_type_id")]
        part_ = self.samples[user_id]["part"]
        qa_ = self.samples[user_id]["answered_correctly"]

        if not self.is_test:
            seq_len = len(q_)
        else:
            start = np.max([0, end - self.max_seq])
            q_ = q_[start:end]
            part_ = part_[start:end]
            qa_ = qa_[start:end]
            seq_len = len(q_)

        q = np.zeros(self.max_seq, dtype=int)
        part = np.zeros(self.max_seq, dtype=int)
        qa = np.zeros(self.max_seq, dtype=int)
        if seq_len >= self.max_seq:
            q[:] = q_[-self.max_seq:]
            part[:] = part_[-self.max_seq:]
            qa[:] = qa_[-self.max_seq:]
        else:
            q[-seq_len:] = q_
            part[-seq_len:] = part_
            qa[-seq_len:] = qa_

        target_id = q[1:]
        part = part[1:]
        label = qa[1:]

        x = np.zeros(self.max_seq - 1, dtype=int)
        x = q[:-1].copy()
        x += (qa[:-1] == 1) * self.n_skill

        return x, target_id, part, label

class FFN(nn.Module):
    def __init__(self, state_size=200):
        super(FFN, self).__init__()
        self.state_size = state_size

        self.lr1 = nn.Linear(state_size, state_size)
        self.relu = nn.ReLU()
        self.lr2 = nn.Linear(state_size, state_size)
        self.dropout = nn.Dropout(0.2)

    def forward(self, x):
        x = self.lr1(x)
        x = self.relu(x)
        x = self.lr2(x)
        return self.dropout(x)

def future_mask(seq_length):
    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')
    return torch.from_numpy(future_mask)

class SAKTModel(nn.Module):
    def __init__(self, n_skill, max_seq=100, embed_dim=128):
        super(SAKTModel, self).__init__()
        self.n_skill = n_skill
        self.embed_dim = embed_dim

        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
        self.e_embedding = nn.Embedding(n_skill + 1, embed_dim)
        self.part_embedding = nn.Embedding(8, embed_dim)

        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)

        self.dropout = nn.Dropout(0.2)
        self.layer_normal = nn.LayerNorm(embed_dim)

        self.ffn = FFN(embed_dim)
        self.pred = nn.Linear(embed_dim, 1)

    def forward(self, x, question_ids, parts):
        device = x.device
        x = self.embedding(x)

        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)
        pos_x = self.pos_embedding(pos_id)
        x = x + pos_x

        e = self.e_embedding(question_ids)
        p = self.part_embedding(parts)
        e = e + p

        x = x.permute(1, 0, 2)  # x: [bs, s_len, embed] => [s_len, bs, embed]
        e = e.permute(1, 0, 2)
        att_mask = future_mask(x.size(0)).to(device)
        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
        att_output = self.layer_normal(att_output + e)
        att_output = att_output.permute(1, 0, 2)  # att_output: [s_len, bs, embed] => [bs, s_len, embed]

        x = self.ffn(att_output)
        x = self.layer_normal(x + att_output)
        x = self.pred(x)

        return x.squeeze(-1), att_weight

def train_epoch(model, train_iterator, val_iterator, optim, criterion, device="cuda"):
    model.train()

    train_loss = []
    num_corrects = 0
    num_total = 0
    labels = []
    outs = []

    tbar = tqdm(train_iterator)
    for item in tbar:
        x = item[0].to(device).long()
        target_id = item[1].to(device).long()
        part = item[2].to(device).long()
        label = item[3].to(device).float()

        optim.zero_grad()
        output, atten_weight = model(x, target_id, part)
        target_idx = (label.view(-1) >= 0).nonzero()
        loss = criterion(output.view(-1)[target_idx], label.view(-1)[target_idx])
        loss.backward()
        optim.step()
        train_loss.append(loss.item())

        output = output[:, -1]
        label = label[:, -1]
        target_idx = (label.view(-1) >= 0).nonzero()

        pred = (torch.sigmoid(output) >= 0.5).long()

        num_corrects += (pred.view(-1)[target_idx] == label.view(-1)[target_idx]).sum().item()
        num_total += len(label)

        labels.extend(label.view(-1)[target_idx].data.cpu().numpy())
        outs.extend(output.view(-1)[target_idx].data.cpu().numpy())

        tbar.set_description('loss - {:.4f}'.format(loss))

    acc = num_corrects / num_total
    auc = roc_auc_score(labels, outs)
    loss = np.mean(train_loss)

    preds = []
    labels = []
    model.eval()
    i = 0
    for d in tqdm(val_iterator):
        x = d[0].to(device).long()
        target_id = d[1].to(device).long()
        part = d[2].to(device).long()
        label = d[3].to(device).long()

        output, atten_weight = model(x, target_id, part)
        target_idx = (label[:, -1] >= 0).nonzero()
        preds.extend(torch.nn.Sigmoid()(output[:, -1]).view(-1)[target_idx].data.cpu().numpy().tolist())
        labels.extend(label[:, -1].view(-1)[target_idx].data.cpu().numpy())
        i += 1
        if i > 100:
            break
    auc_val = roc_auc_score(labels, preds)

    return loss, acc, auc, auc_val

def main(params: dict):
    import mlflow
    logger = get_logger()
    print("start params={}".format(params))
    # df = pd.read_pickle("../input/riiid-test-answer-prediction/train_merged.pickle").head(30_000_000)
    df = pd.read_pickle("../input/riiid-test-answer-prediction/split10/train_0.pickle").sort_values(
        ["user_id", "timestamp"]).reset_index(drop=True)
    if is_debug:
        df = df.head(3000)
        df.to_csv("aaa.csv")
    df = df.sort_values(["user_id", "timestamp"]).reset_index(drop=True)

    df = df[["user_id", "content_id", "content_type_id", "part", "answered_correctly"]]

    train_idx = []
    val_idx = []
    np.random.seed(0)
#    for _, w_df in df[df["content_type_id"] == 0].groupby("user_id"):
    for _, w_df in df.groupby("user_id"):
        if np.random.random() < 0.1:
            # all val
            val_idx.extend(w_df.index.tolist())
        else:
            train_num = int(len(w_df) * 0.9)
            train_idx.extend(w_df[:train_num].index.tolist())
            val_idx.extend(w_df[train_num:].index.tolist())

    df["is_val"] = 0
    df["is_val"].loc[val_idx] = 1

    ff_for_transformer = FeatureFactoryForTransformer(
        column_config={("content_id", "content_type_id"): {"type": "category"},
                       "part": {"type": "category"}},
        dict_path="../feature_engineering/",
        sequence_length=params["max_seq"],
        logger=logger)
    n_skill = len(ff_for_transformer.embbed_dict[("content_id", "content_type_id")])

    group = ff_for_transformer.all_predict(df)
    dataset_val = SAKTDataset(group,
                              is_test=True,
                              n_skill=n_skill,
                              max_seq=params["max_seq"])

    dataloader_val = DataLoader(dataset_val, batch_size=1024, shuffle=False, num_workers=1)

    ## all_predict
    xs = []
    target_ids = []
    parts = []
    labels = []
    for d in tqdm(dataloader_val):
        xs.extend(d[0].to(device).long().data.cpu().numpy().tolist())
        target_ids.extend(d[1].to(device).long().data.cpu().numpy().tolist())
        parts.extend(d[2].to(device).long().data.cpu().numpy().tolist())
        labels.extend(d[3].to(device).long().data.cpu().numpy().tolist())
    print(xs[0])
    print(target_ids[0])
    print(parts[0])
    print(labels[0])

    ## partial_predict
    ff_for_transformer = FeatureFactoryForTransformer(
        column_config={("content_id", "content_type_id"): {"type": "category"},
                       "part": {"type": "category"}},
        dict_path="../feature_engineering/",
        sequence_length=params["max_seq"],
        logger=logger)

    ff_for_transformer.fit(df.loc[train_idx])

    xs_test = []
    target_ids_test = []
    parts_test = []
    labels_test = []
    for i in tqdm(range(len(df.loc[val_idx]))):
        w_df = df.loc[val_idx[i:i+1]]
        group = ff_for_transformer.partial_predict(w_df.copy())

        dataset_val = SAKTDataset(group,
                                  predict_mode=True,
                                  n_skill=n_skill,
                                  max_seq=params["max_seq"])

        dataloader_val = DataLoader(dataset_val, batch_size=1024, shuffle=False, num_workers=1)
        for d in dataloader_val:
            xs_test.extend(d[0].to(device).long().data.cpu().numpy().tolist())
            target_ids_test.extend(d[1].to(device).long().data.cpu().numpy().tolist())
            parts_test.extend(d[2].to(device).long().data.cpu().numpy().tolist())
            labels_test.extend(d[3].to(device).long().data.cpu().numpy().tolist())
        ff_for_transformer.fit(w_df.copy())

    pd.DataFrame(xs).to_csv("all_xs.csv")
    pd.DataFrame(xs_test).to_csv("partial_xs.csv")
    pd.DataFrame(labels).to_csv("all_labels.csv")
    pd.DataFrame(labels_test).to_csv("partial_labels.csv")
    df.loc[val_idx].to_csv("raw_data.csv")
    assert xs[0] == xs_test[0]
    assert xs == xs_test
    assert target_ids == target_ids_test
    assert parts == parts_test

if __name__ == "__main__":
    params = {"embed_dim": 256,
              "max_seq": 10,
              "lr": 1e-3}
    main(params)


# coding=utf-8
import re

from proxy import Proxy
from .basespider import BaseSpider
from ...items  import IpProxyItem
from datetime import datetime

class Socks_proxySpider(BaseSpider):
    name = 'Socks_proxy'

    def __init__(self, *a, **kwargs):
        super(Socks_proxySpider, self).__init__(*a, **kwargs)

        self.urls = [
            #'https://free-proxy-list.net/uk-proxy.html',
            'https://www.socks-proxy.net/',
        ]
        self.headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:50.0) Gecko/20100101 Firefox/50.0',}
        '''
        self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'en-US,en;q=0.5',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Host': 'www.us-proxy.org',
            'If-Modified-Since': 'Tue, 24 Jan 2017 03:32:01 GMT',
            'Referer': 'http://www.sslproxies.org/',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:50.0) Gecko/20100101 Firefox/50.0',
        }
        '''
        self.init()

    def parse_page(self, response):
        try:
            pattern = pattern = re.compile('
(.*?)	(.*?)	(.*?)	(.*?)	(.*?)	(.*?)	(.*?)	(.*?)
',       re.S)

            infos = re.findall(pattern, response.text)
            if len(infos)==0:
                print("%s empty,please check"%Socks_proxySpider.name)
            for i in infos:
                item = IpProxyItem()
                item['ip'] = i[0]
                item['port'] = i[1]
                #country = re.findall("flags/(.*?).png", i[0])
                item['country'] = i[3] + "_" + i[4]
                anonymity = i[5]
                anonymity = anonymity.strip()
                if 'Elite' in anonymity or 'elite' in anonymity : #'高匿'
                    anonymity = 1
                elif anonymity == 'Transparent':
                    anonymity = 3
                elif anonymity == 'Anonymous' :
                    anonymity = 2
                else:
                    anonymity = 0 #unkown
                item['anonymity'] = anonymity
                Https = i[6]
                item['https'] = Https
                httptype ='https' if i[6].strip()=='Yes' else 'http'
                item['httptype'] = httptype
                item['speed'] = 0
                item['alive'] = 0
                item['valid_time'] = "0000-00-00 00:00:00"
                item['valid_count'] = 0
                item['save_time'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                item['source'] = 'Socks_proxy'
                yield item
        except Exception as e:
            print(e)
            self.logger.warning(e)


def output(f, case, *results):
    print("Case #{0:d}: {1:d}".format(case+1, *results), file=f)

def getNextChunkSize(line, l, i):
    #print ("getNextChunkSize: START", line, l, i)
    if not (i < l):
        #print ("getNextChunkSize: END", 0)
        return 0
    size = 1
    while i+size < l and line[i+size] == line[i+size-1]:
        size += 1
    #print ("getNextChunkSize: END", size)
    return size

def findSwaps(line):
    #print ("findSwaps: START", line)
    count = 0
    i = 0
    l = len(line)
    if i < l and line[0] == '-':
        i += getNextChunkSize(line, l, i)
        count += 1
    while i < l:
        sizePositive = getNextChunkSize(line, l, i)
        sizeNegative = getNextChunkSize(line, l, i+sizePositive)
        i += sizePositive + sizeNegative
        if sizeNegative:
            count += 2
    #print ("findSwaps: END", count)
    return count

def main():
    fName = 'R1P2_large0'
    with open(fName + '.txt') as fIn, open(fName + '.out.txt', 'w') as fOut:
        fIn.readline()
        for i, line in enumerate(fIn):
            output( fOut, i, findSwaps(line.strip()) )

if __name__ == '__main__':
    main()


import os
import torch
import pandas as pd
from torch_geometric.utils import to_undirected
from torch_geometric.data import Data, InMemoryDataset, download_url

class KarateClub(InMemoryDataset):
    url = 'https://raw.githubusercontent.com/benedekrozemberczki/karateclub/master/dataset/node_level'
    available_datasets = {
        'twitch',
        'facebook',
        'github',
        'deezer',
        'lastfm',
        'wikipedia'
    }

    def __init__(self, root, name, transform=None, pre_transform=None):
        self.name = name.lower()
        assert self.name in self.available_datasets

        super().__init__(root, transform, pre_transform)
        self.data, self.slices = torch.load(self.processed_paths[0])

    @property
    def raw_dir(self):
        return os.path.join(self.root, self.name, 'raw')

    @property
    def raw_file_names(self):
        return ['edges.csv', 'features.csv', 'target.csv']

    @property
    def processed_dir(self):
        return os.path.join(self.root, self.name, 'processed')

    @property
    def processed_file_names(self):
        return 'data.pt'

    def download(self):
        for part in ['edges', 'features', 'target']:
            download_url(f'{self.url}/{self.name}/{part}.csv', self.raw_dir)

    def process(self):
        target_file = os.path.join(self.raw_dir, self.raw_file_names[2])
        y = pd.read_csv(target_file)['target']
        y = torch.from_numpy(y.to_numpy(dtype=int))
        num_nodes = len(y)

        edge_file = os.path.join(self.raw_dir, self.raw_file_names[0])
        edge_index = pd.read_csv(edge_file)
        edge_index = torch.from_numpy(edge_index.to_numpy()).t().contiguous()
        edge_index = to_undirected(edge_index, num_nodes)  # undirected edges

        feature_file = os.path.join(self.raw_dir, self.raw_file_names[1])
        x = pd.read_csv(feature_file).drop_duplicates()
        x = x.pivot(index='node_id', columns='feature_id', values='value').fillna(0)
        x = x.reindex(range(num_nodes), fill_value=0)
        x = torch.from_numpy(x.to_numpy()).float()

        data = Data(x=x, edge_index=edge_index, y=y, num_nodes=num_nodes)

        if self.pre_transform is not None:
            data = self.pre_transform(data)

        torch.save(self.collate([data]), self.processed_paths[0])

    def __repr__(self):
        return f'KarateClub-{self.name}()'


import json
import requests
from flask_babel import gettext
from config import TRANSLATE_API_KEY

def yandex_translate(text, sourceLang, destLang):
    if TRANSLATE_API_KEY == "":
        return gettext('Error: translation service not configured.')
    try:
        # translate
        params = {'key': TRANSLATE_API_KEY,
                  'text': text.encode("utf-8"),
                  'lang': sourceLang + '-' + destLang}
        response = requests.post('https://translate.yandex.net/api/v1.5/tr.json/translate', data=params)
        response = response.json()
        return response['text'][0]
    except:
    #    print (response)
        return gettext('Error: Unexpected error.')


import time

import logging; logger = logging.getLogger("robot." + __name__)

from robots.lowlevel import *
from robots.action import *

class PR2StateManager:
    """ This class helps with the state of the PR2 robot.

    Currently provides:
     * getjoint(name) to get the value of a given joint
     * distance2obstacle() that returns the distance in meter to the closest obstacle in
     front of the robot.
     """

    def __init__(self, robot):
        self.robot = robot

    @helper("state")
    def isseen(self, obj):
        """ Returns true if an object is currently seen by the
        robot.

        The semantics of "seen" can vary: in the current implementation,
        it means that the 2D ARToolkit tag has been seen in the
        last 2 seconds.

        :param object: the object (SPARK ID) to check
        :returns: true if the object is visible, false otherwise.
        """
        if not obj:
            return False

        try:
            spark = self.robot.poco_modules["spark"]
        except KeyError:
            logger.warning("'isseen' requires 'spark' Genom module, but it"
                           " does not seem to be started. Assuming object"
                           "not visible.")
            return False

        state = spark.poster("PositionsInfopositionsStatus")

        lastseen = -1
        for i in range(len(state)):
            if state[i] == obj:
                lastseen = int(state[i+5])

        if lastseen == -1:
            raise Exception("Object " + obj + " is not a special SPARK object (cf hri_knowledge.cpp:117. Argh!)")

        if lastseen == 0:
            return False
        else:
            seenago = int(time.time()) - lastseen
            logger.debug("%s last seen %dsec ago" % (obj, seenago))
            if seenago < 2:
                return True
            return False

    @helper("state")
    def getjoint(self, name):

        if self.robot.supports(ROS):
            import rospy
            from sensor_msgs.msg import JointState

            data = rospy.wait_for_message("/joint_states", JointState)
            idx = data.name.index(name)
            return data.position[idx]
        else:
            return 0.0

    @helper("state")
    def fingerpressed(self, hand = "right"):
        """ Returns true if the pression sensors of the gripper (right by
        default) detect something.

        Warning: no real calibration made! This only takes one pressure sensor
        as input, and is based on experimental testing.
        """

        import roslib; roslib.load_manifest('pyrobots_actionlib')
        import rospy
        from pr2_msgs.msg import PressureState

        data = rospy.wait_for_message("/pressure/" + hand[0] + "_gripper_motor", PressureState)
        av1 = sum(data.l_finger_tip) / len(data.l_finger_tip)
        av2 = sum(data.r_finger_tip) / len(data.r_finger_tip)
        av = (av1 + av2) / 2
        if av > 3500:
            return True
        else:
            return False

    @helper("state")
    def distance2obstacle(self):

        import rospy
        from sensor_msgs.msg import LaserScan

        data = rospy.wait_for_message("/base_scan", LaserScan)

        nb_scans = int((data.angle_max - data.angle_min) / data.angle_increment)

        return data.ranges[nb_scans / 2]


# 부분집합 중 원소의 합이 10인 부분집합을 모두 구해라.

def backtrack(a,k,input):
    c=[False]*2 # True /False를 받아줄 배열
    ncands = 2 # 후보자 군은 2개/ True , False

    if k==input: # 자리수가 부분집합을 구하려는 input과 같으면
        process_solution(k) # 출력하는 함수로 출력한다.
    else: # 같지 않으면..
        k+=1 # 자리수를 하나 더하고
        make_candidates(k,input,c,ncands) # c 배열 t/f 넣어주는 함수
        for i in range(ncands):
            a[k] = c[i]
            backtrack(a,k,input)

def make_candidates(k,n,c,ncands): # c 배열에 t/f 를 넣어준다.
    c[0] = True
    c[1] = False
    # ncands = 2

def process_solution(k): # 결과를 출력하는 함수
    for i in range(0,k+1): # t/f 만 정해줬고, 어떤 수를 출력할지 안정해 줬으니
        if a[i] == True:  # T 인 것을 출력하기 위해서 선언
            print(i, end=' ')
    print()
a = [False]*11
backtrack(a,0,10)


"""For a given a string(str) and a character X, write a function to remove all the occurrences of X from the given string.
The input string will remain unchanged if the given character(X) doesn't exist in the input string.
Input Format:
The first line of input contains a string without any leading and trailing spaces.

The second line of input contains a character(X) without any leading and trailing spaces."""

def removeRepeatOccurChar(str, chr):
    if len(str) == 0:
        return str
    if chr not in str:
        return str
    ans = " "

    for i in range(len(str)):
        if str[i] == chr:
            continue
        else:
            ans += str[i]
    return ans

str = input()
chr = input()
ans = removeRepeatOccurChar(str, chr)
print(ans)


#!/usr/bin/python3

"""
@file: data_controller.py
@brief: 数据控制器
@author: feihu1996.cn
@date: 18-08-20
@version: 1.0
"""

from collections import defaultdict

from utils.response import Response
from utils.items import item_dict
from utils.logger import Logger
from controllers.base_controller import BaseController
from models.data_model import DataModel
from models.user_model import UserModel
from models.auth_model import AuthModel

class DataController(BaseController):
    """
    数据控制器
    """
    def __init__(self, *args, **kwargs):
        super(DataController, self).__init__(*args, **kwargs)
        self.data = DataModel()
        self.user = UserModel()
        self.auth = AuthModel()

    def get_data(self):
        """
        获取图表数据
        """
        dims = self.args.get('dims', None)
        if not dims:
            return Response.responseJson(Response.INPUT_EMPTY, msg="输入指标为空")

        # 验证用户是否有权限查看当前指标数据
        isAuth = self.auth_require(dims_name=dims)
        Logger.debug_logger().debug(isAuth)
        if not isAuth:
            return Response.responseJson(Response.NO_AUTH)

        params = {
            "start_date": self.args.get("start_date", ""),
            "end_date": self.args.get("end_date", "")
        }

        # 调用模型方法获取数据
        results = self.data.get_data(dims=dims, args=params)

        print(results)

        # 格式化数据
        option = {
            "name": dims,
            "type": "bar",
            "data": results,
            "title": item_dict.get(dims, dims)
        }
        results = self.format_data(option=option)

        #option = {
        #    "title": "hpshSystem",
        #    "xAxis": ["衬衫","羊毛衫","雪纺衫","裤子","高跟鞋","袜子"],
        #    "series": [{
        #        "name": '销量',
        #        "data": [5, 20, 36, 10, 10, 20]
        #    }]
        #}

        return Response.responseJson(Response.SUCCESS, data=results, msg="获取数据成功")

    def format_data(self, option=None):
        """
        格式化数据
        """
        data = option.get("data", [])
        name = option.get("name", "")
        e_type = option.get("type", "bar")
        title = option.get("title", name)

        dict_data = defaultdict(list)
        for item in data:
            key = item.get("fdate", None)
            dict_data[key].append(item.get(name))

        # 对dict_data进行升序排序
        dict_data = sorted(dict_data.items(), key=lambda x: x[0], reverse=False)

        axis_x = []
        axis_y = []

        for key, obj in dict_data:
            axis_x.append(key)
            axis_y.append(sum([item for item in obj if isinstance(item, (float, int))]))

        results = {
            "xAxis": axis_x,
            "title": title,
            "series": [{
                "name": name,
                "data": axis_y,
                "type": e_type
            }]
        }

        return results

    def auth_require(self, dims_name=None):
        """
            根据用户级别控制其可以查看的指标数据
        """
        # 获取用户工号、等级信息
        fwork_id = self.session.get("fwork_id", None)
        user_info = self.user.get_user_info(fwork_id=fwork_id)
        flevel_id = user_info.get("flevel_id", 0)

        # 获取指标的等级要求信息
        auth_level = self.auth.get_auth_level(dims_name=dims_name)
        auth_level_id = auth_level.get("flevel_id", 100000)

        print("flevel_id:"+str(flevel_id))
        print("auth_level_id:"+str(auth_level_id))

        # 比较是否有权限
        if int(flevel_id) < int(auth_level_id):
            return False

        return True


from lxml import etree
import sys
import ftp_knocker
from targetDB import Database
from models.storageCom import DataManage

__author__ = 'pr0c'

class Menu(object):

    def run_menu():
        getItRight = 0 #point system to make sure user uses an option given
        while getItRight == 0:
            menuChoice = input("\n\nHow shall we proceed?\n"+
                "Type (e) to start an exploit on the ftp servers found\n"+
                "Type (p) to print the database to a file in csv format\n"+
                "Type (a) to scan a network again\n"+
                "Type (x) to exit the program\n"+
                ":/>\t")
            if menuChoice.upper() == 'E' or 'EXPLOIT':
                # give the option to bruteforce ftp server username and password
                getItRight+=1
            elif menuChoice.upper() == 'P' or 'PRINT':
                _print_targets()
                getItRight+=1
            elif menuChoice.upper() == 'A' or 'AGAIN':
                ftp_knocker.main()
                getItRight+=1
            elif menuChoice.upper() == 'X' or 'EXIT':
                getItRight+=1
                sys.exit()
            else:
                x = 0

    def _print_targets():
        """
        print xml

        root of tree with each element just an icrement of the next|| childElementId == childElementId++
        childSubElement = IP or MAC or Hostname
        subElementText = IP.ip_addr or MAC.mac_addr or Hostname.hostname

        """
        x = 0
        entries = Database.find()
        #root = etree.Element('targets')
        #roo.append(etree.Element(str(x)))
        for entry in entries:
            print(entry)
            #print("IP: {}, Mac Address: {}, Hostname: {}".format(entry['ip_addr'], entry['mac_addr'], entry['hostname']))

    def _view_blog():
        blog_to_see = input("Enter the ID of the blog you'd like to read: ")
        blog = DataManage.from_mongo(blog_to_see)
        posts = blog.get_posts()
        for post in posts:
            print("Date: {}, Title: {}\n\n{}".format(post['created_date'], post['title'], post['content']))


import numpy as np
import tensorflow as tf

#load data with numpy and predict your final score
xy = np.loadtxt('data/data-01-test-score.csv', delimiter=',', dtype=np.float32)
x_data = xy[:, 0:-1]
y_data = xy[:, [-1]]

X = tf.placeholder(tf.float32, shape=[None, 3])
Y = tf.placeholder(tf.float32, shape=[None, 1])
W = tf.Variable(tf.random_normal([3, 1]), name = "weight")
b = tf.Variable(tf.random_normal([1]), name = "bias")

hypothesis = tf.matmul(X, W) +b
cost = tf.reduce_mean(tf.square(hypothesis-Y))

optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-5)
train = optimizer.minimize(cost)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

for i in range(2001):
    y_pred, cost_val, _ = sess.run([hypothesis, cost, train], feed_dict={X:x_data, Y:y_data})

print("당신의 예측 점수는 ", sess.run(hypothesis, feed_dict={X:[[100,100,100]]}), "점 입니다.")


import pandas as pd
from collections import Counter
from math import log2
import datetime

def base_entropy_masked(seq_list, base_set, base_idx):
    # entropy analysis
    base_list = [seq[base_idx] for seq in seq_list]
    freq_dict = Counter(base_list)
    n_seq = 0
    total_null = 0
    for key in freq_dict:
        if key in ['-', 'N']:
            total_null += freq_dict[key]
            continue
        n_seq += freq_dict[key]
    H = 0
    for base in base_set:
        if freq_dict[base] == 0 or base in ['-', 'N']:
            continue
        P = freq_dict[base]/n_seq
        H -= log2(P) * P
    return H, total_null/len(base_list)

import os
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation
from Bio import SeqIO

# get all sequence records for the specified genbank file
def annotate_ISM_positions(mapped_reference_index, reference_genbank_name="data/covid-19-genbank.gb", output_dir='figures'):
    recs = [rec for rec in SeqIO.parse(reference_genbank_name, "genbank")]

    gene_dict = {}
    for rec in recs:
        feats = [feat for feat in rec.features if feat.type == "CDS"]
        for feat in feats:
            key = (feat.location.start.position, feat.location.end.position)
            content = '{}: {}'.format(feat.qualifiers['protein_id'][0], feat.qualifiers['product'][0])
            gene_dict[key] = content

    with open('{}/ISM_gene_in_reference.csv'.format(output_dir), 'w+') as fw:
        output = '{:^20} | {:^10} | {:^70}'.format('Reference position', 'Entropy', 'Gene')
        print(output)
        print('-'*20 + '   ' + '-'*10 + '   ' + '-'* 70)
        fw.write(output + '\n')
        for index, entropy in mapped_reference_index:
            result = []
            for key in gene_dict:
                if index > key[0] and index <= key[1]:
                    result.append(gene_dict[key])
            output = '{:>20} | {:.8f} | {:<70}'.format(index, entropy, ', '.join(result))
            print(output + '\n')
            fw.write(output + '\n')

def regional_analysis(df, region):
    df_tmp = df[df['country/region'] == region]
    total = df_tmp.sum()['count']
    ISM_list = list(df_tmp['ISM'].values)
    dict_freq = {}
    for ISM in ISM_list:
        freq = df_tmp[df_tmp['ISM'] == ISM]['count'].values[0]
        date = df_tmp[df_tmp['ISM'] == ISM]['date'].values[0]
        dict_freq[ISM] = (date, freq)
    return dict_freq
def ISM_filter(dict_freq, threshold):
    res_dict = {'OTHER': [0, 0]}
    total = sum([dict_freq[ISM][1] for ISM in dict_freq])
    for ISM in dict_freq:
        if dict_freq[ISM][1]/total < threshold:
            res_dict['OTHER'] = [0, res_dict['OTHER'][1] + dict_freq[ISM][1]]
        else:
            res_dict[ISM] = [dict_freq[ISM][0], dict_freq[ISM][1] + res_dict.get(ISM, [0, 0])[1]]
    if res_dict['OTHER'][1] == 0:
        del res_dict['OTHER']
    return res_dict
def statewise_analysis(df, state):
    df_tmp = df[df['division'] == state]
    total = df_tmp.sum()['count']
    ISM_list = list(df_tmp['ISM'].values)
    dict_freq = {}
    for ISM in ISM_list:
        freq = df_tmp[df_tmp['ISM'] == ISM]['count'].values[0]
        date = df_tmp[df_tmp['ISM'] == ISM]['date'].values[0]
        dict_freq[ISM] = (date, freq)
    return dict_freq
def regional_typical_ISM_bar(ISM, region_list, region_raw_count):
    res = []
    for region in region_list:
        if ISM in region_raw_count[region]:
            total = sum([region_raw_count[region][item][1] for item in region_raw_count[region]])
            res.append(region_raw_count[region][ISM][1]/total)
        else:
            res.append(0)
    return res

def time_subset(ISM_df, start, end):
    # start: exclusive, end: inclusive
    filter_date_1 = pd.to_datetime(start, format='%Y-%m-%d')
    filter_date_2 = pd.to_datetime(end, format='%Y-%m-%d')
    return ISM_df[(filter_date_1 < ISM_df['date']) & (ISM_df['date'] <= filter_date_2)]

def frequency_count(df):
    df_date = df.groupby(['country/region','ISM']).agg({'date': 'min'}).reset_index()
    df_ISM = df.groupby('country/region')['ISM'].value_counts().to_frame()
    df_ISM = df_ISM.rename(columns={'ISM': 'count'}).reset_index()
    df_ISM_date = df_ISM.join(df_date.set_index(['country/region','ISM']), on = ['country/region','ISM'],how = 'left')
    return df_ISM_date

def regional_timeseries_analysis(df, region):
    df_tmp = df[df['country/region'] == region]
    total = df_tmp.sum()['count']
    ISM_list = list(df_tmp['ISM'].values)
    dict_freq = {}
    for ISM in ISM_list:
        freq = df_tmp[df_tmp['ISM'] == ISM]['count'].values[0]
        date = df_tmp[df_tmp['ISM'] == ISM]['date'].values[0]
        dict_freq[ISM] = (date, freq)
    return dict_freq

def typical_ISM_regional_growth(ISM, region, count_list, date_list):
    ISM_growth_list = []
    for i in range(len(count_list)):
        regional_dict_freq = count_list[i][region]
        if ISM in regional_dict_freq and regional_dict_freq[ISM][1] != 0:
            count = regional_dict_freq[ISM][1]
            freq = count/sum([regional_dict_freq[ISM][1] for ISM in regional_dict_freq])
        else:
            count, freq = 0, 0
        ISM_growth_list.append((count, freq))

    return ISM_growth_list

def typical_ISM_total_count(ISM_df, start_date, ISM):
    df_tmp = time_subset(ISM_df, '2019-11-01', str(start_date))
    if df_tmp.shape[0] == 0:
        return 0
    df_tmp_tmp = frequency_count(df_tmp)
    return df_tmp_tmp[df_tmp_tmp['ISM'] == ISM]['count'].sum()


import pygame; pygame.init()
from same_adventure.models.character import Player, Orc
from same_adventure.models.scenario import Scenario
from same_adventure.config.window import Window
from same_adventure.models.collision import person_collision

scenario = Scenario('res/bg.png', 'res/music.mp3')
window = Window('Same adventure', (500, 480), pygame.time.Clock(), scenario)

# Objects in window
objects = [Player(250, 400, 64, 64), Orc(100, 410, 60, 60, 450)]
kills = 0

scenario.play_music()

run = True
while run:
    window.clock.tick(45) # 60 images per second

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False

    for obj in objects:
        if type(obj).__name__ == 'Player':
            for bullet in obj.bullets:
                if bullet not in objects:
                    objects.append(bullet)
        if type(obj).__name__ == 'Orc':
            if not obj.visible:
                objects.remove(obj); kills += 1
                for kill in range(0, kills + 1):
                    objects.append(Orc(100 + (5 * kill), 410, 60, 60, 450))

    person_collision(objects, window.display)
    window.redraw_game_window(objects)

    for obj in objects:
        if type(obj).__name__ == 'Player':
            for bullet in obj.bullets:
                if bullet.to_remove:
                    objects.remove(bullet)
                    obj.remove_bullet(bullet)
                if bullet.is_collision(objects, window.display):
                    obj.score += 1

pygame.quit()


import pdfplumber
import tempfile
import pathlib
from PIL import Image, ImageDraw, ImageFont
import pytesseract
from pdf2image import convert_from_path
import re
import os
import pandas as pd
import xlwt
import openpyxl
import xlsxwriter

def test():

    titanicdf = pd.read_csv(r"C:\Users\Brett\Desktop\titanic.csv")
    print(titanicdf[["Age", "Sex"]].head(20))
    return

def pdfdispatch():

    srcpath = pathlib.Path(r'C:\Users\Brett\Desktop\Final Offers')
    targetpath = pathlib.Path(r'C:\Users\Brett\Desktop\test')

    # Make sure paths exists and are directories
    if srcpath.is_dir() == True and targetpath.is_dir() == True:
        # Process PDF files within source directory
        newfilepaths = []
        for xpath in srcpath.iterdir():         #iterate through filepaths in directory
            if xpath.suffix.upper() == ".PDF":  #use of upper() makes it case insensitive

                print(xpath.resolve())          #print full path
                #pdfdirecttotext(xpath, targetpath)
                newimgpath = pdftoimage(xpath, targetpath)
                newpdfpath = createsearchablepdf(newimgpath, targetpath)
                newfilepaths.append({'newpdfpath': newpdfpath, 'newimagepath': newimgpath})

        for x in newfilepaths:
            #imagetotext(xpath, targetpath)
            highlights = pdfexcavate(x['newpdfpath'])
            highlightpdfimage(highlights, x['newpdfpath'], x['newimagepath'])

    else:
        print('Source or target paths do not exist, or are not directories.')

    return

def createsearchablepdf(srcpath, targetpath):

    #srcpath = pathlib.Path(r'C:\Users\Brett\Desktop\test\Wood Partners 03.13.20 PG 1 Searchable1.png')
    #targetpath = pathlib.Path(r'C:\Users\Brett\Desktop\test\Wood Partners 03.13.20 PG 1 Searchable1.pdf')
    im = Image.open(srcpath.resolve())
    newpdf = pytesseract.image_to_pdf_or_hocr(im, extension='pdf')
    newpdffilepath = pathlib.Path.joinpath(targetpath, srcpath.stem + ' from_image.pdf')
    with open(newpdffilepath, 'w+b') as f:
        f.write(newpdf)  # pdf type is bytes by default
    #pytesseract.image_to_string(Image.open(imgpath.resolve())))
    #pytesseract.image_to_string(im)

    ## ocrmypdf.ocr()
    ## ocrmypdf.hocrtransform(srcpath.resolve(), targetpath.resolve())
    return newpdffilepath

def pdftoimage(xpath, targetpath):

    # save temp image files in temp dir, delete them after we are finished

    with tempfile.TemporaryDirectory(dir=targetpath) as temp_dir:
        # convert pdf to multiple image
        images = convert_from_path(xpath.resolve(), output_folder=temp_dir)
        # save images to temporary directory
        temp_images = []
        for i in range(len(images)):
            image_path = f'{temp_dir}\{i}.jpg'
            images[i].save(image_path, 'JPEG')
            temp_images.append(image_path)
         #   image2searchablepdf(image_path)

        # read images into pillow.Image
        imgs = list(map(Image.open, temp_images))
        # find minimum width of images
        min_img_width = min(i.width for i in imgs)
        # find total height of all images
        total_height = 0
        for i, img in enumerate(imgs):
            total_height += imgs[i].height
        # create new image object with width and total height
        merged_image = Image.new(imgs[0].mode, (min_img_width, total_height))
        # paste images together one by one
        y = 0

        for img in imgs:
            merged_image.paste(img, (0, y))
            y += img.height

        # save merged image
        targetfile = targetpath.joinpath(xpath.stem + '.jpg')
        merged_image.save(targetfile, 'JPEG')

        return targetfile

def imagetotext(imgpath, targetdirpath):

    # Open the file in append mode so that
    # All contents of all images are added to the same file
    targetfilepath = pathlib.Path.joinpath(targetdirpath, imgpath.stem + '.txt')
    f = open(targetfilepath, "a")

    # Recognize the text as string in image using pytesserct
    text = str(pytesseract.image_to_string(Image.open(imgpath.resolve())))
    print(text.upper())
    text = text.replace('-\n', '')  #replaces hypens added at line endings

    # Finally, write the processed text to the file.
    f.write(text)

    # Close the file after writing all the text.
    f.close()
    return

def pdfdirecttotext(pdfpath, targetdirpath):

    #this code pulls text directly from searchable pdf's; doesn't work with scanned pdfs
    # pdffile = open(r'C:\Users\Brett\Desktop\Final Offers\Anton 03.13.20.pdf', 'rb')
    # pdfname = r'C:\Users\Brett\Desktop\Final Offers\MBK 03.13.20.pdf'
    # outputfolder = r'C:\Users\Brett\Desktop\test'

    targetfilepath = pathlib.Path.joinpath(targetdirpath, pdfpath.stem + 'DIRECT.txt')
    f = open(targetfilepath, "a")

    with pdfplumber .open(pdfpath.resolve()) as pdf:
        for page in pdf.pages:
            f.write(str(page.extract_text()))

    f.close()
    # pdfread = p2.PdfFileReader(pdffile)
    # x = pdfread.getPage(2)
    # print(x.extractText())
    return

def regexprocessing(pdfpath, mirrorimagepath):
    highlights = []
    fulltext, wordregistry = pdfexcavate(pdfpath)
    myregexdict, searchthesepattersndict = regexdict()

    results = pattern_search(searchthesepattersndict, fulltext, wordregistry)

    bidsummary = []
    # purchase price algorithm

    for r in results:

        if r['pattern_name']== 'purchaseprice' and r['sequence']==0:  #for now, we're using the first match in the pattern
            z = re.search(myregexdict['dollar_amount']['regex'], r['matches_with'])
            if z:
                bidsummary.append({
                    'pattern_name':r['pattern_name'],
                    'sequence': r['sequence'],
                    'matches_with': r['matches_with'],
                    'result': z.group(0)
                })
                print("found purhcase price:", z.group(0))
                print(myregexdict['dollar_amount']['regex'], r['matches_with'])

        # deposit algorithm
        if r['pattern_name'] == 'deposit' and r['sequence']==0:
            z = re.search(myregexdict['dollar_amount']['regex'], r['matches_with'])
            if z:
                bidsummary.append(
                    {'pattern_name': r['pattern_name'], 'sequence': r['sequence'], 'matches_with': r['matches_with'], 'result': z.group(0)})

        # dd period algorithm
        if r['pattern_name']== 'dd' and r['sequence']==0:
            z = re.search(myregexdict['time_period']['regex'], r['matches_with'])
            if z:
                bidsummary.append({'pattern_name':r['pattern_name'], 'sequence': r['sequence'], 'matches_with': r['matches_with'], 'result': z.group(0)})

        # closing period algorithm
        if r['pattern_name']== 'closing' and r['sequence']==0:
            z = re.search(myregexdict['time_period']['regex'], r['matches_with'])
            if z:
                bidsummary.append({'pattern_name':r['pattern_name'], 'sequence': r['sequence'], 'matches_with': r['matches_with'], 'result': z.group(0)})

    bidsummarydf = pd.DataFrame(bidsummary)

    print(bidsummarydf)

    # writer = pd.ExcelWriter(r"C:\Users\Brett\Desktop\bidsummary.xlsx", engine='xlsxwriter')
    # bidsummarydf.to_excel(writer, sheet_name='Sheet1')
    # writer.save()

        #highlightpdfimage(results, pdfpath, mirrorimagepath)
    return

def regexdict():
    #predefined regex code:

    dollar_amount  = """\$ {0,}[0-9]{1,3}(,[0-9]{3}){0,}"""
    first_dollaramt_insentence = """([^.])+""" + dollar_amount
    numberswrittenout = """((one|eleven|ten|two|twelve|twenty|three|thirteen|thirty|fourteen|forty|four|fifteen|fifty|five|sixteen|sixty|six|seventeen|seventy|seven|eighteen|eighty|eight|nineteen|ninety|nine|hundred|thousand|million|billion)[- ]{0,1})+"""
    time_period = """(""" + numberswrittenout +"""|\(?[0-9]{1,3}[) -])+((business|calendar)[ -])?(day|month|week|year)s?"""
    first_timeperiod_insentence = """([^.])+""" + time_period
    monthsofyear_regex = """(january|jan\.{0,1}|february|feb\.{0,1}|march|mar\.{0,1}|april|apr\.{0,1}|may|june|jun\.{0,1}|july|jul\.{0,1}|august|aug\.{0,1}|september|sept\.{0,1}|october|oct\.{0,1}|november|nov\.{0,1}|december|dec\.{0,1})"""
    date_slashtype_regex = """(\d{1,2}/\d{1,2}/\d{4})"""
    date_writtenout_regex = """(""" + monthsofyear_regex + """ \d{1,2}, \d{4}""" + """)"""
    date_eithertype_regex = """(""" + date_writtenout_regex + """|""" + date_slashtype_regex + """)"""
    first_date_or_timeperiod_insentence = """([^.])+""" + """(""" + time_period + """|""" +date_eithertype_regex + """)"""
    deposit_regex = """((initial|first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth) )?deposit"""
    price_regex = """((purchase) )?price""" #################################
    dd_regex = """(dd|due diligence|feasibility|inspection)( period)?"""
    closing_regex = """((closing)( (period|date))?|close of escrow)"""

    myregexdict = dict(
        dollar_amount = {'regex': dollar_amount},
        first_dollaramt_insentence = {'regex': first_dollaramt_insentence}, #############################################
        numberswrittenout = {'regex': numberswrittenout},
        time_period = {'regex': time_period},
        first_timeperiod_insentence = {'regex': first_timeperiod_insentence},
        monthsofyear_regex = {'regex': monthsofyear_regex},
        date_slashtype_regex = {'regex': date_slashtype_regex},
        date_writtenout_regex = {'regex': date_writtenout_regex},
        date_eithertype_regex = {'regex': date_eithertype_regex},
        first_date_or_timeperiod_insentence = {'regex': first_date_or_timeperiod_insentence},
        deposit_regex = {'regex': deposit_regex},
        price_regex = {'regex': price_regex},
        dd_regex = {'regex': dd_regex},
        closing_regex = {'regex': closing_regex}
    )

    searchthesepatternsdict = dict(
        purchaseprice = {'regex': price_regex + first_dollaramt_insentence},
        dateeither={'regex': date_eithertype_regex},
        dd = {'regex': dd_regex + first_date_or_timeperiod_insentence},
        closing = {'regex': closing_regex + first_date_or_timeperiod_insentence},
        deposit = {'regex': deposit_regex + first_dollaramt_insentence},
 #       writtendates = {'regex': date_writtenout_regex}
 #       financingcontingency = {'regex': 'price'},
 #       alldollaramounts = {'regex': dollar_amount},
  #      dd ={'regex': 'due diligence |feasibility|inspection period'},
   #     longhand_numbers = {'regex': numberswrittenout},
        time_period = {'regex': time_period}
        )

    return myregexdict, searchthesepatternsdict

def pdfexcavate(pdfpath):

    # use pdfplumber's page.extract_words() function to get a list of nested dictionaries
    # containing each word in a PDF and its location.  We perform regex searches on a string
    # composed of these words.  We will use the position of a match within the string to trace words back
    # the nested dictionaries from which they arose, which contain their position on the page, allowing us to highlight
    # them in the document image.

    with pdfplumber .open(pdfpath.resolve()) as pdf:
        startindex = 0
        wordregistry = []
        for page in pdf.pages:
            for wd in page.extract_words():
                wd['text'] = wd['text'] + ' ' #adds a space after each word
                wd['pdfpage'] = page.page_number
                wd['startindex'] = startindex #the starting point of this particular word in the fulltext string created later
                wordlength = len(wd['text'])
                wd['endindex'] = startindex + wordlength -1  #the end of this word within the fulltext string
                wordregistry.append(wd) #appends words with their x,y coordinates from each page of the PDF
                startindex = wd['endindex'] + 1

        #string together all the words, separating by spaces.
        fulltext=''
        for w in wordregistry:
            fulltext = fulltext + w['text']
            #print(w['startindex'], w['text'], w['x0'], w['bottom'], w['x1'], w['top'], w['startindex'], w['endindex'])

        return fulltext, wordregistry

def pattern_search(patterns, fulltext, wordregistry):

    results = []  #load with information used to apply highlights to the document image
    match_id = 0
    for p in patterns:
        sequence = 0
        for match in re.finditer(patterns[p]['regex'], fulltext, re.IGNORECASE):
                match_id+=1
                s = match.start()
                e = match.end()

                #cross reference s and e from fulltext string to positions on the pdf document
                #NOTE THAT A MATCH MAY BE A SUBSET OF A WORD, IN WHICH CASE THE STARTING POINT OF THE MATCH
                #WILL COME AFTER THE STARTING POINT OF THE WORD!  A MATCH MAY ALSO STRADDLE MULTIPLE WORDS

                x0, y0, x1, y1 = 0, 0, 0, 0
                for word in wordregistry:
                    success = 0
                    #test if this word contains the beginning of a match
                    if match.start() >= word['startindex'] and match.start() <= word['endindex']:
                        #estimate starting position on document
                        x0 = word['x0']
                        y0 = word['bottom']
                        if match.end() <= word['endindex']:
                            x1 = word['x1']
                            y1 = word['top']
                        else:
                            x1 = word['x1']
                            y1 = word['top']
                        success=1

                    if match.start() < word['startindex'] and match.end() >= word['startindex']:
                        x0 = word['x0']
                        y0 = word['bottom']
                        if match.end() <= word['endindex']:
                            x1 = word['x1']
                            y1 = word['top']
                        else:
                            x1 = word['x1']
                            y1 = word['top']
                        success=1

                    if success == 1:
                        if len(word['text']) > 0:
                            extension = (x1 - x0) / len(word['text'])  ##since the edge of the last character is typically beyond x1, we need to extend the highlight slightly
                        else:
                            extension = 1

                    if success == 1:
                        results.append({
                            'pattern_name': p,
                            'sequence': sequence,
                            'match_id': match_id,
                            'matches_with': fulltext[s:e],
                            'x0': x0, 'y0': y0,
                            'x1': x1 + extension,'y1': y1
                        })
                        sequence+=1
    return results

def highlightpdfimage(highlights, pdfpath, mirrorimagepath):

    base = Image.open(mirrorimagepath.resolve()).convert('RGBA')
    overlay = Image.new('RGBA', base.size, (255, 255, 255, 0))
    with pdfplumber.open(pdfpath.resolve()) as pdf:
        for page in pdf.pages:
            if page.width > 0:
                scale = base.size[0]/page.width # I'm using this to translate pdf plumber coordinates to image coordinates
            else: scale = 1  #defaults to 1 in case there is no page width

            count =0
            for h in highlights:
                    shape = [(h['x0']*scale, h['y0']*scale), (h['x1']*scale, h['y1']*scale)]
                    d = ImageDraw.Draw(overlay)
                    if count%2==0:
                        d.rectangle(shape, fill=(255,0,0,32)) # last number in fill specifies opacity!
                    else:
                        d.rectangle(shape, fill=(0, 0, 255, 32))  # last number in fill specifies opacity!

                    d = ImageDraw.Draw(base, 'RGBA')
                    count += 1

            #font = ImageFont.truetype("arial.ttf", 270)
            # draw text, half opacity
            #d.text((10, 10), "Hello", font=font, fill=(255, 0, 0, 255))
            # draw text, full opacity
            #d.text((10, 60), "World", fill=(255, 0, 0, 255))
            out = Image.alpha_composite(base, overlay)
            #out.show()
            out.save(pathlib.Path.joinpath(mirrorimagepath.parent, mirrorimagepath.stem + ' highlighted.png'), 'PNG')

    return

## testing to run on-demmand to compare results

#bidinterpreter.pdfextract.createsearchablepdf()
#bidinterpreter.pdfextract.pdfdispatch()
#bidinterpreter.pdfextract.imagetotext()
#------
# upload_path = "/home/dave/projects/bidinterpreter/uploads/1"
# target_pdf = f"{upload_path}/Griffis 2019.4.19 LOI Courtyards at 65th.pdf.png.processed.pdf"
# target_img = f"{upload_path}/Griffis 2019.4.19 LOI Courtyards at 65th.pdf.png"

# highlights = regexprocessing(pathlib.Path(target_pdf), pathlib.Path(target_img))
#-----
# highlightpdfimage(highlights, pathlib.Path(target_pdf), pathlib.Path(target_img))

#bidinterpreter.pdfextract.regexdict()
#bidinterpreter.pdfextract.test()


# This is how you work with PDFs with Python
import PyPDF2

with open('dummy.pdf', 'rb') as file:         # 'rb' stands for 'Read-Binary'
    reader = PyPDF2.PdfFileReader(file)       # Read from PDF file
    page = reader.getPage(0)                  # Get page info
    print(page)                               # Print page info
    page.rotateCounterClockwise(90)           # Rotate page
    writer = PyPDF2.PdfFileWriter()           # Write to PDF file
    writer.addPage(page)                      # Add page to PDF file
    with open('tilt.pdf', 'wb') as new_file:  # 'wb' stands for 'Write-Binary
        writer.write(new_file)                # Write to file to save changes


#!/usr/bin/python3
#!/home/pi/.local/lib/python3.7/site-packages

import sys
sys.path.append("/home/pi/.local/lib/python3.7/site-packages")
sys.path.append("/home/pi")
from picamera import PiCamera
from time import sleep
import datetime
import gpiozero as gpz
import json
#from skimage.io import imread #read images
#from skimage.transform import resize
from PIL import Image

import psutil
#SVM dependancies:
import numpy as np
from sklearn import svm
from skimage.feature import hog
import numpy as np
import pickle
from PIL import Image
import cv2
import random
import os

####Device Parameters##############################
#Modify the config.json file w/ correct parameters#
###################################################
with open('/home/pi/ML_Corn/config.json') as f:   #
    device_params = json.load(f)                  #
                                                  #
devicekey = device_params['Hologram ID']          #
cameraID = device_params['Camera ID']             #
crop = device_params['Crop']                      #
treatment = device_params['Treatment']            #
testML = device_params['test ML']
mode = device_params['mode']
time_zone = device_params['timezone']
farm = device_params['farm']
rep = device_params['rep']
version = device_params['ver']
###################################################

tz_splitted = time_zone.split("/")

from Hologram.HologramCloud import HologramCloud
credentials = {'devicekey':devicekey} #'6r)^]p]Q'} #Hologram device key from hologram.io
try:
    hologram = HologramCloud(credentials, network='cellular',authentication_type='csrpsk') #Connect to Hologram CLoud, change network to cellular to connect to LTE
except:
    print("Could not connect or open socket to Hologram Cloud");
sum_RSSI = 0.0
sum_quality = 0.0
num_samples = 5
camera = PiCamera()
curr = datetime.datetime.now()
currDate = curr.strftime("%d_%m_%Y")
currTime = curr.strftime("%H_%M_%S")
timeStamp = curr.strftime("%d_%m_%Y_%H_%M_%S")
file = '/home/pi/images/' + curr.strftime("%d_%m_%Y_%H_%M_%S") + '.jpg'

camera.resolution=(2592,1944)
camera.rotation = 180
camera.capture(file)
camera.close()

cpu = gpz.CPUTemperature()
cpu_temp = cpu.temperature
print("CPU Temperature:", cpu_temp)

disk = psutil.disk_usage('/')
disk_percent_used = disk.percent
disk_free = disk.free / 2**30
################################################
#IMPORTANT: If you want to utilize the SVM ML model on images taken in the field
#            change the "test ML" value to: false  instead of true
if testML=="true":
    #Test Corn Images:
    if (crop == "corn"):
        street = os.listdir('/home/pi/ML_Corn/CornTestImages')
        randomImage = random.randrange(len(street)) #pick a random test image
        file = '/home/pi/ML_Corn/CornTestImages/'+street[randomImage]
        with open('/home/pi/ML_Corn/testImageLabels.json') as f:
            correct_labels = json.load(f)
        actual_label = correct_labels[street[randomImage]]
        im = cv2.imread(file)
    else: #test soybean images
        testImages = np.load("/home/pi/ML_Corn/soybeanTestImages.npy")
        testLabels = np.load("/home/pi/ML_Corn/soybeanTestLabels.npy")
        randomImage = random.randrange(len(testLabels)) #pick a random test image
        actual_label = np.argmax(testLabels[randomImage])#one-hot encoded
        im = testImages[randomImage]
else:#read image just captured by camera
    im = cv2.imread(file)
###SVM model for Corn, TFLite model for soybeans
if (crop == "corn"):
    im = Image.fromarray(im,"RGB")
    #print("Resizing image...")
    im_final = im.resize((324,216))#Model was trained on 324(w)x216(h) images
    #print("Captured and Resized Image!")
    ########Convert RGB Image to LUX representation #############
    #print("Converting Image to LUX format...")
    img0 = np.asarray(im_final)
    LUX = np.zeros((img0.shape[0],img0.shape[1], 3),dtype=np.float32 )
    row = img0.shape[0]-1 #img0 = image we are manipulating. row = height of image. columns = width of image
    columns = img0.shape[1]-1
    for i in range(row):
        for j in range(columns):

          r = img0[i][j][2]
          g = img0[i][j][1]
          b = img0[i][j][0]

          l = ((((r+1)**0.3)) * (((g+1)**0.6)) * (((b+1)**0.1)) - 1).astype(np.float32) #r.val[0] = r,need to replace power function
          M = 256
          if r>l:
            u = ((M/2)*((r+1)/(l+1))).astype(np.float32)
            x = ((M/2)*((b+1)/(l+1))).astype(np.float32)
          else:
            u = (M-((M/2)*((l+1)/(r+1)))).astype(np.float32)
            x = (M-((M/2)*((l+1)/(b+1)))).astype(np.float32)

          LUX[i][j][0] = int(l)
          LUX[i][j][1] = int(u)
          LUX[i][j][2] = int(x)
    #print("Finished converting image to LUX!")
    ######Extract Histogram of Gradients from LUX Image ##########
    #print("Extracting HOG Features...")
    pixels_per_cell = (9,9)
    cells_per_block = (24,36)
    fd = hog(LUX, orientations=9, pixels_per_cell=pixels_per_cell,cells_per_block=cells_per_block,  block_norm='L2', transform_sqrt=False, visualize= False, multichannel=True) #16,16
    ######Load the Model and make a prediction#############
    #print("Loading SVM model and making prediction...")
    svm_file = '/home/pi/ML_Corn/svm_tuned_modelFINAL.sav'
    loaded_model = pickle.load(open(svm_file, 'rb'))
    fd = fd.reshape(1,-1) #required if only one sample
    result = loaded_model.predict_proba(fd)
    #print("Evaluating Image: ", file)
    #print("Predicted Water Stress Probabilites:",result)
    waterStressLevel = int(np.argmax(result))+1
    #print("Predicted WS:", waterStressLevel)
    try:
        print("Actual WS: ", actual_label)
    except NameError:
        pass
else: #DL model for soybeans
    #TFlite Dependancies: don't import unless in Soybeans
    from tensorflow import lite as tflite
    import tensorflow as tf
    #im = Image.fromarray(im,"RGB")
    im_final = cv2.resize(im,(224,224))#Model was trained on 224x224 images
    #Preprocess Image for Mobilenet_V2
    #print(im_final)
    im_final=tf.keras.applications.mobilenet_v2.preprocess_input(im_final)
    # Load TFLite model and allocate tensors.
    im_final= np.array(im_final,dtype=np.float32)
    #print("allocating tensors")
    interpreter = tflite.Interpreter(model_path="/home/pi/ML_Corn/soybeanTFLiteModel.tflite")
    interpreter.allocate_tensors()
    # Get input and output tensors.
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    Xtest = np.array(im_final, dtype = np.float32)

    #test model
    input_data = np.expand_dims(Xtest,axis=0)
    interpreter.set_tensor(input_details[0]['index'], input_data)
    #print("invoking interpreter")
    interpreter.invoke()
    # The function `get_tensor()` returns a copy of the tensor data.
    # Use `tensor()` in order to get a pointer to the tensor.
    output_data = interpreter.get_tensor(output_details[0]['index'])
    results = np.squeeze(output_data)
    #print(results)
    waterStressLevel = int(np.argmax(results))
    percentConfident = results[waterStressLevel]*100
    #print("Predicted WS Level: ", waterStressLevel)
    #print("Confidence: ",percentConfident)
    try:
        print("Actual WS: ", actual_label)
    except NameError:
        pass
if (crop =="corn"):
    if testML=="true":
        data = {
                    "DEV_ID": devicekey,
                    #"CAM_ID": cameraID,
                    "CPU_TEMP": cpu_temp,
                    "DATE":currDate,
                    "TIME":currTime,
                    "SD_free":round(disk_free,2),
                    "FARM":farm,
                    "REP":rep,
                    "CROP": crop,
                    "TRT": treatment,
                    "MODE": mode,
                    "ZONE": tz_splitted[1],
                    "FILE":street[randomImage],
                    "P_WS_1": round(result[0][0],4),
                    "P_WS_2": round(result[0][1],4),
                    "P_WS_3": round(result[0][2],4),
                    "P_WS":waterStressLevel,
                    "A_WS": actual_label,
                    "VER": version,
                }
    else:
        data = {
                    "DEV_ID": devicekey,
                    #"CAM_ID": cameraID,
                    "CPU_TEMP": cpu_temp,
                    "DATE":currDate,
                    "TIME":currTime,
                    "SD_free":round(disk_free,2),
                    "FARM":farm,
                    "REP":rep,
                    "CROP": crop,
                    "TRT": treatment,
                    "MODE": mode,
                    "ZONE": tz_splitted[1],
                    "FILE":timeStamp,
                    "P_WS_1": round(result[0][0],4),
                    "P_WS_2": round(result[0][1],4),
                    "P_WS_3": round(result[0][2],4),
                    "P_WS":waterStressLevel,
                    "VER": version,
                }
else:#different payloads for Soybean cams
    if testML=="true":
        data = {
                    "DEV_ID": devicekey,
                    #"CAM_ID": cameraID,
                    "CPU_TEMP": cpu_temp,
                    "DATE":currDate,
                    "TIME":currTime,
                    "SD_free":round(disk_free,2),
                    "FARM":farm,
                    "REP":rep,
                    "CROP": crop,
                    "TRT": treatment,
                    "MODE": mode,
                    "ZONE": tz_splitted[1],
                    "FILE":"Test Image",
                    "P_WS_0": round(float(results[0]),4),
                    "P_WS_1": round(float(results[1]),4),
                    "P_WS_2": round(float(results[2]),4),
                    "P_WS_3": round(float(results[3]),4),
                    "P_WS_4": round(float(results[4]),4),
                    "P_WS_5": round(float(results[5]),4),
                    "P_WS":waterStressLevel,
                    "A_WS": int(actual_label),
                    "VER": version,
                }
    else:
        data = {
                    "DEV_ID": devicekey,
                    #"CAM_ID": cameraID,
                    "CPU_TEMP": cpu_temp,
                    "DATE":currDate,
                    "TIME":currTime,
                    "SD_free":round(disk_free,2),
                    "FARM":farm,
                    "REP":rep,
                    "CROP": crop,
                    "TRT": treatment,
                    "MODE": mode,
                    "ZONE": tz_splitted[1],
                    "FILE":timeStamp,
                    "P_WS_0": round(float(results[0]),4),
                    "P_WS_1": round(float(results[1]),4),
                    "P_WS_2": round(float(results[2]),4),
                    "P_WS_3": round(float(results[3]),4),
                    "P_WS_4": round(float(results[4]),4),
                    "P_WS_5": round(float(results[5]),4),
                    "P_WS":waterStressLevel,
                    "VER": version,
                }
#print(data)
with open('data.txt', 'a') as outfile:
            json.dump(data, outfile)
            outfile.write('\n')

try:
    recv = hologram.sendMessage(data) # Send message to hologram cloud
    print("Recieved Code:",recv)
    print("0 Means Succesful Transmission")
except:
    print("Not network")

sleep(20)


import numpy as np
import dataloader

def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0)

def beam_search_decoder(encoder_model, decoder_model, target_i2c, target_w2i, input_seq, decoding_json, k=10):
    """ Beam search decoder
        reference from: https://github.com/mmehdig/lm_beam_search/blob/master/beam_search.py
    """
    try:
        decoder_model.get_layer(name='attention')
        has_attention = True
    except:
        has_attention = False

    D = dataloader.load_dict_from_json(decoding_json)
    #D = dataloader.load_dict_from_json('word_models/word_decoding.json')
    max_decoder_seq_length = D['max_decoder_seq_length'] - 2
    ##num_unique_target_chars = 4 #don't need this
    #states_value = encoder_model.predict(input_seq)
    vals = encoder_model.predict(input_seq)

    if has_attention:
        states_value = vals[1:]
        enc_outs = vals[0]
    else:
        states_value = vals

    stop_condition = False
    decoded_sentence = ''
    ##target_seq = np.zeros((1,1))
    ##target_seq[0,0] = char_ind
    cumlogprob = 0.0

    # (log(1), initialize_of_zeros)
    k_beam = [(cumlogprob, np.array([target_w2i['START_']] + [target_w2i['_END']] * max_decoder_seq_length).reshape(1, -1))]
    ended = []

    # l : point on target sentence to predict
    for l in range(max_decoder_seq_length):
        all_k_beams = []

        for prob, sent_predict in k_beam:
            if has_attention:
                output_tokens = decoder_model.predict([sent_predict] + states_value + [enc_outs])[0]
            else:
                output_tokens = decoder_model.predict([sent_predict] + states_value)[0]

        # top k!
        possible_k = output_tokens[0, l, :].argsort()[-k:][::-1]

        # add to all possible candidates for k-beams
        all_k_beams += [
            (
                (sum(np.log(output_tokens[0, i, sent_predict[0,i+1]]) for i in range(l)) + np.log(output_tokens[0,l,next_wid])) / (l+1),
                np.array(list(sent_predict[0,:l+1]) + [next_wid] + [target_w2i['_END']] * (max_decoder_seq_length -l-1)).reshape(1,-1)
            )
            for next_wid in possible_k
        ]

        k_beam = sorted(all_k_beams, key=lambda x: (x[0], x[1]))[-k:]

        # check if there is sentence that has already ended
        to_pop = []
        for i in reversed(range(len(k_beam))):
            sent = k_beam[i][1]
            pred_token = sent[0,l+1]

            #if pred_token in [target_w2i['_END'], target_w2i['entity.n.01']]:
            if pred_token == target_w2i['_END']:
                ended.append(k_beam[i])
                del k_beam[i]

            elif pred_token in sent[0,:l+1]:
                sent[0,l+1] = target_w2i['_END']
                ended += [(k_beam[i][0], np.array(sent).reshape(1,-1))]
                del k_beam[i]

    k_beam = sorted(k_beam + ended, key=lambda x: (x[0], x[1]), reverse=True)[:k]

    #scores = softmax([beam[0] for beam in k_beam])
    k_sents = [' '.join([target_i2c[token] for token in beam[1][0][1:]]) for beam in k_beam]

    return list(set(k_sents))


import codecs
import sys

def character_split(input_file, output_file):
	input_data = codecs.open(input_file, 'r', 'utf-8')
	output_data = codecs.open(output_file, 'w', 'utf-8')
	for line in input_data.readlines():
		words = line.split()
		if not words:
			output_data.write('\n')
		elif words[2] == 'B' or words[2] == 'M':
			output_data.write(words[0])
		elif words[2] == 'E' or words[2] == 'S':
			output_data.write(words[0]+'\t')
	input_data.close()
	output_data.close()

if __name__ == '__main__':
	if len(sys.argv) != 3:
		print("pls use: python make_crf_test_data.py input output")
		sys.exit()
	input_file = sys.argv[1]
	output_file = sys.argv[2]
	character_split(input_file, output_file)


"""Web Scraper für Zeitungsartikel"""
import datetime
from urllib.request import urlopen
import pymongo as p
from bs4 import BeautifulSoup
from bs4 import SoupStrainer

def get_element(soup, tag, klasse):
    """Return an Element(found by class) out of given soup"""
    element = soup.find(tag, {"class" : klasse})
    if element is not None:
        return element
    else:
        return "NotFound"

def filter_text(element):
    """Return text from given element"""
    text = ""
    for content in element.contents:
        if isinstance(content, str):
            text += content
        else:
            text += filter_text(content)
    return text.replace("mehr...", "").replace("\n", "")

def get_news_from_pnp():
    """Zeitungsartikel der PNP holen und speichern"""
    __url = urlopen("http://www.pnp.de/")

    soup = BeautifulSoup(__url, 'html.parser',
                         parse_only=SoupStrainer("div", {"class" : "em_cnt_box_body"}))

    client = p.MongoClient("10.10.11.202:27017")
    db_con = client.Chat.news

    alle_artikel = list()

    for div in soup.find_all("div", {"class" : "em_cnt_box_body"}):
        ort = get_element(div, "h4", "em_cnt_aufmacher_dz")
        titel = get_element(div, "h3", "em_cnt_aufmacher_hz")
        bild_url = get_element(div, "img", "em_img_border")
        text = get_element(div, "p", "em_text")
        url = get_element(div, "a", "em_left")
        if(ort != "NotFound" and
           titel != "NotFound" and
           bild_url != "NotFound" and
           text != "NotFound" and
           url != "NotFound"):
            artikel = {
                "von" : "pnp",
                "ort" : ort.string,
                "titel" : titel.string,
                "bild_url" : bild_url["src"],
                "text" : text.string,
                "url" : "http://www.pnp.de" +  url["href"],
                "datumHinzug" : str(getattr(datetime.datetime.utcnow(), "year")) +
                                "-" + str(getattr(datetime.datetime.utcnow(), "month")) +
                                "-" + str(getattr(datetime.datetime.utcnow(), "day")),
                "zaehler" : 0
            }
            alle_artikel.append(artikel)

    zaehler = len(alle_artikel) - 1
    while zaehler >= 0:
        if db_con.find({"von" : "pnp"}).count() == 0:
            artikel_zaehler = 1
        else:
            artikel_zaehler = (db_con.find({"von" : "pnp"}).sort(
                "zaehler", -1))[0].get("zaehler") + 1
        alle_artikel[zaehler]["zaehler"] = artikel_zaehler
        if db_con.find({"url" : alle_artikel[zaehler]["url"]}).count() == 0:
            db_con.insert_one(alle_artikel[zaehler])
        zaehler = zaehler - 1

def get_news_from_postillion():
    """Zeitungsartikel von Postillion holen und speichern"""
    url = urlopen("http://www.der-postillon.com/")
    soup = BeautifulSoup(url, 'html.parser', parse_only=SoupStrainer("div",
                                                                     {"class" : "main section"}))
    client = p.MongoClient("10.10.11.202:27017")
    db_con = client.Chat.news

    alle_artikel = list()

    for div in soup.find_all("div", {"class" : "post-outer"}):
        post = get_element(div, "div", "post-body")

#        titel = div.h3.a.string
#        print(titel)
#        text = filter_text(post)
#        print(text)
#        bild_url = post.a.img["src"]
#        print(bild_url)
#        url = div.h3.a["href"]
#        print(url)

        artikel = {
            "von" : "postillion",
            "titel" : div.h3.a.string,
            "text" : filter_text(post),
            "bild_url" : post.a.img["src"],
            "url" : div.h3.a["href"],
            "datumHinzug" : str(getattr(datetime.datetime.utcnow(), "year")) +
                            "-" + str(getattr(datetime.datetime.utcnow(), "month")) +
                            "-" + str(getattr(datetime.datetime.utcnow(), "day")),
            "zaehler" : 0,
            "ort" : ""
            }
        alle_artikel.append(artikel)

    zaehler = len(alle_artikel) - 1
    while zaehler >= 0:
        if db_con.find({"von" : "postillion"}).count() == 0:
            artikel_zaehler = 1
        else:
            artikel_zaehler = (db_con.find(
                {"von" : "postillion"}).sort("zaehler", -1))[0].get("zaehler") + 1
        alle_artikel[zaehler]["zaehler"] = artikel_zaehler
        if(db_con.find({"url" : alle_artikel[zaehler]["url"]}).count() == 0 and
           "Newsticker" not in alle_artikel[zaehler]["titel"]):
            db_con.insert_one(alle_artikel[zaehler])
        zaehler = zaehler - 1
get_news_from_pnp()
get_news_from_postillion()


#!/usr/bin/env python

import sys
import re

#from matplotlib import rc
#
#rc('text', usetex=True)
#rc('font', **{'family':'serif', 'serif':['Computer Modern Roman'],
#                                'monospace': ['Computer Modern Typewriter']})

##################################################
# CONFIG
##################################################

# which tasks to show (return true for desired tasks)
def myfilter(x):
    return True

fillcols = ['#ff0000', '#00ff00', '#0000ff', '#ffff00', '#ff00ff', '#00ffff']
edgecols = ['#000000', '#000000', '#000000', '#000000', '#000000', '#000000']
coldict = dict()
nextcol = 0

# which color to use for each task (fillcolor, edgecolor)
def getColor(text):
    global nextcol

    spl = text.split();
    if len(spl) > 1:
        text = spl[0]

    # hard-code task names to colors
    if text.startswith("read"): return ['#000000', '#000000']

    # distribute colors cyclically
    if text in coldict:
        colidx = coldict[text]
    else:
        colidx = nextcol
        nextcol = (nextcol + 1) % len(fillcols)
        coldict[text] = colidx
    return [fillcols[colidx], edgecols[colidx]]

# if any text is to be attached to the task
def getText(text):
    return []

def getExtra(text):
  m = re.match(r'.* tasks: ([0-9]+)', text)
  if m:
    return m.group(1)
  return []

##################################################
def load_file(filename):
    fh = open(filename, "r")
    out = list()
    pattern = re.compile('([0-9]+): ([0-9]+) ([0-9]+) (.*)')
    mpipattern = re.compile('([0-9]+) ([0-9]+): ([0-9]+) ([0-9]+) (.*)')
    cpattern = re.compile('(.*) \((.*)\)')
    while True:
        line = fh.readline()
        if not line:
          break
        if line.startswith('LOG 2'):
            continue
        # [thread id]: [start] [length] [name [perf]]
        g = pattern.match(line)
        if g != None:
          name = g.group(4)
          gg = cpattern.match(name)
          cache = 0
          if gg != None:
              name = gg.group(1)
              cache = int(gg.group(2))
          out.append({'name': name.strip(),
                      'procid': 0,
                      'threadid': int(g.group(1)),
                      'start': int(g.group(2)),
                      'length': int(g.group(3)),
                      'end': int(g.group(2)) + int(g.group(3)),
                      'cache': cache})
          continue

        # [node number] [thread id]: [start] [length] [name [perf]]
        g = mpipattern.match(line)
        if g != None:
          name = g.group(5)
          gg = cpattern.match(name)
          cache = 0
          if gg != None:
            name = gg.group(1)
            cache = int(gg.group(2))
          out.append({'name': name.strip(),
                      'procid': int(g.group(1)),
                      'threadid': int(g.group(2)),
                      'start': int(g.group(3)),
                      'length': int(g.group(4)),
                      'end': int(g.group(3)) + int(g.group(4)),
                      'cache': cache})
          continue

        # [thread id] [start] [length] [name]
        w = line.split()
        if len(w) == 4:
          out.append({'name': w[3],
                      'procid': 0,
                      'threadid': int(w[0]),
                      'start':  float(w[1])*1e6,
                      'length': float(w[2])*1e6,
                      'end': (float(w[1])+float(w[2]))*1e6,
                      'cache': 0})
          continue

        # parse error
        print("Error parsing line: ", line)
    fh.close()
    return out

def getMedian(numericValues):
  theValues = sorted(numericValues)

  if len(theValues) % 2 == 1:
    return theValues[(len(theValues)+1)//2-1]
  else:
    lower = theValues[len(theValues)//2-1]
    upper = theValues[len(theValues)//2]

    return (float(lower + upper)) / 2

##################################################
# plot

import matplotlib.pyplot as plt

def drawBox(x0,x1,y0,y1,col):
    plt.fill([x0, x1, x0], [y0, (y0+y1)/2, y1], fc=col[0], ec=col[1], linewidth=.5)

def drawText(x,y,text):
    plt.text(x,y,text,horizontalalignment='center',verticalalignment='center',fontsize=taskfontsize)

def drawTask(x0,x1,y0,y1,orgtext):
    drawBox(x0,x1,y0,y1,getColor(orgtext))
    text = getText(orgtext)
    if len(text) == 2:
        drawText((x0+x1)/2,(y0+y1)/2 + barHeight/4,text[0])
        drawText((x0+x1)/2,(y0+y1)/2 - barHeight/4,text[1])
    elif len(text) == 3:
        drawText((x0+x1)/2,(y0+y1)/2 + barHeight/4,text[0])
        drawText((x0+x1)/2,(y0+y1)/2,text[1])
        drawText((x0+x1)/2,(y0+y1)/2 - barHeight/4,text[2])
    elif len(text) == 1:
        drawText((x0+x1)/2,(y0+y1)/2,text[0])
    extra = getExtra(orgtext)
    if len(extra) > 0:
        drawText(x0, y1+height*0.1, extra)

def drawPlot(tasks):
    height = 1.0
    barHeight = height * 0.8

    for task in tasks:
        x0 = task['start']
        x1 = x0 + task['length']
        y0 = task['threadid'] * height - barHeight / 2.0
        y1 = y0 + barHeight
        drawTask(x0, x1, y0, y1, task['name'])

    padding = barHeight/2
    plt.ylim([ -barHeight/2 - padding, (numThreads-1)*height + barHeight/2 + padding]);
    yticks=range(0, numThreads)
    plt.yticks(yticks, yticks);
    plt.xlabel(r'Time')#,fontsize=labelfontsize)
    plt.ylabel(r'Thread')#,fontsize=labelfontsize)

##################################################

filename = "schedule.dat"
if len(sys.argv) > 1:
    filename = sys.argv[1]

#taskfontsize = 16
#labelfontsize = 16
#tickfontsize = 16
timeScale = 1.0/1000000.0; # cycles -> Mcycles

tasks = load_file(filename)

# filter out uninteresting tasks

tasks = [x for x in tasks if myfilter(x)]

# normalize start time

starttime = min([x['start'] for x in tasks])
for task in tasks:
  task['start'] = task['start']-starttime
  task['end'] = task['end']-starttime

# scale time

for task in tasks:
  task['start'] = task['start']*timeScale
  task['length'] = task['length']*timeScale
  task['end'] = task['end']*timeScale

# sort by processor (must be numbered properly)

procids = dict(zip([x['procid'] for x in tasks], [x['procid'] for x in tasks]))
numProcs = len(procids)
tasksperproc=[]
for i in sorted(procids):
  tasksperproc.append( [t for t in tasks if t['procid'] == i] )

# true threadids -> logical threadids

threadsPerProc = 0

for i in procids:
  ltasks = tasksperproc[i]
  threadids = dict(zip([x['threadid'] for x in ltasks], [x['threadid'] for x in ltasks]))
  numThreads = len(threadids)
  threadsPerProc = max(numThreads, threadsPerProc)
  threadidmap = dict(zip(sorted(threadids.keys()), range(0, len(threadids))))
  for task in ltasks:
    task['threadid'] = threadidmap[task['threadid']]

# calculate time wasted between tasks

mtd = list()

for p in procids:
  for i in range(0, numThreads):
    t = [ x for x in tasksperproc[p] if x['threadid'] == i ];
    for j in range(1, len(t)):
      mtd.append( t[j]['start']-(t[j-1]['start']+t[j-1]['length']) )

totaltime = sum([x['length'] for x in tasks])
endtime = max([x['end'] for x in tasks])
print('N= ', len(tasks), \
      ' Total= ', totaltime, \
      ' End= ', endtime, \
      ' Par= ', "{0:.2f}".format(totaltime/float(endtime)), \
      ' DistMin= ', min(mtd), \
      ' DistMed= ', getMedian(mtd), \
      ' perf= ', sum([x['cache'] for x in tasks]))

drawPlot(tasks)
plt.show()