from raybot import config
from raybot.model import db, POI, Location
from raybot.bot import bot, dp
from raybot.util import h, HTML, split_tokens, get_buttons, get_map, get_user, tr, DOW
from raybot.actions.poi import POI_EDIT_CB, POI_LIST_CB
from raybot.actions.messages import broadcast_str, broadcast
import re
import os
import logging
import random
import humanized_opening_hours as hoh
from aiosqlite import DatabaseError
from string import ascii_lowercase
from datetime import datetime
from typing import Dict, Union
from collections import Counter
from aiogram import types
from aiogram.utils.callback_data import CallbackData
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.utils.exceptions import TelegramAPIError, MessageToDeleteNotFound

HOUSE_CB = CallbackData('ehouse', 'hid')
FLOOR_CB = CallbackData('efloor', 'floor')
BOOL_CB = CallbackData('boolattr', 'attr', 'value')
PHOTO_CB = CallbackData('ephoto', 'name', 'which')
TAG_CB = CallbackData('etag', 'tag')
TAG_PAGE_CB = CallbackData('tagpg', 'page')

class EditState(StatesGroup):
    name = State()
    location = State()
    keywords = State()
    confirm = State()
    attr = State()
    comment = State()
    message = State()

def cancel_keyboard():
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton('❌ ' + tr('cancel'), callback_data='cancel')
    )

def location_keyboard():
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton(
            tr(('editor', 'latlon')),
            url='https://zverik.github.io/latlon/#16/53.9312/27.6525'),
        types.InlineKeyboardButton('❌ ' + tr('cancel'), callback_data='cancel'),
    )

def valid_location(loc):
    bbox = config.BBOX
    if not bbox or len(bbox) != 4:
        return True
    return bbox[0] <= loc.lon <= bbox[2] and bbox[1] <= loc.lat <= bbox[3]

@dp.callback_query_handler(state=EditState.all_states, text='cancel')
async def new_cancel(query: types.CallbackQuery, state: FSMContext):
    await delete_msg(query, state)
    await state.finish()
    user = await get_user(query.from_user)
    if user.review:
        kbd = types.InlineKeyboardMarkup().add(types.InlineKeyboardButton(
            '🗒 ' + tr(('review', 'continue')), callback_data='continue_review'))
    else:
        kbd = get_buttons()
    await bot.send_message(
        query.from_user.id,
        tr(('new_poi', 'cancel')),
        reply_markup=kbd
    )

@dp.callback_query_handler(state='*', text='new')
async def new_poi(query: types.CallbackQuery):
    if config.MAINTENANCE:
        await bot.send_message(query.from_user.id, tr('maintenance'))
        return
    await EditState.name.set()
    await bot.send_message(
        query.from_user.id,
        tr(('new_poi', 'name')),
        reply_markup=cancel_keyboard()
    )

@dp.callback_query_handler(POI_EDIT_CB.filter(), state='*')
async def edit_poi(query: types.CallbackQuery, callback_data: Dict[str, str],
                   state: FSMContext):
    if config.MAINTENANCE:
        await bot.send_message(query.from_user.id, tr('maintenance'))
        return
    if callback_data['d'] == '1':
        await delete_msg(query)
    poi = await db.get_poi_by_id(int(callback_data['id']))
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(query.from_user, state)

@dp.message_handler(state=EditState.name)
async def new_name(message: types.Message, state: FSMContext):
    name = message.text.strip()
    if len(name) < 3:
        await message.answer(tr(('new_poi', 'name_too_short')))
        return
    await state.set_data({'name': name})
    await EditState.location.set()
    await message.answer(tr(('new_poi', 'location')), reply_markup=location_keyboard())

def parse_location(message: types.Message):
    ll = re.match(r'^\s*(-?\d+\.\d+),\s*(-?\d+\.\d+)\s*$', message.text or '')
    if message.location:
        return Location(lon=message.location.longitude, lat=message.location.latitude)
    elif ll:
        return Location(lat=float(ll.group(1)), lon=float(ll.group(2)))
    return None

@dp.message_handler(state=EditState.location,
                    content_types=[types.ContentType.TEXT, types.ContentType.LOCATION])
async def new_location(message: types.Message, state: FSMContext):
    loc = parse_location(message)
    if not loc:
        await message.answer(tr(('new_poi', 'no_location')),
                             reply_markup=location_keyboard())
        return
    if not valid_location(loc):
        await message.answer(tr(('new_poi', 'location_out')),
                             reply_markup=location_keyboard())
        return
    await state.update_data(lon=loc.lon, lat=loc.lat)
    await EditState.keywords.set()
    await message.answer(tr(('new_poi', 'keywords')), reply_markup=cancel_keyboard())

@dp.message_handler(state=EditState.keywords)
async def new_keywords(message: types.Message, state: FSMContext):
    keywords = split_tokens(message.text)
    if not keywords:
        await message.answer(tr(('new_poi', 'no_keywords')))
        return
    # Create a POI
    data = await state.get_data()
    poi = POI(
        name=data['name'],
        location=Location(lat=data['lat'], lon=data['lon']),
        keywords=' '.join(keywords)
    )
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(message.from_user, state, tr(('new_poi', 'confirm')))

def format(v, yes=None, no=None, null=None):
    if v is None or v == '':
        return f'{null or tr(("editor", "unknown"))}'
    if isinstance(v, str):
        return h(v)
    if isinstance(v, bool):
        return (yes or tr(('editor', 'bool_yes'))) if v else (no or tr(('editor', 'bool_no')))
    if isinstance(v, (int, float)):
        return v
    if isinstance(v, hoh.OHParser):
        return h(v.field)
    if isinstance(v, Location):
        return f'v.lat, v.lon'
    return str(v)

def new_keyboard():
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton('💾 ' + tr('save'), callback_data='save'),
        types.InlineKeyboardButton('❌ ' + tr('cancel'), callback_data='cancel')
    )

async def print_edit_options(user: types.User, state: FSMContext, comment=None):
    poi = (await state.get_data())['poi']
    lines = []
    m = tr(('editor', 'panel'))
    lines.append(f'{format(poi.name)}')
    lines.append('')
    lines.append(f'/edesc {m["desc"]}: {format(poi.description, null=m["none"])}')
    lines.append(f'/ekey {m["keywords"]}: {format(poi.keywords)}')
    lines.append(f'/etag {m["tag"]}: {format(poi.tag)}')
    lines.append(f'/ehouse {m["house"]}: {format(poi.house_name)}')
    lines.append(f'/efloor {m["floor"]}: {format(poi.floor)}')
    lines.append(f'/eaddr {m["addr"]}: {format(poi.address_part)}')
    lines.append(f'/ehour {m["hours"]}: {format(poi.hours_src)}')
    lines.append(f'/eloc {m["loc"]}: '
                 ''
                 f'{m["loc_browse"]}')
    lines.append(f'/ephone {m["phone"]}: {format("; ".join(poi.phones))}')
    lines.append(f'/ewifi {m["wifi"]}: {format(poi.has_wifi)}')
    lines.append(f'/ecard {m["card"]}: {format(poi.accepts_cards)}')
    if poi.links:
        links = ', '.join([f'{h(l[0])}' for l in poi.links])
    else:
        links = f'{m["none"]}'
    lines.append(f'/elink {m["links"]}: {links}')
    lines.append(f'/ecom {m["comment"]}: {format(poi.comment, null=m["none"])}')
    if poi.photo_out and poi.photo_in:
        photos = m['photo_both']
    elif poi.photo_out:
        photos = m['photo_out']
    elif poi.photo_in:
        photos = m['photo_in']
    else:
        photos = m['none']
    lines.append(f'{m["photo"]}: {photos} ({m["photo_comment"]})')
    if poi.id:
        if poi.delete_reason:
            lines.append(f'{m["deleted"]}: {format(poi.delete_reason)}. '
                         f'{m["restore"]}: /undelete')
        else:
            lines.append(f'🗑 {m["delete"]}: /delete')
        lines.append(f'✉ {m["msg"]}: /msg')

    content = '\n'.join(lines)
    if comment is None:
        comment = tr(('new_poi', 'confirm2'))
    if comment:
        content += '\n\n' + h(comment)
    reply = await bot.send_message(
        user.id, content, parse_mode=HTML, reply_markup=new_keyboard(),
        disable_web_page_preview=True)
    await state.update_data(reply=reply.message_id)

def cancel_attr_kbd():
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton(tr(('editor', 'cancel')), callback_data='cancel_attr')
    )

def edit_loc_kbd(poi):
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton(
            tr(('editor', 'latlon')),
            url='https://zverik.github.io/latlon/#18/'
                f'{poi.location.lat}/{poi.location.lon}"'),
        types.InlineKeyboardButton(tr(('editor', 'cancel')), callback_data='cancel_attr')
    )

def boolean_kbd(attr: str):
    return types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton(tr(('editor', 'bool_true')),
                                   callback_data=BOOL_CB.new(attr=attr, value='true')),
        types.InlineKeyboardButton(tr(('editor', 'bool_false')),
                                   callback_data=BOOL_CB.new(attr=attr, value='false')),
        types.InlineKeyboardButton(tr(('editor', 'bool_none')),
                                   callback_data=BOOL_CB.new(attr=attr, value='null')),
        types.InlineKeyboardButton(tr(('editor', 'cancel')), callback_data='cancel_attr')
    )

def tag_kbd(page: int = 1):
    ROWS = 5
    tags = config.TAGS['suggest_tags']
    kbd = types.InlineKeyboardMarkup(row_width=3)
    if (page - 1) * ROWS * 3 >= len(tags):
        page = 1
    for tag in tags[(page - 1) * ROWS * 3:page * ROWS * 3]:
        kbd.insert(types.InlineKeyboardButton(config.TAGS['tags'].get(tag, [tag])[0],
                                              callback_data=TAG_CB.new(tag=tag)))
    kbd.add(
        types.InlineKeyboardButton(tr(('editor', 'cancel')), callback_data='cancel_attr'),
        types.InlineKeyboardButton(tr(('editor', 'next_page')) + ' ⏭',
                                   callback_data=TAG_PAGE_CB.new(page=str(page + 1)))
    )
    return kbd

@dp.callback_query_handler(TAG_PAGE_CB.filter(), state=EditState.attr)
async def next_page(query: types.CallbackQuery, callback_data: Dict[str, str]):
    kbd = tag_kbd(int(callback_data['page']))
    await bot.edit_message_reply_markup(
        query.from_user.id, query.message.message_id, reply_markup=kbd)

async def delete_msg(source: Union[types.Message, types.CallbackQuery],
                     message_id: Union[int, FSMContext] = None):
    user_id = source.from_user.id
    if isinstance(message_id, FSMContext):
        message_id = (await message_id.get_data()).get('reply')
    if isinstance(source, types.CallbackQuery):
        if isinstance(message_id, list):
            message_id.append(source.message.message_id)
        else:
            message_id = source.message.message_id

    if message_id:
        if not isinstance(message_id, list):
            message_id = [message_id]
        for msg_id in message_id:
            if msg_id:
                try:
                    await bot.delete_message(user_id, msg_id)
                except MessageToDeleteNotFound:
                    pass
                except TelegramAPIError:
                    logging.exception('Failed to delete bot message')

@dp.message_handler(commands='ephoto', state=EditState.confirm)
async def show_photos(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    for photo, where in [(poi.photo_out, 'photo_out'), (poi.photo_in, 'photo_in')]:
        if photo:
            path = os.path.join(config.PHOTOS, photo + '.jpg')
            if os.path.exists(path):
                kbd = types.InlineKeyboardMarkup().add(
                    types.InlineKeyboardButton(
                        '🗑 ' + tr(('editor', 'photo_del')),
                        callback_data=PHOTO_CB.new(name=photo, which='unlink')),
                    types.InlineKeyboardButton(
                        tr(('editor', 'cancel')), callback_data='cancel_attr')
                )
                await message.answer_photo(
                    types.InputFile(path), caption=tr(('editor', where)), reply_markup=kbd)

@dp.message_handler(commands='eout', state=EditState.confirm)
async def suggest_photo_out(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    pois = await db.get_poi_around(poi.location, 10, floor=poi.floor)
    photos = [p.photo_out for p in pois if p.photo_out]
    if not photos:
        await message.answer(tr(('editor', 'no_photos_around')))
        return

    photo_dist = {
        name: min(poi.location.distance(p.location) for p in pois if p.photo_out == name)
        for name in photos
    }
    photo_cnt = Counter(photos)
    photos = sorted(photo_cnt, key=lambda p: (int(photo_dist.get(p, 1000) / 10),
                                              100 - photo_cnt[p]))
    photos = photos[:3]

    kbd = types.InlineKeyboardMarkup(row_width=5)
    media = types.MediaGroup()
    for i, photo in enumerate(photos, 1):
        path = os.path.join(config.PHOTOS, photo + '.jpg')
        if os.path.exists(path):
            file_ids = await db.find_file_ids({photo: os.path.getsize(path)})
            if photo in file_ids:
                media.attach_photo(file_ids[photo])
            else:
                media.attach_photo(types.InputFile(path))
            kbd.insert(types.InlineKeyboardButton(
                str(i), callback_data=PHOTO_CB.new(name=photo, which='out')))
    kbd.insert(types.InlineKeyboardButton(
        tr(('editor', 'cancel')), callback_data='cancel_attr'))
    await delete_msg(message, state)
    msg = await bot.send_media_group(message.from_user.id, media=media)
    msg2 = await message.answer(tr(('editor', 'choose_photo')), reply_markup=kbd)
    replies = msg.message_id if not isinstance(msg, list) else [m.message_id for m in msg]
    replies.append(msg2.message_id)
    await state.update_data(reply=replies)

@dp.message_handler(state=EditState.confirm, content_types=types.ContentType.PHOTO)
async def upload_photo(message: types.Message, state: FSMContext):
    file_id = message.photo[-1].file_id
    name = await db.find_path_for_file_id(file_id)
    downloaded = False
    if not name:
        try:
            f = await bot.get_file(file_id)
            name = (''.join(random.sample(ascii_lowercase, 4)) +
                    datetime.now().strftime('%y%m%d%H%M%S'))
            path = os.path.join(config.PHOTOS, name + '.jpg')
            await f.download(path)
        except TelegramAPIError:
            logging.exception('Image upload fail')
            await message.answer(tr(('editor', 'upload_fail')))
            return
        if not os.path.exists(path):
            await message.answer(tr(('editor', 'upload_fail')))
            return
        await db.store_file_id(name, os.path.getsize(path), file_id)
        downloaded = True

    kbd = types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton(tr(('editor', 'photo_out')),
                                   callback_data=PHOTO_CB.new(name=name, which='out')),
        types.InlineKeyboardButton(tr(('editor', 'photo_in')),
                                   callback_data=PHOTO_CB.new(name=name, which='in')),
        types.InlineKeyboardButton(
            '🗑 ' + tr(('editor', 'photo_del')),
            callback_data=PHOTO_CB.new(name=name, which='del' if downloaded else 'skip'))
    )
    await delete_msg(message, state)
    await message.answer(tr(('editor', 'photo')), reply_markup=kbd)

@dp.callback_query_handler(PHOTO_CB.filter(), state=EditState.confirm)
async def store_photo(query: types.CallbackQuery, callback_data: Dict[str, str],
                      state: FSMContext):
    poi = (await state.get_data())['poi']
    name = callback_data['name']
    path = os.path.join(config.PHOTOS, name + '.jpg')
    if not os.path.exists(path):
        await query.answer(tr(('editor', 'photo_lost')))
        return
    which = callback_data['which']
    if which == 'out':
        poi.photo_out = name
    elif which == 'in':
        poi.photo_in = name
    elif which == 'unlink':
        if poi.photo_out == name:
            poi.photo_out = None
        elif poi.photo_in == name:
            poi.photo_in = None
    elif which == 'del':
        os.remove(path)
        await query.answer(tr(('editor', 'photo_deleted')))
    else:
        await query.answer(tr(('editor', 'photo_forgot')))
    await delete_msg(query)
    await state.set_data({'poi': poi})
    await print_edit_options(query.from_user, state)

@dp.message_handler(commands='msg', state=EditState.confirm)
async def message_intro(message: types.Message, state: FSMContext):
    user = await get_user(message.from_user)
    if user.is_moderator():
        await message.answer(tr(('editor', 'cant_message')))
        return
    await delete_msg(message, state)
    reply = await message.answer(
        tr(('editor', 'message')), reply_markup=cancel_attr_kbd())
    await EditState.message.set()
    await state.update_data(reply=reply.message_id)

async def print_edit_message(message: types.Message, state: FSMContext,
                             attr: str, dash: bool = False, poi_attr: str = None,
                             msg_attr: str = None, kbd=None, value='-',
                             content: str = None):
    reply0 = None
    if value is not None:
        poi = (await state.get_data())['poi']
        if value == '-':
            pvalue = getattr(poi, poi_attr or attr)
        else:
            pvalue = value(poi)
        if pvalue:
            reply0 = (await message.answer(str(pvalue))).message_id

    if not content:
        content = tr(('editor', msg_attr or attr))
        if dash:
            content += ' ' + tr(('editor', 'dash'))
    if not kbd:
        kbd = cancel_attr_kbd()
    await delete_msg(message, state)
    reply = await message.answer(content, reply_markup=kbd, disable_web_page_preview=True)
    await EditState.attr.set()
    await state.update_data(attr=attr, reply=[reply.message_id, reply0])

@dp.message_handler(commands='ename', state=EditState.confirm)
async def edit_name(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'name')

@dp.message_handler(commands='edesc', state=EditState.confirm)
async def edit_desc(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'desc', dash=True, poi_attr='description')

@dp.message_handler(commands='etag', state=EditState.confirm)
async def edit_tag(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'tag', dash=True, kbd=tag_kbd())

@dp.message_handler(commands='ecom', state=EditState.confirm)
async def edit_comment(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'comment', dash=True)

@dp.message_handler(commands='ekey', state=EditState.confirm)
async def edit_keywords(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'keywords')

@dp.message_handler(commands='eaddr', state=EditState.confirm)
async def edit_address(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'address', dash=True, poi_attr='address_part')

@dp.message_handler(commands='eloc', state=EditState.confirm)
async def edit_location(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    await print_edit_message(message, state, 'location', value=None,
                             kbd=edit_loc_kbd(poi))

@dp.message_handler(commands='ephone', state=EditState.confirm)
async def edit_phones(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'phones', dash=True,
                             value=lambda p: '; '.join(p.phones))

@dp.message_handler(commands='ehour', state=EditState.confirm)
async def edit_hours(message: types.Message, state: FSMContext):
    await print_edit_message(message, state, 'hours', dash=True,
                             poi_attr='hours_src')

@dp.message_handler(commands='ehouse', state=EditState.confirm)
async def edit_house(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    houses = await db.get_houses()
    houses.sort(key=lambda h: poi.location.distance(h.location))
    houses = houses[:3]

    # Prepare the map
    map_file = get_map([h.location for h in houses], ref=poi.location)
    # Prepare the keyboard
    kbd = types.InlineKeyboardMarkup(row_width=1)
    for i, house in enumerate(houses, 1):
        prefix = '✅ ' if house == poi.house else ''
        kbd.add(types.InlineKeyboardButton(
            f'{prefix} {i} {house.name}', callback_data=HOUSE_CB.new(hid=house.key)))
    kbd.add(types.InlineKeyboardButton(
        tr(('editor', 'cancel')), callback_data='cancel_attr'))

    # Finally send the reply
    await delete_msg(message, state)
    if map_file:
        await message.answer_photo(types.InputFile(map_file.name),
                                   caption=tr(('editor', 'house')), reply_markup=kbd)
        map_file.close()
    else:
        await message.answer(tr(('editor', 'house')), reply_markup=kbd)

@dp.callback_query_handler(HOUSE_CB.filter(), state=EditState.confirm)
async def update_house(query: types.CallbackQuery, callback_data: Dict[str, str],
                       state: FSMContext):
    poi = (await state.get_data())['poi']
    hid = callback_data['hid']
    poi.house = hid
    h_data = await db.get_poi_by_key(hid)
    if h_data:
        poi.house_name = h_data.name
    await delete_msg(query)
    await state.set_data({'poi': poi})
    await print_edit_options(query.from_user, state)

@dp.message_handler(commands='efloor', state=EditState.confirm)
async def edit_floor(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    floors = await db.get_floors_by_house(poi.house)
    if floors and floors != [None]:
        kbd = types.InlineKeyboardMarkup(row_width=3)
        for floor in floors:
            if floor is not None:
                kbd.insert(types.InlineKeyboardButton(
                    floor, callback_data=FLOOR_CB.new(floor=floor)))
        kbd.insert(types.InlineKeyboardButton(
            tr(('editor', 'cancel')), callback_data='cancel_attr'))
    else:
        kbd = cancel_attr_kbd()
    await print_edit_message(message, state, 'floor', dash=True,
                             kbd=kbd, value=None)

@dp.callback_query_handler(FLOOR_CB.filter(), state=EditState.attr)
async def update_floor(query: types.CallbackQuery, callback_data: Dict[str, str],
                       state: FSMContext):
    poi = (await state.get_data())['poi']
    floor = callback_data['floor']
    poi.floor = floor if floor != '-' else None
    await delete_msg(query)
    await EditState.confirm.set()
    await state.set_data({'poi': poi})
    await print_edit_options(query.from_user, state)

@dp.message_handler(commands='ewifi', state=EditState.confirm)
async def edit_wifi(message: types.Message, state: FSMContext):
    await delete_msg(message, state)
    reply = await message.answer(
        tr(('editor', 'wifi')), reply_markup=boolean_kbd('wifi'))
    await state.update_data(reply=reply.message_id)

@dp.message_handler(commands='ecard', state=EditState.confirm)
async def edit_cards(message: types.Message, state: FSMContext):
    await delete_msg(message, state)
    reply = await message.answer(
        tr(('editor', 'cards')), reply_markup=boolean_kbd('cards'))
    await state.update_data(reply=reply.message_id)

@dp.callback_query_handler(BOOL_CB.filter(), state=EditState.confirm)
async def update_boolean(query: types.CallbackQuery, callback_data: Dict[str, str],
                         state: FSMContext):
    poi = (await state.get_data())['poi']
    attr = callback_data['attr']
    svalue = callback_data['value']
    if svalue == 'null':
        value = None
    else:
        value = svalue == 'true'
    if attr == 'wifi':
        poi.has_wifi = value
    elif attr == 'cards':
        poi.accepts_cards = value
    else:
        query.answer(f'Unknown field {attr}')
    await delete_msg(query)
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(query.from_user, state)

@dp.callback_query_handler(TAG_CB.filter(), state=EditState.attr)
async def update_tag(query: types.CallbackQuery, callback_data: Dict[str, str],
                     state: FSMContext):
    poi = (await state.get_data())['poi']
    poi.tag = callback_data['tag']
    await delete_msg(query)
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(query.from_user, state)

@dp.message_handler(commands='elink', state=EditState.confirm)
async def edit_links(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    if poi.links:
        content = tr(('editor', 'links_have')) + '\n\n'
        content += '\n'.join([f'🔗 {h(l[0])}: {h(l[1])}' for l in poi.links])
    else:
        content = tr(('editor', 'no_links'))
    content += '\n\n' + tr(('editor', 'links'))
    await print_edit_message(message, state, 'links', value=None, content=content)

@dp.message_handler(commands='delete', state=EditState.confirm)
async def delete_poi_prompt(message: types.Message, state: FSMContext):
    poi = (await state.get_data())['poi']
    if poi.delete_reason:
        user = await get_user(message.from_user)
        if not user.is_moderator():
            await message.answer(tr(('editor', 'delete_twice')))
        else:
            await db.delete_poi_forever(user.id, poi)
            await state.finish()
            await message.answer(tr(('editor', 'deleted2')), reply_markup=get_buttons())
    else:
        await message.answer(tr(('editor', 'delete')), reply_markup=cancel_attr_kbd())
        await EditState.attr.set()
        await state.update_data(attr='delete')

@dp.message_handler(commands='undelete', state=EditState.confirm)
async def undelete_poi(message: types.Message, state: FSMContext):
    user = await get_user(message.from_user)
    if not user.is_moderator():
        await message.answer(tr(('editor', 'cant_restore')))
        return
    poi = (await state.get_data())['poi']
    await db.restore_poi(message.from_user.id, poi)
    await state.finish()
    if user.review:
        kbd = types.InlineKeyboardMarkup().add(types.InlineKeyboardButton(
            '🗒 ' + tr(('review', 'continue')), callback_data='continue_review'))
    else:
        kbd = get_buttons()
    await message.answer(tr(('editor', 'restored')), reply_markup=kbd)

@dp.callback_query_handler(text='cancel_attr', state=EditState.all_states)
async def cancel_attr(query: types.CallbackQuery, state: FSMContext):
    await delete_msg(query, state)
    await EditState.confirm.set()
    await print_edit_options(query.from_user, state)

@dp.message_handler(content_types=types.ContentType.LOCATION, state=EditState.attr)
async def store_location(message: types.Message, state: FSMContext):
    data = await state.get_data()
    poi = data['poi']
    attr = data['attr']
    if attr != 'location':
        return
    loc = parse_location(message)
    if not loc:
        await message.answer(tr(('new_poi', 'no_location')), reply_markup=edit_loc_kbd(poi))
        return
    poi.location = loc
    await delete_msg(message, state)
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(message.from_user, state)

RE_URL = re.compile(r'^https?://')
# TODO: use hours_abbr and DOW
RE_HOURS = re.compile(r'^(?:(пн|вт|ср|чт|пт|сб|вс|mo|tu|we|th|fr|sa|su)(?:\s*-?'
                      r'\s*(пн|вт|ср|чт|пт|сб|вс|mo|tu|we|th|fr|sa|su))?\s+)?'
                      r'(\d\d?(?:[:.]\d\d)?)\s*-\s*(\d\d(?:[:.]\d\d)?)'
                      r'(?:\s+об?е?д?\s+(\d\d?(?:[:.]\d\d)?)\s*-\s*(\d\d(?:[:.]\d\d)?))?$')

def parse_hours(s):
    def norm_hour(h):
        if not h:
            return None
        if len(h) < 4:
            h += ':00'
        return h.replace('.', ':').rjust(5, '0')

    if s in ('24', '24/7'):
        return '24/7'

    HOURS_WEEK = {tr(('editor', 'hours_abbr'))[i]: DOW[i] for i in range(7)}
    HOURS_WEEK.update({DOW[i].lower(): DOW[i] for i in range(7)})

    parts = []
    for part in s.split(','):
        m = RE_HOURS.match(part.strip().lower())
        if not m:
            raise ValueError(part)
        wd = 'Mo-Su' if not m.group(1) else HOURS_WEEK[m.group(1)]
        if m.group(2):
            wd += '-' + HOURS_WEEK[m.group(2)]
        h1 = norm_hour(m.group(3))
        h2 = norm_hour(m.group(4))
        l1 = norm_hour(m.group(5))
        l2 = norm_hour(m.group(6))
        if l1:
            wd += f' {h1}-{l1},{l2}-{h2}'
        else:
            wd += f' {h1}-{h2}'
        parts.append(wd)
    return '; '.join(parts)

def parse_link(value):
    parts = value.lower().replace('. ', '.').split(None, 1)
    if len(parts) == 1 and '.' not in parts[0]:
        return parts
    if len(parts) == 1:
        parts = [tr('default_link'), parts[0]]
    REPLACE_TITLE = tr(('editor', 'link_replace'))
    REPLACE_TITLE['instagram'] = tr(('editor', 'instagram'))
    if parts[0] in REPLACE_TITLE:
        parts[0] = REPLACE_TITLE[parts[0]]
    if parts[0] == tr(('editor', 'instagram')) and 'instagram.' not in parts[1]:
        parts[1] = 'https://instagram.com/' + parts[1]
    elif parts[0] == 'vk' and 'vk.' not in parts[1]:
        parts[1] = 'https://vk.com/' + parts[1]
    if '://' not in parts[1]:
        parts[1] = 'https://' + parts[1]
    return parts

@dp.message_handler(state=EditState.attr)
async def store_attr(message: types.Message, state: FSMContext):
    data = await state.get_data()
    poi = data['poi']
    attr = data['attr']
    value = message.text.strip()

    if attr == 'name':
        if value == '-':
            await message.answer(tr(('editor', 'empty_name')), reply_markup=cancel_attr_kbd())
            return
        poi.name = value
    elif attr == 'desc':
        poi.description = None if value == '-' else value
    elif attr == 'comment':
        poi.comment = None if value == '-' else value
    elif attr == 'floor':
        poi.floor = None if value == '-' else value
    elif attr == 'tag':
        if value == '-':
            poi.tag = None
        else:
            parts = [p.strip() for p in re.split(r'[ =]+', value.lower().replace('-', '_'))]
            if len(parts) != 2 or not re.match(r'^[a-z]+$', parts[0]):
                await message.answer(tr(('editor', 'tag_format'), value),
                                     reply_markup=cancel_attr_kbd())
                return
            poi.tag = '='.join(parts)
    elif attr == 'keywords':
        new_kw = split_tokens(value)
        if new_kw:
            old_kw = [] if not poi.keywords else poi.keywords.split()
            poi.keywords = ' '.join(old_kw + new_kw)
    elif attr == 'address':
        poi.address_part = None if value == '-' else value
    elif attr == 'location':
        loc = parse_location(message)
        if not loc:
            await message.answer(tr(('new_poi', 'no_location')),
                                 reply_markup=edit_loc_kbd(poi))
            return
        if not valid_location(loc):
            await message.answer(tr(('new_poi', 'location_out')),
                                 reply_markup=edit_loc_kbd(poi))
            return
        poi.location = loc
    elif attr == 'hours':
        if value == '-':
            poi.hours = None
            poi.hours_src = None
        else:
            try:
                hours = parse_hours(value)
            except ValueError as e:
                await message.answer(tr(('editor', 'hours_format'), e),
                                     reply_markup=cancel_attr_kbd())
                return
            poi.hours_src = hours
            poi.hours = hoh.OHParser(hours)
    elif attr == 'phones':
        if not value or value == '-':
            poi.phones = []
        else:
            poi.phones = [p.strip() for p in re.split(r'[;,]', value)]
    elif attr == 'links':
        if value:
            parts = parse_link(value)
            if parts:
                if len(parts) == 1:
                    poi.links = [l for l in poi.links if l[0] != parts[0]]
                else:
                    found = False
                    for i, l in enumerate(poi.links):
                        if l[0] == parts[0]:
                            found = True
                            l[1] = parts[1]
                    if not found:
                        poi.links.append(parts)
    elif attr == 'delete':
        await db.delete_poi(message.from_user.id, poi, value)
        await state.finish()
        user = await get_user(message.from_user)
        if user.review:
            kbd = types.InlineKeyboardMarkup().add(types.InlineKeyboardButton(
                '🗒 ' + tr(('review', 'continue')), callback_data='continue_review'))
        else:
            kbd = get_buttons()
        await message.answer(tr(('editor', 'deleted')), reply_markup=kbd)
        await broadcast_str(tr(('editor', 'just_deleted'), id=poi.id, reason=value),
                            message.from_user.id)
        return
    else:
        await message.answer(tr(('editor', 'wrong_attr'), attr))

    await delete_msg(message, state)
    await state.set_data({'poi': poi})
    await EditState.confirm.set()
    await print_edit_options(message.from_user, state)

@dp.message_handler(state=EditState.confirm)
async def other_msg(message: types.Message, state: FSMContext):
    await message.answer(tr(('editor', 'other_msg')))

@dp.message_handler(state=EditState.message)
async def send_message(message: types.Message, state: FSMContext):
    user = await get_user(message.from_user)
    poi = (await state.get_data())['poi']
    if poi.id is None:
        # Won't add an abstract message to the queue
        await broadcast(message)
    else:
        await db.add_to_queue(user, poi, message.text)
    await state.finish()
    await message.answer(tr(('editor', 'msg_sent')))

@dp.callback_query_handler(state=EditState.confirm, text='save')
async def new_save(query: types.CallbackQuery, state: FSMContext):
    poi = (await state.get_data())['poi']

    # If not a moderator, mark this as needs check
    user = await get_user(query.from_user)
    if not user.is_moderator() and poi.id is None:
        poi.needs_check = True

    # Send the POI to the database
    poi_id = poi.id
    try:
        if user.is_moderator() or poi.id is None:
            poi_id = await db.insert_poi(query.from_user.id, poi)
            saved = 'saved'
            if not user.is_moderator():
                await broadcast_str(tr(('editor', 'just_added'), id=poi.id, name=poi.name),
                                    query.from_user.id)
        else:
            await db.add_to_queue(user, poi)
            await broadcast_str(tr(('queue', 'added')))
            saved = 'sent'
    except DatabaseError as e:
        await bot.send_message(
            query.from_user.id, tr(('editor', 'error_save'), e), reply_markup=new_keyboard())
        return

    # Reset state and thank the user
    await delete_msg(query, state)
    await state.finish()
    kbd = types.InlineKeyboardMarkup().add(
        types.InlineKeyboardButton('👀 ' + tr(('editor', 'saved_look')),
                                   callback_data=POI_LIST_CB.new(id=poi_id)),
        types.InlineKeyboardButton('➕ ' + tr(('editor', 'saved_add')), callback_data='new')
    )
    if user.review:
        kbd.insert(types.InlineKeyboardButton(
            '🗒 ' + tr(('review', 'continue')), callback_data='continue_review'))
    await bot.send_message(query.from_user.id, tr(('editor', saved)), reply_markup=kbd)


from django.urls import path
from product.views import productdetail, ProductListView, ProductDetailView,export,add_wishlist,my_wishlist,deleteFromWishlist,item

urlpatterns = [
    # path('productdetail/',productdetail,name="productdetail"),
    path('productlist/',ProductListView.as_view(),name="productlist"),
    path('productdetail/<int:pk>/',ProductDetailView.as_view(),name='productdetail'),
    path('export/', export, name='export'),
    path('add-wishlist',add_wishlist,name='add_wishlist'),
    path('my-wishlist',my_wishlist,name='my_wishlist'),
    path('delete-from-wishlist/',deleteFromWishlist,name="delete-from-wishlist"),
    path('delete/',item,name="delete"),

    # path('my-wishlist',my_wishlist,name='my_wishlist'),

]


def extend_course(course, enterprise_customer, request):
        """
        Extend a course with more details needed for the program landing page.

        In particular, we add the following:

        * `course_image_uri`
        * `course_title`
        * `course_level_type`
        * `course_short_description`
        * `course_full_description`
        * `course_effort`
        * `expected_learning_items`
        * `staff`
        """
        course_run_id = course['course_runs'][0]['key']
        try:
            catalog_api_client = CourseCatalogApiServiceClient(enterprise_customer.site)
        except ImproperlyConfigured:
            error_code = 'ENTPEV000'
            LOGGER.error(
                'CourseCatalogApiServiceClient is improperly configured. '
                'Returned error code {error_code} to user {userid} '
                'and enterprise_customer {enterprise_customer} '
                'for course_run_id {course_run_id}'.format(
                    error_code=error_code,
                    userid=request.user.id,
                    enterprise_customer=enterprise_customer.uuid,
                    course_run_id=course_run_id,
                )
            )
            messages.add_generic_error_message_with_code(request, error_code)
            return ({}, error_code)

        course_details, course_run_details = catalog_api_client.get_course_and_course_run(course_run_id)
        if not course_details or not course_run_details:
            error_code = 'ENTPEV001'
            LOGGER.error(
                'User {userid} of enterprise customer {enterprise_customer} encountered an error.'
                'No course_details or course_run_details found for '
                'course_run_id {course_run_id}. '
                'The following error code reported to the user: {error_code}'.format(
                    userid=request.user.id,
                    enterprise_customer=enterprise_customer.uuid,
                    course_run_id=course_run_id,
                    error_code=error_code,
                )
            )
            messages.add_generic_error_message_with_code(request, error_code)
            return ({}, error_code)

        weeks_to_complete = course_run_details['weeks_to_complete']
        course_run_image = course_run_details['image'] or {}
        course.update({
            'course_image_uri': course_run_image.get('src', ''),
            'course_title': course_run_details['title'],
            'course_level_type': course_run_details.get('level_type', ''),
            'course_short_description': course_run_details['short_description'] or '',
            'course_full_description': clean_html_for_template_rendering(course_run_details['full_description'] or ''),
            'expected_learning_items': course_details.get('expected_learning_items', []),
            'staff': course_run_details.get('staff', []),
            'course_effort': ungettext_min_max(
                '{} hour per week',
                '{} hours per week',
                '{}-{} hours per week',
                course_run_details['min_effort'] or None,
                course_run_details['max_effort'] or None,
            ) or '',
            'weeks_to_complete': ungettext(
                '{} week',
                '{} weeks',
                weeks_to_complete
            ).format(weeks_to_complete) if weeks_to_complete else '',
        })
        return course, None


#list in python
myclassmate=["Erick", "Joan", "Michael", "Victor"]
mynos=[3,7,9,2,6,0,8]
mynos.sort()
myclassmate[0]="sylvia"
myclassmate.append("daniel")
myclassmate.insert(2, "christine")
myclassmate.sort()
print(mynos)
print(type(myclassmate))
print(myclassmate)
print("My name is "+myclassmate[0])

#tuple in python
myfavfruits=("mangoes", "Apples", "Pinneapples", "Apples")
print(myfavfruits)

print(myfavfruits[0])

#sets datastrucure
myfavcars={"Toyato","mercedes","nissan","vw"}
myfavcars.add("peugeot")
print(myfavcars)
#dictionaries in python
magari={
    "Name":"Toyota",
    "model": "Premio",
    "year": 2020
}
print(magari)
print(magari["model"])


import numpy as np
import cv2 as cv
from matplotlib import pyplot as plt

img1 = cv.imread('../../images/free-photo-Statue-38-m.jpg', 1)          # the same size as photo below
img2 = cv.imread('../../images/free-photo-twin-fowers-407-m.jpg', 1)

dst = cv.addWeighted(img1, 0.7, img2, 0.5, 0)           # marge two photo with any weight

# cv.imshow('dst',dst)
cv.waitKey(0)
cv.destroyAllWindows()

#***************************************Add two photo with foreground and background************************
print('******************')
img1 = cv.imread('../../images/free-photo-Statue-38-m.jpg', 1)  # small
img2 = cv.imread('../../images/ludzie_krajobraz.jpg', 1)        # big

rows, cols, channels = img1.shape   # get photo params
roi = img2[0:rows, 0:cols]          # make Region Of Interest (ROI) at secont pic

# Now create a mask of logo and create its inverse mask also
img1gray = cv.cvtColor(img1, cv.COLOR_BGR2GRAY)
ret, mask = cv.threshold(img1gray, 120, 255, cv. THRESH_BINARY)     # make color photo to binary photo
mask_inv = cv.bitwise_not(mask)

# black out the area of small pic in ROI
img2_bg = cv.bitwise_and(roi, roi, mask=mask_inv)       # place at big photo binary mask from small photo

# take only region of logo from image
img1_fg = cv.bitwise_and(img1, img1, mask=mask)         # place at small photo binary mask from small photo

# put logo in ROI and modify the main image
# add background and foreground. Black color has value of '0', so add black and blue = blue :)
dst = cv.add(img2_bg, img1_fg)

cv.imshow('img2_bg', img2_bg)
cv.imshow('img1_fg', img1_fg)
cv.imshow('dst', dst)
cv.waitKey(0)

img2[0:rows, 0:cols] = dst

cv.imshow('res', img2)
cv.waitKey(0)
cv.destroyAllWindows()


import logging
import time
import os
from tkinter import image_names
import numpy as np

import torch
from tqdm import tqdm

from fcos_core.config import cfg
from fcos_core.data.datasets.evaluation import evaluate
from ..utils.comm import is_main_process, get_world_size
from ..utils.comm import all_gather
from ..utils.comm import synchronize
from ..utils.timer import Timer, get_time_str
from .bbox_aug import im_detect_bbox_aug
from fcos_core.structures.image_list import to_image_list

DATASET_COUNT = 1
def _foward_detector(model, images, targets=None, return_maps=False):
    generator = model["gnet"]
    predictor_module = model["pnet"]
    images = to_image_list(images)
    features = generator(images.tensors)
    boxes, losses = predictor_module(images, features, targets=targets)
    return boxes

def compute_on_dataset(model, data_loader, device, timer=None):
    if isinstance(model, dict):
        for k in model:
            model[k].eval()
    else:
        model.eval()
    results_dict = {}
    # results_dict2 = {}
    # results2 = []
    cpu_device = torch.device("cpu")
    for _, batch in enumerate(data_loader):
        images, targets, image_ids, img_names = batch
        # print(img_names)
        images = images.to(device)
        with torch.no_grad():
            if timer:
                timer.tic()
            if cfg.TEST.BBOX_AUG.ENABLED:
                output = im_detect_bbox_aug(model, images, device)
            else:
                if isinstance(model, dict):
                    output = _foward_detector(model, images, targets=None, return_maps=False)
                else:
                    output = model(images.to(device))
            if timer:
                torch.cuda.synchronize()
                timer.toc()
            output = [o.to(cpu_device) for o in output]
            # print(output[0].get_field("scores"))
        results_dict.update(
            {img_id: result for img_id, result in zip(image_ids, output)}
        )
        # results_dict2.update(
        #     {img_name: result for img_name, result in zip(img_names, output)}
        # )
        # for result in output:
        #     lab = torch.max(result.get_field("labels")).item()
        #     results2.append(int(lab))
    # def takeSecond(elem):
    #     return elem[1]
    # # Get IDN noise samples:
    # idn = []
    # for img_id, result in results_dict2.items():
    #     lab = torch.max(result.get_field("labels")).item()
    #     idn.append((img_id, lab))
    # idn.sort(key=takeSecond)

    # global DATASET_COUNT
    # results2 = np.array(results2)
    # print(results2)
    # np.save("np_fedini/fedini_{}.npy".format(str(DATASET_COUNT)), results2)    # .npy extension is added if not given
    # print("saved to {}".format("np_fedini/fedini_{}.npy".format(str(DATASET_COUNT))))
    # d = np.load('test3.npy')
    # global DATASET_COUNT
    # if os.path.exists("fedavg_{}.txt".format(str(DATASET_COUNT)):
    #     os.remove("fedavg_{}.txt".format(str(DATASET_COUNT))
    # for p in idn:
    #     with open("fedavg_{}.txt".format(str(DATASET_COUNT)), "a+") as f:
    #         f.write(str(p[0])+"\t"+str(p[1])+"\n")
    # print("Ok")
    # DATASET_COUNT += 1
    return results_dict

def _accumulate_predictions_from_multiple_gpus(predictions_per_gpu):
    all_predictions = all_gather(predictions_per_gpu)
    if not is_main_process():
        return
    # merge the list of dicts
    predictions = {}
    for p in all_predictions:
        predictions.update(p)
    # convert a dict where the key is the index in a list
    image_ids = list(sorted(predictions.keys()))
    if len(image_ids) != image_ids[-1] + 1:
        logger = logging.getLogger("fcos_core.inference")
        logger.warning(
            "Number of images that were gathered from multiple processes is not "
            "a contiguous set. Some images might be missing from the evaluation"
        )

    # convert to a list
    predictions = [predictions[i] for i in image_ids]
    return predictions

def inference(
        model,
        data_loader,
        dataset_name,
        iou_types=("bbox",),
        box_only=False,
        device="cuda",
        expected_results=(),
        expected_results_sigma_tol=4,
        output_folder=None,
):
    # convert to a torch.device for efficiency
    device = torch.device(device)
    num_devices = get_world_size()
    logger = logging.getLogger("fcos_core.inference")
    dataset = data_loader.dataset
    logger.info("Start evaluation on {} dataset({} images).".format(dataset_name, len(dataset)))
    total_timer = Timer()
    inference_timer = Timer()
    total_timer.tic()
    predictions = compute_on_dataset(model, data_loader, device, inference_timer)
    # wait for all processes to complete before measuring the time
    synchronize()
    total_time = total_timer.toc()
    total_time_str = get_time_str(total_time)
    logger.info(
        "Total run time: {} ({} s / img per device, on {} devices)".format(
            total_time_str, total_time * num_devices / len(dataset), num_devices
        )
    )
    total_infer_time = get_time_str(inference_timer.total_time)
    logger.info(
        "Model inference time: {} ({} s / img per device, on {} devices)".format(
            total_infer_time,
            inference_timer.total_time * num_devices / len(dataset),
            num_devices,
        )
    )

    predictions = _accumulate_predictions_from_multiple_gpus(predictions)
    if not is_main_process():
        return

    # if output_folder:
    #     torch.save(predictions, os.path.join(output_folder, "predictions.pth"))

    extra_args = dict(
        box_only=box_only,
        iou_types=iou_types,
        expected_results=expected_results,
        expected_results_sigma_tol=expected_results_sigma_tol,
    )

    return evaluate(dataset=dataset,
                    predictions=predictions,
                    output_folder=output_folder,
                    **extra_args)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 10 11:10:02 2020

@author: zhaoyu
"""

import numpy as np

class GridWorld():
    MOVE_RIGHT = 1
    MOVE_LEFT = -1

    def __init__(self):
        self._l_bnd = 0
        self._r_bnd = 2
        self.end = self._r_bnd+self.MOVE_RIGHT
        self.init()

    def init(self):
        self.pos = self._l_bnd
        return self.pos

    def take_action(self, action):
        if self.pos == self._l_bnd:
            self.pos = self._l_bnd if action==self.MOVE_LEFT else self._l_bnd+self.MOVE_RIGHT
        elif self.pos == 1:
            self.pos -= action
        elif self.pos == self._r_bnd:
            self.pos += action
        return -1

    def choose_action(self, p):
        return self.MOVE_RIGHT if np.random.rand()<=p else self.MOVE_LEFT

    def action(self, p):
        a = self.choose_action(p)
        r = self.take_action(a)
        s = self.get_state()
        return a, r, s

    def game_end(self):
        return True if self.pos == self.end else False

    def get_state(self):
        return self.pos

    def __str__(self):
        return '{0}'.format(self.pos)

if __name__ == '__main__':
    env = GridWorld()
    print('start at pos: ', env)

    params = np.random.rand()
    print(params)
    for _ in range(1000):
        if env.game_end():
            break
        a, s, r = env.action([params])
        print('step:', _, 'arrive at pos: ', env)


import os
import re
import random
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint

from nvidia_tao_pytorch.core.callbacks.loggers import TAOStatusLogger
import nvidia_tao_pytorch.core.loggers.api_logging as status_logging
from nvidia_tao_pytorch.core.hydra.hydra_runner import hydra_runner
from nvidia_tao_pytorch.cv.ocrnet.config.default_config import ExperimentConfig
from nvidia_tao_pytorch.cv.ocrnet.model.pl_ocrnet import OCRNetModel

def run_experiment(experiment_spec: ExperimentConfig):
    """run experiment."""
    if experiment_spec.train.results_dir is not None:
        results_dir = experiment_spec.train.results_dir
    else:
        results_dir = os.path.join(experiment_spec.results_dir, "train")
        experiment_spec.train.results_dir = results_dir

    total_epochs = experiment_spec.train.num_epochs
    os.makedirs(f'{results_dir}', exist_ok=True)
    manual_seed = experiment_spec.train.seed
    import torch
    import torch.backends.cudnn as cudnn
    import numpy as np

    random.seed(manual_seed)
    np.random.seed(manual_seed)
    torch.manual_seed(manual_seed)
    torch.cuda.manual_seed(manual_seed)

    cudnn.benchmark = True
    cudnn.deterministic = True

    status_logger_callback = TAOStatusLogger(results_dir,
                                             append=True,
                                             num_epochs=total_epochs)

    status_logging.set_status_logger(status_logger_callback.logger)

    ocrnet_model = OCRNetModel(experiment_spec)
    clip_grad = experiment_spec.train.clip_grad_norm
    gpus_ids = experiment_spec.train.gpu_ids
    num_gpus = experiment_spec.train.num_gpus
    distributed_strategy = None
    if num_gpus > 1 or len(gpus_ids) > 1:
        distributed_strategy = experiment_spec.train.distributed_strategy

    val_inter = experiment_spec.train.validation_interval
    if num_gpus > 1:
        trainer = Trainer(devices=num_gpus,
                          max_epochs=total_epochs,
                          check_val_every_n_epoch=val_inter,
                          default_root_dir=results_dir,
                          enable_checkpointing=False,
                          strategy=distributed_strategy,
                          accelerator='gpu',
                          num_sanity_val_steps=0,
                          gradient_clip_val=clip_grad)
    else:
        trainer = Trainer(gpus=gpus_ids,
                          max_epochs=total_epochs,
                          check_val_every_n_epoch=val_inter,
                          default_root_dir=results_dir,
                          enable_checkpointing=False,
                          strategy=distributed_strategy,
                          accelerator='gpu',
                          num_sanity_val_steps=0,
                          gradient_clip_val=clip_grad)

    ckpt_inter = experiment_spec.train.checkpoint_interval
    ModelCheckpoint.FILE_EXTENSION = ".pth"
    checkpoint_callback = ModelCheckpoint(every_n_epochs=ckpt_inter,
                                          dirpath=results_dir,
                                          save_on_train_epoch_end=True,
                                          monitor=None,
                                          save_top_k=-1,
                                          filename='ocrnet_{epoch:03d}')
    resume_ckpt = experiment_spec['train']['resume_training_checkpoint_path']
    if resume_ckpt:
        status_logging.get_status_logger().write(
            message=f"Resuming training from checkpoint: {resume_ckpt}",
            status_level=status_logging.Status.STARTED
        )
        resumed_epoch = re.search('epoch=(\\d+)', resume_ckpt)
        if resumed_epoch:
            resumed_epoch = int(resumed_epoch.group(1))
        else:
            resumed_epoch = 0
        status_logger_callback.epoch_counter = resumed_epoch + 1  # make sure callback epoch matches resumed epoch

    trainer.callbacks.append(status_logger_callback)
    trainer.callbacks.append(checkpoint_callback)

    trainer.fit(ocrnet_model)

spec_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Load experiment specification, additially using schema for validation/retrieving the default values.
# --config_path and --config_name will be provided by the entrypoint script.

@hydra_runner(
    config_path=os.path.join(spec_root, "experiment_specs"), config_name="experiment", schema=ExperimentConfig
)
def main(cfg: ExperimentConfig) -> None:
    """Run the training process."""
    try:
        run_experiment(experiment_spec=cfg)
        status_logging.get_status_logger().write(
            status_level=status_logging.Status.SUCCESS,
            message="Training finished successfully."
        )
    except (KeyboardInterrupt, SystemExit):
        status_logging.get_status_logger().write(
            message="Training was interrupted",
            verbosity_level=status_logging.Verbosity.INFO,
            status_level=status_logging.Status.FAILURE
        )
    except Exception as e:
        status_logging.get_status_logger().write(
            message=str(e),
            status_level=status_logging.Status.FAILURE
        )
        raise e

if __name__ == '__main__':
    main()


import vcf
import argparse
from record import Record

def read_phased_record(inf):
    ps = 0
    update_ps = False
    phased_record = dict()
    for line in inf:
        if line[0] == 'B':
            update_ps = True
        elif line[0] == '*':
            continue
        else:
            buff = line.split()
            pos = int(buff[0])
            hap0 = buff[1]
            hap1 = buff[2]
            if hap0 == '-' or hap1 == '-':
                continue
            if update_ps:
                ps = pos
                update_ps = False
            phased_record[pos] = Record(pos, int(hap0), int(hap1), ps)
    return phased_record

def reset_phasing_info(record: vcf.model._Record):
    gt_str = record.samples[0]['GT']
    new_gt_str = gt_str[0] + '/' + gt_str[2]
    record.samples[0].gt_nums = new_gt_str
    if 'PS' in record.FORMAT.split(':'):
        record.samples[0].data = record.samples[0].data._replace(PS=0)

    record.samples[0].data = record.samples[0].data._replace(GT=new_gt_str)

def update_phasing_info_dchap(in_vcf: vcf.Reader, out_vcf: vcf.Writer, phased_record:dict):
    rec: vcf.model._Record
    count = 0
    for rec in in_vcf:
        count += 1
        pos = rec.POS
        if count in phased_record.keys(): #phased
            record = phased_record[count]
            record.finalize_record(rec)
        else:                           #not phased
            reset_phasing_info(rec)

        out_vcf.write_record(rec)

def update_phasing_info(in_vcf: vcf.Reader, out_vcf: vcf.Writer, phased_record:dict):
    rec: vcf.model._Record
    for rec in in_vcf:
        pos = rec.POS
        if pos in phased_record.keys(): #phased
            record = phased_record[pos]
            record.finalize_record(rec)
        else:                           #not phased
            reset_phasing_info(rec)

        out_vcf.write_record(rec)

def main():
    parser = argparse.ArgumentParser('phaseset_to_vcf.py')
    parser.add_argument('-i', '--inf',  help='in phaseset, refhap format', required=True)
    parser.add_argument('-v', '--vcf', help='input vcf', required=True)
    parser.add_argument('-o', '--out', help='output vcf', required=True)
    parser.add_argument('--dchap', help='dchap format', action="store_true")
    options = parser.parse_args()

    inf = open(file=options.inf, mode="r")
    in_vcf = vcf.Reader(filename=options.vcf)
    outf = vcf.Writer(open(options.out, "w"), template=in_vcf)

    phased_record = read_phased_record(inf)
    if options.dchap:
        update_phasing_info_dchap(in_vcf, outf, phased_record)
    else:
        update_phasing_info(in_vcf, outf, phased_record)

    inf.close()
    outf.close()

    return

if __name__ == '__main__':
    main()


from .base import BaseConfigWithSerializer

class Config(BaseConfigWithSerializer):
    def set_defaults(self):
        Config._params.update(
            order="bfs",
            batch_size=32,
            max_num_edges=130,
            min_num_edges=2,
            max_num_nodes=40,
            min_num_nodes=4,
            shuffle=True,
            test_size=0.3,
            embed_dim=32,
            hidden_dim=128,
            num_layers=2,
            dropout=0.3,
            temperature1=0.75,
            temperature2=0.75,
            max_epochs=2000,
            stop_loss=0.0,
            scheduler_class="StepLR",
            scheduler_params={"step_size": 200, "gamma": 0.5},
            optimizer_class="Adam",
            optimizer_params={'lr': 0.001},
            device="gpu"
        )

class GRUConfig(BaseConfigWithSerializer):
    def set_defaults(self):
        Config._params.update(
            batch_size=32,
            max_num_edges=130,
            min_num_edges=2,
            max_num_nodes=40,
            min_num_nodes=4,
            shuffle=True,
            test_size=0.3,
            embed_dim=32,
            hidden_dim=128,
            num_layers=2,
            max_epochs=800,
            dropout=0.3,
            stop_loss=0.1,
            temperature=0.8,
            scheduler_class="StepLR",
            scheduler_params={"step_size": 100, "gamma": 0.5},
            optimizer_class="Adam",
            optimizer_params={'lr': 0.001},
            device="gpu"
        )

class BaselineConfig(BaseConfigWithSerializer):
    def set_defaults(self):
        BaselineConfig._params.update(
            name="ER",
            metric="degree",
            max_num_nodes=40,
            min_num_nodes=4,
            max_num_edges=130,
            min_num_edges=2,
            test_size=0.3,
            device="gpu"
        )

class GraphRNNConfig(BaseConfigWithSerializer):
    def set_defaults(self):
        GraphRNNConfig._params.update(
            max_num_nodes=40,
            min_num_nodes=4,
            max_num_edges=130,
            min_num_edges=2,
            max_num_node=None,  # max number of nodes in a graph
            max_prev_node=None,  # max previous node that looks back
            hidden_size_rnn=64,
            hidden_size_rnn_output=16,  # hidden size for output RNN
            embedding_size_rnn=32,
            embedding_size_rnn_output=8,  # the embedding size for output rnn
            embedding_size_output=32,
            batch_size=32,
            test_size=0.3,
            num_layers=3,
            num_workers=4,  # num workers to load data, default 4
            batch_ratio=32,
            epochs=2000,  # now one epoch means batch_ratio x batch_size
            lr=0.003,
            milestones=[400, 1000],
            lr_rate=0.3,
            device="gpu"
        )

def get_config_class(model_name):
    if model_name == "GRAPHER":
        return Config

    if model_name == "GRAPHRNN":
        return GraphRNNConfig

    if model_name == "GRU":
        return GRUConfig

    if model_name in ["ER", "BA"]:
        return BaselineConfig

    return Config


import re
import requests
import sqlite3
from bs4 import BeautifulSoup
from hudba.models import Genre, Band

def genrescsv():
    genres = Genre.objects.order_by('id')
    url = 'hudba/static/csv/genres.csv'
    f = open(url, 'r')
    lines = f.readlines()
    f.close()
    f = open(url, 'w')
    for number, line in enumerate(lines):
        split = line.split(';')
        correct = 0
        for genre in genres:
            if split[0] == genre.name:
                correct = 1
                break
        if correct == 1:
            f.write(line)
    f.close()

    for genre in genres:
        f = open(url, 'r')
        lines = f.readlines()
        i = 0
        found = 0
        while i < len(lines):
            splitline = lines[i].split(';')
            name = splitline[0]
            if name == genre.name:
                found = 1
                break
            else:
                i += 1
        if found == 0:
            link = "https://cs.wikipedia.org/wiki/" + genre.name
            response = requests.get(url=link)
            soup = BeautifulSoup(response.content, 'html.parser')
            item = soup.select('p')[0]
            para = item.text
            edit = para.replace('\'', ' ')
            finaledit = edit.replace('\"', ' ')
            modify = re.sub(r"\[[^()]*]", "", finaledit)
            csvinput = genre.name + ';' + modify
            f = open(url, 'a', encoding='windows-1250')
            f.write(csvinput)
        f.close()

def bandscsv():
    bands = Band.objects.order_by('id')
    url = 'hudba/static/csv/bands.csv'
    f = open(url, 'r')
    lines = f.readlines()
    f.close()
    f = open(url, 'w')
    for number, line in enumerate(lines):
        split = line.split(';')
        correct = 0
        for band in bands:
            if split[0] == band.title:
                correct = 1
                break
        if correct == 1:
            f.write(line)
    f.close()

    for band in bands:
        f = open(url, 'r')
        lines = f.readlines()
        i = 0
        found = 0
        while i < len(lines):
            splitline = lines[i].split(';')
            name = splitline[0]
            if name == band.title:
                found = 1
                break
            else:
                i += 1
        if found == 0:
            link = "https://cs.wikipedia.org/wiki/" + band.title
            response = requests.get(url=link)
            soup = BeautifulSoup(response.content, 'html.parser')
            csvinput = band.title
            items = [soup.select('p')[0], soup.select('p')[1], soup.select('p')[2]]
            for item in items:
                item = item.text
                item = item.replace('\'', ' ')
                item = item.replace('\"', ' ')
                item = item.replace('¡', ' ')
                modified_item = re.sub(r"\[[^()]*]", "", item)
                csvinput = csvinput + ';' + modified_item
            f = open(url, 'a', encoding='windows-1250')
            f.write(csvinput)
        f.close()

def genressql():
    con = sqlite3.connect('db.sqlite3')
    cur = con.cursor()
    f = open('hudba/static/csv/genres.csv', 'r')
    i = 0
    lines = f.readlines()
    while i < len(lines):
        split = lines[i].split(';')
        article = split[1]
        name = split[0]

        cur.execute("UPDATE hudba_genre SET article = '" + article + "' WHERE name = '" + name + "'")
        i += 1
    con.commit()
    con.close()

def bandssql():
    con = sqlite3.connect('db.sqlite3')
    cur = con.cursor()
    f = open('hudba/static/csv/bands.csv', 'r')
    i = 0
    lines = f.readlines()
    while i < len(lines):
        split = lines[i].split(';')
        article = split[1]
        name = split[0]

        cur.execute("UPDATE hudba_band SET article = '" + article + "' WHERE title = '" + name + "'")
        i += 1

    con.commit()
    con.close()


# -*- coding: utf-8 -*-
"""
Created on Thu Sep  8 11:52:16 2016

@author: rookie
"""

import pandas as pd
import matplotlib as mpl

mpl.style.use('ggplot')

student_dataset = pd.read_csv("students.data",index_col=0)
my_series =student_dataset.G3
my_dataframe = student_dataset[['G3','G2','G1']]

#my_series.plot.hist(alpha=0.5)
my_dataframe.plot.hist(alpha=0.5, normed=False)


import json
import numpy as np
import pandas as pd
import jax
from jax import value_and_grad, grad, jit, random, vmap
from jax.experimental import optimizers, stax
from jax.lib import pytree
from jax.tree_util import tree_flatten, tree_multimap, tree_unflatten
import jax.numpy as jnp
from rtree import index
from scipy.stats import entropy
from scipy import sparse
import time

print_for_debug = False

def get_auxiliary_features(filename):
        _bins = 512
        aux_data = np.load(filename)
        H, xedges, yedges = np.histogram2d(aux_data[:, 0], aux_data[:, 1], bins=_bins)
        x_probs = np.true_divide(H,np.sum(H)) # convert the histogram to probability
        x_probs = x_probs.ravel() # flatten
        ent = entropy(x_probs) # shannon entropy
#         print('Determined feature for dataset ',filename, ' of size ', np.sum(H), ' at binning ', _bins, ' entropy: ', ent)
        return ent

class Log():
    def __init__(self):
        self.log = {}

    def add(self, name, val):
        if name not in self.log:
            self.log[name] = []
        self.log[name].append(float(val))

    def get(self, name):
        return self.log[name][-1]

    def save(self, path='results.json'):
        log_df = pd.DataFrame.from_dict(self.log)
        with open(path, 'w') as f:
            log_df.to_json(f)

class MAE():
    def __init__(self):
        self.name = "mae"

    def call(self, model, params, batch, weights=None):
        inputs, y_true = batch[0], batch[1]
        y_pred = model.apply(params, None, inputs)
        return jnp.average(jnp.abs((y_pred - y_true)**2), axis=0)

    def calc(self, y_true, y_pred):
        return jnp.average(jnp.abs(y_pred - y_true), axis=0)

class SAE():
    def __init__(self, smooth, name):
        self.name = name
        self.smooth = smooth

    def call(self, model, params, batch, weights=None):
        inputs, y_true = batch[0], batch[1]
        y_pred = model.apply(params, None, inputs)
        return jnp.average(jnp.abs(y_pred - y_true)/(jnp.maximum(y_true, self.smooth)), axis=0)

    def calc(self, y_true, y_pred):
        return jnp.average(jnp.abs(y_pred - y_true)/(jnp.maximum(y_true, self.smooth)), axis=0)

def mse_weighted_loss(model, weights, params, batch):
    inputs, y_true, _weights = batch[0], batch[1], batch[2]
    y_pred = model.apply(params, None, inputs)
    return jnp.average(jnp.square(jnp.subtract(y_pred, y_true)), weights=_weights)

def calc_metrics(model, params, batch, metrics, logs, model_query_size,  weights=None):
    inputs, y_true = batch[0], batch[1]
    if inputs.shape[1] == 3:
        x = inputs[:, :2]
    else:
        x = inputs
    y_pred = model.apply(params, None, x)

    if inputs.shape[1] == 3:
        y_pred = y_pred*((inputs[:, 2]/model_query_size)**2)

    for metric in metrics:
        val = metric.calc(y_true, y_pred) # nonunifrom test set
        logs.add(metric.name, val[0])

# TODO::: write new get_train_test_categorical function foir ADULT dataset.

def get_train_test(data_loc, cell_width, max_val, augmented_query_size, utilization_range, test_size, dim, eps, test_query_loc, db, min_vals, max_vals, random_seed, get_cell_aligned_qs = True, k = 3):
    #max_dist_thres = 0.32

    # creates a histogram of dataset

    # push the extents a little larger based on augmented_query_size
    t_queries_x = np.arange(start=-max_val/2-augmented_query_size, stop=max_val/2+augmented_query_size, step=cell_width)
    t_queries_y = np.arange(start=-max_val/2-augmented_query_size, stop=max_val/2+augmented_query_size, step=cell_width)
    total_cells = t_queries_x.shape[0]
    xx, yy =  np.meshgrid(t_queries_x, t_queries_y, indexing='ij')
    grid_corners = np.dstack([xx, yy]).reshape(-1, dim)

    H, xedges, yedges = np.histogram2d(db[:, 0], db[:, 1], bins=[np.append(t_queries_x,t_queries_x[-1]+cell_width), np.append(t_queries_y,t_queries_y[-1]+cell_width)], range=[[-max_val/2, max_val/2], [-max_val/2, max_val/2]])

    grid_val = H.astype(int)
    grid_noise  = 1/eps
    #fixing the random seed so that noise value is the same across different neural networks
    np.random.seed(random_seed)
    grid_val_noisy =  grid_val+np.random.laplace(0, grid_noise, grid_val.shape)

    grid_locs = grid_corners.reshape(total_cells, total_cells, 2)
    total_in_cell = int((max_val+augmented_query_size)/cell_width)
    answer_len = np.floor(augmented_query_size/cell_width).astype(int)
    extra_len =  augmented_query_size - answer_len*cell_width
    qs_begin = grid_locs[:total_in_cell-answer_len, :total_in_cell-answer_len]
    qs_end = grid_locs[:total_in_cell-answer_len, :total_in_cell-answer_len]+augmented_query_size
    qs = (qs_begin+qs_end)/2
    qs = qs.reshape(-1, 2)

    if print_for_debug:
        print('grid_val_noisy', grid_val_noisy[0:2,0:2], grid_val_noisy.shape)
        print('xx', xx[0:2,0:2], xx.shape)
        print('yy', yy[0:2,0:2], yy.shape)
        print('total_cells', total_cells)
        print('total_in_cell', total_in_cell)
        print('augmented_query_size', augmented_query_size)
        print('cell_width', cell_width)
        print('qs', qs[0],qs.shape)
        print('answer_len', answer_len)
        print('extra_len', extra_len)

    if get_cell_aligned_qs:
        if answer_len == 0:
            extra_frac = (extra_len/cell_width)
            ress = np.array([[grid_val_noisy[i+answer_len, j+answer_len]*(extra_frac**2)] for i in range(total_in_cell-answer_len) for j in range(total_in_cell-answer_len)]).reshape(-1, 1)
        else:
            ress = np.array([[np.sum(grid_val_noisy[i:i+answer_len, j:j+answer_len])] for i in range(total_in_cell-answer_len) for j in range(total_in_cell-answer_len)]).reshape(-1, 1)
            if extra_len > 0:
                extra_frac = (extra_len/cell_width)
                extra_res = np.array([[(np.sum(grid_val_noisy[i:i+answer_len+1, j:j+answer_len+1])-np.sum(grid_val_noisy[i:i+answer_len, j:j+answer_len])-grid_val_noisy[i+answer_len, j+answer_len])*extra_frac + grid_val_noisy[i+answer_len, j+answer_len]*(extra_frac**2)] for i in range(total_in_cell-answer_len) for j in range(total_in_cell-answer_len)]).reshape(-1, 1)
                ress = ress+extra_res
        if print_for_debug:
            print('ress', ress[0:5], ress.shape)
        ress = np.clip(ress, 0, None)

    else:
        # Q_new = [Q+i*(c/k) for i in range(k)], so basically when k=1 you get the same answers as the block above
        assert k >= 1, 'what do you mean k is zero'
        start = time.time()
        qsess = []
        for i in range(k):
            _qs = qs + (i * (cell_width/k))
            qsess.append(_qs)
        qs = np.concatenate(qsess)
        ress = get_ress_from_grid(qs, augmented_query_size/2, cell_width, max_val, grid_val_noisy)
        ress = ress.reshape(-1, 1)
        print('Generated', len(ress), 'training queries in ' , time.time()-start, 'seconds')

    if print_for_debug:
        print('ress', ress[0:5], ress.shape)

    utilization_range_begin = augmented_query_size - utilization_range/2
    test_range = np.random.rand(test_size, 1)*(utilization_range)+utilization_range_begin
    test_loc = np.load(test_query_loc)
    test_loc = ((test_loc-min_vals)/(max_vals-min_vals)-0.5)*max_val
    np.random.shuffle(test_loc)
    test_loc = test_loc[:test_size]
    test = np.concatenate([test_loc, test_range], axis=1)
    test_res = np.array([np.sum([np.logical_and.reduce([np.logical_and((db[:, d]<test[i, d]+test[i, -1]/2),(db[:, d]>=test[i, d]-test[i, -1]/2)) for d in range(dim)])]) for i in range(test.shape[0])]).reshape((-1, 1))

    return qs, ress, test, test_res

def get_training_weights(q_w_loc, n, max_vals, min_vals, max_val, qs, ress, augmented_query_size):
    q_w = np.load(q_w_loc)
    q_w = ((q_w-min_vals)/(max_vals-min_vals)-0.5)*max_val

    idx = index.Index() # rtree index intialization
    for i in range(q_w.shape[0]): # insert all workload queries as points into the rtree
        idx.insert(i, (q_w[i][0],q_w[i][1]))

    counts = []
    for i in range(qs.shape[0]): # count for each query how many workload points are in it
        lb = qs[i] - augmented_query_size/2
        ur = qs[i] + augmented_query_size/2
        counts.append( idx.count((lb[0],lb[1],ur[0],ur[1])) )

    counts = np.array(counts)
    counts = counts.reshape((len(counts),1))

    weights = counts/(np.maximum(ress, 0.001*n))
    return weights, counts

# q_range is half auigemented query size
# x is H, the noisy counts in each cell
# test_queries are actually all centers of queries.
def get_ress_from_grid(test_queries, q_range, cell_width, max_val, x):

    test_queries_lb = test_queries - q_range
    test_queries_ru = test_queries + q_range
    test_queries_lu = np.concatenate([test_queries_lb[:, 0:1], test_queries_ru[:, 1:2]], axis=1)
    test_queries_rb = np.concatenate([test_queries_ru[:, 0:1], test_queries_lb[:, 1:2]], axis=1)

    if print_for_debug:
        print("test_queries_lb[0] before clip: ", test_queries_lb[0])
        print("test_queries_ru[0] before clip: ", test_queries_ru[0])
        print("test_queries_lu[0] ",test_queries_lu[0])
        print("test_queries_rb[0] ",test_queries_rb[0])

    max_val = max_val + (q_range*2*2) # umm yeah just extend augemented query size on both sides

    cell_area = (cell_width)**2
    test_indx_lb = np.clip(np.floor((test_queries_lb+max_val/2)/(cell_width)), 0, x.shape[0]-1).astype(int)
    test_indx_ru = np.clip(np.floor((test_queries_ru+max_val/2)/(cell_width)), 0, x.shape[0]-1).astype(int)
    test_indx_lu = np.clip(np.floor(((test_queries_lu+max_val/2))/(cell_width)), 0, x.shape[0]-1).astype(int)
    test_indx_rb = np.clip(np.floor(((test_queries_rb+max_val/2))/(cell_width)), 0, x.shape[0]-1).astype(int)

    if print_for_debug:
        print("test_indx_lb[0] ", ((test_queries_lb[0]+max_val/2))/(cell_width), test_indx_lb[0])
        print("test_indx_ru[0] ", ((test_queries_ru[0]+max_val/2))/(cell_width), test_indx_ru[0])
        print("test_indx_lu[0] ", ((test_queries_lu[0]+max_val/2))/(cell_width), test_indx_lu[0])
        print("test_indx_rb[0] ", ((test_queries_rb[0]+max_val/2))/(cell_width), test_indx_rb[0])

    qs = []
    # qs_sparse = []
    # set num of queries to evaluate

    # num_qs = 10000
    num_qs = len(test_queries)

    start = time.time()

    ress_np = np.zeros(num_qs, dtype=np.float32)
    # last_end = -1
    # batch_size = 5000
    # 10k queries batch size 100 = 14 seeconds
    # 10k queries batch size 500 = 14 seeconds
    # 10k queries batch size 1000 = 14 seeconds
    # batch size makes almost no difference in terms of running time. just set it to something that can fit in memory
    # 5k queries take 20 % memory for 642*642 grid.

    # x_sp = sparse.csr_matrix(x)
    for i in range(num_qs):
        q = np.zeros((x.shape[0], x.shape[1]), dtype=np.float32)
        # q = sparse.csr_matrix((x.shape[0], x.shape[1]), dtype=np.float32)

        lb_intersection = np.floor(((test_queries_lb[i]+max_val/2)/(cell_width)))*cell_width+np.array([cell_width, cell_width])-max_val/2
        rb_intersection = np.floor(((test_queries_rb[i]+max_val/2)/(cell_width)))*cell_width+np.array([0, cell_width])-max_val/2
        lu_intersection = np.floor(((test_queries_lu[i]+max_val/2)/(cell_width)))*cell_width+np.array([cell_width, 0])-max_val/2
        ru_intersection = np.floor(((test_queries_ru[i]+max_val/2)/(cell_width)))*cell_width+np.array([0, 0])-max_val/2

        if i == 0 and print_for_debug:
            print("lb_intersection ", lb_intersection)
            print("rb_intersection ", rb_intersection)
            print("lu_intersection ", lu_intersection)
            print("ru_intersection ", ru_intersection)

        area_lb = np.prod(lb_intersection-test_queries_lb[i])
        area_rb = np.prod(np.abs(rb_intersection-test_queries_rb[i]))
        area_lu = np.prod(np.abs(lu_intersection-test_queries_lu[i]))
        area_ru = np.prod(np.abs(ru_intersection-test_queries_ru[i]))

        if i == 0 and print_for_debug:
            print("area_lb ", area_lb, area_lb/cell_area)
            print("area_rb ", area_rb, area_rb/cell_area)
            print("area_lu ", area_lu, area_lu/cell_area)
            print("area_ru ", area_ru, area_ru/cell_area)

        q[test_indx_lb[i, 0], test_indx_lb[i, 1]] = area_lb/cell_area
        q[test_indx_rb[i, 0], test_indx_rb[i, 1]] = area_rb/cell_area
        q[test_indx_lu[i, 0], test_indx_lu[i, 1]] = area_lu/cell_area
        q[test_indx_ru[i, 0], test_indx_ru[i, 1]] = area_ru/cell_area
        q[test_indx_lb[i, 0]+1:test_indx_rb[i, 0], test_indx_lb[i, 1]+1:test_indx_lu[i, 1]]=1

        if test_indx_rb[i, 0] - test_indx_lb[i, 0]>1:
            portion_mid = (lb_intersection[1]-test_queries_lb[i,1])/(cell_width)
            q[test_indx_lb[i, 0]+1:test_indx_rb[i, 0], test_indx_rb[i, 1]] = portion_mid
        if test_indx_ru[i, 1] - test_indx_rb[i, 1]>1:
            portion_mid = (test_queries_rb[i,0] - rb_intersection[0])/(cell_width)
            q[test_indx_rb[i, 0], test_indx_rb[i, 1]+1:test_indx_ru[i, 1]] = portion_mid
        if test_indx_ru[i, 0] - test_indx_lu[i, 0]>1:
            portion_mid = np.abs(lu_intersection[1]-test_queries_lu[i,1])/(cell_width)
            q[test_indx_lu[i, 0]+1:test_indx_ru[i, 0], test_indx_ru[i, 1]] = portion_mid
        if test_indx_lu[i, 1] - test_indx_lb[i, 1]>1:
            portion_mid = np.abs(test_queries_lb[i,0] - lb_intersection[0])/(cell_width)
            q[test_indx_lb[i, 0], test_indx_lb[i, 1]+1:test_indx_lu[i, 1]] = portion_mid

        # this is slow as shit but memory efficient
        # ress_np[i]= (q).multiply(x_sp).sum()

        ress_np[i]= np.sum((q*x) )

        if i == 0 and print_for_debug:
            rows, cols = np.nonzero(q)
            print("non empty vals len(q[rows, cols])", len(q[rows, cols]))
            print(q[np.nonzero(q)])

    if print_for_debug:
        print('ress_np', ress_np[0:5], len(ress_np))
        print('time taken to generate', num_qs, 'training queries' , time.time()-start, 'seconds')

    # dt = qs_sparse.multiply(sparse.csr_matrix(x))
    # ress_from_sparse = sparse.sum(dt, axis=(1, 2))
    # print('ress_from_sparse', ress_from_sparse)
    return ress_np


from hypothesis import given

from tests.utils import (BoundPortedTrapezoidsPair,
                         equivalence)
from . import strategies

@given(strategies.trapezoids_pairs, strategies.trapezoids_pairs)
def test_basic(first_trapezoids_pair: BoundPortedTrapezoidsPair,
               second_trapezoids_pair: BoundPortedTrapezoidsPair) -> None:
    first_bound, first_ported = first_trapezoids_pair
    second_bound, second_ported = second_trapezoids_pair

    assert equivalence(first_bound == second_bound,
                       first_ported == second_ported)


import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics

data = pd.read_csv('keepers.csv', sep='\t')

data.fillna(value=0, inplace=True)

pd.set_option('display.max_rows', data.shape[0]+1)

#print(data)
#print(data.head())
#print(data.tail())
#print(data.shape)

#sns.pairplot(data, x_vars=feature_cols, y_vars=response_col, size=7, aspect=0.7, kind='reg')
#plt.show()

feature_cols = ['age', 'seasons', 'mins', 'apps', 'minapp', 'starts', 'subs', 'ga', 'ga90', 'sota', 'win', 'draw', 'loss', 'cs', 'csperc', 'saves']
response_col = ['rank']

X = data[feature_cols]
#print(X.shape)
y = data[response_col]
#print(y.shape)

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

#print('X_train:', X_train)
#print('X_train.shape:', X_train.shape)
#print('y_train:', y_train)
#print('y_train.shape:', y_train.shape)
#print('X_test:', X_test)
#print('y_test:', y_test)
#print('X_train.shape, y_train.shape:', X_train.shape, y_train.shape)
#print('X_test.shape, y_test.shape:', X_test.shape, y_test.shape)
#print('isnan(X_train):', np.isnan(X_train))

k_range = range(1, 26)
scores = []
for k in k_range:
    estimator = KNeighborsClassifier(n_neighbors=k)
    estimator.fit(X_train, y_train.values.ravel())
    y_pred = estimator.predict(X_test)
    scores.append(metrics.accuracy_score(y_test, y_pred))

plt.plot(k_range, scores)
plt.xlabel('k')
plt.ylabel('Accuracy')
plt.show()  # Highest=2 & 24

estimator = KNeighborsClassifier(n_neighbors=24)
estimator.fit(X_train, y_train.values.ravel())

y_pred = estimator.predict(X_test)

print('mean_absolute_error:', metrics.mean_absolute_error(y_test, y_pred))
print('mean_squared_error:', metrics.mean_squared_error(y_test, y_pred))
print('root mean_squared_error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))

#print(X_test.iloc[[1]])
#print(y_test.iloc[[1]]) # 29
datum = data.iloc[[88]]
#print(datum)
X = datum[feature_cols]
y_pred = estimator.predict(X)
print('Prediction for 29:', y_pred) # 27

#print(X_test.iloc[[2]])
#print(y_test.iloc[[2]]) # 18
datum = data.iloc[[163]]
#print(datum)
X = datum[feature_cols]
y_pred = estimator.predict(X)
print('Prediction for 18:', y_pred) # 16


# -*- coding: utf-8 -*-

"""
Generative Adversarial Nets(GAN)
paper:Generative Adversarial Nets.arxiv:1406.2661v1
This is the implementation of GAN for MNIST dataset
learned from:https://github.com/wiseodd/generative-models/blob/master/GAN/vanilla_gan/gan_tensorflow_v1.py
"""

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os

from tensorflow.examples.tutorials.mnist import input_data

def xavier_init(size):
    in_dim = size[0]
    xavier_stddev = 1. / tf.sqrt(in_dim /2.)
    return tf.random_normal(shape=size, stddev=xavier_stddev)

def generator(z):
    """
    Generator Net:
        The generator(z) takes 100-dimensional vector and returns 784-dimensional
        vector, which is MNIST image(28*28).z here is the prior for the G(z).In a
        way it learns a mapping between the prior space to Pdata.
    :param z:input noise sampled from N(0,1)
    :return:generated data from noise
    """
    g_w1 = tf.Variable(xavier_init([100, 128]))
    g_b1 = tf.Variable(tf.zeros(shape=[128]))
    g_h1 = tf.nn.relu(tf.matmul(z, g_w1) + g_b1)

    g_w2 = tf.Variable(xavier_init([128, 784]))
    g_b2 = tf.Variable(tf.zeros(shape=[784]))
    g_log_prob = tf.matmul(g_h1, g_w2) + g_b2

    g_prob = tf.nn.sigmoid(g_log_prob)

    theta_g = [g_w1, g_w2, g_b1, g_b2]

    return g_prob, theta_g

d_w1 = tf.Variable(xavier_init([784, 128]))
d_b1 = tf.Variable(tf.zeros(shape=[128]))
d_w2 = tf.Variable(xavier_init([128, 1]))
d_b2 = tf.Variable(tf.zeros(shape=[1]))
theta_d = [d_w1, d_w2, d_b1, d_b2]

def discriminator(x):
    """
    Discriminator Net
    :param x:input data
    :return:is data from x(d_prob) and log likelihood
    """
    d_h1 = tf.nn.relu(tf.matmul(x, d_w1) + d_b1)
    d_logit = tf.matmul(d_h1, d_w2) + d_b2
    d_prob = tf.nn.sigmoid(d_logit)
    return d_prob, d_logit

def sample_z(m,n):
    return np.random.uniform(-1., 1., size=[m, n])

def plot(samples):
    fig = plt.figure(figsize=(4, 4))
    gs = gridspec.GridSpec(4, 4)
    gs.update(wspace=0.05, hspace=0.05)

    for i, sample in enumerate(samples):
        ax = plt.subplot(gs[i])
        plt.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_aspect('equal')
        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')

    return fig

def train(batch_size=128,#mini-batch
          learning_rate=0.001#lr
          ):

    print('...... define model ......')
    # define placeholder
    x = tf.placeholder(tf.float32, shape=[None, 784])
    z_dim = 100
    z = tf.placeholder(tf.float32, shape=[None, 100])#z_dim = 100

    g_sample, theta_g = generator(z)
    d_real, d_logit_real = discriminator(x)
    d_fake, d_logit_fake = discriminator(g_sample)

    # D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake))
    # G_loss = -tf.reduce_mean(tf.log(D_fake))

    # define loss
    d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_real, labels=tf.ones_like(d_logit_real)))
    d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_fake, labels=tf.zeros_like(d_logit_fake)))
    d_loss = d_loss_real + d_loss_fake

    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_logit_fake, labels=tf.ones_like(d_logit_fake)))

    d_solver = tf.train.AdamOptimizer(learning_rate).minimize(d_loss, var_list=theta_d)
    g_solver = tf.train.AdamOptimizer(learning_rate).minimize(g_loss, var_list=theta_g)

    print('...... loading dataset......')
    mnist = input_data.read_data_sets('MNIST/', one_hot=True)

    if not os.path.exists('GAN_OUT/'):
        os.makedirs('GAN_OUT/')

    print('...... start to training ......')
    sess = tf.Session()
    sess.run(tf.global_variables_initializer())

    index = 0
    for epoch in range(1000000):
        if epoch % 1000 == 0:
            samples = sess.run(g_sample,
                               feed_dict={z: sample_z(16, z_dim)})
            fig = plot(samples)
            plt.savefig('GAN_OUT/{0}.png'.format(str(index).zfill(3)), bbox_inches='tight')
            index = index + 1
            plt.close(fig)

        xs, _ = mnist.train.next_batch(batch_size)

        _, d_loss_curr = sess.run([d_solver, d_loss],
                                  feed_dict={x: xs,
                                             z: sample_z(batch_size, z_dim)})
        _, g_loss_curr = sess.run([g_solver, g_loss],
                                  feed_dict={z: sample_z(batch_size, z_dim)})

        if epoch % 1000 == 0:
            print('---epoch:{0}'.format(epoch))
            print('d_loss:{:.4}'.format(d_loss_curr))
            print('g_loss:{:.4}'.format(g_loss_curr))
            print('---------------------------------')

if __name__ == '__main__':
    train()


import re
from collections import defaultdict

txtFile = open('day_15.txt')
lines = txtFile.readlines()

inputSensorMap = defaultdict(lambda: ".")
radarMap = defaultdict(lambda: ".")

minPosX = 9999999999
maxPosX = 0
minPosY = 9999999999
maxPosY = 0

txtOutput = ''

# read input data and get min/max of the x/y coordinates
for line in lines:
    coordinates =  re.findall(r'-?\d+',line)
    sensorPosX, sensorPosY, beaconPosX, beaconPosY = int(coordinates[0]), int(coordinates[1]), int(coordinates[2]), int(coordinates[3])
    inputSensorMap[f"{sensorPosX},{sensorPosY}"] = [int(coordinates[2]), int(coordinates[3]) ]
    radarMap[f"{sensorPosX},{sensorPosY}"] = 'S'
    radarMap[f"{beaconPosX},{beaconPosY}"] = 'B'

    maxPosX = max([maxPosX, sensorPosX, beaconPosX] )
    minPosX = min([minPosX, sensorPosX, beaconPosX] )
    maxPosY = max([maxPosY, sensorPosY, beaconPosY] )
    minPosY = min([minPosY, sensorPosY, beaconPosY] )

#print (inputSensorMap)
#print(minPosX, maxPosX, minPosY, maxPosY)
#print( inputSensorMap.values())

def printMap(minX,maxX, minY, maxY):
    global txtOutput
    global radarMap

    #make x axis labels
    txtOutput = "    "
    for posX in range(minX,maxX+1,1):
         txtOutput += str(posX % 10)
    txtOutput +="\n"

    for posY in range(minY,maxY+1,1):
        #make y-axis labels
        txtOutput +=f'{posY:03d} '
        #make x/y points
        for posX in range(minX,maxX+1,1):
            txtOutput += radarMap[f"{posX},{posY}"]#radarMap[posY][posX]
        txtOutput +="\n"
    print(txtOutput)

def drwRadius(sensX, sensY, mhtRadius):
    newRadius = mhtRadius
    for lvlY in range(0,newRadius,1):
        #upper part
        #right side
        if radarMap[f"{mhtRadius -lvlY+sensX},{sensY-lvlY}"] =='.':
            radarMap[f"{mhtRadius -lvlY+sensX},{(sensY-lvlY)}"] = '#'
        #left side
        if radarMap[f"{sensX-(mhtRadius -lvlY)},{sensY-lvlY}"] =='.':
            radarMap[f"{sensX-(mhtRadius -lvlY)},{(sensY-lvlY)}"] = '#'

        #lower part
        #right side
        if radarMap[f"{(mhtRadius -lvlY)+sensX},{sensY+lvlY}"] =='.':
            radarMap[f"{mhtRadius -lvlY+sensX},{(sensY+lvlY)}"] = '#'
        #left side
        if radarMap[f"{sensX-(mhtRadius -lvlY)},{sensY+lvlY}"] =='.':
            radarMap[f"{sensX-(mhtRadius -lvlY)},{(sensY+lvlY)}"] = '#'

    #peaks of lower and upper part
    if radarMap[f"{sensX},{sensY-newRadius}"] =='.':
        radarMap[f"{sensX},{sensY-newRadius}"]= '#'

    if radarMap[f"{sensX},{sensY+newRadius}"] =='.':
        radarMap[f"{sensX},{sensY+newRadius}"] = '#'

# #Test map view
# for sensorKey, beaconPos in inputSensorMap.items():

#     sensPos = sensorKey.split(",")
#     sensPosX, sensPosY = int(sensPos[0]) , int(sensPos[1])
#     mhtRadius = abs(sensPosX - beaconPos[0]) + abs(sensPosY - beaconPos[1])

#     maxPosX = max([maxPosX, sensPosX+mhtRadius] )
#     minPosX = min([minPosX, sensPosX-mhtRadius] )
#     maxPosY = max([maxPosY, sensPosY+mhtRadius] )
#     minPosY = min([minPosY, sensPosY-mhtRadius] )

#     for radius in range(1,mhtRadius+1,1):
#         drwRadius(sensPosX , sensPosY, radius)
#     #printMap(minPosX, maxPosX, minPosY, maxPosY)

# cntr = 0
# for posX in range(minPosX,maxPosX+1,1):
#     if radarMap[f"{posX},{10}"] == "#":
#         cntr+=1

# printMap(minPosX, maxPosX, minPosY, maxPosY)
# print(cntr)

rangesList = []

def mergeRanges(rngMin, rngMax):

    global rangesList
    tmpRngMax = 0
    tmpRngMin = 0
    fial = False
    if len(rangesList) > 0:
        for rng in rangesList:
            if rngMin <= rng[0] :
                tmpRngMin = rngMin
                if rngMax >= rng[1]:
                    tmpRngMax = rngMax
                    rng[0] = tmpRngMin
                    rng[1] = tmpRngMax
                    fial = False

                elif rngMax <= rng[1] and rngMax >= rng[0]:
                    tmpRngMax = rng[1]
                    rng[0] = tmpRngMin
                    rng[1] = tmpRngMax
                    fial = False
                else:
                    fial = True
            elif rngMin >= rng[0] and rngMin <= rng[1]:
                tmpRngMin = rng[0]
                tmpRngMax = rngMax
                if rngMax < rng[1]:
                    tmpRngMax = rng[1]

                rng[0] = tmpRngMin
                rng[1] = tmpRngMax
                fial = False
            else:
                fial = True

    if len(rangesList) == 0 or fial == True:
        rangesList.append([rngMin, rngMax])

myRanges = []
rngCntr = 0
def mergeRangesByList():

    global rangesList
    global myRanges
    newRangeMerged = False
    tmpRngMin, tmpRngMax = rangesList.pop(0)

    while True:
        newRangeMerged = False

        for rngIdx in range(len(rangesList)):
            if tmpRngMin <= rangesList[rngIdx][0]:
                if tmpRngMax >= rangesList[rngIdx][1]:
                    rangesList.pop(rngIdx)
                    newRangeMerged = True
                    break

                elif tmpRngMax <= rangesList[rngIdx][1] and tmpRngMax >= rangesList[rngIdx][0]:
                    tmpRngMax = rangesList[rngIdx][1]
                    rangesList.pop(rngIdx)
                    newRangeMerged = True
                    break

            elif tmpRngMin >= rangesList[rngIdx][0] and tmpRngMin <= rangesList[rngIdx][1]:
                tmpRngMin = rangesList[rngIdx][0]
                if tmpRngMax < rangesList[rngIdx][1]:
                    tmpRngMax = rangesList[rngIdx][1]

                rangesList.pop(rngIdx)
                newRangeMerged = True
                break

        if newRangeMerged == False: #no connectet range found
            myRanges.append([tmpRngMin, tmpRngMax])
            if len(rangesList) != 0:
                tmpRngMin, tmpRngMax = rangesList.pop(0)
            else:
                break

def getRadiusAtPos(sensX, sensY, mhtRadius, targetPos):

    targetPos = abs(sensY - targetPos) #get absolute distance to the beacon on the y Axis
    lineRange = [(mhtRadius -targetPos+sensX), (sensX-(mhtRadius -targetPos))] #get x position range max to min
    lineRange.sort() #sort elements (min, max)

    #mergeRanges(lineRange[0], lineRange[1])

    return lineRange

lineRangeMinX = 9999999999999999999
lineRangeMaxX = 0
targetLine = 3400528#22 <<-- line with the gap
maxSearchPosY = 4000000

for lineIdx in range(maxSearchPosY): #ca 10 min :(
    targetLine = lineIdx
    myRanges = []
    rangesList = []

    for sensorKey, beaconPos in inputSensorMap.items():
        sensPos = sensorKey.split(",")
        sensPosX, sensPosY = int(sensPos[0]) , int(sensPos[1])
        mhtRadius = abs(sensPosX - beaconPos[0]) + abs(sensPosY - beaconPos[1])

        if abs(sensPosY - targetLine) <= mhtRadius:
            lineRangeX= getRadiusAtPos(sensPosX, sensPosY, mhtRadius, targetLine)
            #print(lineRangeX)
            lineRangeMinX = min( [lineRangeMinX, lineRangeX[0] ] )
            lineRangeMaxX = max( [lineRangeMaxX, lineRangeX[1] ] )
            rangesList.append([lineRangeX[0] , lineRangeX[1]])
            #print(sensorKey, lineRangeX[0] , lineRangeX[1])

    mergeRangesByList()
    if len(myRanges) != 1:
        print(myRanges, lineIdx )
        if  myRanges[0][1] < myRanges[1][1]:
            print("gap at :", myRanges[0][1]+1, lineIdx, ((myRanges[0][1]+1)*maxSearchPosY)+lineIdx )
        else:
            print("gap at :", myRanges[1][1]+1, lineIdx, ((myRanges[1][1]+1)*maxSearchPosY)+lineIdx )

        break

# gtX = 3141837
# gtEerg = 12567351400528 #x * 4000000 + y
# gtY = 12567351400528 -(4000000*gtX) #3400528
# print(gtY)
# if len(myRanges) > 1:
#     if  myRanges[0][1] < myRanges[1][1]:
#         print("gap at :", myRanges[0][1]+1)#(myRanges[1][0] - myRanges[0][1])+myRanges[0][1] )
#     else:
#         print("gap at :", myRanges[1][1]+1)#(myRanges[0][0] - myRanges[1][1])+myRanges[1][1] )

# hh = ''.join(radarMap['11'])
# # print(lineRangeMinX, lineRangeMaxX, hh  )

# # number of al possible positions
# numberPositions = (lineRangeMaxX - lineRangeMinX+1)
# beaconInTargetLn = []

# #subtract beacons from the possible position number
# for sensorKey, beaconPos in inputSensorMap.items():

#     if beaconPos[1] not in beaconInTargetLn:
#         beaconInTargetLn.append( beaconPos[1] )
#         if  beaconPos[1] == targetLine and (beaconPos[0] <= lineRangeMaxX and beaconPos[0] >= lineRangeMinX):
#             numberPositions -=1;

# print(numberPositions, lineRangeMinX, lineRangeMaxX ) #6078701 -1161321 4917380

# #printMap(minPosX, maxPosX, minPosY, maxPosY)