from tkinter import *
from PIL import Image, ImageDraw
import numpy as np
np.random.seed(123)  # for reproducibility
from keras.applications.vgg16 import preprocess_input
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.preprocessing.image import img_to_array
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import cv2
from keras.utils import np_utils
from keras.models import model_from_yaml
import os

import HandwritingNNv2Function as nn
import LetterBreaker
import Prediction

IMAGES = 10

class GUI(Frame):
    def __init__(self, master):
        Frame.__init__(self, master)
        self.master = master
        self.users = ["Hannah", "Hayley", "Ashley"]
        self.mainMenu()

    #resets the GUI so that new things can be displayed on it
    def reset(self):
        for child in self.master.winfo_children():
            child.destroy()

    #the main menu screen
    def mainMenu(self):
        self.reset()
        self.title = Label(self.master, text = "Neural Network Handwriting Analysis").pack(side = TOP)
        self.subtitle = Label(self.master, text = "Can you fool the computer?").pack(side = TOP)
        self.newUserButton = Button(self.master, text = "Create New User", command = self.createNewUser).pack()
        self.determineUserButton = Button(self.master, text = "Determine User", command = self.determineUserScreen).pack()
        #self.debugButton = Button(self.master, text = "DEBUG, straight to the NN baby", command = trainNeuralNetwork([])).pack()
        self.exitbutton = Button(self.master, text = "Exit", fg = "red", command = self.master.destroy).pack(side = BOTTOM)

    #asks the user for a username
    def createNewUser(self):
        self.reset()
        self.title = Label(self.master, text = "Type your name here:").pack(side = TOP)
        self.namebox = Entry(self.master)
        self.namebox.pack()
        self.namebox.focus()
        self.submitbutton = Button(self.master, text = "Submit", command = self.newUserWriting).pack()

    #opens the writing screen 25 times so that 25 images can be created
    def newUserWriting(self):

        username = self.namebox.get()
        self.reset()
        self.users.append(username)
        for i in range(0, IMAGES):
            self.title = Label(self.master, text = "Write something! Anything! In print, and something different each time. (Testing Image {}/{})".format(i+1, IMAGES))
            self.title.grid(row = 0, column = 0, sticky = E+W+N)
            self.writingScreen(username, i)
        self.reset()
        label = Label(self.master, text = "Training Neural Network... \n\n(This may take a bit! Be patient!)").pack()

        data, labels, tempwords = LetterBreaker.imageProcess(self.users, 10)

        nn.trainNeuralNetwork(self.users, data, labels) #########
        self.mainMenu()

    #lets the user draw on the screen while creating an identical image in memory to be saved as a .png
    def writingScreen(self, username, image_number):
        def draw(event):
            x, y = event.x, event.y
            if canvas.old_coords:
                x1, y1 = canvas.old_coords
                #tkinter canvas drawing (visible)
                canvas.create_line(x, y, x1, y1)
                #PIL image draw (in memory)
                invisible_draw.line([x, y, x1, y1], (0,0,0))
            canvas.old_coords = x, y
        def reset_coords(event):
            canvas.old_coords = None

        canvas = Canvas(window, width=400, height=400, bg = "white")
        canvas.grid(row = 1, column = 0, sticky = E+W)
        canvas.old_coords = None

        # PIL create an empty image and draw object to draw on
        # memory only, not visible
        user_image = Image.new("RGB", (400, 400), (255, 255, 255))
        invisible_draw = ImageDraw.Draw(user_image)

        writing = IntVar()
        submit = Button(window, text = "Submit", command = lambda: writing.set(1))
        submit.grid(row = 2, column = 0, sticky = E+W+S)

        self.master.bind('<B1-Motion>', draw)
        self.master.bind('<ButtonRelease-1>', reset_coords)

        submit.wait_variable(writing)

        filename = "{}{}.png".format(username, image_number)
        #user_image.save(filename)
        user_image = np.array(user_image)
        cv2.imwrite(filename, user_image)
        return filename

    #lets the user write something, then calls the NN to figure out which user wrote it
    def determineUserScreen(self):
        self.reset()
        unknown_user_image = self.writingScreen("unknown", 0)
        this_user = self.determineUserNN(unknown_user_image)
        self.reset()
        self.guess = Label(self.master, text = "The AI thinks that {} wrote this!".format(this_user)).pack(side = TOP)
        self.back = Button(self.master, text = "Return to Main Menu", command = self.mainMenu).pack()

    def determineUserNN(self, filename):
        model = self.loadModel()
        temp = ["test"]
        tempdata, templabels, words = LetterBreaker.imageProcess(temp, 1, filename)
        whichUser = Prediction.predictUser(model, words, self.users)
        return whichUser

    def loadModel(self):
        yaml_file = open('model.yaml', 'r')
        loaded_model_yaml = yaml_file.read()
        yaml_file.close()
        loaded_model = model_from_yaml(loaded_model_yaml)
        loaded_model.load_weights("model.h5")
        return loaded_model

#####################################################

window = Tk()
main_menu = GUI(window)
window.mainloop()


#!/usr/bin/python3

import os,sys,re
import pandas as pd
import argparse
import numpy as np
#from Bio import Phylo
#from Bio.Phylo.TreeConstruction import DistanceCalculator
#from Bio.Phylo.TreeConstruction import DistanceMatrix
#from Bio.Phylo.TreeConstruction import DistanceTreeConstructor

def createParser ():
  parser = argparse.ArgumentParser()
  parser.add_argument ('-n', '--name', nargs='?', type = str, default='data.tab', help = 'Name of file with microsats')
  return parser

def main():
	parser = createParser()
	namespace = parser.parse_args(sys.argv[1:])
	infile = namespace.name
	dta = pd.read_csv(infile, sep = '\t', header = 0)

	a = (dta.a1>0)*4 + (dta.a2>0)*2 + (dta.a3>0)

	s1 = sum(a & 0b100000000)/4
	s2 = sum(a & 0b10000000)/2
	s3 = sum(a & 0b1000000)

	b = (dta.b1>0)*4 + (dta.b2>0)*2 + (dta.b3>0)

	s4 = sum(a & 0b100000)/4
	s5 = sum(a & 0b10000)/2
	s6 = sum(a & 0b1000)

	c = (dta.c1>0)*4 + (dta.c2>0)*2 + (dta.c3>0)

	s7 = sum(a & 0b100)/4
	s8 = sum(a & 0b10)/2
	s9 = sum(a & 0b1)

	hoab = sum(((a == 1) & (b == 1)) | ((a == 2) & (b == 2)) | ((a == 4) & (b == 4)))
	heab = sum(((a == 3) & (b == 3)) | ((a == 5) & (b == 5)) | ((a == 6) & (b == 6)))
	hoac = sum(((a == 1) & (c == 1)) | ((a == 2) & (c == 2)) | ((a == 4) & (c == 4)))
	heac = sum(((a == 3) & (c == 3)) | ((a == 5) & (c == 5)) | ((a == 6) & (c == 6)))
	hobc = sum(((b == 1) & (c == 1)) | ((b == 2) & (c == 2)) | ((b == 4) & (c == 4)))
	hebc = sum(((b == 3) & (c == 3)) | ((b == 5) & (c == 5)) | ((b == 6) & (c == 6)))

	print ( hoab,heab, hoac, hoab, hobc, hebc )
	#print(hoa, sep='\n')

if __name__=="__main__":
	main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

# Program: Get Courses from coursehunters.net
#
# Author: Andrey Klimov < ak545 at mail dot ru >
# https://github.com/ak545
#
# Current Version: 0.1.2
# Date: 14-07-2019 (dd-mm-yyyy)
#
# License:
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU General Public License for more details.

import os
import sys
import platform
import argparse
import requests
import re
import itertools
from pathlib import Path
import time
import urllib3

try:
    from bs4 import BeautifulSoup
except ImportError:
    sys.exit("""You need bs4!
                install it from https://pypi.org/project/bs4
                or run pip install bs4.""")

try:
    from colorama import init
    from colorama import Fore, Back, Style
except ImportError:
    sys.exit(
        """You need colorama!
                install it from http://pypi.python.org/pypi/colorama
                or run pip install colorama"""
    )

# Init colorama
init(autoreset=True)

urllib3.disable_warnings()

# Check Python Version
if sys.version_info < (3, 6):
    print("Error. Python version 3.6 or later required to run this script")
    print("Your version:", sys.version)
    sys.exit(-1)

# Глобальные константы
__version__ = "0.1.2"

FR = Fore.RESET
FLW = Fore.LIGHTWHITE_EX
FLG = Fore.LIGHTGREEN_EX
FLR = Fore.LIGHTRED_EX
FLC = Fore.LIGHTCYAN_EX
FLY = Fore.LIGHTYELLOW_EX
FLM = Fore.LIGHTMAGENTA_EX
FLB = Fore.LIGHTBLUE_EX

BLB = Back.LIGHTBLACK_EX
BR = Back.RESET

SDIM = Style.DIM
SNORMAL = Style.NORMAL
SBRIGHT = Style.BRIGHT
SR = Style.RESET_ALL

SEP = os.sep

# Command line parameters
NAMESPACE = None

# User-Agent for requests
S_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.46 Safari/537.36'

STREAM = sys.stderr

# Шаблоны для progress bar
BAR_TEMPLATE = '%s[%s%s] %s ( %s ) ( %s )\r'
MILL_TEMPLATE = '%s %s %i/%i\r'
DOTS_CHAR = '.'
BAR_FILLED_CHAR = '#'
BAR_EMPTY_CHAR = ' '
MILL_CHARS = ['|', '/', '-', '\\']

# How long to wait before recalculating the ETA
ETA_INTERVAL = 1
# How many intervals (excluding the current one)
# to calculate the simple moving average
ETA_SMA_WINDOW = 9

def humanize_bytes(bytesize, precision=2):
    """
    Humanize byte size figures
    :param bytesize: float
    :param precision: int
    :return: string
    """
    abbrevs = (
        (1 << 50, 'PB'),
        (1 << 40, 'TB'),
        (1 << 30, 'GB'),
        (1 << 20, 'MB'),
        (1 << 10, 'kB'),
        (1, 'bytes')
    )
    if bytesize == 1:
        return '1 byte'
    for factor, suffix in abbrevs:
        if bytesize >= factor:
            break
    return '%.*f %s' % (precision, bytesize / factor, suffix)

class MyBar(object):
    """
    Рисует прогресс бар в консоли
    """

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.done()
        return False  # we're not suppressing exceptions

    def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
                 filled_char=BAR_FILLED_CHAR, expected_size=None, every=1, total_length=None):
        self.label = label
        self.width = width
        self.hide = hide
        # Only show bar in terminals by default (better for piping, logging etc.)
        if hide is None:
            try:
                self.hide = not STREAM.isatty()
            except AttributeError:  # output does not support isatty()
                self.hide = True
        self.empty_char = empty_char
        self.filled_char = filled_char
        self.expected_size = expected_size
        self.every = every
        self.total_length = total_length
        self.str_total_length = humanize_bytes(self.total_length)
        self.str_percent = '0.00 %'
        self.start = time.time()
        self.ittimes = []
        self.eta = 0
        self.etadelta = time.time()
        self.etadisp = self.format_time(self.eta)
        self.last_progress = 0
        if (self.expected_size):
            self.show(0)

    def show(self, progress, count=None):
        if count is not None:
            self.expected_size = count
        if self.total_length is None:
            self.total_length = 0
        self.str_total_length = humanize_bytes(self.total_length)
        if self.expected_size is None:
            raise Exception("expected_size not initialized")

        if (progress == self.expected_size):
            self.str_percent = "100 %"
        else:
            self.str_percent = "{:6.2f} %".format(progress * 100 / self.expected_size)

        self.last_progress = progress
        if (time.time() - self.etadelta) > ETA_INTERVAL:
            self.etadelta = time.time()
            self.ittimes = \
                self.ittimes[-ETA_SMA_WINDOW:] + \
                [-(self.start - time.time()) / (progress + 1)]
            self.eta = \
                sum(self.ittimes) / float(len(self.ittimes)) * \
                (self.expected_size - progress)
            self.etadisp = self.format_time(self.eta)
        x = int(self.width * progress / self.expected_size)
        if not self.hide:
            if ((progress % self.every) == 0 or  # True every "every" updates
                    (progress == self.expected_size)):  # And when we're done
                STREAM.write(BAR_TEMPLATE % (
                    self.label, self.filled_char * x,
                    self.empty_char * (self.width - x), self.etadisp, self.str_percent, self.str_total_length))

                STREAM.flush()

    def done(self):
        # self.elapsed = time.time() - self.start
        # elapsed_disp = self.format_time(self.elapsed)
        elapsed_disp = self.format_time(0)
        self.str_percent = "100.00 %"
        if not self.hide:
            # Print completed bar with elapsed time
            STREAM.write(BAR_TEMPLATE % (
                self.label, self.filled_char * self.width,
                self.empty_char * 0, elapsed_disp, self.str_percent, self.str_total_length))

            STREAM.write('\n')
            STREAM.flush()

    def format_time(self, seconds):
        return time.strftime('%H:%M:%S', time.gmtime(seconds))

# Функция рисования progress bar
def bar(it, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR,
        filled_char=BAR_FILLED_CHAR, expected_size=None, every=1, total_length=None):
    """Progress iterator. Wrap your iterables with it."""

    count = len(it) if expected_size is None else expected_size

    with MyBar(label=label, width=width, hide=hide, empty_char=empty_char,
               filled_char=filled_char, expected_size=count, every=every, total_length=total_length) \
            as bar:
        for i, item in enumerate(it):
            yield item
            bar.show(i + 1)

def dots(it, label='', hide=None, every=1):
    """Progress iterator. Prints a dot for each item being iterated"""

    count = 0

    if not hide:
        STREAM.write(label)

    for i, item in enumerate(it):
        if not hide:
            if i % every == 0:  # True every "every" updates
                STREAM.write(DOTS_CHAR)
                sys.stderr.flush()

        count += 1

        yield item

    STREAM.write('\n')
    STREAM.flush()

def mill(it, label='', hide=None, expected_size=None, every=1):
    """Progress iterator. Prints a mill while iterating over the items."""

    def _mill_char(_i):
        if _i >= count:
            return ' '
        else:
            return MILL_CHARS[(_i // every) % len(MILL_CHARS)]

    def _show(_i):
        if not hide:
            if ((_i % every) == 0 or  # True every "every" updates
                    (_i == count)):  # And when we're done

                STREAM.write(MILL_TEMPLATE % (
                    label, _mill_char(_i), _i, count))
                STREAM.flush()

    count = len(it) if expected_size is None else expected_size

    if count:
        _show(0)

    for i, item in enumerate(it):
        yield item
        _show(i + 1)

    if not hide:
        STREAM.write('\n')
        STREAM.flush()

def save_html_dump(html, filename):
    """
    Сохранение отладочного дампа html в файл
    :param html: string  # Содержимое
    :param filename: string  # Имя файла для записи отладочного дампа
    :return: None
    """
    bs = BeautifulSoup(html, 'html.parser')
    dump = []
    dump.append(bs.prettify())
    open_mode = "w+"
    with open(filename, open_mode, encoding="utf-8", newline="\n") as f:
        for item in dump:
            f.write("%s\n" % item)

def get_html(url, error_msg=''):
    """
    Загрузка веб страницы курса

    Возвращает: Строку-дамп HTML-страницы

    :param url: string  # url курса
    :param error_msg: string  # Строка выводимая во время
                              # ошибки при попытке загрузить
                              # HTML страницу курса
    :return: str
    """
    global S_USER_AGENT
    if error_msg.strip() != '':
        error_msg = error_msg + ' невозможен'

    headers = {
        'User-Agent': S_USER_AGENT
    }
    try:
        result = requests.get(url, timeout=10, headers=headers, verify=False)
        result.raise_for_status()
        return result.text
    except requests.exceptions.RequestException as e:
        if error_msg.strip() != '':
            print(f'\r{FLR}{error_msg}         ')
        print(f'Не получилось подключиться к сайту: {FLG}{url}')
        print(f'{e}')
        sys.exit(-1)

def get_videos(url):
    """
    Поиск видео файлов на HTML странице курса

    Возвращает:
    язык курса (string),
    название курса (string),
    описание курса (string),
    список из словарей ["название видео", "ссылка на видео"]
        list[
                {'title': 'item1', 'link': 'item2'},
                {'title': 'item1', 'link': 'item2'},
                ...,
                {'title': 'item1', 'link': 'item2'},
            ]

    :param url: string  # url курса
    :return: string, string, list
    """
    title_course = ''
    title_description = ''
    data = []
    lng = 'en'
    html = get_html(url=url, error_msg='Поиск видео')
    if html:
        # save_html_dump(html=html, filename='dump.txt')
        bs = BeautifulSoup(html, 'html.parser')
        item = bs.find('h1', class_='hero-title')
        if item:
            title_course = item.text.strip()

        item = bs.find('div', class_='course-wrap-description')
        if item:
            title_description = str(item).strip()
            title_description = str(title_description).replace('
', '').replace(
                '
', '').replace('
', "\n").replace('
', "\n").replace(
                '
', '').replace('
', "\n").replace(
                '
', '').replace('
', "\n").replace('
', '- ').replace('
', "\n").replace(
                '', '').replace('', '').replace('', '').replace('', '')

        items = bs.find_all('div', class_='course-box-value')
        if items:
            for item in items:
                if 'Ангельский' in item.text.strip():
                    print(f'Язык     : [ {FLB}Ангельский{FR} ]')
                    lng = 'en'
                if 'Русский' in item.text.strip():
                    print(f'Язык     : [ {FLB}Русский{FR} ]')
                    lng = 'ru'

        items = bs.find_all('li', class_='lessons-item')
        if items:
            for item in items:
                item_title = item.find('div', class_='lessons-name')
                item_href = item.find('link', itemprop="contentUrl")
                if item_href:
                    data.append({
                        'title': item_title.text.strip(),
                        'link': item_href.get('href').strip()
                    })

        item = bs.find('a', class_='section-block-btn btn-outline')
        if item:
            data.append({
                'title': item.text.strip(),
                'link': item.get('href').strip()
            })
    return lng, title_course, title_description, data

def sanitize_filename(s, restricted=False, is_id=False):
    """
    Изменяет строку (s) так, чтобы её можно было использовать,
    как часть имени файла.
    Параметр restricted изменит строку на основании более
    строгого подмножества разрешенных для файловой системы
    символов (accent_chars).
    Параметр is_id позволяет не изменять идентификаторы
    (если это возможно).

    Modifies the string (s) so that it can be used as part of
    the file name.
    The restricted parameter will change the string based on
    a more strict subset of the allowed characters (accent_chars)
    for the file system.
    The is_id parameter allows not to change identifiers
    (if possible).

    :param s: string
    :param restricted: bool
    :param is_id: bool
    :return: string
    """

    def replace_insane(char):
        accent_chars = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñò'
                                'óôõöőøœùúûüűýþÿ',
                                itertools.chain(
                                    'AAAAAA',
                                    ['AE'],
                                    'CEEEEIIIIDNOOOOOOO',
                                    ['OE'],
                                    'UUUUUYP',
                                    ['ss'],
                                    'aaaaaa',
                                    ['ae'],
                                    'ceeeeiiiionooooooo',
                                    ['oe'],
                                    'uuuuuypy'
                                )))
        if restricted and char in accent_chars:
            return accent_chars[char]
        if char == '?' or ord(char) < 32 or ord(char) == 127:
            return ''
        elif char == '"':
            return '' if restricted else '\''
        elif char == ':':
            return '_-' if restricted else ' -'
        elif char in '\\/|*<>':
            return '_'
        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
            return '_'
        if restricted and ord(char) > 127:
            return '_'
        return char

    # Handle timestamps
    s = re.sub(r'[0-9]+(?::[0-9]+)+',
               lambda m: m.group(0).replace(':', '_'), s)
    result = ''.join(map(replace_insane, s))
    if not is_id:
        while '__' in result:
            result = result.replace('__', '_')
        result = result.strip('_')
        # Common case of "Foreign band name - English song title"
        if restricted and result.startswith('-_'):
            result = result[2:]
        if result.startswith('-'):
            result = '_' + result[len('-'):]
        result = result.lstrip('.')
        if not result:
            result = '_'
    return result

def download_file(url, file_out, title, i, total):
    """
    Загрузка файла
    Возвращает: код ошибки
    0 - файл успешно загружен
    1 - файл уже существует
    2 - ошибка загрузки файла
    28 - недостаточно место для сохранения файла

    :param url: string
    :param file_out: string
    :param title: string
    :param i: int
    :param total: int

    :return: int
    """

    global S_USER_AGENT
    headers = {
        'User-Agent': S_USER_AGENT
    }
    print(f'Имя      : [ {i}/{total} ] {FLC}{title}{FR}')
    try:
        r = requests.get(url, stream=True, headers=headers, verify=False)

        if r.status_code < 200 or r.status_code > 299:
            print(f'Статус   : [ {FLM}Error{FR} ]\n'
                  f'Cсылка   : {FLW}{url}{FR}\n'
                  f'Файл     : {FLW}{file_out}{FR}\n'
                  f'Код      : {FLM}{r.status_code}')
            exit(-1)

        total_length = int(r.headers.get('content-length'))
        # Проверка на наличие уже скаченного файла
        if Path(file_out).is_file():
            file_size = os.stat(Path(file_out)).st_size
            if file_size == int(total_length):
                print(f'Статус   : [ {FLG}Уже существует ]\n')
                return 1

        # Скачивание файла
        def_chunk_size = min(total_length, 8192)
        # def_chunk_size = min(total_length, 4096)
        with open(file_out, "wb") as f:
            for chunk in bar(r.iter_content(chunk_size=def_chunk_size), label="Загружено: ",
                             filled_char='#',
                             expected_size=(total_length / def_chunk_size) + 1,
                             total_length=total_length):
                if chunk:
                    f.write(chunk)
                    f.flush()

        print(f'Статус   : [ {FLG}Ok{FR} ]\n')
        return 0

    except Exception as e:
        print(f'Статус   : [ {FLM}Error{FR} ]\n'
              f'Cсылка   : {FLW}{url}{FR}\n'
              f'Файл     : {FLW}{file_out}{FR}\n'
              f'Код      : {FLM}{e}\n')
        if '[Errno 28]' in str(e):  # [Errno 28] No space left on device
            return 28
        else:
            return 2

def process_cli():
    """
    parses the CLI arguments and returns a domain or
        a file with a list of domains etc.
    :return: dict
    """
    parser = argparse.ArgumentParser(
        description="""Get Courses from coursehunters.net
        A simple python script for download the courses from coursehunters.net.
        """,
        usage='%(prog)s [-h][-v][-nb] -u URL [-o DIR]',
        epilog="(c) AK545 (Andrey Klimov) 2019, e-mail: ak545@mail.ru",
        add_help=False
    )
    parent_group = parser.add_argument_group(
        title="Options"
    )
    parent_group.add_argument(
        "-h",
        "--help",
        action="help",
        help="Help"
    )
    parent_group.add_argument(
        "-v",
        "--version",
        action="version",
        help="Display the version number",
        version="%(prog)s version: {}".format(__version__)
    )
    parent_group.add_argument(
        "-u",
        "--url",
        help="URL of the Course (default is None)",
        metavar="URL"
        # required=True
    )
    parent_group.add_argument(
        "-o",
        "--out",
        help="Download dir for the Course (default is None)",
        metavar="DIR"
    )
    parent_group.add_argument(
        "-nb",
        "--no-banner",
        action="store_true",
        default=False,
        help="Do not print banner (default is False)"
    )
    return parser

def print_namespase():
    """
    Print preset options to console
    :return: None
    """
    global NAMESPACE
    print(
        f"\tPreset options\n"
        f"\t-------------------------\n"
        f"\tUrl                      : {NAMESPACE.url}\n"
        f"\tOut Dir                  : {NAMESPACE.out}\n"
        f"\tPrint banner             : {NAMESPACE.no_banner}\n"
        f"\t-------------------------"
    )

def main():
    """
    Main function
    :return: None
    """
    global NAMESPACE

    if NAMESPACE.out:
        # Создание папки для сохранения видео и материалов курса
        try:
            Path(NAMESPACE.out).mkdir(parents=True, exist_ok=True)
        except:
            print(f'Ошибка создания папки:\n{FLY}{NAMESPACE.out}')
            print(f'Проверьте корректность задания параметров.')
            print(f'Не добавляйте в конце параметра обратный слеш "{FLY}\\"')
            sys.exit(-1)
    else:
        NAMESPACE.out = os.getcwd()

    if NAMESPACE.url:
        lng, title_course, title_description, data = get_videos(NAMESPACE.url)
        if title_course != "":
            print(f'Курс     : [ {FLC}{title_course}{FR} ]')
            print()
            title_course = sanitize_filename(title_course)
            NAMESPACE.out = NAMESPACE.out + SEP + '( ' + lng + ' ) ' + title_course
            # Создание папки для сохранения видео и материалов курса
            try:
                Path(NAMESPACE.out).mkdir(parents=True, exist_ok=True)
            except:
                print(f'Ошибка создания папки:\n{FLY}{NAMESPACE.out}')
                print(f'Проверьте корректность задания параметров.')
                print(f'Не добавляйте в конце параметра обратный слеш "{FLY}\\"')
                sys.exit(-1)

        if title_description != "":
            filename = NAMESPACE.out + SEP + "description.txt"
            with open(filename, "w+", encoding="utf-8", newline="\n") as f:
                f.write(title_description)

        data_len = len(data)
        for i, item in enumerate(data, start=1):
            url = str(item.get('link')).strip()
            f = str(item.get('link')).strip().split('/')[-1]
            f = sanitize_filename(f)
            title = str(item.get('title')).strip()
            title = sanitize_filename(title)
            file = os.path.join(str(NAMESPACE.out), str(i) + '. ' + title + ' - ' + f)

            ret = download_file(url, file, title, i, data_len)
            if ret == 28:  # No space left on device
                break
        print()
        print(f'{FLW}Завершена загрузка курса{FR}')
        print()

if __name__ == "__main__":
    print(f"{SR}")

    # Parsing command line
    parser = process_cli()
    NAMESPACE = parser.parse_args(sys.argv[1:])
    if not NAMESPACE.url:
        print(f'{FLM}Error! The following arguments are required: {FLY}-u{FR} / {FLY}--url')
        parser.print_help()
    if not NAMESPACE.no_banner:
        if platform.platform().startswith('Windows'):
            home_path = os.path.join(os.getenv('HOMEDRIVE'),
                                     os.getenv('HOMEPATH'))
        else:
            home_path = os.path.join(os.getenv('HOME'))
        print(
            f"\tPython  : {FLC}{sys.version}{FR}\n"
            f"\tNode    : {FLC}{platform.node()}{FR}\n"
            f"\tHome    : {FLC}{home_path}{FR}\n"
            f"\tOS      : {FLC}{platform.system()}{FR}\n"
            f"\tRelease : {FLC}{platform.release()}{FR}\n"
            f"\tVersion : {FLC}{platform.version()}{FR}\n"
            f"\tArch    : {FLC}{platform.machine()}{FR}\n"
            f"\tCPU     : {FLC}{platform.processor()}{FR}"
        )
    rc = 0
    try:
        main()
    except Exception as e:
        print('Error: %s' % e, file=sys.stderr)
    sys.exit(rc)


import csv
import json
import os
import io, gzip

from django.http import HttpResponse
from django.views import View

from command.lib.db.admin.admin_options import AdminOptions
from command.lib.db.admin.compendium_database import CompendiumDatabase
from command.lib.db.compendium.bio_feature import BioFeature
from command.lib.db.compendium.bio_feature_reporter import BioFeatureReporter
from command.lib.db.compendium.raw_data import RawData
from command.lib.tasks import export_data
from command.lib.utils.decorators import forward_exception_to_http, check_permission
#from command.lib.utils.group_compendium_permission import GroupCompendiumPermission
from command.lib.utils.message import Message
from command.lib.utils.permission import Permission

class ExportDataView(View):

    def get(self, request, operation, *args, **kwargs):
        method = getattr(self, operation)
        return method(request, *args, **kwargs)

    def post(self, request, operation, *args, **kwargs):
        method = getattr(self, operation)
        return method(request, *args, **kwargs)

    @staticmethod
    @forward_exception_to_http
    def read_file(request, *args, **kwargs):
        path = request.GET['path']
        base_dir = AdminOptions.objects.get(option_name='raw_data_directory').option_value

        if not os.path.isfile(base_dir + path):
            return HttpResponse('File ' + os.path.basename(path) + ' no longer exists!',
                                    content_type='text/plain')
        file = open(base_dir + path, 'rb')
        if path.endswith('tsv.gz'):
            response = HttpResponse(file.read(), content_type='application/x-gzip')
            response['Content-Disposition'] = 'attachment; filename="raw_data.tsv.gz"'
        elif path.endswith('hdf5'):
            response = HttpResponse(file.read(), content_type='application/octet-stream')
            response['Content-Disposition'] = 'attachment; filename="raw_data.hdf5"'
        response['Content-Length'] = str(os.stat(base_dir + path).st_size)

        return response

    @staticmethod
    @forward_exception_to_http
    @check_permission(Permission.EXPORT_COMPENDIUM)
    def export_raw_data(request, *args, **kwargs):
        comp_id = request.POST['compendium_id']
        channel_name = request.session['channel_name']
        view = request.POST['view']
        operation = request.POST['operation']

        compendium = CompendiumDatabase.objects.get(id=comp_id)

        base_dir = AdminOptions.objects.get(option_name='raw_data_directory').option_value
        full_dir = os.path.join(base_dir, compendium.compendium_nick_name, 'export_raw_data')

        # check genes, raw_data and mapping
        bio_features_check = BioFeature.objects.using(compendium.compendium_nick_name).count() > 0
        raw_data_check = RawData.objects.using(compendium.compendium_nick_name).count() > 0
        mapping_check = BioFeatureReporter.objects.using(compendium.compendium_nick_name).\
            exclude(bio_feature__isnull=True).count() > 0

        if bio_features_check and raw_data_check and mapping_check:
            export_data.export_raw_data.apply_async(
                (request.user.id, comp_id, full_dir, channel_name, view, operation)
            )
            return HttpResponse(json.dumps({'success': True}),
                                content_type="application/json")
        else:
            error_msg = Message(type='error', title='Cannot export data',
                                message='To export data you need to:'
                                        '
'
                                        '
import biological features;
'
                                        '
import at least one experiment;
'
                                        '
platform mapped (if requested);
'
                                        '
')
            return HttpResponse(json.dumps(error_msg.to_dict()),
                                content_type="application/json")


"""
.. _config:

------
config
------

Allows the central definition of constants that may change between environments
You may create the file `fpesa.cfg` in the directory you start the fpesa
command line utility to change any of the following values

.. literalinclude :: ../fpesa/config/default.cfg
   :language: ini

"""
import os
import configparser
import logging

config = configparser.ConfigParser()
"""
:py:class:`configparser.ConfigParser` object that holds the config
"""

with open(os.path.join(os.path.dirname(__file__), 'default.cfg')) as fo:
    config.read_file(fo)

additional_config = config.read(["fpesa.cfg"])
if additional_config:
    logging.info("read additional configs: {}".format(
        [os.path.abspath(p) for p in additional_config]))
else:
    logging.info("no additional configs found")


import argparse
import itertools
import os
import sys
from torch.profiler import record_function, ProfilerActivity
from dataclasses import dataclass, field
from enum import Enum
from pprint import pprint
from typing import List, Optional
import time

import torch
print(f"torch version {torch.__version__}")
import torchmetrics as metrics
from pyre_extensions import none_throws
from torch.utils.data import DataLoader
from torchrec import EmbeddingBagCollection
from torchrec.datasets.criteo import DEFAULT_CAT_NAMES, DEFAULT_INT_NAMES

from torchrec.models.dlrm import DLRM, DLRM_DCN, DLRM_Projection, DLRMTrain
from torchrec.modules.embedding_configs import EmbeddingBagConfig
from torchrec.optim.apply_optimizer_in_backward import apply_optimizer_in_backward
from torchrec.optim.keyed import CombinedOptimizer, KeyedOptimizerWrapper
from torchrec.optim.optimizers import in_backward_optimizer_filter
from tqdm import tqdm
from jit_trace_able_utils import unpack, SparseArchTraceAbleWrapper
import intel_extension_for_pytorch as ipex
print(f"IPEX version {ipex.__version__}")

# OSS import
try:
    # pyre-ignore[21]
    # @manual=//ai_codesign/benchmarks/dlrm/torchrec_dlrm/data:dlrm_dataloader
    from data_process.dlrm_dataloader import get_dataloader

    # pyre-ignore[21]
    # @manual=//ai_codesign/benchmarks/dlrm/torchrec_dlrm:lr_scheduler
    from lr_scheduler import LRPolicyScheduler

    # pyre-ignore[21]
    # @manual=//ai_codesign/benchmarks/dlrm/torchrec_dlrm:multi_hot
    from multi_hot import Multihot, RestartableMap
except ImportError:
    pass

# internal import
try:
    from .data_process.dlrm_dataloader import get_dataloader  # noqa F811
    from .lr_scheduler import LRPolicyScheduler  # noqa F811
    from .multi_hot import Multihot, RestartableMap  # noqa F811
except ImportError:
    pass

TRAIN_PIPELINE_STAGES = 3  # Number of stages in TrainPipelineSparseDist.
# Optimizer parameters:
ADAGRAD_LR_DECAY = 0
ADAGRAD_INIT_ACC = 0
ADAGRAD_EPS = 1e-8
WEIGHT_DECAY = 0

import logging
logger: logging.Logger = logging.getLogger(__name__)
logger.setLevel(1)

class InteractionType(Enum):
    ORIGINAL = "original"
    DCN = "dcn"
    PROJECTION = "projection"

    def __str__(self):
        return self.value

def load_snapshot(model, snapshot_dir):
    from torchsnapshot import Snapshot
    snapshot = Snapshot(path=snapshot_dir)
    snapshot.restore(app_state={"model": model})

def trace_handler(prof):
    print(prof.key_averages().table(sort_by="self_cpu_time_total"))
    prof.export_chrome_trace(f"{prof._mode}-{prof._dtype}-dlrm_trace_step{prof.step_num}.json")

prof_schedule=torch.profiler.schedule(
    wait=10,
    warmup=10,
    active=1,
    repeat=10
)

def print_memory(stage):
    import os
    import psutil
    print("dlrmv2-memory-usage-log: ", time.time(), stage, psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024)

def fetch_batch(dataloader):
    try:
        batch = dataloader.dataset.load_batch()
    except:
        batch = dataloader.source.dataset.batch_generator._generate_batch()
    return batch

def parse_autocast(dtype: str):
    _dtype = None
    autocast = False
    if dtype == 'bf16':
        autocast = True
        _dtype = torch.bfloat16
    elif dtype == 'fp16':
        autocast= True
        _dtype = torch.float16
    elif dtype == 'int8':
        _dtype = torch.int8
    else:
        assert dtype in ['fp32', 'bf32']
        _dtype = torch.float
    return autocast, _dtype

def convert_int8(args, model, dataloader):
    from torch.ao.quantization import HistogramObserver, PerChannelMinMaxObserver, QConfig
    from intel_extension_for_pytorch.quantization import prepare, convert
    qconfig = QConfig(
        activation=HistogramObserver.with_args(qscheme=torch.per_tensor_symmetric, dtype=torch.qint8, bins=127, upsample_rate=256, quant_min= -127, quant_max=126),
        weight=PerChannelMinMaxObserver.with_args(dtype=torch.qint8, qscheme=torch.per_channel_symmetric)
    )
    batch = fetch_batch(dataloader)
    batch.sparse_features = unpack(batch.sparse_features)
    model = prepare(
        model,
        qconfig,
        example_inputs=(batch.dense_features, batch.sparse_features),
        inplace=True
    )
    if args.calibration:
        # https://github.com/mlcommons/inference/tree/master/recommendation/dlrm_v2/pytorch#calibration-set
        assert args.synthetic_multi_hot_criteo_path, "need real dataset to calibrate"
        batch_idx = list(range(128000))
        batch = dataloader.dataset.load_batch(batch_idx)
        batch.sparse_features = unpack(batch.sparse_features)
        model(batch.dense_features, batch.sparse_features)
        model.save_qconf_summary(qconf_summary=args.int8_configure_dir)
        logger.info("calibration done and save to %s", args.int8_configure_dir)
        exit()
    else:
        model.load_qconf_summary(qconf_summary = args.int8_configure_dir)
        convert(model, inplace=True)
        model.eval()
        torch._C._jit_set_texpr_fuser_enabled(False)
        model = torch.jit.trace(model, (batch.dense_features, batch.sparse_features), check_trace=True)
        model = torch.jit.freeze(model)
        model(batch.dense_features, batch.sparse_features)
        model(batch.dense_features, batch.sparse_features)
        print(model.graph_for(batch.dense_features, batch.sparse_features))
        return model

def ipex_optimize(args, model, optimizer, dataloader):
    example_batch = fetch_batch(dataloader)
    example_batch.sparse_features = unpack(example_batch.sparse_features)
    dense, sparse = example_batch.dense_features, example_batch.sparse_features
    autocast, dtype = parse_autocast(args.dtype)
    auto_kernel_selection = True if args.dtype == 'bf32' else False
    import intel_extension_for_pytorch as ipex
    if dtype == torch.int8:
        assert args.inference_only
        with torch.no_grad():
            model = convert_int8(args, model, dataloader)
    elif args.inference_only:
        with torch.no_grad():
            model = ipex.optimize(
                model=model.eval(),
                dtype=dtype,
                sample_input=(dense, sparse),
                inplace=True,
                auto_kernel_selection=auto_kernel_selection,
                linear_bn_folding=False,
                conv_bn_folding=False
            )
            if args.dtype == 'bf32':
                ipex.set_fp32_math_mode(mode=ipex.FP32MathMode.BF32, device="cpu")
            if args.jit:
                with torch.cpu.amp.autocast(enabled=autocast, dtype=dtype):
                    torch._C._jit_set_texpr_fuser_enabled(False)
                    model = torch.jit.trace(model, (dense, sparse))
                    model = torch.jit.freeze(model)
                    model(dense, sparse)
                    model(dense, sparse)
    else:
        model, optimizer = ipex.optimize(
            model=model,
            optimizer=optimizer,
            dtype=dtype,
            sample_input=(dense, sparse),
            inplace=True,
            auto_kernel_selection=auto_kernel_selection,
            linear_bn_folding=False,
            conv_bn_folding=False
        )
        pass
    if args.dtype == 'bf32':
        ipex.set_fp32_math_mode(mode=ipex.FP32MathMode.BF32, device="cpu")
    return model, optimizer

def parse_args(argv: List[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="torchrec dlrm example trainer")
    parser.add_argument(
        "--epochs",
        type=int,
        default=1,
        help="number of epochs to train",
    )
    parser.add_argument(
        "--batch_size",
        type=int,
        default=32,
        help="batch size to use for training",
    )
    parser.add_argument(
        "--drop_last_training_batch",
        dest="drop_last_training_batch",
        action="store_true",
        help="Drop the last non-full training batch",
    )
    parser.add_argument(
        "--test_batch_size",
        type=int,
        default=None,
        help="batch size to use for validation and testing",
    )
    parser.add_argument(
        "--limit_train_batches",
        type=int,
        default=None,
        help="number of train batches",
    )
    parser.add_argument(
        "--limit_val_batches",
        type=int,
        default=None,
        help="number of validation batches",
    )
    parser.add_argument(
        "--limit_test_batches",
        type=int,
        default=None,
        help="number of test batches",
    )
    parser.add_argument(
        "--dataset_name",
        type=str,
        default="criteo_1t",
        help="dataset for experiment, current support criteo_1tb, criteo_kaggle",
    )
    parser.add_argument(
        "--num_embeddings",
        type=int,
        default=100_000,
        help="max_ind_size. The number of embeddings in each embedding table. Defaults"
        " to 100_000 if num_embeddings_per_feature is not supplied.",
    )
    parser.add_argument(
        "--num_embeddings_per_feature",
        type=str,
        default=None,
        help="Comma separated max_ind_size per sparse feature. The number of embeddings"
        " in each embedding table. 26 values are expected for the Criteo dataset.",
    )
    parser.add_argument(
        "--dense_arch_layer_sizes",
        type=str,
        default="512,256,64",
        help="Comma separated layer sizes for dense arch.",
    )
    parser.add_argument(
        "--over_arch_layer_sizes",
        type=str,
        default="512,512,256,1",
        help="Comma separated layer sizes for over arch.",
    )
    parser.add_argument(
        "--embedding_dim",
        type=int,
        default=64,
        help="Size of each embedding.",
    )
    parser.add_argument(
        "--interaction_branch1_layer_sizes",
        type=str,
        default="2048,2048",
        help="Comma separated layer sizes for interaction branch1 (only on dlrm with projection).",
    )
    parser.add_argument(
        "--interaction_branch2_layer_sizes",
        type=str,
        default="2048,2048",
        help="Comma separated layer sizes for interaction branch2 (only on dlrm with projection).",
    )
    parser.add_argument(
        "--dcn_num_layers",
        type=int,
        default=3,
        help="Number of DCN layers in interaction layer (only on dlrm with DCN).",
    )
    parser.add_argument(
        "--dcn_low_rank_dim",
        type=int,
        default=512,
        help="Low rank dimension for DCN in interaction layer (only on dlrm with DCN).",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Random seed for reproducibility.",
    )
    parser.add_argument(
        "--pin_memory",
        dest="pin_memory",
        action="store_true",
        help="Use pinned memory when loading data.",
    )
    parser.add_argument(
        "--mmap_mode",
        dest="mmap_mode",
        action="store_true",
        help="--mmap_mode mmaps the dataset."
        " That is, the dataset is kept on disk but is accessed as if it were in memory."
        " --mmap_mode is intended mostly for faster debugging. Use --mmap_mode to bypass"
        " preloading the dataset when preloading takes too long or when there is "
        " insufficient memory available to load the full dataset.",
    )
    parser.add_argument(
        "--in_memory_binary_criteo_path",
        type=str,
        default=None,
        help="Directory path containing the Criteo dataset npy files.",
    )
    parser.add_argument(
        "--synthetic_multi_hot_criteo_path",
        type=str,
        default=None,
        help="Directory path containing the MLPerf v2 synthetic multi-hot dataset npz files.",
    )
    parser.add_argument(
        "--learning_rate",
        type=float,
        default=15.0,
        help="Learning rate.",
    )
    parser.add_argument(
        "--shuffle_batches",
        dest="shuffle_batches",
        action="store_true",
        help="Shuffle each batch during training.",
    )
    parser.add_argument(
        "--shuffle_training_set",
        dest="shuffle_training_set",
        action="store_true",
        help="Shuffle the training set in memory. This will override mmap_mode",
    )
    parser.add_argument(
        "--validation_freq_within_epoch",
        type=int,
        default=None,
        help="Frequency at which validation will be run within an epoch.",
    )
    parser.add_argument(
        "--validation_auroc",
        type=float,
        default=None,
        help="Validation AUROC threshold to stop training once reached.",
    )
    parser.add_argument(
        "--evaluate_on_epoch_end",
        action="store_true",
        help="Evaluate using validation set on each epoch end.",
    )
    parser.add_argument(
        "--evaluate_on_training_end",
        action="store_true",
        help="Evaluate using test set on training end.",
    )
    parser.add_argument(
        "--adagrad",
        dest="adagrad",
        action="store_true",
        help="Flag to determine if adagrad optimizer should be used.",
    )
    parser.add_argument(
        "--interaction_type",
        type=InteractionType,
        choices=list(InteractionType),
        default=InteractionType.ORIGINAL,
        help="Determine the interaction type to be used (original, dcn, or projection)"
        " default is original DLRM with pairwise dot product",
    )
    parser.add_argument(
        "--collect_multi_hot_freqs_stats",
        dest="collect_multi_hot_freqs_stats",
        action="store_true",
        help="Flag to determine whether to collect stats on freq of embedding access.",
    )
    parser.add_argument(
        "--multi_hot_sizes",
        type=str,
        default=None,
        help="Comma separated multihot size per sparse feature. 26 values are expected for the Criteo dataset.",
    )
    parser.add_argument(
        "--multi_hot_distribution_type",
        type=str,
        choices=["uniform", "pareto"],
        default=None,
        help="Multi-hot distribution options.",
    )
    parser.add_argument(
        "--lr_warmup_steps",
        type=int,
        default=0,
    )
    parser.add_argument(
        "--lr_decay_start",
        type=int,
        default=0,
    )
    parser.add_argument(
        "--lr_decay_steps",
        type=int,
        default=0,
    )
    parser.add_argument(
        "--print_lr",
        action="store_true",
        help="Print learning rate every iteration.",
    )
    parser.add_argument(
        "--allow_tf32",
        action="store_true",
        help="Enable TensorFloat-32 mode for matrix multiplications on A100 (or newer) GPUs.",
    )
    parser.add_argument(
        "--print_sharding_plan",
        action="store_true",
        help="Print the sharding plan used for each embedding table.",
    )
    parser.add_argument(
        "--print_progress",
        action="store_true",
        help="Print tqdm progress bar during training and evaluation.",
    )
    parser.add_argument(
        "--ipex-optimize",
        action="store_true",
        help="Whether optimize model by ipex.optimize",
    )
    parser.add_argument(
        "--dtype",
        type=str,
        choices=["fp32", "bf16", "fp16", "int8", "bf32"],
        default=None,
        help="Model dtypes.",
    )
    parser.add_argument(
        "--inference-only",
        action="store_true",
        help="Whether only run inference path.",
    )
    parser.add_argument(
        "--int8-configure-dir",
        type=str,
        default="./int8_configure.json",
        help="Int8 recipe location.",
    )
    parser.add_argument(
        "--calibration",
        action="store_true",
        help="Whether calibration only for this run.",
    )
    parser.add_argument(
        "--test_auroc",
        action="store_true",
        help="Test auroc.",
    )
    parser.add_argument(
        "--log-freq",
        type=int,
        default=0,
        help="log-freq to print performance statistics.",
    )
    parser.add_argument(
        "--snapshot-dir",
        type=str,
    )
    parser.add_argument(
        "--profile",
        action="store_true",
        help="Test auroc.",
    )
    parser.add_argument(
        "--jit",
        action="store_true",
        help="Test auroc.",
    )
    parser.add_argument(
        "--benchmark",
        action="store_true",
        help="use same samples to benchmark",
    )
    parser.add_argument(
        "--ipex-merged-emb-cat",
        action="store_true",
        help="whether use ipex customer op for merged_emb_cat",
    )
    parser.add_argument(
        "--share-weight-instance",
        type=int,
        default=0,
        help="if value > 0, will use pytorch throughputbenchmark to share weight with `value` instance",
    )
    return parser.parse_args(argv)

def _evaluate(
    eval_model,
    eval_dataloader: DataLoader,
    stage: str,
    epoch_num: float,
    args,
) -> float:
    """
    Evaluates model. Computes and prints AUROC. Helper function for train_val_test.

    Args:
        limit_batches (Optional[int]): Limits the dataloader to the first `limit_batches` batches.
        eval_pipeline (TrainPipelineSparseDist): pipelined model.
        eval_dataloader (DataLoader): Dataloader for either the validation set or test set.
        stage (str): "val" or "test".
        epoch_num (float): Iterations passed as epoch fraction (for logging purposes).
        print_progress (bool): Whether to print tqdm progress bar.

    Returns:
        float: auroc result
    """
    limit_batches = args.limit_val_batches
    print_progress = args.print_progress
    autocast_enabled, autocast_dtype = parse_autocast(args.dtype)
    log_freq = args.log_freq
    enable_torch_profile = args.profile

    benckmark_batch = fetch_batch(eval_dataloader)
    benckmark_batch.sparse_features = unpack(benckmark_batch.sparse_features)

    def fetch_next(iterator, current_it):
        if args.benchmark:
            if current_it == limit_batches:
                raise StopIteration
            return benckmark_batch
        else:
            with record_function("generate batch"):
                next_batch = next(iterator)
            with record_function("unpack KeyJaggedTensor"):
                next_batch.sparse_features = unpack(next_batch.sparse_features)
            return next_batch

    def eval_step(model, iterator, current_it):
        batch = fetch_next(iterator, current_it)
        with record_function("model forward"):
            t1 = time.time()
            logits = model(batch.dense_features, batch.sparse_features)
            t2 = time.time()
        return logits, batch.labels, t2 - t1

    pbar = tqdm(
        iter(int, 1),
        desc=f"Evaluating {stage} set",
        total=len(eval_dataloader),
        disable=not print_progress,
    )

    logger.info(f"EVAL_START, EPOCH_NUM: {epoch_num}")

    eval_model.eval()
    device = torch.device('cpu')

    iterator = itertools.islice(iter(eval_dataloader), limit_batches)
    # Two filler batches are appended to the end of the iterator to keep the pipeline active while the
    # last two remaining batches are still in progress awaiting results.
    two_filler_batches = itertools.islice(
        iter(eval_dataloader), TRAIN_PIPELINE_STAGES - 1
    )
    iterator = itertools.chain(iterator, two_filler_batches)

    preds = []
    labels = []
    auroc = metrics.AUROC(compute_on_step=False, task='binary').to(device)
    total_t = 0
    it = 1
    ctx1 = torch.no_grad()
    ctx2 = torch.cpu.amp.autocast(enabled=autocast_enabled, dtype=autocast_dtype)
    ctx3 = torch.profiler.profile(activities=[ProfilerActivity.CPU], schedule=prof_schedule, on_trace_ready=trace_handler)
    with ctx1, ctx2:
        if enable_torch_profile:
            p = ctx3.__enter__()
            setattr(p, '_dtype', autocast_dtype)
            setattr(p, '_mode', 'eval')
        while True:
            try:
                logits, label, fw_t = eval_step(eval_model, iterator, it)
                total_t += fw_t
                if log_freq != 0 and it % log_freq == 0:
                    preds = [torch.cat(preds)]
                    labels = [torch.cat(labels)]
                    num_samples = labels[0].shape[0]
                    auroc = ipex._C.roc_auc_score(labels[0].float(), preds[0].squeeze().float())
                    logger.info(f"avg eval time per iter at ITER: {it}, {total_t/it} s, num_samples: {num_samples}, AUROC: {auroc}")
                pred = torch.sigmoid(logits)
                preds.append(pred)
                labels.append(label)
                pbar.update(1)
                it += 1
                if enable_torch_profile:
                    p.step()
            except StopIteration:
                # Dataset traversal complete
                break
        if enable_torch_profile:
            ctx3.__exit__(None, None, None)

    preds = torch.cat(preds)
    labels = torch.cat(labels)
    num_samples = labels.shape[0]
    auroc = ipex._C.roc_auc_score(labels.float(), preds.squeeze().float())

    logger.info(f"AUROC over {stage} set: {auroc}.")
    logger.info(f"Number of {stage} samples: {num_samples}")
    logger.info(f"Throughput: {num_samples/total_t} fps")
    logger.info(f"Final AUROC: {auroc} ")

    return auroc

def _train(
    train_model,
    train_optimizer,
    train_dataloader: DataLoader,
    val_dataloader: DataLoader,
    epoch: int,
    lr_scheduler,
    args,
) -> bool:
    """
    Trains model for 1 epoch. Helper function for train_val_test.

    Args:
        train_pipeline (TrainPipelineSparseDist): pipelined model used for training.
        val_pipeline (TrainPipelineSparseDist): pipelined model used for validation.
        train_dataloader (DataLoader): Training set's dataloader.
        val_dataloader (DataLoader): Validation set's dataloader.
        epoch (int): The number of complete passes through the training set so far.
        lr_scheduler (LRPolicyScheduler): Learning rate scheduler.
        print_lr (bool): Whether to print the learning rate every training step.
        validation_freq (Optional[int]): The number of training steps between validation runs within an epoch.
        validation_auroc (Optional[float]): AUROC level desired for stopping training.
        limit_train_batches (Optional[int]): Limits the training set to the first `limit_train_batches` batches.
        limit_val_batches (Optional[int]): Limits the validation set to the first `limit_val_batches` batches.
        print_progress (bool): Whether to print tqdm progress bar.

    Returns:
        bool: Whether the validation_auroc threshold is reached.
    """
    print_lr = args.print_lr
    validation_freq = args.validation_freq_within_epoch
    validation_auroc = args.validation_auroc
    limit_train_batches = args.limit_train_batches
    print_progress = args.print_progress
    autocast_enabled, autocast_dtype = parse_autocast(args.dtype)
    log_freq = args.log_freq
    enable_torch_profile = args.profile

    train_model.train()
    if autocast_dtype == torch.float16:
        scaler = torch.cpu.amp.GradScaler()

    benckmark_batch = fetch_batch(train_dataloader)
    benckmark_batch.sparse_features = unpack(benckmark_batch.sparse_features)

    def fetch_next(iterator, current_it):
        if args.benchmark:
            if current_it == limit_train_batches:
                raise StopIteration
            return benckmark_batch
        else:
            with record_function("generate batch"):
                next_batch = next(iterator)
            with record_function("unpack KeyJaggedTensor"):
                next_batch.sparse_features = unpack(next_batch.sparse_features)
            return next_batch

    def train_step(model, opt, iterator, current_it):
        next_batch = fetch_next(iterator, current_it)
        t1 = time.time()
        with record_function("zero_grad"):
            opt.zero_grad(set_to_none=True)
        with record_function("fw"):
            if autocast_enabled:
                with torch.cpu.amp.autocast(enabled=autocast_enabled, dtype=autocast_dtype):
                    losses, _ = model(next_batch)
                    loss = torch.sum(losses, dim=0)
            else:
                losses, _ = model(next_batch)
                loss = torch.sum(losses, dim=0)
        with record_function("bw"):
            if autocast_dtype == torch.float16:
                scaler.scale(loss).backward()
            else:
                loss.backward()
        with record_function("optimizer update"):
            if autocast_dtype == torch.float16:
                scaler.step(opt)
                scaler.update()
            else:
                opt.step()
                pass
        train_t = time.time() - t1
        return next_batch.dense_features.shape[0], train_t

    iterator = itertools.islice(iter(train_dataloader), limit_train_batches)
    # Two filler batches are appended to the end of the iterator to keep the pipeline active while the
    # last two remaining batches are still in progress awaiting results.
    two_filler_batches = itertools.islice(
        iter(train_dataloader), TRAIN_PIPELINE_STAGES - 1
    )
    iterator = itertools.chain(iterator, two_filler_batches)

    pbar = tqdm(
        iter(int, 1),
        desc=f"Epoch {epoch}",
        total=len(train_dataloader),
        disable=not print_progress,
    )

    it = 1
    is_success = False
    total_t = 0
    num_samples = 0
    ctx = torch.profiler.profile(activities=[ProfilerActivity.CPU], schedule=prof_schedule, on_trace_ready=trace_handler)
    if enable_torch_profile:
        p = ctx.__enter__()
        setattr(p, '_dtype', autocast_dtype)
        setattr(p, '_mode', 'train')
    for it in itertools.count(1):
        try:
            if  print_lr:
                for i, g in enumerate(train_optimizer.param_groups):
                    logger.info(f"lr: {it} {i} {g['lr']:.6f}")
            samples, train_t = train_step(train_model, train_optimizer, iterator, it)
            if enable_torch_profile:
                p.step()
            num_samples += samples
            total_t += train_t

            if log_freq != 0 and it % log_freq == 0:
                logger.info(f"avg training time per iter at ITER: {it}, {total_t/it} s")
                print_memory(f"memory usage at iter {it}")

            lr_scheduler.step()
            pbar.update(1)
            if validation_freq and it % validation_freq == 0:
                epoch_num = epoch + it / len(train_dataloader)
                auroc_result = _evaluate(
                    train_model.model,
                    val_dataloader,
                    "val",
                    epoch_num,
                    args
                )
                if validation_auroc is not None and auroc_result >= validation_auroc:
                    logger.info("auc = {auroc_result} >= validation_auroc {validation_auroc}, stopped since success")
                    is_success = True
                    break
                train_model.train()
        except StopIteration:
            # Dataset traversal complete
            break
    if enable_torch_profile:
        ctx.__exit__(None, None, None)

    logger.info(f"Total number of iterations: {it - 1}")
    logger.info(f"Throughput: {num_samples/total_t} fps")

    return is_success

def _share_weight_benchmark(
    model,
    data_loader,
    args,
):
    from torch.utils import ThroughputBenchmark
    print_memory("start to init throughput benchmark")
    bench = ThroughputBenchmark(model)
    batch = fetch_batch(data_loader)
    batch.sparse_features = unpack(batch.sparse_features)
    print_memory("start to add input to throughput benchmark")
    bench.add_input(batch.dense_features, batch.sparse_features)
    print_memory("start to run throughput benchmark")
    stats = bench.benchmark(
        num_calling_threads=args.share_weight_instance,
        num_warmup_iters=100,
        num_iters=args.limit_val_batches * args.share_weight_instance,
    )
    print(stats)
    latency = stats.latency_avg_ms
    batch_size = batch.dense_features.shape[0]
    throughput = (1 / latency) * 1000 * batch_size * args.share_weight_instance
    print("Throughput: {:.3f} fps".format(throughput))

@dataclass
class TrainValTestResults:
    val_aurocs: List[float] = field(default_factory=list)
    test_auroc: Optional[float] = None

def train_val_test(
    args: argparse.Namespace,
    model: torch.nn.Module,
    optimizer: torch.optim.Optimizer,
    device: torch.device,
    train_dataloader: DataLoader,
    val_dataloader: DataLoader,
    test_dataloader: DataLoader,
    lr_scheduler: LRPolicyScheduler,
) -> TrainValTestResults:
    """
    Train/validation/test loop.

    Args:
        args (argparse.Namespace): parsed command line args.
        model (torch.nn.Module): model to train.
        optimizer (torch.optim.Optimizer): optimizer to use.
        device (torch.device): device to use.
        train_dataloader (DataLoader): Training set's dataloader.
        val_dataloader (DataLoader): Validation set's dataloader.
        test_dataloader (DataLoader): Test set's dataloader.
        lr_scheduler (LRPolicyScheduler): Learning rate scheduler.

    Returns:
        TrainValTestResults.
    """
    results = TrainValTestResults()

    if args.inference_only:
        if args.share_weight_instance > 0:
            _share_weight_benchmark(
                model.model,
                val_dataloader,
                args
            )
            exit()
        # Mlperf is using val set to test auroc
        # https://github.com/mlcommons/inference/blob/master/recommendation/dlrm_v2/pytorch/python/multihot_criteo.py#L99-L107
        test_auroc = _evaluate(
            model.model,
            val_dataloader,
            "test",
            0,
            args,
        )
        results.test_auroc = test_auroc
        return results

    epoch = 0
    is_success = False
    for epoch in range(args.epochs):
        logger.info(f"EPOCH_START, EPOCH_NUM: {epoch}")
        is_success = _train(
            model,
            optimizer,
            train_dataloader,
            val_dataloader,
            epoch,
            lr_scheduler,
            args,
        )
        if args.evaluate_on_epoch_end:
            val_auroc = _evaluate(
                model.model,
                val_dataloader,
                "val",
                epoch + 1,
                args,
            )
            results.val_aurocs.append(val_auroc)
        # logger.end(
        #     key=mllog_constants.EPOCH_STOP,
        #     metadata={mllog_constants.EPOCH_NUM: epoch},
        # )
        if is_success:
            break

    if is_success:
        pass
        # Run status "aborted" is reported in the case AUROC threshold is not met
        # logger.end(
        #     key=mllog_constants.RUN_STOP,
        #     metadata={
        #         mllog_constants.STATUS: mllog_constants.ABORTED,
        #         mllog_constants.EPOCH_NUM: epoch + 1,
        #     },
        # )

    if args.evaluate_on_training_end:
        test_auroc = _evaluate(
            model.model,
            test_dataloader,
            "test",
            epoch + 1,
            args,
        )
        results.test_auroc = test_auroc

    return results

def main(argv: List[str]) -> None:
    """
    Trains, validates, and tests a Deep Learning Recommendation Model (DLRM)
    (https://arxiv.org/abs/1906.00091). The DLRM model contains both data parallel
    components (e.g. multi-layer perceptrons & interaction arch) and model parallel
    components (e.g. embedding tables). The DLRM model is pipelined so that dataloading,
    data-parallel to model-parallel comms, and forward/backward are overlapped. Can be
    run with either a random dataloader or an in-memory Criteo 1 TB click logs dataset
    (https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/).

    Args:
        argv (List[str]): command line args.

    Returns:
        None.
    """

    args = parse_args(argv)
    for name, val in vars(args).items():
        try:
            vars(args)[name] = list(map(int, val.split(",")))
        except (ValueError, AttributeError):
            pass

    if args.multi_hot_sizes is not None:
        assert (
            args.num_embeddings_per_feature is not None
            and len(args.multi_hot_sizes) == len(args.num_embeddings_per_feature)
            or args.num_embeddings_per_feature is None
            and len(args.multi_hot_sizes) == len(DEFAULT_CAT_NAMES)
        ), "--multi_hot_sizes must be a comma delimited list the same size as the number of embedding tables."
    assert (
        args.in_memory_binary_criteo_path is None
        or args.synthetic_multi_hot_criteo_path is None
    ), "--in_memory_binary_criteo_path and --synthetic_multi_hot_criteo_path are mutually exclusive CLI arguments."
    assert (
        args.multi_hot_sizes is None or args.synthetic_multi_hot_criteo_path is None
    ), "--multi_hot_sizes is used to convert 1-hot to multi-hot. It's inapplicable with --synthetic_multi_hot_criteo_path."
    assert (
        args.multi_hot_distribution_type is None
        or args.synthetic_multi_hot_criteo_path is None
    ), "--multi_hot_distribution_type is used to convert 1-hot to multi-hot. It's inapplicable with --synthetic_multi_hot_criteo_path."

    device: torch.device = torch.device("cpu")
    backend = "gloo"

    pprint(vars(args))
    # logger.event(
    #     key=mllog_constants.GLOBAL_BATCH_SIZE,
    #     value=dist.get_world_size() * args.batch_size,
    # )
    # logger.event(
    #     key=mllog_constants.GRADIENT_ACCUMULATION_STEPS,
    #     value=1,  # Gradient accumulation is not supported in the reference implementation
    # )
    # logger.event(
    #     key=mllog_constants.SEED,
    #     value=args.seed,  # Seeding model is not supported in the reference implementation
    # )

    if args.num_embeddings_per_feature is not None:
        args.num_embeddings = None

    # Sets default limits for random dataloader iterations when left unspecified.
    if (
        args.in_memory_binary_criteo_path is None
        and args.synthetic_multi_hot_criteo_path is None
    ):
        for split in ["train", "val", "test"]:
            attr = f"limit_{split}_batches"
            if getattr(args, attr) is None:
                setattr(args, attr, 10)

    eb_configs = [
        EmbeddingBagConfig(
            name=f"t_{feature_name}",
            embedding_dim=args.embedding_dim,
            num_embeddings=none_throws(args.num_embeddings_per_feature)[feature_idx]
            if args.num_embeddings is None
            else args.num_embeddings,
            feature_names=[feature_name],
        )
        for feature_idx, feature_name in enumerate(DEFAULT_CAT_NAMES)
    ]
    sharded_module_kwargs = {}
    if args.over_arch_layer_sizes is not None:
        sharded_module_kwargs["over_arch_layer_sizes"] = args.over_arch_layer_sizes

    print_memory("start init dlrm ")
    if args.interaction_type == InteractionType.ORIGINAL:
        dlrm_model = DLRM(
            embedding_bag_collection=EmbeddingBagCollection(
                tables=eb_configs, device=torch.device("meta")
            ),
            dense_in_features=len(DEFAULT_INT_NAMES),
            dense_arch_layer_sizes=args.dense_arch_layer_sizes,
            over_arch_layer_sizes=args.over_arch_layer_sizes,
            dense_device=device,
        )
    elif args.interaction_type == InteractionType.DCN:
        if args.ipex_optimize:
            from ipex_optimized_model.model import IPEX_DLRM_DCN
            dcn_init_fn = IPEX_DLRM_DCN
        else:
            dcn_init_fn = DLRM_DCN
        dlrm_model = dcn_init_fn(
            embedding_bag_collection=EmbeddingBagCollection(
                tables=eb_configs, device=torch.device("cpu")
            ),
            dense_in_features=len(DEFAULT_INT_NAMES),
            dense_arch_layer_sizes=args.dense_arch_layer_sizes,
            over_arch_layer_sizes=args.over_arch_layer_sizes,
            dcn_num_layers=args.dcn_num_layers,
            dcn_low_rank_dim=args.dcn_low_rank_dim,
            dense_device=device,
        )
    elif args.interaction_type == InteractionType.PROJECTION:
        dlrm_model = DLRM_Projection(
            embedding_bag_collection=EmbeddingBagCollection(
                tables=eb_configs, device=torch.device("meta")
            ),
            dense_in_features=len(DEFAULT_INT_NAMES),
            dense_arch_layer_sizes=args.dense_arch_layer_sizes,
            over_arch_layer_sizes=args.over_arch_layer_sizes,
            interaction_branch1_layer_sizes=args.interaction_branch1_layer_sizes,
            interaction_branch2_layer_sizes=args.interaction_branch2_layer_sizes,
            dense_device=device,
        )
    else:
        raise ValueError(
            "Unknown interaction option set. Should be original, dcn, or projection."
        )

    train_model = DLRMTrain(dlrm_model)
    if args.test_auroc or args.calibration:
        assert args.snapshot_dir
        print_memory("start loading checkpoint ")
        load_snapshot(train_model, args.snapshot_dir)

    from ipex_optimized_model.model import replace_crossnet, replace_embeddingbag_collection
    replace_crossnet(train_model.model)
    if args.inference_only and args.ipex_merged_emb_cat:
        replace_embeddingbag_collection(train_model.model)
    else:
        for emb in train_model.model.sparse_arch.embedding_bag_collection.embedding_bags.values():
            emb.sparse = True
    # embedding_optimizer = torch.optim.Adagrad if args.adagrad else torch.optim.SGD
    # This will apply the Adagrad optimizer in the backward pass for the embeddings (sparse_arch). This means that
    # the optimizer update will be applied in the backward pass, in this case through a fused op.
    # TorchRec will use the FBGEMM implementation of EXACT_ADAGRAD. For GPU devices, a fused CUDA kernel is invoked. For CPU, FBGEMM_GPU invokes CPU kernels
    # https://github.com/pytorch/FBGEMM/blob/2cb8b0dff3e67f9a009c4299defbd6b99cc12b8f/fbgemm_gpu/fbgemm_gpu/split_table_batched_embeddings_ops.py#L676-L678

    # Note that lr_decay, weight_decay and initial_accumulator_value for Adagrad optimizer in FBGEMM v0.3.2
    # cannot be specified below. This equivalently means that all these parameters are hardcoded to zero.
    optimizer_kwargs = {"lr": args.learning_rate}
    if args.adagrad:
        optimizer_kwargs["eps"] = ADAGRAD_EPS
    # apply_optimizer_in_backward(
    #     embedding_optimizer,
    #     train_model.model.sparse_arch.parameters(),
    #     optimizer_kwargs,
    # )
    model = train_model
    # model.model.sparse_arch = SparseArchTraceAbleWrapper(model.model.sparse_arch)

    # def optimizer_with_params():
    #     if args.adagrad:
    #         return lambda params: torch.optim.Adagrad(
    #             params,
    #             lr=args.learning_rate,
    #             lr_decay=ADAGRAD_LR_DECAY,
    #             weight_decay=WEIGHT_DECAY,
    #             initial_accumulator_value=ADAGRAD_INIT_ACC,
    #             eps=ADAGRAD_EPS,
    #         )
    #     else:
    #         return lambda params: torch.optim.SGD(
    #             params,
    #             lr=args.learning_rate,
    #             weight_decay=WEIGHT_DECAY,
    #         )

    # dense_optimizer = KeyedOptimizerWrapper(
    #     dict(in_backward_optimizer_filter(model.named_parameters())),
    #     optimizer_with_params(),
    # )
    # optimizer = CombinedOptimizer([model.fused_optimizer, dense_optimizer])
    optimizer,  lr_scheduler = None, None
    if not args.inference_only:
        print_memory("start create optimizer ")
        if args.adagrad:
            optimizer = torch.optim.Adagrad(
                model.parameters(),
                lr=args.learning_rate,
                lr_decay=ADAGRAD_LR_DECAY,
                weight_decay=WEIGHT_DECAY,
                initial_accumulator_value=ADAGRAD_INIT_ACC,
                eps=ADAGRAD_EPS,
            )
        else:
            optimizer = torch.optim.SGD(
                model.parameters(),
                lr=args.learning_rate,
                weight_decay=WEIGHT_DECAY,
            )

        lr_scheduler = LRPolicyScheduler(
            optimizer, args.lr_warmup_steps, args.lr_decay_start, args.lr_decay_steps
        )

    # logger.event(
    #     key=mllog_constants.OPT_NAME,
    #     value=mllog_constants.ADAGRAD if args.adagrad else mllog_constants.SGD,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_BASE_LR,
    #     value=args.learning_rate,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_ADAGRAD_LR_DECAY,
    #     value=ADAGRAD_LR_DECAY,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_WEIGHT_DECAY,
    #     value=WEIGHT_DECAY,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_ADAGRAD_INITIAL_ACCUMULATOR_VALUE,
    #     value=ADAGRAD_INIT_ACC,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_ADAGRAD_EPSILON,
    #     value=ADAGRAD_EPS,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_LR_WARMUP_STEPS,
    #     value=args.lr_warmup_steps,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_LR_DECAY_START_STEP,
    #     value=args.lr_decay_start,
    # )
    # logger.event(
    #     key=mllog_constants.OPT_LR_DECAY_STEPS,
    #     value=args.lr_decay_steps,
    # )

    print_memory("start create dataloader ")

    # only create 1 multi-hot sample to save memory
    if args.multi_hot_sizes is not None:
        vb, tb = args.limit_val_batches, args.limit_test_batches
        args.limit_val_batches = 1
        args.limit_test_batches = 1

    val_dataloader = get_dataloader(args, backend, "val")
    test_dataloader = get_dataloader(args, backend, "test")

    if args.multi_hot_sizes is not None:
        args.limit_val_batches, args.limit_test_batches = vb, tb

    if args.inference_only:
        train_dataloader = None
    else:
        train_dataloader = get_dataloader(args, backend, "train")

    # logger.event(
    #     key=mllog_constants.TRAIN_SAMPLES,
    #     value=dist.get_world_size() * len(train_dataloader) * args.batch_size,
    # )

    if args.multi_hot_sizes is not None:
        print_memory("start to create Multihot for dummy data ")
        multihot = Multihot(
            args.multi_hot_sizes,
            args.num_embeddings_per_feature,
            args.batch_size,
            collect_freqs_stats=args.collect_multi_hot_freqs_stats,
            dist_type=args.multi_hot_distribution_type,
        )
        multihot.pause_stats_collection_during_val_and_test(model)
        print_memory("start transfer to multihot dataloader for dummy data ")
        if train_dataloader:
            train_dataloader = RestartableMap(
                multihot.convert_to_multi_hot, train_dataloader
            )

        val_dataloader = RestartableMap(multihot.convert_to_multi_hot, val_dataloader)
        test_dataloader = RestartableMap(multihot.convert_to_multi_hot, test_dataloader)

    if args.ipex_optimize:
        print_memory("start ipex_optimize ")
        model.model, optimizer = ipex_optimize(args, model.model, optimizer, test_dataloader)

    print_memory("start running model")
    train_val_test(
        args,
        model,
        optimizer,
        device,
        train_dataloader,
        val_dataloader,
        test_dataloader,
        lr_scheduler,
    )
    if args.collect_multi_hot_freqs_stats:
        multihot.save_freqs_stats()

if __name__ == "__main__":
    main(sys.argv[1:])


# -*- coding: utf-8 -*-

import os
import re
import sys
import random
import contextlib
from utilities import PROJECT_DIR
from urllib.parse import urlparse

pathjoin = os.path.join

#######################
# PATH UTILS
def list_path(path):
    return os.listdir(path)

def last_dir(path, level=1):

    suffix = ""
    current_level = level
    path = str(path)
    while current_level > 0:
        if suffix != "":
            suffix = '/' + suffix
        suffix = os.path.basename(path) + suffix
        current_level -= 1
        if current_level > 0:
            path = os.path.dirname(path)

    return suffix

def parent_dir(path):
    return os.path.dirname(path)

#######################
# WITH SUFFIXES
def root_path(*suffixes):
    return pathjoin(os.path.abspath(os.sep), *suffixes)

def script_abspath(file, *suffixes):
    return pathjoin(os.path.dirname(os.path.realpath(file)), *suffixes)

def current_dir(*suffixes):
    return pathjoin(os.curdir, *suffixes)

def current_fullpath(*suffixes):
    return pathjoin(os.getcwd(), *suffixes)

def latest_dir(path):
    return next(reversed(list(os.path.split(path))))

#######################
# RANDOM
def random_name(lenght=10):
    import string

    return ''.join(
        random.choice(
            # string.ascii_uppercase
            string.ascii_lowercase
            + string.digits
        )
        for _ in range(lenght)
    )

def random_element(mylist):
    """ Recover a random element from a list """
    if not isinstance(mylist, list):
        return None
    if len(mylist) < 1:
        return None
    index = random.randint(0, len(mylist) - 1)
    # log.debug("Random index: %s", index)
    return mylist.pop(index)

#######################
# OTHERS
def module_from_package(package):
    return package.split('.')[::-1][0]

def project_dir(project, *suffixes):
    return current_dir(PROJECT_DIR, project, *suffixes)

def get_api_url(request_object, production=False):
    """ Get api URL and PORT

    Usefull to handle https and similar
    unfiltering what is changed from nginx and container network configuration

    Warning: it works only if called inside a Flask endpoint
    """

    api_url = request_object.url_root

    if production:
        parsed = urlparse(api_url)
        if parsed.port is not None and parsed.port == 443:
            removed_port = re.sub(r':[\d]+$', '', parsed.netloc)
            api_url = parsed._replace(scheme="https", netloc=removed_port).geturl()

    return api_url

def ask_yes_or_no(question, error='Unknown', print_function=None):

    if print_function is None:
        print_function = print

    answer = 'unknown'
    possible_answers = ['yes', 'y', 'no', 'n']

    while True:
        print_function(question)
        try:
            answer = input('(y/n) ')
        except BaseException as e:
            raise e
        finally:
            if answer == 'unknown' or answer.strip() == '':
                print("\nInterrupted!!\n")
                # log.warning("Interrupted by the user")

        if answer not in possible_answers:
            print("Please answer one of the following: %s" % possible_answers)
        else:
            if answer.strip().startswith('y'):
                break
            else:
                print('USER INTERRUPT:\t' + error)
                sys.exit(1)

@contextlib.contextmanager
def nooutput():
    """ Thanks Alex: https://stackoverflow.com/a/1810086 """
    savestderr = sys.stderr
    savestdout = sys.stdout

    class Devnull(object):
        def write(self, _):
            pass

        def flush(self):
            pass

    sys.stdout = Devnull()
    sys.stderr = Devnull()
    try:
        yield
    finally:
        sys.stdout = savestdout
        sys.stderr = savestderr


import subprocess
import shlex
import re
from optparse import OptionParser
import numpy
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from matplotlib.ticker import FormatStrFormatter
import sys
import os
import fnmatch
import locale

nodeKind = ['o-', 's--', 'D-.', '^:', 'X-', 'V--', '>-.', '<:']

matplotlib.rcParams['ps.fonttype'] = 42
matplotlib.rcParams['pdf.fonttype'] = 42

def main ():
	files = ["GC.dat"]
	for f in files:
		Cassandra = []
		EC = []
		CC = []

		infile = open (f, "r")
		for line in infile.readlines ():
			if line == "\n" or line.startswith ("#"):
				continue

			d = line.split ()
			if (len(d) == 6):
				Cassandra.append ((float(d[0])/10000,float (d[1])/100))
				EC.append ((float(d[2])/10000,float (d[3])/100))
				CC.append ((float(d[4])/10000,float (d[5])/100))
			elif (len(d) == 4):
				EC.append ((float(d[0])/10000,float (d[1])/100))
				CC.append ((float(d[2])/10000,float (d[3])/100))
			elif (len(d) == 2):
				CC.append ((float(d[0])/10000,float (d[1])/100))

		(cx,cy) = zip(*Cassandra)
		(ecx,ecy) = zip(*EC)
		(ccx,ccy) = zip(*CC)

		plt.xlabel ("Operations (X 10,000)")
		plt.ylabel ("Latency (X 100 ms)")
		plt.grid (True)
		font = {'family' : 'normal', 'weight' : 'normal', 'size' : 20}
		matplotlib.rc('font', **font)

		plt.subplots_adjust (bottom=0.12)
		plt.plot (cx, cy, nodeKind [0], label="No Sum", linewidth=2.0, ms=10)
		plt.plot (ecx, ecy, nodeKind [1], label="Mem Only", linewidth=2.0, ms=10)
		plt.plot (ccx, ccy, nodeKind [2], label="Mem & Disk", linewidth=2.0, ms=10)
		plt.xlim(xmin = 0)
		plt.legend (bbox_to_anchor=(1, 0.75))
		print ("saving fig " + f.replace("dat","pdf"))
		plt.savefig (f.replace("dat","pdf"))
		plt.close ()

main ()


from flask import Blueprint, request
from modules.common.gestor_personas import gestor_personas
from modules.common.gestor_comun import exportar

reportes_exel = Blueprint('routes_reportes', __name__)

@reportes_exel.route('/personas/generar_exel', methods=['POST', 'GET'])
def generar_exel_personas():
    personas=gestor_personas().obtener_todo()
    personas_data=[]
    for persona in personas:
        pd={}
        pd["Nombre"] = persona.nombre
        pd["Apellido"] = persona.apellido
        pd["email"] = persona.email
        pd["Edad"] = persona.age
        pd["Fecha nacimiento"]=persona.birthdate.strftime('%d/%m/%Y')
        pd["Genero"]=persona.genero.nombre
        pd["Pais"]=persona.lugar.pais.nombre
        pd["Provincia"]=persona.lugar.provincia.nombre
        pd["Ciudad"]=persona.lugar.ciudad.nombre
        pd["Barrio"]=persona.lugar.barrio.nombre
        personas_data.append(pd)

    return exportar.exportar_excel(personas_data)


from cmath import inf
import sys
from itertools import accumulate
input = sys.stdin.readline

N = int(input())
trees = [[0] + list(map(int, input().split())) for _ in range(N)]
trees.insert(0, [0] * (N+1))

for i, tree in enumerate(trees):
    trees[i] = list(accumulate(tree))

for i, tree in enumerate(list(zip(*trees))):
    trees[i] = list(accumulate(tree))

trees = list(zip(*trees))

res = -inf
for i in range(N):
    for j in range(1, N-i+1):
        for k in range(1, N-i+1):
            v = trees[i+j][i+k] - trees[j-1][i+k] - trees[i+j][k-1] + trees[j-1][k-1]
            res = max(res, v)

print(res)


def lowest(num, den):
    # Finding gcd of both terms
    common_factor = gcd(num, den)
    # Converting both terms into simpler terms by dividing them by common factor
    den = int(den / common_factor)
    num = int(num / common_factor)
    return (num, den)
#Greatest Common Denominator code
def gcd(a, b):
    if (a == 0):
        return b
    return gcd(b % a, a)

def add(num1, den1, num2, den2):
    # Finding gcd of den1 and den2
    den3 = gcd(den1, den2)
    # Denominator of final fraction obtained finding LCM of den1 and den2 LCM * GCD = a * b
    den3 = (den1 * den2) / den3
    # Changing the fractions to have same denominator Numerator of the final fraction obtained
    num3 = ((num1) * (den3 / den1) + (num2) * (den3 / den2))
    # Calling function to convert final fraction into it's simplest form
    lowest(den3, num3)
    return (num3, den3)
#Multiplies the fractions
def multiply(frac1, frac2):
    dev1 = (frac1[0] * frac1[1])
    dev2 = (frac2[0] * frac2[1])
    return (dev1, dev2)
#Divides the fractions
def divide(frac1, frac2):
    dev1 = (frac1[0] * frac2[1])
    dev2 = (frac1[1] * frac2[0])
    return (dev1, dev2)
#Asks for input and computes fractions
def main():
    frac1 = (float(input("What is the numerator? ")) ,float(input("What is the denominator? ")))
    frac2 = (float(input("What is the numerator of the second fraction?")) ,float(input("What is the denominator? ")))
    redfrac1 = lowest(frac1[0], frac1[1])
    redfrac2 = lowest(frac2[0], frac2[1])
    print("Reduced fractions to", redfrac1, " ", redfrac2)
    print("Sum of the fractions: ", add(redfrac1[0], redfrac1[1], redfrac2[0], redfrac2[1]))
    print("Multiplication of the fractions: ", multiply(frac1, frac2))
    print("Division of the first by the second: ", divide(frac1, frac2))

#Boilerplate
if __name__ == "__main__":
    main()


import datetime
import json

import django_rq

from server.apis.guidebox import GuideBox
from server.models import Channel, Content

def inital_database_population_of_content():
    g = GuideBox()
    q = django_rq.get_queue('low')
    result = []

    total_shows = g.get_total_number_of_shows()

    for i in range(0, total_shows, 250):
        show_list = g.get_content_list(i)
        shows_dict = json.loads(show_list)

        for show in shows_dict['results']:
            q.enqueue(g.save_content, show)

            result.append(show)

    return result

def refresh_database(d):
    g = GuideBox()
    q = django_rq.get_queue('high')

    last = datetime.date(2016,1,1)
    milis = time.mktime(last.timetuple())
    milis = int(milis)
    new_shows = g.get_new_shows(milis)
    # for i in range

def inital_database_population_of_channels():
    g = GuideBox()
    q = django_rq.get_queue('low')
    result = []

    total_channels = g.get_total_number_of_channels()

    for i in range(0, total_channels, 24):
        channel_list = g.get_channel_list('all', i)
        channel_dict = json.loads(channel_list)

        for chan in channel_dict['results']:
            q.enqueue(g.save_channel, chan)

            result.append(chan)

    return result

def connect_content_channel_task():
    g = GuideBox()
    q_low = django_rq.get_queue('low')
    q_high = django_rq.get_queue('high')
    print('im splitting stuff')

    all_channels = Channel.objects.all()
    count = 0
    for i in [all_channels[i:i + 5] for i in range(0, len(all_channels), 5)]:
        count += 5
        if count % 2 == 0:
            q_high.enqueue(g.connect_channels_shows, i)
        else:
            q_low.enqueue(g.connect_channels_shows, i)

def add_available_sources_to_shows():
    g = GuideBox()
    q_high = django_rq.get_queue('high')

    # all_shows = Content.objects.all()
    all_shows = Content.objects.all()
    # print(all_shows)
    for show in [all_shows[i: i + 20]  for i in range(0, len(all_shows), 20)]:
        shows = [x.id for x in show]
        q_high.enqueue(g.add_additional_channels_for_show, shows)

def add_detail_to_shows():
    g = GuideBox()
    q_low = django_rq.get_queue('low')

    all_shows = Content.objects.all()

    all_shows = [show for show in all_shows if show.guidebox_data is not None and 'detail' not in show.guidebox_data]
    for shows_chunk in [all_shows[i: i+20] for i in range(0, len(all_shows), 20)]:
        q_low.enqueue(g.process_shows_for_content_detail, shows_chunk)


#!/usr/bin/python

__author__ = 'Jan Nash'
__version__ = "0.2"
__license__ = "MIT"

VERSION = "%s v" + __version__

def parse_args():
    """
    Parses command-line arguments and returns the parsed namespace-object.
    """
    import argparse
    import os

    parser = argparse.ArgumentParser(version=VERSION)

    parser.add_argument(
        '-p', '--path',
        action='store', default=os.getcwd(), dest='project_source_root_path',
        help='The path to the Xcode-project\'s source-root folder that will be walked. '
             'Defaults to the current working directory.')

    parser.add_argument(
        '-u', '--update',
        action='store_true', dest='update',
        help='If set, the keys of missing translations will be written to the strings-files.')

    parser.add_argument(
        '-d', '--debug',
        action='store_true', default=False, dest='debug',
        help='Set logging level to DEBUG. Beware, it\'s gonna be verbose :).')

    parser.add_argument(
        '-c', '--custom-macros',
        action='store', nargs='*', dest='custom_macros', default=(),
        help='Custom macros that are used in the project. For help regarding the format of those custom macros '
             'see documentation of nslocapysation.classes.localized_string.LocalizedString')

    parser.add_argument(
        '-i', '--ignore',
        action='store', nargs='*', dest='ignore_language_codes', default=(),
        help='Language codes for which missing translations should be disregarded.'
             'This is useful if you are lazy and use the UI-string of that language as key.')

    return parser.parse_args()

def main(cmd_args):
    import logging
    from nslocapysation import constants
    from nslocapysation.tools.collect_m_file_and_lproj_dir_paths import collect_implementation_file_and_lproj_dir_paths
    from nslocapysation.tools.collect_localized_strings import collect_localized_strings
    from nslocapysation.tools.collect_localizable_strings_files import collect_localizable_strings_files
    from nslocapysation.tools.check_translations import check_translations
    from nslocapysation.classes.ns_localized_string_macro import NSLocalizedStringMacro

    if cmd_args.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    file_path_dict = collect_implementation_file_and_lproj_dir_paths(cmd_args.project_source_root_path)

    implementation_file_paths = file_path_dict[constants.IMPLEMENTATION_FILE_PATHS_KEY]

    localization_file_paths = file_path_dict[constants.LPROJ_DIR_PATHS_KEY]

    custom_macros = [NSLocalizedStringMacro(format_=fmt) for fmt in cmd_args.custom_macros]

    localized_strings = collect_localized_strings(implementation_file_paths=implementation_file_paths,
                                                  custom_macros=custom_macros)

    localizable_strings_files = collect_localizable_strings_files(localization_dir_paths=localization_file_paths)

    check_translations(translation_files=localizable_strings_files,
                       localized_strings=localized_strings,
                       update=cmd_args.update,
                       ignore_language_codes=cmd_args.ignore_language_codes)

if __name__ == '__main__':
    cl_args = parse_args()
    main(cl_args)


# -*- coding: utf-8 -*-

import six
from django.core.exceptions import NON_FIELD_ERRORS, ValidationError
from django.forms.fields import Field
from django.forms.forms import DeclarativeFieldsMetaclass
from django.forms.utils import ErrorDict, ErrorList

class BaseDeserializer(Field):
    """
    The main implementation of Deserializers logic. Note that this class is
    slightly different from the classic Form.
    """

    __slots__ = ()

    empty_values = (None, "")

    def __init__(self, data=None, **kwargs):
        super(BaseDeserializer, self).__init__(**kwargs)
        self._init_data(data)

    def _init_data(self, data):
        # type:(Optional[Dict]) -> None
        self._data = data
        self._errors = None  # Stores the errors after `full_clean()` has been called.
        self._cleaned_data = None

    @classmethod
    def fields(cls):
        # type:() -> Iterator[str, Field]
        for name in cls.declared_fields:
            yield name, cls.declared_fields[name]

    @property
    def errors(self):
        # type:() -> ErrorDict
        """Return an ErrorDict for the data provided for the deserializer."""
        if self._errors is None:
            self.full_clean()
        return self._errors

    def is_valid(self):
        """Return True if the deserializer has no errors, or False otherwise."""
        return not self.errors

    def _get_error_dict(self, field_name, error):
        # type:(str, ValidationError) -> Dict
        if not hasattr(error, "error_dict"):
            # In case the error comes from simple ("flat") Field
            return {field_name or NON_FIELD_ERRORS: error.messages}

        # Nested field case (the Field itself is a Deserializer instance)
        return {field_name: dict(error)}

    def add_error(self, field_name, error):
        # type:(str, ValidationError) -> None
        """
        Updates the content of `self._errors`. Each `self._errors` is a `dict` instance.
        """
        error = self._get_error_dict(field_name, error)
        for field_name, errors in error.items():
            self._errors[field_name] = errors

    def full_clean(self):
        """
        Clean all of self._data and populate self._errors and self._cleaned_data.
        """
        self._errors = ErrorDict()
        self._cleaned_data = {}
        self._clean_fields()

    def _clean_fields(self):
        if self._data in self.empty_values:
            self._cleaned_data = self._data
            return
        if not isinstance(self._data, dict):
            error_list = ErrorList(["This field should be an object."])
            raise ValidationError(error_list)
        for name, field in self.declared_fields.items():
            value = self._data.get(name)
            try:
                value = field.clean(value)
                self._cleaned_data[name] = value
                if hasattr(self, "post_clean_{}".format(name)):
                    value = getattr(self, "post_clean_{}".format(name))(value)
                    self._cleaned_data[name] = value
            except ValidationError as e:
                if field.required:
                    self.add_error(name, e)

    def clean(self, data):
        """
        Overriden version of superclass Field.clean(). Called in case of nested
        Deserializers. Performs the validation and cleaning process on the data
        parameter. Returns the cleaned data to the parent deserializer.
        In case of error, Raises `ValidationError` with the whole ErrorDict()
        of its errors in order to be catched in the parent.
        """

        # Raises an error if `data` is empty and the current (nested)
        # deserializer is required.
        self.validate(data)

        self._init_data(data)
        if self.errors:
            raise ValidationError(self.errors)
        return self._cleaned_data

    @property
    def data(self):
        # type:() -> dict
        if not self._cleaned_data:
            self.full_clean()
        return self._cleaned_data

class Deserializer(six.with_metaclass(DeclarativeFieldsMetaclass, BaseDeserializer)):
    pass

class AllPassDeserializer(Deserializer):
    """Deserializer that accepts any given data. the clean always returns
    the data received as input.
    """

    def _clean_fields(self):
        for name, value in self._data.items():
            self._cleaned_data[name] = value


# coding: utf-8

import pytest
import os

from hubblestack.hec.dq import DiskQueue
from hubblestack.hec.dq import QueueTypeError, QueueCapacityError

TEST_DQ_DIR = os.environ.get('TEST_DQ_DIR', '/tmp/dq.{0}'.format(os.getuid()))

@pytest.fixture
def samp():
    return tuple('one two three four five'.split())

@pytest.fixture
def dq():
    return DiskQueue(TEST_DQ_DIR, fresh=True)

@pytest.fixture
def dqc():
    return DiskQueue(TEST_DQ_DIR + ".bz2", fresh=True, compression=9)

def _test_disk_queue(dq):
    borked = False

    dq.put('one', testinator=3)
    dq.put('two', testinator=4)
    dq.put('three', testinator=5)

    if not dq.compression:
        # NOTE: with the huffman headers (or whatever), the size of the dq is
        # probably quite a lot larger than the expected 13. the test is
        # essentially meaningless unless the uncompressed message was large
        # enough… then we could test to see if the dq len was smaller than
        # expected or something… let's just skip this for the compressed dq
        assert len(dq) == 13

    assert dq.peek() == ('one', {'testinator': 3})
    assert dq.get() == ('one', {'testinator': 3})
    assert dq.peek() == ('two', {'testinator': 4})

    if not dq.compression:
        assert len(dq) == 9

    assert dq.getz() == ('two three', {'testinator': 5})
    assert len(dq) == 0

    dq.put('one')
    dq.put('two')
    dq.put('three')

    assert dq.getz(8) == ('one two', {})
    assert dq.getz(8) == ('three', {})

def test_disk_queue(dq):
    _test_disk_queue(dq)

def test_disk_queue_with_compression(dqc):
    _test_disk_queue(dqc)

def _test_pop(samp,q):
    for i in samp:
        q.put(i)
    for i in samp:
        assert q.peek() == (i, {})
        q.pop()

def test_dq_pop(samp,dq):
    _test_pop(samp,dq)

def test_disk_queue_put_estimator(dq):
    for item in ['hi-there-{}'.format(x) for x in range(20)]:
        pre = dq.cn, dq.sz
        dq.put(item)
        post = dq.cn, dq.sz
        assert (pre[0]+1, pre[1]+len(item)) == post
        dq._count()
        more = dq.cn, dq.sz
        assert post == more