# -*- coding: utf-8 -*-

import logging
import datetime

import pytz

__all__ = ['timezones']

logger = logging.getLogger('django')

def is_standard_time(time_zone, date_time):
    try:
        dst_delta = time_zone.dst(date_time, is_dst=False)
    except TypeError:
        dst_delta = time_zone.dst(date_time)

    return dst_delta == datetime.timedelta(0)

def utc_offset(time_zone, fixed_dt=None):
    tz = pytz.timezone(time_zone)
    now = fixed_dt or datetime.datetime.now()

    for __ in range(72):
        if is_standard_time(time_zone=tz, date_time=now):
            break

        now += datetime.timedelta(days=30)
    else:
        logger.warning(
            'Standard Time not found for %s, will use DST.' % time_zone)

    return tz.localize(now, is_dst=False).strftime('%z')

def offset_to_int(offset):
    assert offset[0] in ('-', '+')

    sign, hour, minutes = offset[0], offset[1:3], offset[3:5]
    utc_offset_int = int(hour) + int(minutes) / 100

    if sign == '-':
        utc_offset_int *= -1

    return utc_offset_int

def timezones_by_offset():
    return sorted(
        ((utc_offset(tz), tz)
         for tz in pytz.common_timezones),
        key=lambda x: (offset_to_int(x[0]), x[1]))

def timezone_format(time_zone, offset):
    zone_parts = time_zone.split('/')
    zone = zone_parts[0]

    if len(zone_parts) > 1:
        zone_label = ', '.join(zone_parts[1:]).replace('_', ' ')
    else:
        zone_label = zone

    return zone, '(UTC{}) {}'.format(offset, zone_label)

def timezones():
    """
    Result format::

        [
            ("Africa", [
                ("Africa/Abidjan", "(UTC...) Abidjan"),
                ("Africa/Accra", "(UTC...) Accra"),
                #...
            ]),
            ("America", [
                ("America/Argentina/Buenos_Aires",
                 "(UTC...) Argentina, Buenos Aires"),
                #...
            ]),
            #...
        ]
    """
    timezones_cache = {}

    for offset, time_zone in timezones_by_offset():
        zone, pretty_time_zone = timezone_format(time_zone, offset)
        (timezones_cache
         .setdefault(zone, [])
         .append((time_zone, pretty_time_zone)))

    return sorted(
        timezones_cache.items(),
        key=lambda x: x[0])


import pyautogui
from random import random
import pyscreenshot as ImageGrab
import math
import cv2 as cv
from utilities import inventory as inv
from utilities.core import get_bounding_rect_from_pts

def get_pickup_rects(o_img, cnts):  # returns bounding rect(s) of item(s) to pickup
    items = []
    line_pts = []
    for c in cnts:
        # compute the center of each contour
        M = cv.moments(c)
        if M["m00"] != 0:
            cX = int(M["m10"] / M["m00"])
            cY = int(M["m01"] / M["m00"])
            line_pts.append((cX, cY))
    # cv.rectangle(output,(minx,miny),(maxx,maxy),(0,255,255),2)
    maxslope = 0
    item_split_indices = [0]
    for x in range(1, len(line_pts) - 2):
        x1, y1 = (line_pts[x - 1])
        x2, y2 = (line_pts[x])
        if (x2 - x1) != 0:
            m = (y2 - y1) / (x2 - x1)
            if m > maxslope:
                maxslope = m
            if (m <= 0.05) and (m >= -0.05):
                cv.line(o_img, line_pts[x - 1], line_pts[x], (0, 255, 255), 2)
            else:
                # split lines based on slope and dist
                # print(item_split_indices)
                # print(math.dist((x1,y1),(x2,y2)))
                if (math.dist((x1, y1), (x2, y2))) >= 40 and x != 1:
                    item_split_indices.append(x)
    if len(item_split_indices) == 1:
        rect = get_bounding_rect_from_pts(line_pts[1:len(line_pts)])
        if rect not in items:
            items.append(rect)
    elif (len(item_split_indices) > 1):
        for i in range(0, len(item_split_indices) - 1):
            if i == 0:
                rect = get_bounding_rect_from_pts(line_pts[1:item_split_indices[i + 1]])
            elif (i + 1 <= len(item_split_indices) + 1):
                rect = get_bounding_rect_from_pts(line_pts[item_split_indices[i]:item_split_indices[i + 1]])
            if rect not in items:
                items.append(rect)
    for item in items:
        cv.rectangle(o_img, item[0], item[1], (0, 0, 255), 1)
    return items

def bury_bones(min_bone_count):
    inv_img, bones = inv.get_item_rects(inv.get_icon(526))
    inv_img2, big_bones = inv.get_item_rects(inv.get_icon(532))
    if len(bones) + len(big_bones) > min_bone_count:
        for rect in bones:
            rect[0]
            pt = ((rect[0][0] + 625 + rect[1][0] + 625) / 2, (rect[0][1] + 485 + rect[1][1] + 485) / 2)
            pyautogui.moveTo(pt)
            pyautogui.click(clicks=1, interval=random() * 3, button='left')
        for rect in big_bones:
            rect[0]
            pt = ((rect[0][0] + 625 + rect[1][0] + 625) / 2, (rect[0][1] + 485 + rect[1][1] + 485) / 2)
            pyautogui.moveTo(pt)
            pyautogui.click(clicks=1, interval=random() * 3, button='left')

def start(targets, items, players, target_mask, selected):
    combat_img = ImageGrab.grab(bbox=[10, 50, 140, 69])
    enemy_name = pytesseract.image_to_string(combat_img)
    for name in marked_enemy_names:
        if enemy_name != None:
            if similar(enemy_name, name) > 0.7:
                enemy_name = name

    min_dist = 3000
    other_players_target = None

    for r in targets:
        for p in players:
            dist = math.sqrt(math.pow(r[0] - p[0][0], 2) + math.pow(r[1] - p[0][1], 2))
            if p in items:
                players.remove(p)

            if dist < 200:
                if r in targets:
                    if (dist < min_dist):
                        if r[2] > 30:
                            other_players_target = r
                    targets.remove(r)
                continue
        if r[2] > 10 and r in targets:
            cv.rectangle(target_mask, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]), (0, 255, 255), 2)

    if other_players_target is not None:
        if len(players) >= 1:
            if players[0] not in items:
                cv.line(target_mask, (other_players_target[0], other_players_target[1]),
                        (players[0][0][0], players[0][0][1]), (0, 255, 255), 2)
        cv.rectangle(target_mask, (other_players_target[0], other_players_target[1]), (
        other_players_target[0] + other_players_target[2], other_players_target[1] + other_players_target[3]),
                     (0, 40, 255), 2)

    if random() > 0.95:
        bury_bones(0)
    if enemy_name not in marked_enemy_names:
        if len(items) >= 1:
            # item pickup
            item = items[0]
            min = item[0]
            max = item[1]
            pyautogui.moveTo((min[0] + max[0]) / 2, ((min[1] + max[1]) / 2) + 10)
            pyautogui.click(clicks=2, interval=random(), button='left')
        elif len(rects[4]) > 0:
            # need combat panel screengrab and imagetotext
            # if in combat dont move cursor or click
            #
            if selected != targets[0]:
                selected = targets[0]
                pyautogui.moveTo((selected[0] + selected[2] / 2), (selected[1] + selected[3] / 2))
                pyautogui.click(button='left', clicks=1, interval=random())


from collections import OrderedDict

# from datetime import datetime
from django.conf import settings
from django.db import models
from django.utils import timezone

from jsonfield import JSONField

# Create your models here.

class fhir_Consent(models.Model):
    """ Store User:application consent in fhir format
    """
    user = models.ForeignKey(settings.AUTH_USER_MODEL)
    application = models.ForeignKey(settings.OAUTH2_PROVIDER_APPLICATION_MODEL)
    consent = JSONField(load_kwargs={'object_pairs_hook': OrderedDict})
    created = models.DateTimeField(blank=True, null=True)
    revoked = models.DateTimeField(blank=True, null=True)
    valid_until = models.DateTimeField(blank=True, null=True)
    key = models.TextField(max_length=250, blank=True, null=True)

    def save(self, *args, **kwargs):
        ''' On save, update timestamps '''
        if not self.id:
            self.created = timezone.now()
        # Update the key field
        self.key = self.user.username + ":" + self.application.name + "["
        self.key += self.created.strftime('%Y-%m-%dT%H:%M.%S') + "]"

        if self.valid_until:
            # print("\nChecking valid_until"
            #       " still valid:%s\nType:%s" % (self.valid_until,
            #                                   type(self.valid_until)))
            if self.valid_until <= timezone.now():
                if not self.revoked:
                    self.revoked = self.valid_until

        return super(fhir_Consent, self).save(*args, **kwargs)

    def revoke_consent(self, confirm=False, *args, **kwargs):
        if confirm is True:
            if not self.revoked:
                self.revoked = timezone.now()

        return super(fhir_Consent, self).save(*args, **kwargs)

    def status(self):
        consent_status = None
        if self.revoked:
            consent_status = "REVOKED"
        else:
            consent_status = "VALID"
        return consent_status

    def granted(self):
        if self.created and self.revoked:
            valid = False
        else:
            valid = True
        return valid

    def __str__(self):
        name = '%s %s (%s)' % (self.user.first_name,
                               self.user.last_name,
                               self.user.username)
        return ("%s:%s" % (name, self.application.name))

    def __unicode__(self):
        name = '%s %s (%s)' % (self.user.first_name,
                               self.user.last_name,
                               self.user.username)
        return ("%s:%s" % (name, self.application.name))


import os
import sys
import json
import logging

from time import time

from PyQt5.Qt import PYQT_VERSION_STR
from PyQt5.QtCore import (
    QT_VERSION_STR, QStandardPaths, QSysInfo, QLocale, QLibraryInfo,
    QTranslator
)

from novelwriter.error import logException, formatException
from novelwriter.common import splitVersionNumber, formatTimeStamp, NWConfigParser
from novelwriter.constants import nwFiles, nwUnicode

logger = logging.getLogger(__name__)

class Config:

    LANG_NW   = 1
    LANG_PROJ = 2

    def __init__(self):

        # Set Application Variables
        self.appName   = "novelWriter"
        self.appHandle = self.appName.lower()

        # Config Error Handling
        self.hasError  = False  # True if the config class encountered an error
        self.errData   = []     # List of error messages

        # Set Paths
        self.cmdOpen   = None   # Path from command line for project to be opened on launch
        self.confPath  = None   # Folder where the config is saved
        self.confFile  = None   # The config file name
        self.dataPath  = None   # Folder where app data is stored
        self.lastPath  = None   # The last user-selected folder (browse dialogs)
        self.appPath   = None   # The full path to the novelwriter package folder
        self.appRoot   = None   # The full path to the novelwriter root folder
        self.appIcon   = None   # The full path to the novelwriter icon file
        self.assetPath = None   # The full path to the novelwriter/assets folder
        self.pdfDocs   = None   # The location of the PDF manual, if it exists

        # Runtime Settings and Variables
        self.confChanged = False  # True whenever the config has chenged, false after save

        # General
        self.guiTheme    = ""     # GUI theme
        self.guiSyntax   = ""     # Syntax theme
        self.guiIcons    = ""     # Icon theme
        self.guiFont     = ""     # Defaults to system default font
        self.guiFontSize = 11     # Is overridden if system default is loaded
        self.guiScale    = 1.0    # Set automatically by Theme class
        self.lastNotes   = "0x0"  # The latest release notes that have been shown

        self.setDefaultGuiTheme()
        self.setDefaultSyntaxTheme()
        self.setDefaultIconTheme()

        # Localisation
        self.qLocal     = QLocale.system()
        self.guiLang    = self.qLocal.name()
        self.qtLangPath = QLibraryInfo.location(QLibraryInfo.TranslationsPath)
        self.nwLangPath = None
        self.qtTrans    = {}

        # Sizes
        self.winGeometry   = [1200, 650]
        self.prefGeometry  = [700, 615]
        self.treeColWidth  = [200, 50, 30]
        self.novelColWidth = [200, 50]
        self.projColWidth  = [200, 60, 140]
        self.mainPanePos   = [300, 800]
        self.docPanePos    = [400, 400]
        self.viewPanePos   = [500, 150]
        self.outlnPanePos  = [500, 150]
        self.isFullScreen  = False

        # Features
        self.hideVScroll = False  # Hide vertical scroll bars on main widgets
        self.hideHScroll = False  # Hide horizontal scroll bars on main widgets
        self.emphLabels  = True   # Add emphasis to H1 and H2 item labels

        # Project
        self.autoSaveProj = 60  # Interval for auto-saving project in seconds
        self.autoSaveDoc  = 30  # Interval for auto-saving document in seconds

        # Text Editor
        self.textFont        = None   # Editor font
        self.textSize        = 12     # Editor font size
        self.textWidth       = 600    # Editor text width
        self.textMargin      = 40     # Editor/viewer text margin
        self.tabWidth        = 40     # Editor tabulator width

        self.focusWidth      = 800    # Focus Mode text width
        self.hideFocusFooter = False  # Hide document footer in Focus Mode
        self.showFullPath    = True   # Show full document path in editor header
        self.autoSelect      = True   # Auto-select word when applying format with no selection

        self.doJustify       = False  # Justify text
        self.showTabsNSpaces = False  # Show tabs and spaces in edior
        self.showLineEndings = False  # Show line endings in editor
        self.showMultiSpaces = True   # Highlight multiple spaces in the text

        self.doReplace       = True   # Enable auto-replace as you type
        self.doReplaceSQuote = True   # Smart single quotes
        self.doReplaceDQuote = True   # Smart double quotes
        self.doReplaceDash   = True   # Replace multiple hyphens with dashes
        self.doReplaceDots   = True   # Replace three dots with ellipsis

        self.scrollPastEnd   = 25     # Number of lines to scroll past end of document
        self.autoScroll      = False  # Typewriter-like scrolling
        self.autoScrollPos   = 30     # Start point for typewriter-like scrolling

        self.wordCountTimer  = 5.0    # Interval for word count update in seconds
        self.bigDocLimit     = 800    # Size threshold for heavy editor features in kilobytes
        self.incNotesWCount  = True   # The status bar word count includes notes

        self.highlightQuotes = True   # Highlight text in quotes
        self.allowOpenSQuote = False  # Allow open-ended single quotes
        self.allowOpenDQuote = True   # Allow open-ended double quotes
        self.highlightEmph   = True   # Add colour to text emphasis

        self.stopWhenIdle    = True   # Stop the status bar clock when the user is idle
        self.userIdleTime    = 300    # Time of inactivity to consider user idle

        # User-Selected Symbols
        self.fmtApostrophe   = nwUnicode.U_RSQUO
        self.fmtSingleQuotes = [nwUnicode.U_LSQUO, nwUnicode.U_RSQUO]
        self.fmtDoubleQuotes = [nwUnicode.U_LDQUO, nwUnicode.U_RDQUO]
        self.fmtPadBefore    = ""
        self.fmtPadAfter     = ""
        self.fmtPadThin      = False

        # Spell Checking
        self.spellLanguage = None

        # Search Bar Switches
        self.searchCase     = False
        self.searchWord     = False
        self.searchRegEx    = False
        self.searchLoop     = False
        self.searchNextFile = False
        self.searchMatchCap = False

        # Backup
        self.backupPath      = ""
        self.backupOnClose   = False
        self.askBeforeBackup = True

        # State
        self.showRefPanel = True  # The reference panel for the viewer is visible
        self.viewComments = True  # Comments are shown in the viewer
        self.viewSynopsis = True  # Synopsis is shown in the viewer

        # Check Qt5 Versions
        verQt = splitVersionNumber(QT_VERSION_STR)
        self.verQtString = QT_VERSION_STR
        self.verQtMajor  = verQt[0]
        self.verQtMinor  = verQt[1]
        self.verQtPatch  = verQt[2]
        self.verQtValue  = verQt[3]

        verQt = splitVersionNumber(PYQT_VERSION_STR)
        self.verPyQtString = PYQT_VERSION_STR
        self.verPyQtMajor  = verQt[0]
        self.verPyQtMinor  = verQt[1]
        self.verPyQtPatch  = verQt[2]
        self.verPyQtValue  = verQt[3]

        # Check Python Version
        self.verPyString = sys.version.split()[0]
        self.verPyMajor  = sys.version_info[0]
        self.verPyMinor  = sys.version_info[1]
        self.verPyPatch  = sys.version_info[2]
        self.verPyHexVal = sys.hexversion

        # Check OS Type
        self.osType    = sys.platform
        self.osLinux   = False
        self.osWindows = False
        self.osDarwin  = False
        self.osUnknown = False
        if self.osType.startswith("linux"):
            self.osLinux = True
        elif self.osType.startswith("darwin"):
            self.osDarwin = True
        elif self.osType.startswith("win32"):
            self.osWindows = True
        elif self.osType.startswith("cygwin"):
            self.osWindows = True
        else:
            self.osUnknown = True

        # Other System Info
        self.hostName  = "Unknown"
        self.kernelVer = "Unknown"

        # Packages
        self.hasEnchant = False  # The pyenchant package

        # Recent Cache
        self.recentProj = {}

        return

    ##
    #  Methods
    ##

    def pxInt(self, theSize):
        """Used to scale fixed gui sizes by the screen scale factor.
        This function returns an int, which is always rounded down.
        """
        return int(theSize*self.guiScale)

    def rpxInt(self, theSize):
        """Used to un-scale fixed gui sizes by the screen scale factor.
        This function returns an int, which is always rounded down.
        """
        return int(theSize/self.guiScale)

    ##
    #  Config Actions
    ##

    def initConfig(self, confPath=None, dataPath=None):
        """Initialise the config class. The manual setting of confPath
        and dataPath is mainly intended for the test suite.
        """
        logger.debug("Initialising Config ...")
        if confPath is None:
            confRoot = QStandardPaths.writableLocation(QStandardPaths.ConfigLocation)
            self.confPath = os.path.join(os.path.abspath(confRoot), self.appHandle)
        else:
            logger.info("Setting config from alternative path: %s", confPath)
            self.confPath = confPath

        if dataPath is None:
            if self.verQtValue >= 50400:
                dataRoot = QStandardPaths.writableLocation(QStandardPaths.AppDataLocation)
            else:
                dataRoot = QStandardPaths.writableLocation(QStandardPaths.DataLocation)
            self.dataPath = os.path.join(os.path.abspath(dataRoot), self.appHandle)
        else:
            logger.info("Setting data path from alternative path: %s", dataPath)
            self.dataPath = dataPath

        logger.verbose("Config path: %s", self.confPath)
        logger.verbose("Data path: %s", self.dataPath)

        # Check Data Path Subdirs
        dataDirs = ["syntax", "themes"]
        for dataDir in dataDirs:
            dirPath = os.path.join(self.dataPath, dataDir)
            if not os.path.isdir(dirPath):
                try:
                    os.mkdir(dirPath)
                    logger.info("Created folder: %s", dirPath)
                except Exception:
                    logger.error("Could not create folder: %s", dirPath)
                    logException()

        self.confFile = self.appHandle+".conf"
        self.lastPath = os.path.expanduser("~")
        self.appPath = getattr(sys, "_MEIPASS", os.path.abspath(os.path.dirname(__file__)))
        self.appRoot = os.path.abspath(os.path.join(self.appPath, os.path.pardir))

        if os.path.isfile(self.appRoot):
            # novelWriter is packaged as a single file, so the app and
            # root paths are the same, and equal to the folder that
            # contains the single executable.
            self.appRoot = os.path.dirname(self.appRoot)
            self.appPath = self.appRoot

        # Assets
        self.assetPath = os.path.join(self.appPath, "assets")
        self.appIcon   = os.path.join(self.assetPath, "icons", "novelwriter.svg")

        # Internationalisation
        self.nwLangPath = os.path.join(self.assetPath, "i18n")

        logger.debug("Assets: %s", self.assetPath)
        logger.verbose("App path: %s", self.appPath)
        logger.verbose("Last path: %s", self.lastPath)

        # If the config folder does not exist, create it.
        # This assumes that the os config folder itself exists.
        if not os.path.isdir(self.confPath):
            try:
                os.mkdir(self.confPath)
            except Exception as exc:
                logger.error("Could not create folder: %s", self.confPath)
                logException()
                self.hasError = True
                self.errData.append("Could not create folder: %s" % self.confPath)
                self.errData.append(formatException(exc))
                self.confPath = None

        # Check if config file exists
        if self.confPath is not None:
            if os.path.isfile(os.path.join(self.confPath, self.confFile)):
                # If it exists, load it
                self.loadConfig()
            else:
                # If it does not exist, save a copy of the default values
                self.saveConfig()

        # If the data folder does not exist, create it.
        # This assumes that the os data folder itself exists.
        if self.dataPath is not None:
            if not os.path.isdir(self.dataPath):
                try:
                    os.mkdir(self.dataPath)
                except Exception as exc:
                    logger.error("Could not create folder: %s", self.dataPath)
                    logException()
                    self.hasError = True
                    self.errData.append("Could not create folder: %s" % self.dataPath)
                    self.errData.append(formatException(exc))
                    self.dataPath = None

        # Host and Kernel
        if self.verQtValue >= 50600:
            self.hostName  = QSysInfo.machineHostName()
            self.kernelVer = QSysInfo.kernelVersion()

        # Load recent projects cache
        self.loadRecentCache()

        # Check the availability of optional packages
        self._checkOptionalPackages()

        if self.spellLanguage is None:
            self.spellLanguage = "en"

        # Look for a PDF version of the manual
        pdfDocs = os.path.join(self.assetPath, "manual.pdf")
        if os.path.isfile(pdfDocs):
            logger.debug("Found manual: %s", pdfDocs)
            self.pdfDocs = pdfDocs

        logger.debug("Config initialisation complete")

        return True

    def initLocalisation(self, nwApp):
        """Initialise the localisation of the GUI.
        """
        self.qLocal = QLocale(self.guiLang)
        QLocale.setDefault(self.qLocal)
        self.qtTrans = {}

        langList = [
            (self.qtLangPath, "qtbase"),  # Qt 5.x
            (self.nwLangPath, "qtbase"),  # Alternative Qt 5.x
            (self.nwLangPath, "nw"),      # novelWriter
        ]
        for lngPath, lngBase in langList:
            for lngCode in self.qLocal.uiLanguages():
                qTrans = QTranslator()
                lngFile = "%s_%s" % (lngBase, lngCode.replace("-", "_"))
                if lngFile not in self.qtTrans:
                    if qTrans.load(lngFile, lngPath):
                        logger.debug("Loaded: %s", os.path.join(lngPath, lngFile))
                        nwApp.installTranslator(qTrans)
                        self.qtTrans[lngFile] = qTrans

        return

    def listLanguages(self, lngSet):
        """List localisation files in the i18n folder. The default GUI
        language 'en_GB' is British English.
        """
        if lngSet == self.LANG_NW:
            fPre = "nw_"
            fExt = ".qm"
            langList = {"en_GB": QLocale("en_GB").nativeLanguageName().title()}
        elif lngSet == self.LANG_PROJ:
            fPre = "project_"
            fExt = ".json"
            langList = {"en_GB": QLocale("en_GB").nativeLanguageName().title()}
        else:
            return []

        for qmFile in os.listdir(self.nwLangPath):
            if not os.path.isfile(os.path.join(self.nwLangPath, qmFile)):
                continue
            if not qmFile.startswith(fPre) or not qmFile.endswith(fExt):
                continue
            qmLang = qmFile[len(fPre):-len(fExt)]
            qmName = QLocale(qmLang).nativeLanguageName().title()
            if qmLang and qmName and qmLang != "en_GB":
                langList[qmLang] = qmName

        return sorted(langList.items(), key=lambda x: x[0])

    def loadConfig(self):
        """Load preferences from file and replace default settings.
        """
        logger.debug("Loading config file")
        if self.confPath is None:
            return False

        theConf = NWConfigParser()
        cnfPath = os.path.join(self.confPath, self.confFile)
        try:
            with open(cnfPath, mode="r", encoding="utf-8") as inFile:
                theConf.read_file(inFile)
        except Exception as exc:
            logger.error("Could not load config file")
            logException()
            self.hasError = True
            self.errData.append("Could not load config file")
            self.errData.append(formatException(exc))
            return False

        # Main
        cnfSec = "Main"
        self.guiTheme    = theConf.rdStr(cnfSec, "theme", self.guiTheme)
        self.guiSyntax   = theConf.rdStr(cnfSec, "syntax", self.guiSyntax)
        self.guiIcons    = theConf.rdStr(cnfSec, "icons", self.guiIcons)
        self.guiFont     = theConf.rdStr(cnfSec, "guifont", self.guiFont)
        self.guiFontSize = theConf.rdInt(cnfSec, "guifontsize", self.guiFontSize)
        self.lastNotes   = theConf.rdStr(cnfSec, "lastnotes", self.lastNotes)
        self.guiLang     = theConf.rdStr(cnfSec, "guilang", self.guiLang)
        self.hideVScroll = theConf.rdBool(cnfSec, "hidevscroll", self.hideVScroll)
        self.hideHScroll = theConf.rdBool(cnfSec, "hidehscroll", self.hideHScroll)

        # Sizes
        cnfSec = "Sizes"
        self.winGeometry   = theConf.rdIntList(cnfSec, "geometry", self.winGeometry)
        self.prefGeometry  = theConf.rdIntList(cnfSec, "preferences", self.prefGeometry)
        self.treeColWidth  = theConf.rdIntList(cnfSec, "treecols", self.treeColWidth)
        self.novelColWidth = theConf.rdIntList(cnfSec, "novelcols", self.novelColWidth)
        self.projColWidth  = theConf.rdIntList(cnfSec, "projcols", self.projColWidth)
        self.mainPanePos   = theConf.rdIntList(cnfSec, "mainpane", self.mainPanePos)
        self.docPanePos    = theConf.rdIntList(cnfSec, "docpane", self.docPanePos)
        self.viewPanePos   = theConf.rdIntList(cnfSec, "viewpane", self.viewPanePos)
        self.outlnPanePos  = theConf.rdIntList(cnfSec, "outlinepane", self.outlnPanePos)
        self.isFullScreen  = theConf.rdBool(cnfSec, "fullscreen", self.isFullScreen)

        # Project
        cnfSec = "Project"
        self.autoSaveProj = theConf.rdInt(cnfSec, "autosaveproject", self.autoSaveProj)
        self.autoSaveDoc  = theConf.rdInt(cnfSec, "autosavedoc", self.autoSaveDoc)
        self.emphLabels   = theConf.rdBool(cnfSec, "emphlabels", self.emphLabels)

        # Editor
        cnfSec = "Editor"
        self.textFont        = theConf.rdStr(cnfSec, "textfont", self.textFont)
        self.textSize        = theConf.rdInt(cnfSec, "textsize", self.textSize)
        self.textWidth       = theConf.rdInt(cnfSec, "width", self.textWidth)
        self.textMargin      = theConf.rdInt(cnfSec, "margin", self.textMargin)
        self.tabWidth        = theConf.rdInt(cnfSec, "tabwidth", self.tabWidth)
        self.focusWidth      = theConf.rdInt(cnfSec, "focuswidth", self.focusWidth)
        self.hideFocusFooter = theConf.rdBool(cnfSec, "hidefocusfooter", self.hideFocusFooter)
        self.doJustify       = theConf.rdBool(cnfSec, "justify", self.doJustify)
        self.autoSelect      = theConf.rdBool(cnfSec, "autoselect", self.autoSelect)
        self.doReplace       = theConf.rdBool(cnfSec, "autoreplace", self.doReplace)
        self.doReplaceSQuote = theConf.rdBool(cnfSec, "repsquotes", self.doReplaceSQuote)
        self.doReplaceDQuote = theConf.rdBool(cnfSec, "repdquotes", self.doReplaceDQuote)
        self.doReplaceDash   = theConf.rdBool(cnfSec, "repdash", self.doReplaceDash)
        self.doReplaceDots   = theConf.rdBool(cnfSec, "repdots", self.doReplaceDots)
        self.scrollPastEnd   = theConf.rdInt(cnfSec, "scrollpastend", self.scrollPastEnd)
        self.autoScroll      = theConf.rdBool(cnfSec, "autoscroll", self.autoScroll)
        self.autoScrollPos   = theConf.rdInt(cnfSec, "autoscrollpos", self.autoScrollPos)
        self.fmtSingleQuotes = theConf.rdStrList(cnfSec, "fmtsinglequote", self.fmtSingleQuotes)
        self.fmtDoubleQuotes = theConf.rdStrList(cnfSec, "fmtdoublequote", self.fmtDoubleQuotes)
        self.fmtPadBefore    = theConf.rdStr(cnfSec, "fmtpadbefore", self.fmtPadBefore)
        self.fmtPadAfter     = theConf.rdStr(cnfSec, "fmtpadafter", self.fmtPadAfter)
        self.fmtPadThin      = theConf.rdBool(cnfSec, "fmtpadthin", self.fmtPadThin)
        self.spellLanguage   = theConf.rdStr(cnfSec, "spellcheck", self.spellLanguage)
        self.showTabsNSpaces = theConf.rdBool(cnfSec, "showtabsnspaces", self.showTabsNSpaces)
        self.showLineEndings = theConf.rdBool(cnfSec, "showlineendings", self.showLineEndings)
        self.showMultiSpaces = theConf.rdBool(cnfSec, "showmultispaces", self.showMultiSpaces)
        self.wordCountTimer  = theConf.rdFlt(cnfSec, "wordcounttimer", self.wordCountTimer)
        self.bigDocLimit     = theConf.rdInt(cnfSec, "bigdoclimit", self.bigDocLimit)
        self.incNotesWCount  = theConf.rdBool(cnfSec, "incnoteswcount", self.incNotesWCount)
        self.showFullPath    = theConf.rdBool(cnfSec, "showfullpath", self.showFullPath)
        self.highlightQuotes = theConf.rdBool(cnfSec, "highlightquotes", self.highlightQuotes)
        self.allowOpenSQuote = theConf.rdBool(cnfSec, "allowopensquote", self.allowOpenSQuote)
        self.allowOpenDQuote = theConf.rdBool(cnfSec, "allowopendquote", self.allowOpenDQuote)
        self.highlightEmph   = theConf.rdBool(cnfSec, "highlightemph", self.highlightEmph)
        self.stopWhenIdle    = theConf.rdBool(cnfSec, "stopwhenidle", self.stopWhenIdle)
        self.userIdleTime    = theConf.rdInt(cnfSec, "useridletime", self.userIdleTime)

        # Backup
        cnfSec = "Backup"
        self.backupPath      = theConf.rdStr(cnfSec, "backuppath", self.backupPath)
        self.backupOnClose   = theConf.rdBool(cnfSec, "backuponclose", self.backupOnClose)
        self.askBeforeBackup = theConf.rdBool(cnfSec, "askbeforebackup", self.askBeforeBackup)

        # State
        cnfSec = "State"
        self.showRefPanel   = theConf.rdBool(cnfSec, "showrefpanel", self.showRefPanel)
        self.viewComments   = theConf.rdBool(cnfSec, "viewcomments", self.viewComments)
        self.viewSynopsis   = theConf.rdBool(cnfSec, "viewsynopsis", self.viewSynopsis)
        self.searchCase     = theConf.rdBool(cnfSec, "searchcase", self.searchCase)
        self.searchWord     = theConf.rdBool(cnfSec, "searchword", self.searchWord)
        self.searchRegEx    = theConf.rdBool(cnfSec, "searchregex", self.searchRegEx)
        self.searchLoop     = theConf.rdBool(cnfSec, "searchloop", self.searchLoop)
        self.searchNextFile = theConf.rdBool(cnfSec, "searchnextfile", self.searchNextFile)
        self.searchMatchCap = theConf.rdBool(cnfSec, "searchmatchcap", self.searchMatchCap)

        # Path
        cnfSec = "Path"
        self.lastPath = theConf.rdStr(cnfSec, "lastpath", self.lastPath)

        # Check Certain Values for None
        self.spellLanguage = self._checkNone(self.spellLanguage)

        # If we're using straight quotes, disable auto-replace
        if self.fmtSingleQuotes == ["'", "'"] and self.doReplaceSQuote:
            logger.info("Using straight single quotes, so disabling auto-replace")
            self.doReplaceSQuote = False

        if self.fmtDoubleQuotes == ['"', '"'] and self.doReplaceDQuote:
            logger.info("Using straight double quotes, so disabling auto-replace")
            self.doReplaceDQuote = False

        # Check deprecated settings
        if self.guiIcons in ("typicons_colour_dark", "typicons_grey_dark"):
            self.guiIcons = "typicons_dark"
        elif self.guiIcons in ("typicons_colour_light", "typicons_grey_light"):
            self.guiIcons = "typicons_light"

        return True

    def saveConfig(self):
        """Save the current preferences to file.
        """
        logger.debug("Saving config file")
        if self.confPath is None:
            return False

        theConf = NWConfigParser()

        theConf["Main"] = {
            "timestamp":   formatTimeStamp(time()),
            "theme":       str(self.guiTheme),
            "syntax":      str(self.guiSyntax),
            "icons":       str(self.guiIcons),
            "guifont":     str(self.guiFont),
            "guifontsize": str(self.guiFontSize),
            "lastnotes":   str(self.lastNotes),
            "guilang":     str(self.guiLang),
            "hidevscroll": str(self.hideVScroll),
            "hidehscroll": str(self.hideHScroll),
        }

        theConf["Sizes"] = {
            "geometry":    self._packList(self.winGeometry),
            "preferences": self._packList(self.prefGeometry),
            "treecols":    self._packList(self.treeColWidth),
            "novelcols":   self._packList(self.novelColWidth),
            "projcols":    self._packList(self.projColWidth),
            "mainpane":    self._packList(self.mainPanePos),
            "docpane":     self._packList(self.docPanePos),
            "viewpane":    self._packList(self.viewPanePos),
            "outlinepane": self._packList(self.outlnPanePos),
            "fullscreen":  str(self.isFullScreen),
        }

        theConf["Project"] = {
            "autosaveproject": str(self.autoSaveProj),
            "autosavedoc":     str(self.autoSaveDoc),
            "emphlabels":      str(self.emphLabels),
        }

        theConf["Editor"] = {
            "textfont":        str(self.textFont),
            "textsize":        str(self.textSize),
            "width":           str(self.textWidth),
            "margin":          str(self.textMargin),
            "tabwidth":        str(self.tabWidth),
            "focuswidth":      str(self.focusWidth),
            "hidefocusfooter": str(self.hideFocusFooter),
            "justify":         str(self.doJustify),
            "autoselect":      str(self.autoSelect),
            "autoreplace":     str(self.doReplace),
            "repsquotes":      str(self.doReplaceSQuote),
            "repdquotes":      str(self.doReplaceDQuote),
            "repdash":         str(self.doReplaceDash),
            "repdots":         str(self.doReplaceDots),
            "scrollpastend":   str(self.scrollPastEnd),
            "autoscroll":      str(self.autoScroll),
            "autoscrollpos":   str(self.autoScrollPos),
            "fmtsinglequote":  self._packList(self.fmtSingleQuotes),
            "fmtdoublequote":  self._packList(self.fmtDoubleQuotes),
            "fmtpadbefore":    str(self.fmtPadBefore),
            "fmtpadafter":     str(self.fmtPadAfter),
            "fmtpadthin":      str(self.fmtPadThin),
            "spellcheck":      str(self.spellLanguage),
            "showtabsnspaces": str(self.showTabsNSpaces),
            "showlineendings": str(self.showLineEndings),
            "showmultispaces": str(self.showMultiSpaces),
            "wordcounttimer":  str(self.wordCountTimer),
            "bigdoclimit":     str(self.bigDocLimit),
            "incnoteswcount":  str(self.incNotesWCount),
            "showfullpath":    str(self.showFullPath),
            "highlightquotes": str(self.highlightQuotes),
            "allowopensquote": str(self.allowOpenSQuote),
            "allowopendquote": str(self.allowOpenDQuote),
            "highlightemph":   str(self.highlightEmph),
            "stopwhenidle":    str(self.stopWhenIdle),
            "useridletime":    str(self.userIdleTime),
        }

        theConf["Backup"] = {
            "backuppath":      str(self.backupPath),
            "backuponclose":   str(self.backupOnClose),
            "askbeforebackup": str(self.askBeforeBackup),
        }

        theConf["State"] = {
            "showrefpanel":    str(self.showRefPanel),
            "viewcomments":    str(self.viewComments),
            "viewsynopsis":    str(self.viewSynopsis),
            "searchcase":      str(self.searchCase),
            "searchword":      str(self.searchWord),
            "searchregex":     str(self.searchRegEx),
            "searchloop":      str(self.searchLoop),
            "searchnextfile":  str(self.searchNextFile),
            "searchmatchcap":  str(self.searchMatchCap),
        }

        theConf["Path"] = {
            "lastpath": str(self.lastPath),
        }

        # Write config file
        cnfPath = os.path.join(self.confPath, self.confFile)
        try:
            with open(cnfPath, mode="w", encoding="utf-8") as outFile:
                theConf.write(outFile)
            self.confChanged = False
        except Exception as exc:
            logger.error("Could not save config file")
            logException()
            self.hasError = True
            self.errData.append("Could not save config file")
            self.errData.append(formatException(exc))
            return False

        return True

    def loadRecentCache(self):
        """Load the cache file for recent projects.
        """
        if self.dataPath is None:
            return False

        self.recentProj = {}

        cacheFile = os.path.join(self.dataPath, nwFiles.RECENT_FILE)
        if not os.path.isfile(cacheFile):
            return True

        try:
            with open(cacheFile, mode="r", encoding="utf-8") as inFile:
                theData = json.load(inFile)

            for projPath, theEntry in theData.items():
                self.recentProj[projPath] = {
                    "title": theEntry.get("title", ""),
                    "time": theEntry.get("time", 0),
                    "words": theEntry.get("words", 0),
                }

        except Exception as exc:
            self.hasError = True
            self.errData.append("Could not load recent project cache")
            self.errData.append(formatException(exc))
            return False

        return True

    def saveRecentCache(self):
        """Save the cache dictionary of recent projects.
        """
        if self.dataPath is None:
            return False

        cacheFile = os.path.join(self.dataPath, nwFiles.RECENT_FILE)
        cacheTemp = os.path.join(self.dataPath, nwFiles.RECENT_FILE+"~")

        try:
            with open(cacheTemp, mode="w+", encoding="utf-8") as outFile:
                json.dump(self.recentProj, outFile, indent=2)
        except Exception as exc:
            self.hasError = True
            self.errData.append("Could not save recent project cache")
            self.errData.append(formatException(exc))
            return False

        if os.path.isfile(cacheFile):
            os.unlink(cacheFile)
        os.rename(cacheTemp, cacheFile)

        return True

    def updateRecentCache(self, projPath, projTitle, wordCount, saveTime):
        """Add or update recent cache information on a given project.
        """
        self.recentProj[os.path.abspath(projPath)] = {
            "title": projTitle,
            "time": int(saveTime),
            "words": int(wordCount),
        }
        return True

    def removeFromRecentCache(self, thePath):
        """Trying to remove a path from the recent projects cache.
        """
        if thePath in self.recentProj:
            del self.recentProj[thePath]
            logger.verbose("Removed recent: %s", thePath)
            self.saveRecentCache()
        else:
            logger.error("Unknown recent: %s", thePath)
            return False
        return True

    ##
    #  Setters
    ##

    def setConfPath(self, newPath):
        """Set the path and filename to the config file.
        """
        if newPath is None:
            return True
        if not os.path.isfile(newPath):
            logger.error("File not found, using default config path instead")
            return False
        self.confPath = os.path.dirname(newPath)
        self.confFile = os.path.basename(newPath)
        return True

    def setDataPath(self, newPath):
        """Set the data path.
        """
        if newPath is None:
            return True
        if not os.path.isdir(newPath):
            logger.error("Path not found, using default data path instead")
            return False
        self.dataPath = os.path.abspath(newPath)
        return True

    def setLastPath(self, lastPath):
        """Set the last used path (by the user).
        """
        if lastPath is None or lastPath == "":
            self.lastPath = ""
        else:
            self.lastPath = os.path.dirname(lastPath)
        return True

    def setWinSize(self, newWidth, newHeight):
        """Set the size of the main window, but only if the change is
        larger than 5 pixels. The OS window manager will sometimes
        adjust it a bit, and we don't want the main window to shrink or
        grow each time the app is opened.
        """
        newWidth = int(newWidth/self.guiScale)
        newHeight = int(newHeight/self.guiScale)
        if abs(self.winGeometry[0] - newWidth) > 5:
            self.winGeometry[0] = newWidth
            self.confChanged = True
        if abs(self.winGeometry[1] - newHeight) > 5:
            self.winGeometry[1] = newHeight
            self.confChanged = True
        return True

    def setPreferencesSize(self, newWidth, newHeight):
        """Sat the size of the Preferences dialog window.
        """
        self.prefGeometry[0] = int(newWidth/self.guiScale)
        self.prefGeometry[1] = int(newHeight/self.guiScale)
        self.confChanged = True
        return True

    def setTreeColWidths(self, colWidths):
        """Set the column widths of the main project tree.
        """
        self.treeColWidth = [int(x/self.guiScale) for x in colWidths]
        self.confChanged = True
        return True

    def setNovelColWidths(self, colWidths):
        """Set the column widths of the novel tree.
        """
        self.novelColWidth = [int(x/self.guiScale) for x in colWidths]
        self.confChanged = True
        return True

    def setProjColWidths(self, colWidths):
        """Set the column widths of the Load Project dialog.
        """
        self.projColWidth = [int(x/self.guiScale) for x in colWidths]
        self.confChanged = True
        return True

    def setMainPanePos(self, panePos):
        """Set the position of the main GUI splitter.
        """
        self.mainPanePos = [int(x/self.guiScale) for x in panePos]
        self.confChanged = True
        return True

    def setDocPanePos(self, panePos):
        """Set the position of the main editor/viewer splitter.
        """
        self.docPanePos = [int(x/self.guiScale) for x in panePos]
        self.confChanged = True
        return True

    def setViewPanePos(self, panePos):
        """Set the position of the viewer meta data splitter.
        """
        self.viewPanePos = [int(x/self.guiScale) for x in panePos]
        self.confChanged = True
        return True

    def setOutlinePanePos(self, panePos):
        """Set the position of the outline details splitter.
        """
        self.outlnPanePos = [int(x/self.guiScale) for x in panePos]
        self.confChanged = True
        return True

    def setShowRefPanel(self, checkState):
        """Set the visibility state of the reference panel.
        """
        self.showRefPanel = checkState
        self.confChanged = True
        return self.showRefPanel

    def setViewComments(self, viewState):
        """Set the visibility state of comments in the viewer.
        """
        self.viewComments = viewState
        self.confChanged = True
        return self.viewComments

    def setViewSynopsis(self, viewState):
        """Set the visibility state of synopsis comments in the viewer.
        """
        self.viewSynopsis = viewState
        self.confChanged = True
        return self.viewSynopsis

    ##
    #  Default Setters
    ##

    def setDefaultGuiTheme(self):
        """Reset the GUI theme to default value.
        """
        self.guiTheme = "default"

    def setDefaultSyntaxTheme(self):
        """Reset the syntax theme to default value.
        """
        self.guiSyntax = "default_light"

    def setDefaultIconTheme(self):
        """Reset the icon theme to default value.
        """
        self.guiIcons = "typicons_light"

    ##
    #  Getters
    ##

    def getWinSize(self):
        return [int(x*self.guiScale) for x in self.winGeometry]

    def getPreferencesSize(self):
        return [int(x*self.guiScale) for x in self.prefGeometry]

    def getTreeColWidths(self):
        return [int(x*self.guiScale) for x in self.treeColWidth]

    def getNovelColWidths(self):
        return [int(x*self.guiScale) for x in self.novelColWidth]

    def getProjColWidths(self):
        return [int(x*self.guiScale) for x in self.projColWidth]

    def getMainPanePos(self):
        return [int(x*self.guiScale) for x in self.mainPanePos]

    def getDocPanePos(self):
        return [int(x*self.guiScale) for x in self.docPanePos]

    def getViewPanePos(self):
        return [int(x*self.guiScale) for x in self.viewPanePos]

    def getOutlinePanePos(self):
        return [int(x*self.guiScale) for x in self.outlnPanePos]

    def getTextWidth(self, focusMode=False):
        if focusMode:
            return self.pxInt(max(self.focusWidth, 200))
        else:
            return self.pxInt(max(self.textWidth, 200))

    def getTextMargin(self):
        return self.pxInt(max(self.textMargin, 0))

    def getTabWidth(self):
        return self.pxInt(max(self.tabWidth, 0))

    def getErrData(self):
        """Compile and return error messages from the initialisation of
        the Config class, and clear the error buffer.
        """
        errMessage = "<br>".join(self.errData)
        self.hasError = False
        self.errData = []
        return errMessage

    ##
    #  Internal Functions
    ##

    def _packList(self, inData):
        """Pack a list of items into a comma-separated string for saving
        to the config file.
        """
        return ", ".join([str(inVal) for inVal in inData])

    def _checkNone(self, checkVal):
        """Return a NoneType if the value corresponds to None, otherwise
        return the value unchanged.
        """
        if checkVal is None:
            return None
        if isinstance(checkVal, str):
            if checkVal.lower() == "none":
                return None
        return checkVal

    def _checkOptionalPackages(self):
        """Cheks if we have the optional packages used by some features.
        """
        try:
            import enchant  # noqa: F401
            self.hasEnchant = True
            logger.debug("Checking package 'pyenchant': OK")
        except Exception:
            self.hasEnchant = False
            logger.debug("Checking package 'pyenchant': Missing")

        return

# END Class Config


import numpy as np
from .BaseNB import BaseNaiveBayes

# 高斯贝叶斯
class GaussianNaiveBayes(BaseNaiveBayes):

    # 训练
    def fit(self, X: np.ndarray, y: np.ndarray):
        """
        X: train dataset, shape = (n_samples, n_features)
        y: target, shape = (n_samples, )
        """
        # 计算 y 的先验概率
        y_prior_proba = []
        self.classes_ = np.unique(y)
        for c in self.classes_:
            c_count = (y == c).sum()
            y_prior_proba.append(c_count / np.size(y))
        self._y_prior_proba = np.array(y_prior_proba)
        # 计算连续变量 x 的高斯分布参数
        features = X.shape[1]
        self._theta = np.zeros((np.size(self.classes_), features))
        self._sigma = np.zeros((np.size(self.classes_), features))
        for i in range(np.size(self.classes_)):
            x_c = X[y == self.classes_[i]]
            self._theta[i, :] = np.mean(x_c, axis=0)
            self._sigma[i, :] = np.var(x_c, axis=0)
        return self

    def _joint_log_likelihood(self, X: np.ndarray):
        jll = []
        for i in range(np.size(self.classes_)):
            log_prior = np.log(self._y_prior_proba[i])
            # 高斯公式取对数
            x_given_y = - 0.5 * np.sum(np.log(2. * np.pi * self._sigma[i, :]))
            x_given_y -= 0.5 * np.sum(((X - self._theta[i, :]) ** 2) / (self._sigma[i, :]), axis=1)
            jll.append(log_prior + x_given_y)
        jll = np.array(jll).T
        return jll

    def __repr__(self):
        return "<GaussianNaiveBayes>"

if __name__ == "__main__":
    from sklearn.datasets import load_iris
    from sklearn.metrics import accuracy_score
    from sklearn.model_selection import train_test_split
    from sklearn.naive_bayes import GaussianNB

    test = np.array([[1, 'S'], [1, 'M'], [1, 'M'], [1, 'S'], [1, 'S'], [2, 'S'], [2, 'M'],
                    [2, 'M'], [2, 'L'], [2, 'L'], [3, 'L'], [3, 'M'], [3, 'M'], [3, 'L'], [3, 'L']])
    iris = load_iris()
    X = iris.data
    y = iris.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
    gnb = GaussianNaiveBayes().fit(X_train, y_train)
    log_proba = gnb.predict_log_proba(X_test)
    proba = gnb.predict_proba(X_test)
    y_pred = gnb.predict(X_test)
    print(accuracy_score(y_test, y_pred))


import matplotlib.pyplot as plt

class Drawer():

    def __init__(self,y_pred, y_test, target_names,X_test,eigenfaces):
        self.y_pred=y_pred
        self.y_test=y_test
        self.target_names=target_names
        self.X_test=X_test
        self.eigenfaces=eigenfaces

    def plot_gallery(self,images, titles, h, w, n_row=3, n_col=4):
        """Helper function to plot a gallery of portraits"""
        plt.figure(figsize=(1.8 * n_col, 2.4 * n_row))
        plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35)
        for i in range(n_row * n_col):
            plt.subplot(n_row, n_col, i + 1)
            plt.imshow(images[i].reshape((h, w)), cmap=plt.cm.gray)
            plt.title(titles[i], size=12)
            plt.xticks(())
            plt.yticks(())

    # plot the result of the prediction on a portion of the test set

    # def title(self):
    #     pred_name = self.target_names[self.y_pred[self.i]].rsplit(' ', 1)[-1]
    #     true_name = self.target_names[self.y_test[self.i]].rsplit(' ', 1)[-1]
    #     return 'predicted: %s\ntrue:      %s' % (pred_name, true_name)

    def show(self,h,w):
        prediction_titles = []

        for i in range(self.y_pred.shape[0]):
            pred_name = self.target_names[self.y_pred[i]].rsplit(' ', 1)[-1]
            true_name = self.target_names[self.y_test[i]].rsplit(' ', 1)[-1]
            prediction_titles.append('predicted: %s\ntrue:      %s' % (pred_name, true_name))

        self.plot_gallery(self.X_test, prediction_titles, h, w)

        # plot the gallery of the most significative eigenfaces

        eigenface_titles = ["eigenface %d" % i for i in range(self.eigenfaces.shape[0])]
        self.plot_gallery(self.eigenfaces, eigenface_titles, h, w)
        plt.show()


import os
from collections import deque
from typing import Dict, List, Optional, Any
import langchain
import openai
import pinecone
from langchain.chains import LLMChain
from langchain.chains.base import Chain
from langchain.agents import AgentType, ZeroShotAgent, Tool, AgentExecutor, initialize_agent
from langchain.llms import OpenAI, LlamaCpp, BaseLLM
from langchain.prompts import PromptTemplate
from langchain.memory import ConversationBufferMemory, ReadOnlySharedMemory
from langchain.utilities import GoogleSearchAPIWrapper
from langchain.vectorstores.base import VectorStore
from pydantic import BaseModel, Field
from langchain.embeddings import OpenAIEmbeddings
import faiss
from langchain.vectorstores import FAISS
from langchain.docstore import InMemoryDocstore

# Define your embedding model
embeddings_model = OpenAIEmbeddings()

# Initialize the vectorstore as empty
embedding_size = 1536
index = faiss.IndexFlatL2(embedding_size)
vectorstore = FAISS(embeddings_model.embed_query, index, InMemoryDocstore({}), {})

# Initialize our LLM
llm = OpenAI(temperature=0)

class TaskCreationChain(LLMChain):
    """Chain to generates tasks."""

    @classmethod
    def from_llm(cls, llm: BaseLLM, verbose: bool = True) -> LLMChain:
        """Get the response parser."""
        task_creation_template = (
            "You are a task creation AI that uses the result of an execution agent"
            " to create new tasks with the following objective: {objective},"
            " The last completed task has the result: {result}."
            " This result was based on this task description: {task_description}."
            " These are incomplete tasks: {incomplete_tasks}."
            " Based on the result, create new tasks to be completed"
            " by the AI system that do not overlap with incomplete tasks."
            " Return the tasks as an array."
        )
        prompt = PromptTemplate(
            template=task_creation_template,
            input_variables=[
                "result",
                "task_description",
                "incomplete_tasks",
                "objective",
            ],
        )
        return cls(prompt=prompt, llm=llm, verbose=verbose)

class TaskPrioritizationChain(LLMChain):
    """Chain to prioritize tasks."""

    @classmethod
    def from_llm(cls, llm: BaseLLM, verbose: bool = True) -> LLMChain:
        """Get the response parser."""
        task_prioritization_template = (
            "You are a task prioritization AI tasked with cleaning the formatting of and reprioritizing"
            " the following tasks: {task_names}."
            " Consider the ultimate objective of your team: {objective}."
            " Do not remove any tasks. Return the result as a numbered list, like:"
            " #. First task"
            " #. Second task"
            " Start the task list with number {next_task_id}."
        )
        prompt = PromptTemplate(
            template=task_prioritization_template,
            input_variables=["task_names", "next_task_id", "objective"],
        )
        return cls(prompt=prompt, llm=llm, verbose=verbose)

class ExecutionChain(AgentExecutor):
    """Chain to execute tasks."""

    @classmethod
    def from_llm(cls, llm: BaseLLM, verbose: bool = True) -> AgentExecutor:
        """Get the response parser."""

        template = """This is a conversation between a human and a bot:

            {chat_history}

            Write a summary of the conversation for {input}:
        """
        prompt = PromptTemplate(template=template, input_variables=["chat_history", "input"])
        memory = ConversationBufferMemory(memory_key="chat_history")
        read_memory = ReadOnlySharedMemory(memory=memory)
        summary_chain = LLMChain(memory=read_memory, prompt=prompt, llm=llm, verbose=True)
        search = GoogleSearchAPIWrapper()
        tools = [
            Tool(
                name = "Search",
                func=search.run,
                description="useful for when you need to answer questions about current events"
            ),
            Tool(
                name = "Summary",
                func=summary_chain.run,
                description="useful for when you summarize a conversation. The input to this tool should be a string, representing who will read this summary."
            )
        ]
        prefix = """
            You are an AI who performs one task based on the following objective: {objective}.
            Take into account these previously completed tasks: {context}.
        """
        suffix = """
             Your task: {task}.
             Response:
             {agent_scratchpad}
        """
        prompt = ZeroShotAgent.create_prompt(
            tools,
            prefix=prefix,
            suffix=suffix,
            input_variables=["objective", "context", "task", "agent_scratchpad"],
        )
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        agent = ZeroShotAgent(llm_chain=llm_chain, tools=tools, verbose=verbose)
        return cls.from_agent_and_tools(agent=agent, tools=tools, verbose=verbose, memory=memory)

todo_template = """
    You are an expert in walking through your thoughts. You are able to take the main objective, as well as the tasks, and create step by step observations. Here is the objective: {objective}
"""
todo_prompt = PromptTemplate(template=todo_template, input_variables=["objective"])
todo_chain = LLMChain(llm=OpenAI(temperature=0), prompt=todo_prompt)
search = GoogleSearchAPIWrapper()
tools = [
    Tool(
        name="Search",
        func=search.run,
        description="useful for when you need to answer questions about current events",
    ),
    Tool(
        name="TODO",
        func=todo_chain.run,
        description="useful for when you need to come up brainstorming for the current task at hand. Input: an objective to create a todo list for as well as the task. Output: a rational thought process behind the objective and task, enough to help you craft a perfect response.",
    ),
]

prefix = """You are an AI who performs one task based on the following objective: {objective}. Take into account these previously completed tasks: {context}."""
suffix = """Question: {task}
{agent_scratchpad}"""
zeroshot_prompt = ZeroShotAgent.create_prompt(
    tools,
    prefix=prefix,
    suffix=suffix,
    input_variables=["objective", "task", "context", "agent_scratchpad"],
)

def get_next_task(
    task_creation_chain: LLMChain,
    result: Dict,
    task_description: str,
    task_list: List[str],
    objective: str,
) -> List[Dict]:
    """Get the next task."""
    incomplete_tasks = ", ".join(task_list)
    response = task_creation_chain.run(
        result=result,
        task_description=task_description,
        incomplete_tasks=incomplete_tasks,
        objective=objective,
    )
    new_tasks = response.split("\n")
    return [{"task_name": task_name} for task_name in new_tasks if task_name.strip()]

def prioritize_tasks(
    task_prioritization_chain: LLMChain,
    this_task_id: int,
    task_list: List[Dict],
    objective: str,
) -> List[Dict]:
    """Prioritize tasks."""
    task_names = [t["task_name"] for t in task_list]
    next_task_id = int(this_task_id) + 1
    response = task_prioritization_chain.run(
        task_names=task_names, next_task_id=next_task_id, objective=objective
    )
    new_tasks = response.split("\n")
    prioritized_task_list = []
    for task_string in new_tasks:
        if not task_string.strip():
            continue
        task_parts = task_string.strip().split(".", 1)
        if len(task_parts) == 2:
            task_id = task_parts[0].strip()
            task_name = task_parts[1].strip()
            prioritized_task_list.append({"task_id": task_id, "task_name": task_name})
    return prioritized_task_list

def _get_top_tasks(vectorstore, query: str, k: int) -> List[str]:
    """Get the top k tasks based on the query."""
    results = vectorstore.similarity_search_with_score(query, k=k)
    if not results:
        return []
    sorted_results, _ = zip(*sorted(results, key=lambda x: x[1], reverse=True))
    return [str(item.metadata["task"]) for item in sorted_results]

def execute_task(
    vectorstore, execution_chain: LLMChain, objective: str, task: str, k: int = 5
) -> str:
    """Execute a task."""
    context = _get_top_tasks(vectorstore, query=objective, k=k)
    return execution_chain.run(objective=objective, context=context, task=task)

class BabyAGI(Chain, BaseModel):
    """Controller model for the BabyAGI agent."""

    task_list: deque = Field(default_factory=deque)
    task_creation_chain: TaskCreationChain = Field(...)
    task_prioritization_chain: TaskPrioritizationChain = Field(...)
    execution_chain: ExecutionChain = Field(...)
    task_id_counter: int = Field(1)
    vectorstore: VectorStore = Field(init=False)
    max_iterations: Optional[int] = None

    class Config:
        """Configuration for this pydantic object."""

        arbitrary_types_allowed = True

    def add_task(self, task: Dict):
        self.task_list.append(task)

    def print_task_list(self):
        print("\033[95m\033[1m" + "\n*****TASK LIST*****\n" + "\033[0m\033[0m")
        for t in self.task_list:
            print(str(t["task_id"]) + ": " + t["task_name"])

    def print_next_task(self, task: Dict):
        print("\033[92m\033[1m" + "\n*****NEXT TASK*****\n" + "\033[0m\033[0m")
        print(str(task["task_id"]) + ": " + task["task_name"])

    def print_task_result(self, result: str):
        print("\033[93m\033[1m" + "\n*****TASK RESULT*****\n" + "\033[0m\033[0m")
        print(result)

    @property
    def input_keys(self) -> List[str]:
        return ["objective"]

    @property
    def output_keys(self) -> List[str]:
        return []

    def _call(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
        """Run the agent."""
        objective = inputs["objective"]
        first_task = inputs.get("first_task", "Make a todo list")
        self.add_task({"task_id": 1, "task_name": first_task})
        num_iters = 0
        while True:
            if self.task_list:
                self.print_task_list()

                # Step 1: Pull the first task
                task = self.task_list.popleft()
                self.print_next_task(task)

                # Step 2: Execute the task
                result = execute_task(
                    self.vectorstore, self.execution_chain, objective, task["task_name"]
                )
                this_task_id = int(task["task_id"])
                self.print_task_result(result)

                # Step 3: Store the result in Pinecone
                result_id = f"result_{task['task_id']}"
                self.vectorstore.add_texts(
                    texts=[result],
                    metadatas=[{"task": task["task_name"]}],
                    ids=[result_id],
                )

                # Step 4: Create new tasks and reprioritize task list
                new_tasks = get_next_task(
                    self.task_creation_chain,
                    result,
                    task["task_name"],
                    [t["task_name"] for t in self.task_list],
                    objective,
                )
                for new_task in new_tasks:
                    self.task_id_counter += 1
                    new_task.update({"task_id": self.task_id_counter})
                    self.add_task(new_task)
                self.task_list = deque(
                    prioritize_tasks(
                        self.task_prioritization_chain,
                        this_task_id,
                        list(self.task_list),
                        objective,
                    )
                )
            num_iters += 1
            if self.max_iterations is not None and num_iters == self.max_iterations:
                print(
                    "\033[91m\033[1m" + "\n*****TASK ENDING*****\n" + "\033[0m\033[0m"
                )
                break
        return {}

    @classmethod
    def from_llm(
        cls, llm: BaseLLM, vectorstore: VectorStore, verbose: bool = False, **kwargs
    ) -> "BabyAGI":
        """Initialize the BabyAGI Controller."""
        task_creation_chain = TaskCreationChain.from_llm(llm, verbose=verbose)
        task_prioritization_chain = TaskPrioritizationChain.from_llm(llm, verbose=verbose)
        llm_chain = LLMChain(llm=llm, prompt=zeroshot_prompt)
        tool_names = [tool.name for tool in tools]
        agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names)
        agent_executor = AgentExecutor.from_agent_and_tools(
            agent=agent, tools=tools, verbose=True
        )

        return cls(
            task_creation_chain=task_creation_chain,
            task_prioritization_chain=task_prioritization_chain,
            execution_chain=agent_executor,
            vectorstore=vectorstore,
            **kwargs,
        )

# Logging of LLMChains
verbose = False
# If None, will keep on going forever
max_iterations: Optional[int] = None

baby_agi = BabyAGI.from_llm(
    llm=llm, vectorstore=vectorstore, verbose=verbose, max_iterations=max_iterations
)

baby_agi({"objective": 'Write a cohesive and articulated story about a man named Jack who conquers the world.'})


import sqlite3

class sqlcommands:

    def Open(self):
        if self.bOpen is False:
            self.conn = sqlite3.connect(self.db)
            self.curs = self.conn.cursor()
            self.bOpen = True
        return True

    def __init__(self, table):
        self.db = './~newsqlcommands.sqlt3'
        self.conn = None
        self.curs = None
        self.bOpen = False
        self.fields = None
        self.table_name = table

    def CreateTable(self):
        sqlCommand = f"""CREATE TABLE if not EXISTS {self.table_name} (
        EntryNumber INTEGER PRIMARY KEY,
        TimeIn TEXT,
        DateIn TEXT,
        TimeOut TEXT,
        DateOut TEXT);"""
        self.curs.execute(sqlCommand)

    def NewIn(self, field):
        if self.bOpen:
            self.curs.execute(f"INSERT INTO {self.table_name} (DateIn, TimeIn) VALUES (date('now', 'localtime'), '{field}');")
            self.conn.commit()
            self.fields = field
            return True
        return False

    def NewOut(self, field):
        if self.bOpen:
            self.curs.execute(f"UPDATE {self.table_name} SET DateOut = date('now', 'localtime'), TimeOut = '{field}' WHERE TimeIn = '{self.fields}';")
            self.conn.commit()
            return True
        return False

    def FillMissing(self, time_out, date_out, ifempty):
        if self.bOpen:
            self.curs.execute(f"UPDATE {self.table_name} SET DateOut = '{date_out}', TimeOut = '{time_out}' WHERE TimeIn = '{ifempty}';")
            self.conn.commit()
            return True
        return False

    def GetAll(self):
        self.curs.execute("SELECT TimeIn, TimeOut FROM " + self.table_name + ";")
        ans = self.curs.fetchall()
        return ans

    def GetLast(self):
        self.curs.execute(f"SELECT TimeIn FROM {self.table_name} WHERE ROWID IN ( SELECT max( ROWID ) FROM {self.table_name});")
        nochar = "!@,(){}[]' "
        naked = ""
        self.fields = self.curs.fetchone()
        for each in self.fields:
            for char in nochar:
                each = str(each).replace(char,"")
            naked = naked + each
        self.fields = naked
        return self.fields

    def GetSome(self, count):
        self.curs.execute("SELECT * FROM " + self.table_name + " ORDER BY DateIn LIMIT 5 OFFSET %s" % count + ";")
        ans = self.curs.fetchall()
        return ans

    def GetPrimary(self):
        self.curs.execute("SELECT TimeIn FROM " + self.table_name + ";")
        ans = self.curs.fetchall()
        return ans

    def CountDateIn(self):
        self.curs.execute("SELECT COUNT(DateIn) FROM " + self.table_name + ";")
        ans = self.curs.fetchone()
        return ans

    def DelTable(self):
        if self.bOpen:
            self.curs.execute("DROP TABLE IF EXISTS " + self.table_name + ";")
            return True
        return False

    def GetOne(self, TimeIn, inout):
        if self.bOpen:
            if inout == "TimeIn": self.curs.execute(f"Select DateIn from {self.table_name} where {inout} = '{TimeIn}';")
            else: self.curs.execute(f"Select DateOut from {self.table_name} where {inout} = '{TimeIn}';")
            Lalist = self.curs.fetchone()
            self.curs.execute(f"Select {inout} from {self.table_name} where {inout} = '{TimeIn}';")
            Lelist = self.curs.fetchone()
            self.curs.execute(f"SELECT TimeIn FROM {self.table_name} WHERE TimeOut is null;")
            saved = self.curs.fetchone()
            return Lelist, Lalist, saved
        return None

    def DelEm(self, TimeIns):
        if self.bOpen:
            if len(TimeIns) != 0:
                self.curs.execute("DELETE from " + self.table_name + " where " + " or ".join(("TimeIn = " + str(n) for n in TimeIns)))
                self.conn.commit()
                return True
        return False

    def UpdateOne(self, new, time, oldday, oldtime, inout):
        if self.bOpen:
            if inout == "TimeIn":
                self.curs.execute(f"UPDATE {self.table_name} SET DateIn = '{new} ', {inout} = '{time}' WHERE DateIn = '{oldday}' and {inout} = '{oldtime}';")
            else:
                self.curs.execute(f"UPDATE {self.table_name} SET DateOut = '{new} ', {inout} = '{time}' WHERE DateOut = '{oldday}' and {inout} = '{oldtime}';")
            self.conn.commit()
            return True
        return False

    def DelOne(self, TimeIn):
        if self.bOpen:
            self.curs.execute("DELETE from " + self.table_name + " where TimeIn = '" + TimeIn + "';")
            return True
        return False

    def End(self):
        if self.bOpen:
            self.conn.commit()
            self.bOpen = False
        return True


import logging
import os
import gzip
import filetype
import multiprocessing as mp
import pandas as pd
from moonstone.normalization.reads.read_downsize import DownsizePair

logger = logging.getLogger(__name__)

def pair_up(seq_files_info):
    paired_list = []
    query = None

    for key in seq_files_info:
        if key not in paired_list:  # Any one pairs should generate ONLY one loop.
            query = seq_files_info[key][0]  # set the query to the header of the sequence file
            file_counter = 0  # Confirm that we find two files
            fwd_reads_file = None  # reset the file results
            rev_reads_file = None

            for key_a in seq_files_info:  # Loop through again to find the matching header
                if query in seq_files_info[key_a][0]:  # If the headers match it is either forward or reverse reads.
                    if seq_files_info[key_a][1] == '1':
                        fwd_reads_file = key_a
                        file_counter += 1
                    if seq_files_info[key_a][1] == '2':
                        rev_reads_file = key_a
                        file_counter += 1
                    if fwd_reads_file and rev_reads_file:
                        logger.info('\nForward file = %s\nReverse file = %s' % (fwd_reads_file, rev_reads_file))
                        paired_list.append(fwd_reads_file)
                        paired_list.append(rev_reads_file)

    logger.info(f'List of Paired Reads Files:\n{paired_list}')
    return paired_list

def plot_reads(file_info_dict):
    logger.info('Generating plot of number of reads')
    # generate a dataframe from the file information dictionary
    # to include filename as the index
    files: list = []
    reads: list = []
    for key in file_info_dict:
        files.append(key)
        reads.append(file_info_dict[key][2])

    df = pd.DataFrame(index=files, data=reads, columns=['reads'])
    return df

class DownsizeDir:
    """Used to downsize all reads in a given directory to the same number of reads. Reads are downsized by
    random selection of raw reads generating a subset from which alpha diversity can be calculated.
    Note that removal of data, while useful for diversity assessment, is no longer considered good practice.
    https://doi.org/10.1371/journal.pcbi.1003531
    """
    def __init__(self, n=1000, processes=1, seed=62375, in_dir='./', out_dir=''):
        logger.info(f'Starting instance of {__class__.__name__} in {__name__}.')
        self.in_dir = in_dir
        self.downsize_to = n
        self.seed = seed

        if out_dir:
            self.out_dir = out_dir
        else:
            self.out_dir = in_dir + 'downsized/'
            logger.info('No output directory specified.\nCreating default: %s ' % self.out_dir)
        if not os.path.exists(self.out_dir):
            os.mkdir(self.out_dir)
        else:
            logger.info('Looks like %s exists.' % self.out_dir)

        if processes > mp.cpu_count():
            logger.warning('Number of requested processes [%i] is greater that the number of system CPUs [%i]' %
                           (processes, mp.cpu_count()))
            self.processes = mp.cpu_count()
            logger.info('Number of processes set to maximum number of detected CPUs [%i].' % self.processes)
        else:
            self.processes = processes
            logger.info('Number of processes set to %i ' % self.processes)

    def detect_seq_reads(self):
        """The provided directory might contain files that are not sequence reads.
        This module attempts to identify ONLY files with sequence data."""
        logger.info(f'Detecting sequence files in {self.in_dir}')
        seq_files = [f for f in os.listdir(self.in_dir) if os.path.isfile(self.in_dir + f)]
        logger.info(f'List of Sequencing Files Found:\n{seq_files}')
        return seq_files

    def read_info(self, seq_file):
        """Gather information on the number of reads for each of the sequence reads in the given directory.
        Number of reads can be plotted or reported. Files names and headers are used to match pairs.
        Both compressed and gzipped files are accepted.

        Function returns a dictionary where the filename is the key and the value is a list of information:
        {file: [header, F/R, Number of reads, format]}
        # e.g. {'forward.fastq': ['@A00709:44:HYG57DSXX:2:1101:10737:1266', '1', 100257, 'Uncompressed/FASTQ']"""

        seq_files_info = {}
        detect_type = filetype.guess(self.in_dir + seq_file)
        if detect_type:
            if detect_type.mime == 'application/gzip':
                logger.info('Detected gzipped file for %s' % seq_file)
                file = gzip.open(self.in_dir + seq_file, 'r')
                read_num = sum(1 for _ in gzip.open(self.in_dir + seq_file)) // 4
                header = file.readline().decode().split(' ')[0]
                file.seek(0, 0)
                pair = file.readline().decode().split(' ')[1][0]
                file.close()
                seq_files_info[seq_file] = [header, pair, read_num, detect_type.mime]
                return seq_files_info

        if not detect_type:
            logger.info('Assuming uncompressed fastq file for %s' % seq_file)
            file = open(self.in_dir + seq_file, 'r')
            read_num = sum(1 for _ in open(self.in_dir + seq_file)) // 4
            header = file.readline().split(' ')[0]
            file.seek(0, 0)
            pair = file.readline().split(' ')[1][0]
            file.close()
            seq_files_info[seq_file] = [header, pair, read_num, 'Uncompressed/FASTQ']
            return seq_files_info

    def down_dir_pair(self):
        files_to_downsize = self.detect_seq_reads()
        logging.info('Found %i files.' % len(files_to_downsize))

        '''This is a quick but efficient multiprocessing implementation to handle retrieving information from files
        in the target directory. The Pool is created, the number of workers = the class 'processes' attribute.
        Results are expected as a dictionary, so the resulting 'list of dictionaries' is converted with handy
        dict comprehension.
        '''
        with mp.Pool(processes=self.processes) as pool:
            results = pool.map(self.read_info, files_to_downsize, chunksize=1)
        file_info_dict = {k: v for result in results for k, v in result.items()}

        list_to_downsize = pair_up(file_info_dict)

        worker_parameters = []
        for k in range(len(list_to_downsize)//2):  # number of files divided by 2: one instance per pair
            worker_parameters.append({'raw_file_f': list_to_downsize[k * 2],
                                      'raw_file_r': list_to_downsize[k * 2 + 1],
                                      'read_info': file_info_dict[list_to_downsize[k * 2]],
                                      'in_dir': self.in_dir, 'out_dir': self.out_dir, 'n': self.downsize_to
                                      })

        with mp.Pool(processes=self.processes) as pool:
            check = pool.map(self.instantiate, worker_parameters, chunksize=1)  # noqa

        logger.info('Done!')

    def instantiate(self, wp):
        logger.info('Instantiating with parameters: %s' % wp)
        instance = DownsizePair(**wp)
        instance.downsize_pair()


import os
import json
import requests
from bs4 import BeautifulSoup
from datetime import datetime as dt
import pandas as pd
from __common__ import user_choice

def __load_json__(file):
    """
    """
    try:
        with open(file) as rf:
            data = json.load(rf)
        return data
    except:
        return {}

def __get_links__(url, base_url):
    """
    """
    r = requests.get(url)
    soup = BeautifulSoup(r.text)
    link_dict = {}
    ignore = ['[To Parent Directory]', 'DUPLICATE']
    for link in soup.findAll('a'):
        if link.text in ignore :
            pass
        else:
            link_dict[link.text] = base_url+link.get('href')
    return link_dict

def __gen_file_name__(data_keys, resource_type):
    """
    """
    if resource_type == 'Archive':
        i = -1
    else:
        i = -2
    name = set(map(lambda x: '-'.join(x.split('_')[:i]).lower(),
                   data_keys))
    name = list(name)
    if len(name) > 1:
        print("More than one file type detected!")
        return None
    else:
        if resource_type == 'Archive':
            name = '-'.join([name[0],resource_type.lower()])
        else:
            name = name[0]
        return name + '.csv'

def __gen_tracker_df__(data_files, data_keys, resource_type):
    """
    """
    if resource_type == 'Archive':
        timestamp = pd.to_datetime(list(map(lambda x:
                                   x.split('_')[-1][:-4],
                                   data_keys)))
        version = resource_type
    else:
        timestamp = pd.to_datetime(list(map(lambda x:
                                    x.split('_')[-2],
                                    data_keys)))
        version = list(map(lambda x: 'V'+x.split('_')[-1][:-4],
                                      data_keys))
    data_dict = {
        'TIMESTAMP': timestamp,
        'VERSION': version,
        'DOWNLOADED': False,
        'DOWNLOAD_DATE': pd.to_datetime('19000101'),
        'URL': list(data_files.values())
    }
    return pd.DataFrame(data_dict)

def __merge_trackers__(old_df, new_df):
    """
    Add anything from new_df that is not in old_df already
    """
    merge_cols = ['TIMESTAMP', 'VERSION']
    old_temp = old_df.copy()[merge_cols]
    new_temp = new_df.copy()[merge_cols]
    merge_df = pd.merge(old_temp, new_temp, on=merge_cols,
                        how='outer', indicator=True)
    # rows no longer in CURRENT NEM reports
    old_rows = merge_df[merge_df["_merge"]=="left_only"]
    old_rows = old_rows[merge_cols].merge(old_df, on=merge_cols)
      # keep only old rows that have been downloaded
      # else they need to extracted from ARCHIVE or older records
    old_rows = old_rows[old_rows.DOWNLOADED]
    # for common rows, must keep old_df records
    com_rows = merge_df[merge_df["_merge"]=="both"]
    com_rows = com_rows[merge_cols].merge(old_df, on=merge_cols)
    # for new rows, use latest records
    new_rows = merge_df[merge_df["_merge"]=="right_only"]
    new_rows = new_rows[merge_cols].merge(new_df, on=merge_cols)
    return pd.concat([old_rows, com_rows, new_rows],
                     ignore_index=True)

class NEM_tracker:
    """
    Class object for tracking various resources on NEM website.
    Can potentially be extended to other websites too.
    """

    def __init__(self, data_dir, base_url="http://nemweb.com.au"):
        """
        """
        self.data_dir = data_dir
        self.base_url = base_url
        self.resource_path = os.path.join(data_dir, 'resources.json')
        self.resources = __load_json__(self.resource_path)
        self.selected_resource = None
        self.tracker_dir = os.path.join(data_dir, 'resource_trackers')
        if os.path.isdir(self.tracker_dir):
            pass
        else:
            os.makedirs(self.tracker_dir)

    def resources_report(self):
        """
        """
        for k, v in self.resources.items():
            print('\n%s\nLast update: %s'%(k, v['last_update']))

    def update_resource(self, resource):
        """
        """
        url = self.base_url + resource
        data_files = __get_links__(url, self.base_url)
        data_keys = list(data_files.keys())
        resource_type = resource.split('/')[2]
        file_name = __gen_file_name__(data_keys, resource_type)
        self.resources[resource] = {
            'url': url,
            'type': resource_type,
            'tracker_file': file_name,
            'last_update': dt.now().strftime('%Y-%m-%d-%H:%M:%S')
        }
        with open(self.resource_path, 'w') as wf:
            json.dump(self.resources, wf)
        new_df = __gen_tracker_df__(data_files, data_keys,
                                    resource_type)
        file_path = os.path.join(self.tracker_dir, file_name)
        if file_name in os.listdir(self.tracker_dir):
            old_df = pd.read_csv(file_path, parse_dates=[
                'TIMESTAMP', 'DOWNLOAD_DATE'
            ])
            tracker_df = __merge_trackers__(old_df, new_df)
        else:
            tracker_df = new_df
        tracker_df.to_csv(file_path, index=False)

    def add_resources(self, resources):
        """
        For adding new resources.
        Resources must be specified as the relative URL
        path from the domain to the directory where all
        relevant data files are stored.
        E.g. /Reports/Current/Next_Day_Intermittent_DS/
        """
        if type(resources) is not list:
            resources = [resources]
        else:
            pass
        for resource in resources:
            self.update_resource(resource)

    def bulk_update(self):
        """
        For updating existing resources.
        """
        for resource in self.resources.keys():
            self.update_resource(resource)

    def select_resource(self, resource=None):
        """
        Select a resource for further processing.
        """
        d = {}
        if resource is None:
            res_list = self.resources.keys()
            name = user_choice(res_list)
        else:
            name = resource
        d['name'] = name
        tracker_file = self.resources[name]['tracker_file']
        d['tracker_path'] = os.path.join(self.tracker_dir,
                                         tracker_file)
        d['resource_dir'] = os.path.join(self.data_dir,
                                         tracker_file[:-4])
        self.selected_resource = d


import os
import sys
import csv
from collections import Counter, defaultdict

import pandas as pd
from statsmodels.stats.inter_rater import aggregate_raters, fleiss_kappa
#from pptx import Presentation

# configure Django so we can use models from the annotate app
sys.path.append('/home/nejl/Dropbox/projects/tator/repo/tator')
os.environ['DJANGO_SETTINGS_MODULE'] = 'tagit.settings'
import django
django.setup()

from django.contrib.auth.models import User

from annotate.models import Query, Annotation, UserResponse
from templates import slide_template

# TODO: need to add sampling method that samples equally from dividing the
# probability mass of the distribution into thirds: top most frequent, middle,
# and bottom.

# TODO: fix analysis to have a global collection filter and then make sure adding
# annotations to queries in a different collection does not change the results

def split_data_frame_by_prob(df, column, nbins):
    # splits a dataframe into 'nbins' of equal probability mass
    # using column specified by 'coilumn'

    df = df.sort_values(column, ascending=False)
    values = df[column]

    bin_probability = 1/nbins
    total = sum(values)
    cutoffs = []
    cumulative_total = 0
    next_bin_probability = bin_probability

    for i, count in enumerate(values):
        cumulative_total += count

        if cumulative_total/total < next_bin_probability:
            continue

        cutoffs.append(i)
        next_bin_probability += bin_probability

    start = 0
    new_dfs = []
    while cutoffs:
        cutoff = cutoffs.pop(0)
        if len(cutoffs) == 0:
            # last item; get the rest
            new_dfs.append(df[start:])
        else:
            new_dfs.append(df[start:cutoff])
        start = cutoff

    return new_dfs

def load_queries(path):
    with open(path) as csvfile:
        df = pd.read_csv(csvfile, delimiter=';')
    return df

def clean_queries(df):
    """Returns the input DataFrame of queries cleaned"""
    # filter out queries with length less than 2 characters long
    df = df[df['querystring'].str.len() > 1]
    return df

def split_num(num, splits):
    """Returns the number 'num' divided into a list of numbers of size 'splits' """
    splits = [int(num/splits)+1]*(num%splits) + [int(num/splits)]*(splits-num%splits)
    assert sum(splits) == num
    return splits

def import_queries(path, collection, sample='first', limit=None, allow_dupes=False):
    df = load_queries(path)
    df = clean_queries(df)

    if not allow_dupes:
        # remove existing queries from candidate queries to sample
        existing = [query.text for query in Query.objects.all()]
        df = df[~df['querystring'].isin(existing)]

    if limit is not None:
        if sample == 'first':
            df = df[:limit]
        elif sample == 'random':
            df = df.sample(limit)
        elif sample == 'proportional':
            df = df.sample(limit, weights='countqstring')
        elif sample == 'split':
            split_size = 3
            splits = split_data_frame_by_prob(df, 'countqstring', split_size)
            sizes = split_num(limit, split_size)
            sub_samples = []
            for size, split_df in zip(sizes, splits):
                sub_samples.append(split_df.sample(size, weights='countqstring'))
            df = pd.concat(sub_samples)
            assert len(df) == limit
        else:
            print('Unknown sampling method')
            return

    for i, values in enumerate(df.values.tolist()):
         text, count = values
         Query.objects.create(text=text, count=count, collection=collection)

    print("Added {} queries to the database.\n".format(i+1))
    print(df.describe())

def pretty_print_counter(counter, reverse=False):
    lines = []
    for key, value in sorted(counter.items(), reverse=reverse):
        lines.append("{}: {}".format(key, value))
    return "\n".join(lines)

def get_user_results(username, collection=None):
    # for each user, display the number of results
    # user
    lines = ["*** Annotator: {} ***".format(username)]
    lines.append("===================================\n")
    responses = UserResponse.objects.filter(user__username=username)

    if collection is not None:
        responses = responses.filter(query__collection=collection)

    annotations = [r for r in responses if r.annotation]
    skipped = [r for r in responses if r.skipped]

    lines.append("{} Skipped Queries:\n".format(len(skipped)))
    for response in skipped:
        line ='    "{}"\n    --- "{}"'.format(response.query.text,
                                              response.skipped.description)
        lines.append(line)

    lines.append("\n{} Annotations:\n".format(len(annotations)))

    lines.append(Annotation._meta.get_field('is_geo').verbose_name)
    q1 = Counter(r.annotation.is_geo for r in annotations)
    lines.append(pretty_print_counter(q1, reverse=True))

    lines.append("")

    lines.append(Annotation._meta.get_field('loc_type').verbose_name)
    q2 = Counter(r.annotation.loc_type for r in annotations)
    lines.append(pretty_print_counter(q2, reverse=True))

    lines.append("")

    lines.append(Annotation._meta.get_field('query_type').verbose_name)
    q3 = Counter(r.annotation.query_type for r in annotations)
    lines.append(pretty_print_counter(q3))

    return "\n".join(lines)

def do_iaa_pairs(user_pairs, questions=(1,2,3), collection=None, level='fine'):
    results = defaultdict(list)
    for question in questions:
        for users in user_pairs:
            kappa = get_iaa(question, users=users, collection=collection, level=level)
            results[question].append(kappa)
    return results

def print_iaa_pairs(results, user_pairs):
    print('    '+'   '.join(', '.join(user) for user in user_pairs))
    for question, kappas in results.items():
        ks = ''.join("{:0<5.3}                    ".format(k) for k in kappas)
        print("Q{}: {}".format(question, ks))

def get_iaa(question_num, queries=None, users=None, collection=None, level='fine'):
    data  = get_annotations(question_num, queries=queries, users=users, level=level, collection=collection)
    #n_cat = Annotation.get_num_categories(question_num)
    results = aggregate_raters(data, n_cat=None)
    kappa = fleiss_kappa(results[0])
    return kappa

def get_annotations(question_num, queries=None, users=None, level='fine', collection=None):
    assert level in ('fine', 'coarse')

    queries = Query.objects.exclude(responses__skipped__isnull=False).distinct()

    if collection is not None:
        queries = queries.filter(collection=collection)

    if queries is not None:
        queries = queries.filter(pk__in=queries)

    data = []
    for query in queries:
        # get all non-skipped results
        responses = query.responses.exclude(skipped__isnull=False)

        if users is not None:
            # restrict annotations to supplied users
            responses = responses.filter(user__username__in=users)

        results = [r.annotation.get_question(question_num) for r in responses]

        if question_num in (2,3) and level == 'coarse':
            # use course grained agreement
            results = [r[0] for r in results]

        if results:
            data.append(results)
    return data

def show_agreement(question_num, users, collection=None, skip_agree=True):
    lines = []
    queries = Query.objects.exclude(responses__skipped__isnull=False).distinct()

    if collection is not None:
        queries = queries.filter(collection=collection)

    queries = sorted(queries, key=lambda x:x.pk)
    users.sort()
    col_width = max(len(u) for u in users) + 2

    lines.append("".join("{u:{width}}".format(u=u, width=col_width)
                         for u in users))
    agree = 0
    disagree = 0

    for query in queries:
        responses  = query.responses.order_by('user__username')
        answers = [r.annotation.get_question(question_num) for r in responses]
        if skip_agree and len(set(answers)) <= 1:
            # all annotators agree, skip
            agree += 1
            continue
        disagree += 1
        line = "".join("{a:<{width}}".format(a=a, width=col_width)
                       for a in answers) + query.text
        lines.append(line)

    start = [
        "Question {}:".format(question_num),
        "Number all agree: {}".format(agree),
        "Number with some disagreement: {}".format(disagree),
        ""
    ]

    return "\n".join(start + lines)

def get_results(users):
    queries = Query.objects.exclude(responses__skipped__isnull=False).distinct()
    queries = sorted(queries, key=lambda x:x.pk)
    users.sort()

    rest_cols = ["Q{}_{}".format(num, user) for user in users for num in (1,2,3)]
    header = ['id', 'query'] + rest_cols
    rows = [header]

    for query in queries:
        row = [query.pk, query.text]
        responses = query.responses.order_by('user__username')
        for response in responses:
            row.append(response.annotation.get_question(1))
            row.append(response.annotation.get_question(2))
            row.append(response.annotation.get_question(3))
        rows.append(row)
    return rows

def export_results_csv(users, outfile='annotations.csv'):
    results = get_results(users)
    with open(outfile, 'w', encoding='utf8', newline='') as csvfile:
        writer = csv.writer(csvfile, delimiter=',')
        writer.writerows(results)

def make_slides_latex(users, csv=None, outfile='slides/slides.tex'):
    if csv is None:
        results = get_results(users)
    else:
        with open(csv, encoding='utf8') as csvfile:
            results = list(csv.reader(csvfile))

    lines = []
    header = results[0]
    for i, query in enumerate(results[1:]):
        row1 = r"Q1 & {} & {} & {}\\".format(query[2], query[5], query[8])
        row2 = r"Q2 & {} & {} & {}\\".format(query[3], query[6], query[9])
        row3 = r"Q3 & {} & {} & {}\\".format(query[4], query[7], query[10])
        rows = "\n".join([row1, row2, row3])
        title = "Query {}".format(i+1)
        slide = slide_template.format(title=title, query=query[1], rows=rows)
        lines.append(slide)

    with open(outfile, 'w', encoding='utf8') as texfile:
        texfile.write('\n'.join(lines))

def make_slides_pptx(users, csv=None):
    """Not finished. Used latex instead"""
    if csv is None:
        results = get_results(users)
    else:
        with open(csv, encoding='utf8') as csvfile:
            results = list(csv.reader(csvfile))

    header = results[0]

    prs = Presentation()
    slide_layout = prs.slide_layouts[1]

    for i, query in enumerate(results[1:]):
        slide = prs.slides.add_slide(slide_layout)
        slide.shapes.title.text = 'Query {}'.format(i+1)

        body_shape = slide.shapes.placeholders[1]
        tf = body_shape.text_frame

        p = tf.paragraphs[0]
        p.text = query[1]
        p.level = 0

    prs.save('test.pptx')