from builtins import staticmethod
from datetime import datetime

from sklearn.metrics import roc_curve, auc, accuracy_score
from torch import nn
from PIL import Image

from classes.Utils import Utils
from old.GlobalSettings import GlobalSettings

import matplotlib.pyplot as plt
import matplotlib.ticker
import matplotlib.patches as patches

import numpy as np
import os
import seaborn as sns
import torch

import pandas as pd

class ModelEvaluation:
    @staticmethod
    def train_valid_acc_loss_graph(history, model_name, save, training_evaluation_directory):
        history = pd.DataFrame(history, columns=['train_acc', 'train_loss', 'valid_acc', 'valid_loss'])

        fig = plt.figure(figsize=(15, 6))
        locator = matplotlib.ticker.MultipleLocator(1)
        fig.suptitle(model_name)

        subfig = fig.add_subplot(122)
        fig.gca().xaxis.set_major_locator(locator)
        subfig.plot(history['train_acc'], label="Training")
        subfig.plot(history['valid_acc'], label="Validation")
        subfig.set_title('Model Accuracy')
        subfig.set_xlabel('Epoch')
        subfig.set_ylabel('Percentage')
        subfig.legend(loc='upper left')

        subfig = fig.add_subplot(121)
        fig.gca().xaxis.set_major_locator(locator)
        subfig.plot(history['train_loss'], label="Training")
        subfig.plot(history['valid_loss'], label="Validation")
        subfig.set_title('Model Loss')
        subfig.set_xlabel('Epoch')
        subfig.set_ylabel('Loss')
        subfig.legend(loc='upper right')

        plt.show()

        if save:
            path = os.path.join(training_evaluation_directory, model_name)
            path = Utils.create_folder_if_not_exists(path)

            plt.savefig(os.path.join(path, 'valid_acc_loss_graph_' + datetime.now().strftime("%Y%m%d_%H%M%S") + '.png'),
                        bbox_inches='tight')

    @staticmethod
    def plot_roc_curve(model, dataloader, classes):
        plt.figure()

        fpr = dict()
        tpr = dict()
        roc_auc = dict()
        for i in range(len(classes)):
            actuals, probabilities = test_class_probabilities(model, dataloader, i)
            fpr[i], tpr[i], threshold = roc_curve(actuals, probabilities)
            roc_auc[i] = auc(fpr[i], tpr[i])

        # plt.figure(figsize=(6,6))
        for i in range(len(classes)):
            plt.plot(fpr[i], tpr[i], label='ROC curve of class {0} (area = {1:0.2f})'
                                           ''.format(classes[i], roc_auc[i]))  # roc_auc_score

        # # # Compute micro-average ROC curve and ROC area
        # all_labels = sum([i for i in range(len(classes))] * len(p))
        # fpr["micro"], tpr["micro"], _ = roc_curve(all_labels, sum(p, []))
        # roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

        plt.plot([0, 1], [0, 1], 'k--')
        # plt.grid()
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title('Receiver operating characteristic')
        plt.legend(loc="lower right")
        # plt.tight_layout()
        plt.show()

    @staticmethod
    def get_predictions(model, dataloader):
        model.to(torch.device("cuda:0"))
        model.eval()
        labels = []
        predictions = []
        probabilities = []
        for i, batch in enumerate(dataloader):
            data, label = batch  # ignore label
            data = data.cuda()
            label = label.cuda()
            outputs = model(data)
            _, preds = torch.max(outputs.data, 1)
            labels.extend(label.tolist())
            predictions.extend(preds.tolist())
            m = torch.nn.Softmax(dim=1)
            outputs = m(outputs)
            probabilities.extend(outputs.tolist())

        return labels, predictions, probabilities

    @staticmethod
    def predict_fingerprint(pil_image, model, class_names, all_classes, sliding_window_height, sliding_window_width, step,
                probability_threshold, tensor_height, tensor_width):
        model.eval()
        model.to('cuda:0')

        colors = {'bifurcation': 'lime', 'ending': 'r', 'nothing': 'y', 'dot': 'm', 'overlap': 'b'}

        image_width = tensor_width
        image_height = tensor_height

        def pr(model, img_tensor):
            with torch.no_grad():
                outputs = model(img_tensor)
                _, predicted = torch.max(outputs.data, 1)

                m = torch.nn.Softmax(dim=1)
                outputs = m(outputs)
                # print(outputs)
                return predicted, outputs[0][predicted]

        image = pil_image

        fig, ax = plt.subplots(1)

        stepSize = step
        w_width, w_height = sliding_window_width, sliding_window_height  # window size
        for x in range(0, image.shape[1] - w_width, stepSize):
            for y in range(0, image.shape[0] - w_height, stepSize):
                window = image[x:x + w_width, y:y + w_height, :]
                # plt.imshow(window)
                # plt.show()
                img_tensor = process_image(window, image_width, image_height)
                # plt.imshow(img_tensor.permute(1, 2, 0))
                # plt.show()
                img_tensor = img_tensor.view(1, 3, image_height, image_width).cuda()

                pred, prob = pr(model, img_tensor)
                predicted_class_string = all_classes[pred]
                if predicted_class_string in class_names and prob > probability_threshold:
                    color = colors[predicted_class_string] if (predicted_class_string in colors) else 'k'
                    ax.add_patch(
                        patches.Rectangle((x, y), w_width, w_height, edgecolor=color, fill=False))

        legend_elements = []
        for class_name in all_classes:
            color = colors[class_name] if (class_name in colors) else 'k'

            legend_elements.append(patches.Rectangle((0, 0), 20, 20, color=color, label=class_name))

        ax.legend(handles=legend_elements, loc='upper right')
        ax.imshow(pil_image)
        plt.show()

    @staticmethod
    def predict(pil_image, model, all_classes, tensor_height, tensor_width):
        model.eval()
        model.to('cuda:0')

        colors = {'bifurcation': 'lime', 'ending': 'r', 'nothing': 'y', 'dot': 'm', 'overlap': 'b'}

        image_width = tensor_width
        image_height = tensor_height

        def pr(model, img_tensor):
            with torch.no_grad():
                outputs = model(img_tensor)
                _, predicted = torch.max(outputs.data, 1)

                m = torch.nn.Softmax(dim=1)
                outputs = m(outputs)
                # print(outputs)
                return predicted, outputs[0][predicted]

        image = pil_image

        # fig, ax = plt.subplots(1)

        img_tensor = process_image(image, image_width, image_height)
        img_tensor = img_tensor.view(1, 3, image_height, image_width).cuda()

        pred, prob = pr(model, img_tensor)
        return all_classes[pred]

    @staticmethod
    def get_accuracy(dataset_dataloader_test, model):
        correct = 0
        total = 0
        with torch.no_grad():
            for data in dataset_dataloader_test:
                images, labels = data
                outputs = model(images)
                _, predicted = torch.max(outputs.data, 1)
                total += labels.size(0)
                correct += (predicted == labels).sum().item()

        return 100 * correct / total

    @staticmethod
    def get_accuracy_classes(dataset_dataloader_test, model):
        classes = dataset_dataloader_test.dataset.classes

        class_correct = list(0. for i in range(len(classes)))
        class_total = list(0. for i in range(len(classes)))
        with torch.no_grad():
            for data in dataset_dataloader_test:
                images, labels = data
                outputs = model(images)
                _, predicted = torch.max(outputs, 1)
                c = (predicted == labels).squeeze()
                for i in range(len(classes)):
                    label = labels[i]
                    class_correct[label] += c[i].item()
                    class_total[label] += 1

        accuracy_classes = ''
        for i in range(len(classes)):
            accuracy_classes += 'Accuracy of %5s : %2d %%' % (
                classes[i], 100 * class_correct[i] / class_total[i])

def test_class_probabilities(model, test_loader, n_class):
    model.eval()
    actuals = []
    probabilities = []
    with torch.no_grad():
        for i, batch in enumerate(test_loader):
            inputs, labels = batch

            outputs = model(inputs).squeeze()

            prediction = outputs.argmax(dim=1, keepdim=True)
            updated_list = [int(bool) for bool in (labels == n_class).tolist()]
            actuals.extend(updated_list)

            m = torch.nn.Softmax(dim=1)
            outputs = m(outputs)
            probabilities.extend(outputs[:, n_class].tolist())

    return actuals, probabilities  # [i.item() for i in actuals], [i.item() for i in probabilities]

@staticmethod
def get_top1_accuracy(results, save_on_disk):
    sns.lmplot(y='top1', x='n_train', data=results, height=6)
    plt.xlabel('Image Count')
    plt.ylabel('Accuracy (%)')
    plt.title(str(GlobalSettings.CNN_MODEL.value[0]).upper() + '\nTop 1 Accuracy vs Number of Training Images')
    plt.ylim(-5, 105)

    if save_on_disk:
        plt.savefig(GlobalSettings.SAVE_FOLDER + '/' + str(GlobalSettings.CNN_MODEL.value[0]) + '/top1.png',
                    bbox_inches='tight')
    else:
        plt.show()

def process_image(pil_image, img_width, img_height):
    # Resize
    img = Image.fromarray(pil_image).resize((img_height, img_width))

    # Convert to numpy, transpose color dimension and normalize
    img = np.array(img).transpose((2, 0, 1)) / 256

    # Standardization
    means = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1))
    stds = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1))

    img = img - means
    img = img / stds

    img_tensor = torch.Tensor(img)

    return img_tensor

@staticmethod
def accuracy(output, target, topk=(1,)):
    """Compute the topk accuracy(s)"""
    output = output.to('cuda')
    target = target.to('cuda')

    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        # Find the predicted classes and transpose
        _, pred = output.topk(k=maxk, dim=1, largest=True, sorted=True)
        pred = pred.t()

        # Determine predictions equal to the targets
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []

        # For each k, find the percentage of correct
        for k in topk:
            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size).item())
        return res

@staticmethod
def evaluate(model, test_loader, criterion, topk=(1, GlobalSettings.NUM_CLASSES)):
    """Measure the performance of a trained PyTorch app_model

    Params
    --------
        app_model (PyTorch app_model): trained cnn for inference
        test_loader (PyTorch DataLoader): test dataloader
        topk (tuple of ints): accuracy to measure

    Returns
    --------
        results (DataFrame): results for each category

    """

    classes = []
    losses = []
    # Hold accuracy results
    acc_results = np.zeros((len(test_loader.dataset), len(topk)))
    i = 0

    model.eval()
    with torch.no_grad():

        # Testing loop
        for data, targets in test_loader:
            data, targets = data.to('cuda'), targets.to('cuda')

            # Raw app_model output
            out = model(data)
            # Iterate through each example
            for pred, true in zip(out, targets):
                # Find topk accuracy
                acc_results[i, :] = ModelEvaluation.accuracy(pred.unsqueeze(0), true.unsqueeze(0), topk)
                classes.append(model.idx_to_class[true.item()])
                # Calculate the loss
                loss = criterion(pred.view(1, len(os.listdir(GlobalSettings.TRAIN_DIR))), true.view(1))
                losses.append(loss.item())
                i += 1

    results = pd.DataFrame(acc_results, columns=[f'top{i}' for i in topk])
    results['class'] = classes
    results['loss'] = losses
    results = results.groupby(classes).mean()

    return results.reset_index().rename(columns={'index': 'class'})

def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a bit so that plots are updated

def visualize_model(model, dataset_dataloader_test, num_images=9):
    classes = dataset_dataloader_test.dataset.classes

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    was_training = model.training
    model.eval()
    images_so_far = 0
    fig = plt.figure()

    with torch.no_grad():
        for i, (inputs, labels) in enumerate(dataset_dataloader_test):
            inputs = inputs.to(device)
            labels = labels.to(device)

            outputs = model(inputs)
            _, preds = torch.max(outputs, 1)

            for j in range(inputs.size()[0]):
                images_so_far += 1
                ax = plt.subplot(num_images // 2, 2, images_so_far)
                ax.axis('off')
                ax.set_title('{}'.format(classes[preds[j]]))
                ModelEvaluation.imshow(inputs.cpu().data[j])

                if images_so_far == num_images:
                    model.train(mode=was_training)
                    return
        model.train(mode=was_training)
        plt.show()

@staticmethod
def confusion_matrix(self):
    nb_samples = 20
    nb_classes = 4
    output = torch.randn(nb_samples, nb_classes)
    pred = torch.argmax(output, 1)
    target = torch.randint(0, nb_classes, (nb_samples,))

    conf_matrix = torch.zeros(nb_classes, nb_classes)
    for t, p in zip(target, pred):
        conf_matrix[t, p] += 1

    print('Confusion matrix\n', conf_matrix)

    TP = conf_matrix.diag()
    for c in range(nb_classes):
        idx = torch.ones(nb_classes).byte()
        idx[c] = 0
        # all non-class samples classified as non-class
        TN = conf_matrix[idx.nonzero()[:,
                         None], idx.nonzero()].sum()  # conf_matrix[idx[:, None], idx].sum() - conf_matrix[idx, c].sum()
        # all non-class samples classified as class
        FP = conf_matrix[idx, c].sum()
        # all class samples not classified as class
        FN = conf_matrix[c, idx].sum()

        print('Class {}\nTP {}, TN {}, FP {}, FN {}'.format(
            c, TP[c], TN, FP, FN))

@staticmethod
def confusion_matrix_image():
    import numpy as np
    import matplotlib.pyplot as plt

    from sklearn import svm, datasets
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import plot_confusion_matrix

    # import some data to play with
    iris = datasets.load_iris()
    X = iris.data
    y = iris.target
    class_names = iris.target_names

    # Split the data into a training set and a test set
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

    # Run classifier, using a model that is too regularized (C too low) to see
    # the impact on the results
    classifier = svm.SVC(kernel='linear', C=0.01).fit(X_train, y_train)

    np.set_printoptions(precision=2)

    # Plot non-normalized confusion matrix
    titles_options = [("Confusion matrix, without normalization", None),
                      ("Normalized confusion matrix", 'true')]
    for title, normalize in titles_options:
        disp = plot_confusion_matrix(classifier, X_test, y_test,
                                     display_labels=class_names,
                                     cmap=plt.cm.Blues,
                                     normalize=normalize)
        disp.ax_.set_title(title)

        print(title)
        print(disp.confusion_matrix)

    plt.show()


import shelve

# SHELVE basically doesn't require its mode to be specified, it is generally READ/WRITE by nature

with shelve.open('ShelfTest') as fruit:
    fruit['orange'] = "a sweet, orange, citrus fruit"
    fruit['apple'] = "good for making cider"
    fruit['lemon'] = "a sour, yellow citrus fruit"
    fruit['grape'] = "a small, sweet fruit"
    fruit['lime'] = "a sour, green citrus fruit"

    print(fruit["lemon"])
    print(fruit["grape"])

print(fruit)

# This is same as the above code but it is done using KEY and VALUE PAIRS
with shelve.open('ShelfTest') as fruit:
    fruit = {"orange": "a sweet, orangish fruit",
             "apple": "good for making cider",
             "lemon": "a sour, yellow citrus fruit",
             "grape": "a small, sweet fruit",
             "lime": "a sour, green citrus fruit"}

print(fruit["lemon"])
print(fruit["apple"])

print(fruit)

# Another alternative instead of using WITH command, we can directly
# declare a variable FRUIT and pass the SHELVE in it and it will give the same results
fruit = shelve.open('ShelfTest')
fruit['orange'] = "a sweet, orange, citrus fruit"
fruit['apple'] = "good for making cider"
fruit['lemon'] = "a sour, yellow citrus fruit"
fruit['grape'] = "a small, sweet fruit"
fruit['lime'] = "a sour, green citrus fruit"

print(fruit["lemon"])
print(fruit["grape"])

# But in this approach we have to close the shelve manually
fruit.close()

print(fruit)


#!/usr/bin/python2
import r2pipe
import sys

if len(sys.argv) != 2:
    raise SystemExit

exe = sys.argv[1]
r2 = r2pipe.open(exe)
mvtext = r2.cmdj('pdj 2 @ section.multiverse_text')

jump = mvtext[0]
guard = mvtext[1]
print(exe + " & " + jump['disasm'] + " & " + guard['opcode'])


import time

import pika
from pika.adapters import SelectConnection

pika.log.setup(color=True)

class Worker(object):
    def on_connected(self, connection):
        pika.log.info("Connected to server")
        connection.channel(self.on_channel_open)

    def on_channel_open(self, channel_):
        self.channel = channel_
        pika.log.info("Received channel")
        self.channel.queue_declare(queue="test", durable=True,
                              exclusive=False, auto_delete=False,
                              callback=self.on_queue_declared)

    def on_queue_declared(self, frame):
        pika.log.info("Queue declared!")
        for x in xrange(0,10):
            message = "Hello world #%i: %.8f" % (x, time.time())
            pika.log.info("Sending message: %s" % message)
            self.channel.basic_publish(exchange='',
                                  routing_key='test',
                                  body=message,
                                  properties=pika.BasicProperties(
                                  content_type="text/plain",
                                  delivery_mode=1))
        self.connection.close()

if __name__ == "__main__":
    worker = Worker()
    parameters = pika.ConnectionParameters('localhost')
    worker.connection = SelectConnection(parameters, worker.on_connected)
    try:
        worker.connection.ioloop.start()
    except KeyboardInterrupt:
       worker.connection.close()


import json

wrestler_edges = []

with open('scrapedmatchdata_all.json', 'r') as json_file:
    text = json_file.read()
    data = json.loads(text)

    print(data)

#test = {
        #"promotion": "World Championship Wrestling",
        #"wrestlers": [
            # "Billy Kidman",
            # "Rey Mysterio Jr.",
            # "3-Count",
            # "Evan Karagias",
            # "Shannon Moore",
            # "The Jung Dragons",
            # "Kaz Hayashi",
            # "Yang"
        #],
    #     "match type": "WCW Cruiserweight Tag Team Title #1 Contendership Three Way: ",
    #     "event title": "WCW Monday NITRO #286 - Night Of Champions",
    #     "date": "26.03.2001",
    #     "location": " @ Panama City Beach, Florida, USA",
    #     "match card": "Billy Kidman & Rey Mysterio Jr. defeat 3-Count (Evan Karagias & Shannon Moore) and The Jung Dragons (Kaz Hayashi & Yang) (3:40)"
    # }

for a_row in data:
    matchdate = a_row['date']
    matchdate = matchdate[6: ]
    matchdate = int(matchdate)
    if matchdate > 1994 and matchdate < 2002:
        for wrestler_a in a_row['wrestlers']:
            for wrestler_b in a_row['wrestlers']:
                if wrestler_a != wrestler_b:
                    #print(wrestler_a, ",", wrestler_b)
                    wrestler_edges.append([wrestler_a, wrestler_b, "undirected", a_row['promotion'], matchdate])

import csv
with open('wrestler_edge_full_yeardate.csv', 'w') as fp:
    csvwriter = csv.writer(fp, delimiter=",")
    csvwriter.writerows(wrestler_edges)


# -*- coding: utf-8 -*-
#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2018      Paul Culley
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
#
# Place Cleanup Gramplet.
#
# pylint: disable=attribute-defined-outside-init
#------------------------------------------------------------------------
#
# Python modules
#
#------------------------------------------------------------------------
from urllib.request import urlopen, URLError, quote
from xml.dom.minidom import parseString
import os
import sys
import ctypes
import locale
import socket

#------------------------------------------------------------------------
#
# GTK modules
#
#------------------------------------------------------------------------
from gi.repository import Gtk

#------------------------------------------------------------------------
#
# Gramps modules
#
#------------------------------------------------------------------------
from gramps.gen.merge.mergeplacequery import MergePlaceQuery
from gramps.gui.dialog import ErrorDialog, WarningDialog
from gramps.gen.plug import Gramplet
from gramps.gen.db import DbTxn
from gramps.gen.lib import Citation
from gramps.gen.lib import Place, PlaceName, PlaceType, PlaceRef, Url, UrlType
from gramps.gen.lib import Note, NoteType, Repository, RepositoryType, RepoRef
from gramps.gen.lib import StyledText, StyledTextTag, StyledTextTagType
from gramps.gen.lib import Source, SourceMediaType
from gramps.gen.datehandler import get_date
from gramps.gen.config import config
from gramps.gen.constfunc import win
from gramps.gui.display import display_url
from gramps.gui.autocomp import StandardCustomSelector
from gramps.gen.display.place import displayer as _pd
from gramps.gen.utils.location import located_in

#------------------------------------------------------------------------
#
# Internationalisation
#
#------------------------------------------------------------------------
from gramps.gen.const import GRAMPS_LOCALE as glocale
try:
    _trans = glocale.get_addon_translator(__file__)
except ValueError:
    _trans = glocale.translation
_ = _trans.gettext

#------------------------------------------------------------------------
#
# Constants
#
#------------------------------------------------------------------------
WIKI_PAGE = ('https://gramps-project.org/wiki/index.php/'
             'Addon:PlaceCleanupGramplet')
PREFS_WIKI = ('https://gramps-project.org/wiki/index.php/'
              'Addon:PlaceCleanupGramplet#Preferences')
#-------------------------------------------------------------------------
#
# configuration
#
#-------------------------------------------------------------------------

GRAMPLET_CONFIG_NAME = "place_cleanup_gramplet"
CONFIG = config.register_manager(GRAMPLET_CONFIG_NAME)
CONFIG.register("preferences.geo_userid", '')
CONFIG.register("preferences.web_links", True)
CONFIG.register("preferences.add_cit", True)
CONFIG.register("preferences.keep_enclosure", True)
CONFIG.register("preferences.keep_lang", "en fr de pl ru da es fi sw no")
CONFIG.load()

#------------------------------------------------------------------------
#
# PlaceCleanup class
#
#------------------------------------------------------------------------
class PlaceCleanup(Gramplet):
    """
    Gramplet to cleanup places.
    Can Look for place that needs attention, or work on current place.
    Can search your own places, and merge current with another
    Can search GeoNames data on web and load data to a place.
    Data includes, Lat/Lon, enclosed by, type, postal code, and alternate
    names.
    """
    def init(self):
        """
        Initialise the gramplet.
        """
        self.keepweb = CONFIG.get("preferences.web_links")
        self.addcitation = CONFIG.get("preferences.add_cit")
        self.geonames_id = CONFIG.get("preferences.geo_userid")
        self.keep_enclosure = CONFIG.get("preferences.keep_enclosure")
        allowed_languages = CONFIG.get("preferences.keep_lang")
        self.allowed_languages = allowed_languages.split()
        self.incomp_hndl = ''  # a last used handle for incomplete places
        self.matches_warn = True  # Display the 'too many matches' warning?
        root = self.__create_gui()
        self.gui.get_container_widget().remove(self.gui.textview)
        self.gui.get_container_widget().add(root)
        root.show_all()

    def __create_gui(self):
        """
        Create and display the GUI components of the gramplet.
        """
        self.top = Gtk.Builder()  # IGNORE:W0201
        # Found out that Glade does not support translations for plugins, so
        # have to do it manually.
        base = os.path.dirname(__file__)
        glade_file = base + os.sep + "placecleanup.glade"
        # This is needed to make gtk.Builder work by specifying the
        # translations directory in a separate 'domain'
        try:
            localedomain = "addon"
            localepath = base + os.sep + "locale"
            if hasattr(locale, 'bindtextdomain'):
                libintl = locale
            elif win():  # apparently wants strings in bytes
                localedomain = localedomain.encode('utf-8')
                localepath = localepath.encode('utf-8')
                libintl = ctypes.cdll.LoadLibrary('libintl-8.dll')
            else:  # mac, No way for author to test this
                libintl = ctypes.cdll.LoadLibrary('libintl.dylib')

            libintl.bindtextdomain(localedomain, localepath)
            libintl.textdomain(localedomain)
            libintl.bind_textdomain_codeset(localedomain, "UTF-8")
            # and finally, tell Gtk Builder to use that domain
            self.top.set_translation_domain("addon")
        except (OSError, AttributeError):
            # Will leave it in English
            print("Localization of PlaceCleanup failed!")

        self.top.add_from_file(glade_file)
        # the results screen items
        self.results_win = self.top.get_object("results")
        self.alt_store = self.top.get_object("alt_names_liststore")
        self.alt_selection = self.top.get_object("alt_names_selection")
        self.res_lbl = self.top.get_object("res_label")
        self.find_but = self.top.get_object("find_but")
        self.top.connect_signals({
            # for results screen
            "on_res_ok_clicked"      : self.on_res_ok_clicked,
            "on_res_cancel_clicked"  : self.on_res_cancel_clicked,
            "on_keep_clicked"        : self.on_keep_clicked,
            "on_prim_clicked"        : self.on_prim_clicked,
            "on_disc_clicked"        : self.on_disc_clicked,
            "on_alt_row_activated"   : self.on_alt_row_activated,
            "on_latloncheck"         : self.on_latloncheck,
            "on_postalcheck"         : self.on_postalcheck,
            "on_typecheck"           : self.on_typecheck,
            "on_idcheck"             : self.on_idcheck,
            # Preferences screen item
            "on_pref_help_clicked"   : self.on_pref_help_clicked,
            # main screen items
            "on_find_clicked"        : self.on_find_clicked,
            "on_prefs_clicked"       : self.on_prefs_clicked,
            "on_select_clicked"      : self.on_select_clicked,
            "on_edit_clicked"        : self.on_edit_clicked,
            "on_next_clicked"        : self.on_next_clicked,
            "on_res_row_activated"   : self.on_select_clicked,
            "on_res_sel_changed"     : self.on_res_sel_changed,
            "on_title_entry_changed" : self.on_title_entry_changed,
            "on_help_clicked"        : self.on_help_clicked})
        # main screen items
        self.res_store = self.top.get_object("res_names_liststore")
        self.res_selection = self.top.get_object("res_selection")
        self.mainwin = self.top.get_object("main")
        return self.mainwin

    # ======================================================
    # gramplet event handlers
    # ======================================================
    def on_help_clicked(self, dummy):
        ''' Button: Display the relevant portion of GRAMPS manual'''
        display_url(WIKI_PAGE)

    def on_res_sel_changed(self, res_sel):
        """ Selecting a row in the results list """
        self.top.get_object("select_but").set_sensitive(
            res_sel.get_selected())

    def on_title_entry_changed(self, dummy):
        ''' Occurs during edits of the Title box on the main screen.
        we use this to reset the GeoNames search row, as the user may be
        trying another search term.'''
        self.reset_main()

    def on_save(self, *args, **kwargs):
        CONFIG.set("preferences.geo_userid", self.geonames_id)
        CONFIG.set("preferences.web_links", self.keepweb)
        CONFIG.set("preferences.add_cit", self.addcitation)
        CONFIG.set("preferences.keep_enclosure", self.keep_enclosure)
        CONFIG.set("preferences.keep_lang", ' '.join(self.allowed_languages))
        CONFIG.save()

    def db_changed(self):
        self.dbstate.db.connect('place-update', self.update)
        self.main()
        if not self.dbstate.db.readonly:
            self.connect_signal('Place', self.update)

    def main(self):
        self.reset_main()
        if self.gui.get_child().get_child() == self.results_win:
            self.gui.get_child().remove(self.results_win)
            self.gui.get_child().add(self.mainwin)
        active_handle = self.get_active('Place')
        self.top.get_object("edit_but").set_sensitive(False)
        self.top.get_object("find_but").set_sensitive(False)
        self.top.get_object("title_entry").set_sensitive(False)
        if active_handle:
            self.place = self.dbstate.db.get_place_from_handle(active_handle)
            self.mainwin.hide()
            if self.place:
                self.set_has_data(True)
                title = _pd.display(self.dbstate.db, self.place)
                item = self.top.get_object("title_entry")
                item.set_text(title)
                self.top.get_object("edit_but").set_sensitive(True)
                self.top.get_object("find_but").set_sensitive(True)
                self.top.get_object("title_entry").set_sensitive(True)
            else:
                self.set_has_data(False)
            self.mainwin.show()
        else:
            self.set_has_data(False)

    def reset_main(self):
        """ Reset the main Gui to default clear """
        self.res_store.clear()
        self.res_lbl.set_text(_('No\nMatches'))
        self.find_but.set_label(_("Find"))
        self.start_row = 0
        self.geo_stage = False
        self.top.get_object("select_but").set_sensitive(False)

    def on_find_clicked(self, dummy):
        """ find a matching place.  First try in the db, then try in the
        GeoNames. """
        self.res_store.clear()
        self.top.get_object("select_but").set_sensitive(False)
        if self.geo_stage:
            self.search_geo()
        else:
            self.geo_stage = True
            item = self.top.get_object("title_entry")
            title = item.get_text()
            self.places = self.lookup_places_by_name(title)
            for index, place in enumerate(self.places):
                # make sure the place found isn't self, or a place
                # enclosed by the working place
                if place.handle != self.place.handle and not located_in(
                        self.dbstate.db, place.handle, self.place.handle):
                    title = _pd.display(self.dbstate.db, place)
                    self.res_store.append(row=(index, title,
                                               str(place.place_type)))
            if len(self.res_store) > 0:
                self.res_lbl.set_text(_('%s\nLocal\nMatches') %
                                      len(self.res_store))
                self.find_but.set_label(_("Find GeoNames"))
            else:
                self.search_geo()

    def search_geo(self):
        """ find a matching place in the geonames, if possible """
        self.res_store.clear()
        if not self.geonames_id:
            ErrorDialog(_('Need to set GeoNames ID'),
                        msg2=_('Use the Help button for more information'),
                        parent=self.uistate.window)
            return
        # lets get a preferred language
        fmt = config.get('preferences.place-format')
        placef = _pd.get_formats()[fmt]
        self.lang = placef.language
        if len(self.lang) != 2:
            self.lang = 'en'
        if self.lang not in self.allowed_languages:
            self.allowed_languages.append(self.lang)
        # now lets search for a place in GeoNames
        item = self.top.get_object("title_entry")
        title = quote(item.get_text().lower().replace('co.', 'county'))
        adm = self.top.get_object('adm_check').get_active()
        ppl = self.top.get_object('ppl_check').get_active()
        spot = self.top.get_object('spot_check').get_active()
        geo_url = (
            'http://api.geonames.org/search?q=%s'
            '&maxRows=10&style=SHORT&lang=en&isNameRequired=True'
            '%s%s%s&username=%s&startRow=%s' %
            (title,
             '&featureClass=A' if adm else '',
             '&featureClass=P' if ppl else '',
             '&featureClass=S' if spot else '',
             self.geonames_id, self.start_row))
        dom = self.get_geo_data(geo_url)
        if not dom:
            return

        g_names = dom.getElementsByTagName('geoname')
        if not g_names:
            WarningDialog(_('No matches were found'),
                          msg2=_('Try changing the Title, or use the "Edit"'
                                 ' button to finish this level of the place'
                                 ' manually.'),
                          parent=self.uistate.window)
            return
        # let's check the total results; if too many, warn user and set up for
        # another pass through the search.
        value = dom.getElementsByTagName('totalResultsCount')
        if value:
            totalresults = int(value[0].childNodes[0].data)
            self.res_lbl.set_text(_("%s\nGeoNames\nMatches") % totalresults)
            if totalresults > 10:
                self.start_row += 10
                if self.matches_warn:
                    self.matches_warn = False
                    WarningDialog(
                        _('%s matches were found') % totalresults,
                        msg2=_('Only 10 matches are shown.\n'
                               'To see additional results, press the'
                               ' search button again.\n'
                               'Or try changing the Title with'
                               ' more detail, such as a country.'),
                        parent=self.uistate.window)
        index = 0
        self.places = []
        for g_name in g_names:
            # got a match, now get its hierarchy for display.
            value = g_name.getElementsByTagName('geonameId')
            geoid = value[0].childNodes[0].data
            value = g_name.getElementsByTagName('fcl')
            fcl = value[0].childNodes[0].data
            value = g_name.getElementsByTagName('fcode')
            try:
                _type = fcl + ':' + value[0].childNodes[0].data
            except IndexError:
                _type = fcl + ':'
            geo_url = ('http://api.geonames.org/hierarchy?geonameId=%s'
                       '&lang=%s&username=%s' %
                       (geoid, self.lang, self.geonames_id))
            hier = self.get_geo_data(geo_url)
            if not hier:
                return
            h_names = hier.getElementsByTagName('geoname')
            h_name_list = []
            h_geoid_list = []
            for h_name in h_names:
                value = h_name.getElementsByTagName('fcl')
                fcl = value[0].childNodes[0].data
                if fcl not in 'APS':
                    # We don't care about Earth or continent (yet)
                    continue
                value = h_name.getElementsByTagName('name')
                h_name_list.append(value[0].childNodes[0].data)
                value = h_name.getElementsByTagName('geonameId')
                h_geoid_list.append('GEO' + value[0].childNodes[0].data)
            # make sure that this place isn't already enclosed by our place.
            bad = self.place.gramps_id in h_geoid_list[:-1]
            # assemble a title for the result
            h_name_list.reverse()
            h_geoid_list.reverse()
            if bad:
                title = ('' + _(', ').join(h_name_list) + '')
            else:
                title = _(', ').join(h_name_list)
            row = (index, title, _type)
            self.res_store.append(row=row)
            self.places.append(('GEO' + geoid, title, h_geoid_list,
                                h_name_list, bad))
            index += 1
            while Gtk.events_pending():
                Gtk.main_iteration()

    def get_geo_data(self, geo_url):
        """ Get GeoNames data from web with error checking """
        print(geo_url)
        try:
            with urlopen(geo_url, timeout=20) as response:
                data = response.read()
        except URLError as err:
            try:
                txt = err.read().decode('utf-8')
            except:
                txt = ''
            ErrorDialog(_('Problem getting data from web'),
                        msg2=str(err) +'\n' + txt,
                        parent=self.uistate.window)
            return None
        except socket.timeout:
            ErrorDialog(_('Problem getting data from web'),
                        msg2=_('Web request Timeout, you can try again...'),
                        parent=self.uistate.window)
            return None

        dom = parseString(data)
        status = dom.getElementsByTagName('status')
        if status:
            err = status[0].getAttribute("message")
            ErrorDialog(_('Problem getting data from GeoNames'),
                        msg2=err,
                        parent=self.uistate.window)
            return None
        return dom

    def on_next_clicked(self, dummy):
        """ find a incomplete place in the db, if possible """
        self.reset_main()
        place = self.find_an_incomplete_place()
        if place:
            self.set_active('Place', place.handle)

    def on_select_clicked(self, *dummy):
        """ If the selected place is mergable, merge it, otherwise Open
        completion screen """
        model, _iter = self.res_selection.get_selected()
        if not _iter:
            return
        (index, ) = model.get(_iter, 0)
        place = self.places[index]
        if not isinstance(place, Place):
            # we have a geoname_id
            if place[4]:
                return
            # check if we might already have it in db
            t_place = self.dbstate.db.get_place_from_gramps_id(place[0])
            if not t_place or t_place.handle == self.place.handle:
                # need to process the GeoNames ID for result
                self.gui.get_child().remove(self.mainwin)
                self.gui.get_child().add(self.results_win)
                if not self.geoparse(*place):
                    return
                self.res_gui()
                return
            else:
                # turns out we already have this place, under different name!
                place = t_place
        # we have a Gramps Place, need to merge
        if place.handle == self.place.handle:
            # found self, nothing to do.
            return
        if(located_in(self.dbstate.db, place.handle, self.place.handle) or
           located_in(self.dbstate.db, self.place.handle, place.handle)):
            # attempting to create a place loop, not good!
            ErrorDialog(_('Place cycle detected'),
                        msg2=_("One of the places you are merging encloses "
                               "the other!\n"
                               "Please choose another place."),
                        parent=self.uistate.window)
            return
        # lets clean up the place name
        self.place.name.value = self.place.name.value.split(',')[0].strip()
        place_merge = MergePlaceQuery(self.dbstate, place, self.place)
        place_merge.execute()
        # after merge we should select merged result
        self.set_active('Place', place.handle)

    adm_table = {
        # note the True/False in the following indicates the certainty that the
        # entry is correct.  If it is only sometimes correct, and the name
        # might have a different type embedded in it, then use False.
        'US': {'ADM1': (PlaceType.STATE, True),
               'ADM2': (PlaceType.COUNTY, False),
               'ADM3': (PlaceType.TOWN, False)},
        'CA': {'ADM1': (PlaceType.PROVINCE, True),
               'ADM2': (PlaceType.REGION, False)},
        'GB': {'ADM1': (PlaceType.COUNTRY, True),
               'ADM2': (PlaceType.REGION, True),
               'ADM3': (PlaceType.COUNTY, False),
               'ADM4': (PlaceType.BOROUGH, False)},
        'FR': {'ADM1': (PlaceType.REGION, True),
               'ADM2': (PlaceType.DEPARTMENT, True)},
        'DE': {'ADM1': (PlaceType.STATE, True),
               'ADM2': (PlaceType.COUNTY, False),
               'ADM3': ('Amt', False)}}

    def geoparse(self, geoid, title, h_geoid_list, h_name_list, *dummy):
        """ get data for place and parse out g_name dom structure into the
        NewPlace structure """
        geo_url = ('http://api.geonames.org/get?geonameId=%s&style=FULL'
                   '&username=%s' % (geoid.replace('GEO', ''),
                                     self.geonames_id))
        dom = self.get_geo_data(geo_url)
        if not dom:
            return False

        g_name = dom.getElementsByTagName('geoname')[0]
        self.newplace = NewPlace(title)
        self.newplace.geoid = geoid
        self.newplace.gramps_id = geoid
        value = g_name.getElementsByTagName('lat')
        self.newplace.lat = str(value[0].childNodes[0].data)
        value = g_name.getElementsByTagName('lng')
        self.newplace.long = str(value[0].childNodes[0].data)
        value = g_name.getElementsByTagName('toponymName')
        topname = value[0].childNodes[0].data
        new_place = PlaceName()
        new_place.set_value(topname)
        new_place.set_language("")
        # make sure we have the topname in the names list and default to
        # primary
        self.newplace.add_name(new_place)
        self.newplace.name = new_place
        # lets parse the alternative names
        alt_names = g_name.getElementsByTagName('alternateName')
        for a_name in alt_names:
            pattr = a_name.getAttribute("lang")
            value = a_name.childNodes[0].data
            if pattr == "post":
                if self.newplace.code:
                    self.newplace.code += " " + value
                else:
                    self.newplace.code = value
            elif pattr == "link":
                url = Url()
                url.set_path(value)
                url.set_description(value)
                url.set_type(UrlType(UrlType.WEB_HOME))
                self.newplace.links.append(url)
            elif pattr not in ['iata', 'iaco', 'faac', 'wkdt', 'unlc']:
                new_place = PlaceName()
                new_place.set_language(pattr)
                new_place.set_value(value)
                self.newplace.add_name(new_place)
            if a_name.hasAttribute('isPreferredName') and (
                    pattr and pattr == self.lang):
                # if not preferred lang, we use topo name, otherwise
                # preferred name for lang
                self.newplace.name = new_place
        # Try to deduce PlaceType:
        #   If populated place, set as City. Long description could over-ride
        #   Parse long description, looking for keyword (Region, County, ...)
        #   Top-level must be a country.
        #   Children of USA are States.
        #   Children of Canada are Provinces.
        #
        value = g_name.getElementsByTagName('fcl')
        fcl = value[0].childNodes[0].data
        value = g_name.getElementsByTagName('fcode')
        try:
            fcode = value[0].childNodes[0].data
        except IndexError:
            fcode = ''
        value = g_name.getElementsByTagName('countryCode')
        try:
            countrycode = value[0].childNodes[0].data
        except IndexError:
            countrycode = ''
        self.newplace.place_type = PlaceType(PlaceType.UNKNOWN)
        ptype = PlaceType()
        # scan thorough names looking for name portion that matches a Placetype
        for name in self.newplace.names:
            for tname in name.value.split(' '):
                ptype.set_from_xml_str(tname.capitalize())
                if ptype != PlaceType.CUSTOM:
                    self.newplace.place_type = ptype
                    break
                # see if it is a translated PlaceType
                ptype.set(tname.capitalize())
                if ptype != PlaceType.CUSTOM:
                    self.newplace.place_type = ptype
                    break
                # see if it is an already added custom type
                cust_types = self.dbstate.db.get_place_types()
                if tname.capitalize() in cust_types:
                    self.newplace.place_type = ptype
                    break
            else:
                # Continue if the inner loop wasn't broken.
                continue
            # Inner loop was broken, break the outer.
            break
        if fcl == 'P':
            self.newplace.place_type = PlaceType(PlaceType.CITY)
        elif fcode == 'PRSH':
            self.newplace.place_type = PlaceType(PlaceType.PARISH)
        elif 'PCL' in fcode:
            self.newplace.place_type = PlaceType(PlaceType.COUNTRY)
        elif 'ADM' in fcode:
            if countrycode in self.adm_table:
                _ptype = self.adm_table[countrycode].get(fcode[:4])
                if _ptype and (_ptype[1] or
                               self.newplace.place_type.is_default()):
                    self.newplace.place_type = PlaceType(_ptype[0])
        # save a parent for enclosing
        if len(h_geoid_list) > 1:
            # we have a parent
            self.newplace.parent_names = h_name_list[1:]
            self.newplace.parent_ids = h_geoid_list[1:]
        return True

    def on_edit_clicked(self, dummy):
        """User wants to jump directly to the results view to finish off
        the place, possibly because a place was not found"""
#         if ',' in self.place.name.value:
#             name = self.place.name.value
#         else:
        name = self.place.name.value
        self.newplace = NewPlace(name)
        names = name.split(',')
        names = [name.strip() for name in names]
        self.newplace.name = PlaceName()
        self.newplace.name.value = names[0]
        self.newplace.gramps_id = self.place.gramps_id
        self.newplace.lat = self.place.lat
        self.newplace.long = self.place.long
        self.newplace.code = self.place.code
        if self.place.place_type == PlaceType.UNKNOWN:
            self.newplace.place_type = PlaceType(PlaceType.UNKNOWN)
            if any(i.isdigit() for i in self.newplace.name.value):
                self.newplace.place_type = PlaceType(PlaceType.STREET)
            ptype = PlaceType()
            for tname in self.newplace.name.value.split(' '):
                # see if it is an English PlaceType
                ptype.set_from_xml_str(tname.capitalize())
                if ptype != PlaceType.CUSTOM:
                    self.newplace.place_type = ptype
                    break
                # see if it is a translated PlaceType
                ptype.set(tname.capitalize())
                if ptype != PlaceType.CUSTOM:
                    self.newplace.place_type = ptype
                    break
                # see if it is an already added custom type
                cust_types = self.dbstate.db.get_place_types()
                if tname.capitalize() in cust_types:
                    self.newplace.place_type = ptype
        else:
            self.newplace.place_type = self.place.place_type
        self.newplace.add_name(self.newplace.name)
        self.newplace.add_name(self.place.name)
        self.newplace.add_names(self.place.alt_names)
        if self.place.placeref_list:
            # If we already have an enclosing place, use it.
            parent = self.dbstate.db.get_place_from_handle(
                self.place.placeref_list[0].ref)
            self.newplace.parent_ids = [parent.gramps_id]
        elif len(names) > 1:
            # we have an enclosing place, according to the name string
            self.newplace.parent_names = names[1:]
        self.gui.get_child().remove(self.mainwin)
        self.gui.get_child().add(self.results_win)
        self.res_gui()

# Results view

    def res_gui(self):
        """ Fill in the results display with values from new place."""
        self.alt_store.clear()
        # Setup sort on 'Inc" column so Primary is a top with checks next
        self.alt_store.set_sort_func(0, inc_sort, None)
        self.alt_store.set_sort_column_id(0, 0)
        # Merge old name and alt names into new set
        self.newplace.add_name(self.place.name)
        self.newplace.add_names(self.place.alt_names)
        # Fill in ohter fields
        self.top.get_object('res_title').set_text(self.newplace.title)
        self.top.get_object('primary').set_text(self.newplace.name.value)
        self.on_idcheck()
        self.on_latloncheck()
        self.on_postalcheck()
        self.on_typecheck()
        # Fill in names list
        for index, name in enumerate(self.newplace.names):
            if self.newplace.name == name:
                inc = 'P'
            elif name.lang in self.allowed_languages or (
                    name.lang == 'abbr' or name.lang == 'en' or not name.lang):
                inc = '\u2714'  # Check mark
            else:
                inc = ''
            row = (inc, name.value, name.lang, get_date(name), index)
            self.alt_store.append(row=row)

# Results dialog items

    def on_res_ok_clicked(self, dummy):
        """ Accept changes displayed and commit to place.
        Also find or create a new enclosing place from parent. """
        # do the names
        namelist = []
        for row in self.alt_store:
            if row[0] == 'P':
                self.place.name = self.newplace.names[row[4]]
            elif row[0] == '\u2714':
                namelist.append(self.newplace.names[row[4]])
        self.place.alt_names = namelist
        # Lat/lon/ID/code/type
        self.place.lat = self.top.get_object('latitude').get_text()
        self.place.long = self.top.get_object('longitude').get_text()
        self.place.gramps_id = self.top.get_object('grampsid').get_text()
        self.place.code = self.top.get_object('postal').get_text()
        self.place.place_type.set(self.type_combo.get_values())
        # Add in URLs if wanted
        if self.keepweb:
            for url in self.newplace.links:
                self.place.add_url(url)
        # Enclose in the next level place
        next_place = False
        parent = None
        if not self.keep_enclosure or not self.place.placeref_list:
            if self.newplace.parent_ids:
                # we might have a parent with geo id 'GEO12345'
                parent = self.dbstate.db.get_place_from_gramps_id(
                    self.newplace.parent_ids[0])
            if not parent and self.newplace.parent_names:
                # make one, will have to be examined/cleaned later
                parent = Place()
                parent.title = ', '.join(self.newplace.parent_names)
                name = PlaceName()
                name.value = parent.title
                parent.name = name
                if self.newplace.parent_ids:
                    parent.gramps_id = self.newplace.parent_ids[0]
                with DbTxn(_("Add Place (%s)") % parent.title,
                           self.dbstate.db) as trans:
                    self.dbstate.db.add_place(parent, trans)
                    next_place = True
            if parent:
                if located_in(self.dbstate.db, parent.handle,
                              self.place.handle):
                    # attempting to create a place loop, not good!
                    ErrorDialog(_('Place cycle detected'),
                                msg2=_("The place you chose is enclosed in the"
                                       " place you are workin on!\n"
                                       "Please cancel and choose another "
                                       "place."),
                                parent=self.uistate.window)
                    return
                # check to see if we already have the enclosing place
                already_there = False
                for pref in self.place.placeref_list:
                    if parent.handle == pref.ref:
                        already_there = True
                        break
                if not already_there:
                    placeref = PlaceRef()
                    placeref.set_reference_handle(parent.handle)
                    self.place.set_placeref_list([placeref])
        # Add in Citation/Source if wanted
        if self.addcitation and self.newplace.geoid:
            src_hndl = self.find_or_make_source()
            cit = Citation()
            cit.set_reference_handle(src_hndl)
            cit.set_page("GeoNames ID: %s" %
                         self.newplace.geoid.replace('GEO', ''))
            with DbTxn(_("Add Citation (%s)") % "GeoNames",
                       self.dbstate.db) as trans:
                self.dbstate.db.add_citation(cit, trans)
            self.place.add_citation(cit.handle)
        # We're finally ready to commit the updated place
        with DbTxn(_("Edit Place (%s)") % self.place.title,
                   self.dbstate.db) as trans:
            self.dbstate.db.commit_place(self.place, trans)
        # Jump to enclosing place to clean it if necessary
        if next_place:
            self.set_active('Place', parent.handle)
            self.place = parent
            # if geoparse fails, leave us at main view
            if self.newplace.parent_ids and \
                self.geoparse(self.newplace.parent_ids[0],
                              _(", ").join(self.newplace.parent_names),
                              self.newplace.parent_ids,
                              self.newplace.parent_names):
                # geoparse worked, lets put up the results view
                self.gui.get_child().remove(self.mainwin)
                self.gui.get_child().add(self.results_win)
                self.res_gui()
                return
        self.reset_main()
        if self.gui.get_child().get_child() == self.results_win:
            self.gui.get_child().remove(self.results_win)
            self.gui.get_child().add(self.mainwin)

    def on_res_cancel_clicked(self, dummy):
        """ Cancel operations on this place. """
        self.gui.get_child().remove(self.results_win)
        self.gui.get_child().add(self.mainwin)

    def on_keep_clicked(self, dummy):
        """ Keep button clicked.  Mark selected names rows to keep. """
        model, rows = self.alt_selection.get_selected_rows()
        for row in rows:
            if model[row][0] == 'P':
                continue
            model[row][0] = '\u2714'

    def on_prim_clicked(self, dummy):
        """ Primary button clicked.  Mark first row in selection as Primary
        name, any previous primary as keep """
        model, rows = self.alt_selection.get_selected_rows()
        if not rows:
            return
        # Clear prior primary
        for row in model:
            if row[0] == 'P':
                row[0] = '\u2714'
        # mark new one.
        self.top.get_object('primary').set_text(model[rows[0]][1])
        model[rows[0]][0] = 'P'

    def on_disc_clicked(self, dummy):
        """ Discard button clicked.  Unmark selected rows. """
        model, rows = self.alt_selection.get_selected_rows()
        for row in rows:
            if model[row][0] == 'P':
                continue
            model[row][0] = ''

    def on_alt_row_activated(self, *dummy):
        """ Toggle keep status for selected row.  Seems this only works for
        last selected row."""
        model, rows = self.alt_selection.get_selected_rows()
        for row in rows:
            if model[row][0] == 'P':
                continue
            if model[row][0] == '':
                model[row][0] = '\u2714'
            else:
                model[row][0] = ''

    def on_latloncheck(self, *dummy):
        """ Check toggled; if active, load lat/lon from original place, else
        use lat/lon from gazetteer """
        obj = self.top.get_object("latloncheck")
        if not dummy:
            # inititlization
            obj.set_sensitive(True)
            obj.set_active(False)
        place = self.newplace
        if self.place.lat and self.place.long:
            if obj.get_active():
                place = self.place
        else:
            obj.set_sensitive(False)
        self.top.get_object('latitude').set_text(place.lat)
        self.top.get_object('longitude').set_text(place.long)

    def on_postalcheck(self, *dummy):
        """ Check toggled; if active, load postal from original place, else
        use postal from gazetteer """
        obj = self.top.get_object("postalcheck")
        if not dummy:
            # inititlization
            obj.set_sensitive(True)
            obj.set_active(False)
        place = self.newplace
        if self.place.code:
            if obj.get_active():
                place = self.place
            obj.set_sensitive(True)
        else:
            obj.set_sensitive(False)
        self.top.get_object('postal').set_text(place.code)

    def on_typecheck(self, *dummy):
        """ Check toggled; if active, load type from original place, else
        use type from gazetteer """
        obj = self.top.get_object("typecheck")
        combo = self.top.get_object('place_type')
        additional = sorted(self.dbstate.db.get_place_types(),
                            key=lambda s: s.lower())
        self.type_combo = StandardCustomSelector(PlaceType().get_map(), combo,
                                                 PlaceType.CUSTOM,
                                                 PlaceType.UNKNOWN,
                                                 additional)
        if not dummy:
            # inititlization
            obj.set_sensitive(True)
            obj.set_active(False)
        place = self.newplace
        if(self.place.place_type and
           self.place.place_type != PlaceType.UNKNOWN):
            if obj.get_active():
                place = self.place
        else:
            obj.set_sensitive(False)
        self.type_combo.set_values((int(place.place_type),
                                    str(place.place_type)))

    def on_idcheck(self, *dummy):
        """ Check toggled; if active, load gramps_id from original place, else
        use geonamesid from gazetteer """
        obj = self.top.get_object("idcheck")
        if not dummy:
            # inititlization
            obj.set_sensitive(True)
            obj.set_active(False)
        place = self.newplace
        if self.place.gramps_id:
            if obj.get_active():
                place = self.place
        else:
            obj.set_sensitive(False)
        self.top.get_object('grampsid').set_text(place.gramps_id)

    def find_or_make_source(self):
        """ Find or create a source.
        returns handle to source."""
        for hndl in self.dbstate.db.get_source_handles():
            if self.dbstate.db.get_raw_source_data(hndl)[2] == 'GeoNames':
                return hndl
        # No source found, lets add one with associated repo and note
        repo = Repository()
        repo.set_name("www.geonames.org")
        rtype = RepositoryType(RepositoryType.WEBSITE)
        repo.set_type(rtype)
        url = Url()
        url.set_path('http://www.geonames.org/')
        url.set_description(_('GeoNames web site'))
        url.set_type(UrlType(UrlType.WEB_HOME))
        repo.add_url(url)
        url = Url()
        url.set_path('marc@geonames.org')
        url.set_description(_('GeoNames author'))
        url.set_type(UrlType(UrlType.EMAIL))
        repo.add_url(url)

        note_txt = StyledText(_(
            'GeoNames was founded by Marc Wick. You can reach him at '))
        note_txt += StyledText('marc@geonames.org' + '\n')
        note_txt += StyledText(_(
            'GeoNames is a project of Unxos GmbH, Weingartenstrasse 8,'
            ' 8708 Männedorf, Switzerland.\nThis work is licensed under a '))
        note_txt += linkst(
            _('Creative Commons Attribution 3.0 License'),
            'https://creativecommons.org/licenses/by/3.0/legalcode')

        new_note = Note()
        new_note.set_styledtext(note_txt)
        new_note.set_type(NoteType.REPO)
        src = Source()
        src.title = 'GeoNames'
        src.author = 'Marc Wick'
        repo_ref = RepoRef()
        mtype = SourceMediaType(SourceMediaType.ELECTRONIC)
        repo_ref.set_media_type(mtype)
        with DbTxn(_("Add Souce/Repo/Note (%s)") % "GeoNames",
                   self.dbstate.db) as trans:

            self.dbstate.db.add_note(new_note, trans)
            repo.add_note(new_note.get_handle())
            self.dbstate.db.add_repository(repo, trans)
            repo_ref.set_reference_handle(repo.handle)
            src.add_repo_reference(repo_ref)
            self.dbstate.db.add_source(src, trans)
        return src.handle

# Preferences dialog items

    def on_prefs_clicked(self, dummy):
        """ Button: display preference dialog """
        top = self.top.get_object("pref_dialog")
        top.set_transient_for(self.uistate.window)
        parent_modal = self.uistate.window.get_modal()
        if parent_modal:
            self.uistate.window.set_modal(False)
        keepweb = self.top.get_object("keepweb")
        keepweb.set_active(self.keepweb)
        addcit = self.top.get_object("addcitation")
        addcit.set_active(self.addcitation)
        # for some reason you can only set the radiobutton to True
        self.top.get_object(
            "enc_radio_but_keep" if self.keep_enclosure
            else "enc_radio_but_repl").set_active(True)
        geoid = self.top.get_object("geonames_id_entry")
        geoid.set_text(self.geonames_id)
        keepalt = self.top.get_object("keep_alt_entry")
        keepalt.set_text(' '.join(self.allowed_languages))
        top.show()
        top.run()
        if self.uistate.window and parent_modal:
            self.uistate.window.set_modal(True)
        self.geonames_id = geoid.get_text()
        self.addcitation = addcit.get_active()
        self.keepweb = keepweb.get_active()
        self.keep_enclosure = self.top.get_object(
            "enc_radio_but_keep").get_active()
        self.allowed_languages = keepalt.get_text().split()
        top.hide()

    def on_pref_help_clicked(self, dummy):
        ''' Button: Display the relevant portion of GRAMPS manual'''
        display_url(PREFS_WIKI)

    def lookup_places_by_name(self, search_name):
        """ In local db.  Only completed places are matched.
        We may want to try some better matching algorithms, possibly
        something from difflib"""
        search_name = search_name.lower().split(',')
        places = []
        for place in self.dbstate.db.iter_places():
            if (place.get_type() != PlaceType.UNKNOWN and
                (place.get_type() == PlaceType.COUNTRY or
                 (place.get_type() != PlaceType.COUNTRY and
                  place.get_placeref_list()))):
                # valid place, get all its names
                for name in place.get_all_names():
                    if name.get_value().lower() == search_name[0]:
                        places.append(place)
                        break
        return places

    def find_an_incomplete_place(self):
        """ in our db.  Will return with a place (and active set to place)
        or None if no incomplete places, in which case active will be the same.
        Will also find unused places, and offer to delete."""
        p_hndls = self.dbstate.db.get_place_handles()
        if not p_hndls:
            return None  # in case there aren't any
        # keep handles in an order to avoid inconsistant
        # results when db returns them in different orders.
        p_hndls.sort()
        # try to find the handle after the previously scanned handle in the
        # list.
        found = False
        for indx, hndl in enumerate(p_hndls):
            if hndl > self.incomp_hndl:
                found = True
                break
        if not found:
            indx = 0
        # now, starting from previous place, look for incomplete place
        start = indx
        while True:
            hndl = p_hndls[indx]
            place_data = self.dbstate.db.get_raw_place_data(hndl)
            p_type = place_data[8][0]  # place_type
            refs = list(self.dbstate.db.find_backlink_handles(hndl))
            if(p_type == PlaceType.UNKNOWN or
               not refs or
               p_type != PlaceType.COUNTRY and
               not place_data[5]):  # placeref_list
                # need to get view to this place...
                self.set_active("Place", hndl)
                self.incomp_hndl = hndl
                if not refs:
                    WarningDialog(
                        _('This Place is not used!'),
                        msg2=_('You should delete it, or, if it contains '
                               'useful notes or other data, use the Find to '
                               'merge it into a valid place.'),
                        parent=self.uistate.window)
                return self.dbstate.db.get_place_from_handle(hndl)
            indx += 1
            if indx == len(p_hndls):
                indx = 0
            if indx == start:
                break
        return None

class NewPlace():
    """ structure to store data about a found place"""
    def __init__(self, title):
        self.title = title
        self.gramps_id = ''
        self.lat = ''
        self.long = ''
        self.code = ''
        self.place_type = None
        self.names = []  # all names, including alternate, acts like a set
        self.name = PlaceName()
        self.links = []
        self.geoid = ''
        self.parent_ids = []   # list of gramps_ids in hierarchical order
        self.parent_names = [] # list of names in hierarchical order

    def add_name(self, name):
        """ Add a name to names list without repeats """
        if ',' in name.value:
            name.value = name.value.split(',')[0]
            return
        if name not in self.names:
            self.names.append(name)

    def add_names(self, names):
        """ Add names to names list without repeats """
        for name in names:
            self.add_name(name)

#------------------------------------------------------------------------
#
# Functions
#
#------------------------------------------------------------------------

def linkst(text, url):
    """ Return text as link styled text
    """
    tags = [StyledTextTag(StyledTextTagType.LINK, url, [(0, len(text))])]
    return StyledText(text, tags)

def inc_sort(model, row1, row2, user_data):
    value1 = model.get_value(row1, 0)
    value2 = model.get_value(row2, 0)
    if value1 == value2:
        return 0
    if value1 == 'P':
        return -1
    if value2 == 'P':
        return 1
    if value2 > value1:
        return 1
    else:
        return -1

# if found a matching place in our db, merge
# if not, lookup on web, present list of candidates
# start with simple search
#   extend each result with heirarchy query
# Web button, if pressed again, more answers.

# Alternative name Lang field:
#   unlc:Un/locodes
#   post:for postal codes
#   iata,icao,faac: for airport codes
#   fr_1793:French Revolution names
#   abbr:abbreviation
#   link: to a website (mostly to wikipedia)
#   wkdt: for the wikidata id
#   otherwise 2/3 char ISO639 lang code
#   empty: just an alternative name
# Alternative name other fields:
#   isPreferredName="true"
#   isShortName="true"
#   isColloquial="true"
#   isHistoric="true"


import ase
from ase.io import write
from ase.build import fcc100, fcc111, fcc110

element = 'Au'   # element name

# Build Au(111) surface slab that is DFT sized
slab = fcc111(element, size=(3,3,6), a=4.16, vacuum=10, periodic=True) # choose miller indices, element, periodic replications (size), PBE lattice constant, vacuum, and periodicity
#print(slab.get_positions()) # print positions in terminal for sanity
write('Au111_slab_3x3x6.xyz', slab, format='extxyz') # write slab using ase write to whatever file type

# Build Au(110) surface slab that is DFT sized
slab = fcc110(element, size=(3,2,10), a=4.16, vacuum=10, periodic=True)
write('Au110_slab_3x2x10.xyz', slab, format='extxyz')

# Build Au(100) surface slab that is DFT sized
slab = fcc100(element, size=(3,3,6), a=4.16, vacuum=10, periodic=True)
write('Au100_slab_3x3x6.xyz', slab, format='extxyz')


import pandas as pd
import numpy as np
import math
import random
from sklearn import preprocessing

def sigmoid(x):
    return 1 / (1 + math.exp(-x))

# read csv
df = pd.read_csv('\\email_spam.csv')

# convert strings to numerical values
keywords = ['yes', 'no', 'HTML', 'Plain', 'none',
           'big', 'small']

mapping = [1,0,0,1,0,1,2]

df = df.replace(keywords,mapping)

# normalize df
min_max_scaler = preprocessing.MinMaxScaler()
np_scaled = min_max_scaler.fit_transform(df)
df = pd.DataFrame(np_scaled)

# split to train and test sets for holdout crossvalidation

train_ratio = 0.8
df = df.drop(df.columns[[0]], axis=1)

df.insert(df.shape[1], 'bias', 1)

# shuffle data array
dataset = df.values
random.shuffle(dataset)

response_col = 0

rows = df.shape[0]
trainrows = int(train_ratio*rows)

trainX = dataset[1:trainrows, response_col+1 : dataset.shape[1]]
trainY = dataset[1:trainrows, response_col]

testX = dataset[trainrows:dataset.shape[0], response_col+1: dataset.shape[1]]
testY = dataset[trainrows:dataset.shape[0], response_col]

# here's where the magic happens

epochs = 1000
step_size = 0.01
params_num = testX.shape[1]
params = np.zeros((params_num, 1))
params[:,0] = np.random.uniform(low=-0.5, high=0.5, size=(params_num,))

for i in range(epochs):

    random.shuffle(testX)

    sig_out = [0] * trainX.shape[0]
    diff = [0] * trainX.shape[0]
    gradient = np.zeros((trainX.shape[1], 1))
    data = np.zeros((trainX.shape[1], trainX.shape[0]))
    sig_der = np.zeros((trainX.shape[0], trainX.shape[0]))

    # row iterator loop
    for j in range(trainX.shape[0]):
        # compute gradient vector of negative log likelihood

        # compute sigmoid outputs
        sig_out[j] = sigmoid(np.dot(trainX[j], params[:,]))
        diff[j] = sig_out[j] - trainY[j]

        data[:,j] = trainX[j].transpose()
        gradient[:, 0] = gradient[:,0] + np.multiply(trainX[j].transpose(), diff[j])

    print("Epoch %d" % i)
    print("Train RMSE %0.4f" % np.sqrt(np.dot(diff[j], diff[j]) / len(diff)))
    # compute Hessian
    sig_der = np.diag(np.multiply(sig_out, np.subtract(1, sig_out)))
    hess = np.matmul(np.matmul(data,sig_der), np.transpose(data))

    # invert Hessian
    hess = np.linalg.inv(hess)

    # do the weight update
    params[:,] = params[:,]  - step_size* np.matmul(hess, gradient)

    # do testing
    sig_out_test = [0] * testX.shape[0]
    diff_test = [0] * testX.shape[0]
    for k in range(testX.shape[0]):
        # compute sigmoid outputs
        sig_out_test[k] = sigmoid(np.dot(testX[k], params[:, ]))
        diff_test[k] = sig_out[k] - testY[k]

    print("Test RMSE %0.4f" % np.sqrt(np.dot(diff_test, diff_test) / len(diff_test)))


from __future__ import annotations

import io
from configparser import ConfigParser, MissingSectionHeaderError

# Local - make sure that all imports here are carefully done in bash.py first
from .bass import dirs, get_path_from_ini
from .bolt import GPath, Path, decoder, deprint, os_name, top_level_dirs
from .env import get_legacy_ws_game_info, get_local_app_data_path, \
    get_personal_path, shellMakeDirs, is_case_sensitive, \
    get_case_sensitivity_advice
from .exception import BoltError
from .loot_parser import LOOTParser

mopy_dirs_initialized = bash_dirs_initialized = False
#--Config Helper files (LOOT Master List, etc.)
lootDb: LOOTParser | None = None

def _get_ini_option(ini_parser, option_key) -> str | None:
    if not ini_parser:
        return None
    # logic for getting the path from the ini - get(section, key,
    # fallback=default). section is case sensitive - key is not
    return ini_parser.get('General', option_key, fallback=None)

def _get_cli_ini_path(my_docs_path, cli_switch, ini_path_key, game_info,
                      fallback, fail_msg, not_exists_msg):
    if my_docs_path:
        my_docs_path = GPath(my_docs_path)
        sErrorInfo = _('Folder path specified on command line '
                       '(%(cli_switch)s)') % {'cli_switch': cli_switch}
    else:
        my_docs_path = get_path_from_ini(ini_path_key)
        if my_docs_path:
            sErrorInfo = _('Folder path specified in bash.ini '
                           '(%(bash_ini_setting)s)') % {
                             'bash_ini_setting': f's{ini_path_key}'}
        else:
            my_docs_path, sErrorInfo = fallback(game_info)
    if my_docs_path is None:
        raise BoltError('\n'.join([fail_msg,
            _('Additional info: %(error_info)s') % {'error_info': sErrorInfo},
        ]))
    #  If path is relative, make absolute ##: only do that in the cli_switch case
    if not my_docs_path.is_absolute():
        my_docs_path = dirs['app'].join(my_docs_path)
    #  Error check
    if not my_docs_path.exists():
        raise BoltError('\n'.join([not_exists_msg % {'folder': my_docs_path},
            _('Additional info: %(error_info)s') % {'error_info': sErrorInfo},
        ]))
    return my_docs_path

def getOblivionModsPath(game_info):
    ob_mods_path = get_path_from_ini('OblivionMods')
    if ob_mods_path:
        return ob_mods_path, ['[General]', 'sOblivionMods']
    ws_info = get_legacy_ws_game_info(game_info)
    if not ws_info.installed:
        # Currently the standard location, next to the game install
        ob_mods_path = GPath(GPath(u'..').join(
            f'{game_info.bash_root_prefix} Mods'))
        ob_mods_path = dirs['app'].join(ob_mods_path)
        src = u'Relative Path'
    else:
        # New location for Windows Store games,
        # Documents\Wrye Bash\{game} Mods
        ob_mods_path = dirs[u'personal'].join(
            u'Wrye Bash', f'{game_info.bash_root_prefix} Mods')
        src = u'My Documents'
    return ob_mods_path, src

def _get_ini_path(ini_key, dir_key, *args):
    idata_path = get_path_from_ini(ini_key)
    if idata_path:
        src = ['[General]', f's{ini_key}']
    else:
        idata_path = dirs[dir_key].join(*args)
        src = 'Relative Path'
    return idata_path, src

def init_dirs(personal, localAppData, game_info):
    """Initialize bass.dirs dictionary. We need the bash.ini and the game
    being set, so this is called upon setting the game. Global structures
    that need info on Bash / Game dirs should be initialized here and set
    as globals in module scope. It may be called two times if restoring
    settings fails."""
    if not mopy_dirs_initialized:
        raise BoltError(u'init_dirs: Mopy dirs uninitialized')
    # Any warnings found during this stage can be added here as strings
    init_warnings = []
    #--Oblivion (Application) Directories
    dirs[u'app'] = game_info.gamePath
    dirs[u'defaultPatches'] = (
        dirs[u'mopy'].join(u'Bash Patches', game_info.bash_patches_dir)
        if game_info.bash_patches_dir else u'')
    dirs[u'taglists'] = dirs[u'mopy'].join(u'taglists', game_info.taglist_dir)
    # Determine the user's personal (i.e. My Documents) folder. Attempt to pull
    # from, in order:
    #  - CLI
    #  - bash.ini
    #  - Windows only:
    #    - SHGetKnownFolderPath
    #  - Linux only:
    #    - Proton prefix (For Windows games installed via Steam's Proton)
    #    - XDG_DOCUMENTS_DIR
    #    - ~/Documents
    dirs['personal'] = personal = _get_cli_ini_path(personal, '-p',
        'PersonalPath', game_info, get_personal_path,
        _('Failed to determine personal folder.'), _(
            'Personal folder does not exist: %(folder)s'))
    if game_info.uses_personal_folders:
        dirs[u'saveBase'] = personal.join(u'My Games', game_info.my_games_name)
    else:
        dirs[u'saveBase'] = dirs[u'app']
    deprint(f'My Games location set to {dirs[u"saveBase"]}')
    # Determine the user's AppData\Local (i.e. %LOCALAPPDATA%) folder. Attempt
    # to pull from, in order:
    #  - CLI
    #  - bash.ini
    #  - Windows only:
    #    - SHGetKnownFolderPath
    #  - Linux only:
    #    - Proton prefix (For Windows games installed via Steam's Proton)
    #    - XDG_DATA_HOME
    #    - ~/.local/share
    dirs['local_appdata'] = localAppData = _get_cli_ini_path(localAppData,
        '-l', 'LocalAppDataPath', game_info, get_local_app_data_path,
        _('Failed to determine LocalAppData folder.'),
        _('LocalAppData folder does not exist: %(folder)s'))
    # AppData for the game, depends on if it's a WS game or not.
    ws_info = get_legacy_ws_game_info(game_info)
    if ws_info.installed:
        version_info = ws_info.get_installed_version()
        dirs[u'userApp'] = localAppData.join(
            u'Packages', version_info.full_name, u'LocalCache', u'Local',
            game_info.appdata_name)
    else:
        dirs[u'userApp'] = localAppData.join(game_info.appdata_name)
    deprint(f'LocalAppData location set to {dirs[u"userApp"]}')
    # Use local copy of the oblivion.ini if present
    # see: http://en.uesp.net/wiki/Oblivion:Ini_Settings
    # Oblivion reads the Oblivion.ini in the directory where it exists
    # first, and only if bUseMyGamesDirectory is non-existent or set to 1 does
    # it then look for My Documents\My Games\Oblivion.ini. In other words,
    # both can exist simultaneously, and only the value of bUseMyGamesDirectory
    # in the Oblivion.ini directory where Oblivion.exe is run from will
    # actually matter.
    # Utumno: not sure how/if this applies to other games
    first_ini_name = game_info.Ini.dropdown_inis[0]
    data_oblivion_ini = dirs[u'app'].join(first_ini_name)
    game_ini_path = dirs[u'saveBase'].join(first_ini_name)
    dirs[u'mods'] = dirs[u'app'].join(game_info.mods_dir)
    if data_oblivion_ini.is_file():
        ##: use GameIni here
        oblivionIni = ConfigParser(allow_no_value=True, strict=False)
        try:
            try:
                # Try UTF-8 first, will also work for ASCII-encoded files
                oblivionIni.read(data_oblivion_ini, encoding='utf8')
            except UnicodeDecodeError:
                # No good, this is a nonstandard encoding
                with data_oblivion_ini.open(u'rb') as ins:
                    ini_contents = ins.read()
                oblivionIni.read_file(io.StringIO(decoder(ini_contents)))
        except MissingSectionHeaderError:
            # Probably not actually a game INI - might be reshade
            init_warnings.append(_(
                'The global INI file in your game directory (%(global_ini)s) '
                'does not appear to be a valid game INI. It might come from '
                'an incorrectly installed third party tool. Consider deleting '
                'it and validating your game files.') % {
                'global_ini': data_oblivion_ini})
        # is bUseMyGamesDirectory set to 0?
        if _get_ini_option(oblivionIni, 'bUseMyGamesDirectory') == '0':
            game_ini_path = data_oblivion_ini
            # Set the save game folder to the Oblivion directory
            dirs[u'saveBase'] = dirs[u'app']
            # Set the data folder to sLocalMasterPath
            dirs['mods'] = dirs['app'].join(_get_ini_option(oblivionIni,
                u'SLocalMasterPath') or game_info.mods_dir)
    # Check and warn if the Data folder is case-sensitive
    if is_case_sensitive(dirs['mods']):
        ci_warn = _(
            'The %(data_folder)s folder is case sensitive. This will cause '
            'serious problems for Wrye Bash, like BAIN not working if the '
            'case differs between a mod-added file and an existing version of '
            'that file in the Data folder.') % {
            'data_folder': game_info.mods_dir}
        init_warnings.append(ci_warn + '\n\n' + get_case_sensitivity_advice())
    # these are relative to the mods path so they must be set here
    dirs[u'patches'] = dirs[u'mods'].join(u'Bash Patches')
    dirs[u'tag_files'] = dirs[u'mods'].join(u'BashTags')
    dirs[u'ini_tweaks'] = dirs[u'mods'].join(u'INI Tweaks')
    #--Mod Data, Installers
    oblivionMods, oblivionModsSrc = getOblivionModsPath(game_info)
    dirs[u'bash_root'] = oblivionMods
    deprint(f'Game Mods location set to {oblivionMods}')
    dirs['modsBash'], modsBashSrc = _get_ini_path('BashModData', 'bash_root',
                                                  'Bash Mod Data')
    if game_info.check_legacy_paths:
        mpath = dirs['modsBash']
        old_path = dirs['app'].join(game_info.mods_dir, 'Bash')
        if not mpath.is_dir() and old_path.is_dir():
            dirs['modsBash'], modsBashSrc = old_path, 'Relative path'
    deprint(f'Bash Mod Data location set to {dirs[u"modsBash"]}')
    dirs[u'installers'] = oblivionMods.join(u'Bash Installers')
    if game_info.check_legacy_paths:
        ipath = dirs['installers']
        old_path = dirs['app'].join('Installers')
        dirs['installers'] = (old_path, ipath)[
            ipath.is_dir() or not old_path.is_dir()]
    deprint(f'Installers location set to {dirs[u"installers"]}')
    dirs['bainData'], bainDataSrc = _get_ini_path('InstallersData',
                                                  'installers', 'Bash')
    deprint(f'Installers bash data location set to {dirs[u"bainData"]}')
    dirs[u'bsaCache'] = dirs[u'bainData'].join(u'BSA Cache')
    dirs[u'converters'] = dirs[u'installers'].join(u'Bain Converters')
    dirs[u'dupeBCFs'] = dirs[u'converters'].join(u'--Duplicates')
    dirs[u'corruptBCFs'] = dirs[u'converters'].join(u'--Corrupt')
    # create bash user folders, keep these in order
    dir_keys = (u'modsBash', u'installers', u'converters', u'dupeBCFs',
                u'corruptBCFs', u'bainData', u'bsaCache')
    deprint(u'Checking if WB directories exist and creating them if needed:')
    try:
        for dir_key in dir_keys:
            wanted_dir = dirs[dir_key]
            deprint(f' - {wanted_dir}')
            shellMakeDirs([wanted_dir])
    except NotADirectoryError as e:
        # NotADirectoryError is thrown by shellMakeDirs if any of the
        # directories cannot be created due to residing on a non-existing
        # drive (in posix if permission is denied). Find which keys are
        # causing the errors
        msg = _dirs_err_msg(e, dir_keys, bainDataSrc, modsBashSrc,
                            oblivionMods, oblivionModsSrc)
        raise BoltError(msg)
    loot_gname = game_info.loot_dir
    loot_folder = dirs['local_appdata'].join('LOOT')
    # Since LOOT v0.18, games are stored in LOOT\games\<game>, try that first
    loot_path = loot_folder.join('games', loot_gname)
    if not loot_path.is_dir():
        # Fall back to the 'legacy' path (LOOT\<game>)
        loot_path = loot_folder.join(loot_gname)
    loot_master_path = loot_path.join('masterlist.yaml')
    loot_user_path = loot_path.join('userlist.yaml')
    loot_tag_path = dirs['taglists'].join('taglist.yaml')
    global lootDb
    lootDb = LOOTParser(loot_master_path, loot_user_path, loot_tag_path)
    global bash_dirs_initialized
    bash_dirs_initialized = True
    return game_ini_path, init_warnings

def _dirs_err_msg(e, dir_keys, bainDataSrc, modsBashSrc, oblivionMods,
                  oblivionModsSrc):
    badKeys = set()  # List of dirs[key] items that are invalid
    # First, determine which dirs[key] items are causing it
    for dir_key in dir_keys:
        if dirs[dir_key] in e.failed_paths:
            badKeys.add(dir_key)
    # Now, work back from those to determine which setting created those
    if os_name == 'posix':
        m = _("Please check the settings for the following paths in your "
              "bash.ini, the drive does not exist or you don't have write "
              "permissions")
    else:
        m = _(u'Please check the settings for the following paths in your '
              u'bash.ini, the drive does not exist')
    msg = _(u'Error creating required Wrye Bash directories.') + f'  {m}:\n\n'
    relativePathError = []
    if u'modsBash' in badKeys:
        if isinstance(modsBashSrc, list):
            msg += (' '.join(modsBashSrc) + f'\n    {dirs[u"modsBash"]}\n')
        else:
            relativePathError.append(dirs[u'modsBash'])
    if {u'installers', u'converters', u'dupeBCFs', u'corruptBCFs'} & badKeys:
        # All derived from oblivionMods -> getOblivionModsPath
        if isinstance(oblivionModsSrc, list):
            msg += (u' '.join(oblivionModsSrc) + f'\n    {oblivionMods}\n')
        else:
            relativePathError.append(oblivionMods)
    if {u'bainData', u'bsaCache'} & badKeys:
        # Both derived from 'bainData' -> getBainDataPath
        # Sometimes however, getBainDataPath falls back to oblivionMods,
        # So check to be sure we haven't already added a message about that
        if bainDataSrc != oblivionModsSrc:
            if isinstance(bainDataSrc, list):
                msg += u' '.join(bainDataSrc) + f'\n    {dirs[u"bainData"]}\n'
            else:
                relativePathError.append(dirs[u'bainData'])
    if relativePathError:
        msg += u'\n' + _(u'A path error was the result of relative paths.')
        msg += u'  ' + _(u'The following paths are causing the errors, '
                         u'however usually a relative path should be fine.')
        msg += u'  ' + _(u'Check your setup to see if you are using '
                         u'symbolic links or NTFS Junctions') + u':\n\n'
        msg += u'\n'.join([f'{x}' for x in relativePathError])
    return msg

def init_dirs_mopy():
    dirs[u'mopy'] = Path.getcwd()
    dirs[u'bash'] = dirs[u'mopy'].join(u'bash')
    dirs[u'compiled'] = dirs[u'bash'].join(u'compiled')
    dirs[u'l10n'] = dirs[u'bash'].join(u'l10n')
    dirs[u'db'] = dirs[u'bash'].join(u'db')
    dirs[u'templates'] = dirs[u'mopy'].join(u'templates')
    dirs[u'images'] = dirs[u'bash'].join(u'images')
    from . import archives
    if os_name == u'nt': # don't add local directory to binaries on linux
        archives.exe7z = dirs[u'compiled'].join(archives.exe7z).s
    global mopy_dirs_initialized
    mopy_dirs_initialized = True

def getLocalSaveDirs():
    """Return a list of possible local save directories, NOT including the
    base directory."""
    baseSaves = dirs[u'saveBase'].join(u'Saves')
    # Path.ilist returns [] for non existent dirs
    localSaveDirs = [x for x in top_level_dirs(baseSaves) if
                     x not in (u'Bash', u'Mash')]
    # Filter out non-encodable names
    bad = set()
    for folder in localSaveDirs:
        try:
            folder.encode(u'cp1252')
        except UnicodeEncodeError:
            bad.add(folder)
    localSaveDirs = sorted(x for x in localSaveDirs if x not in bad)
    return localSaveDirs


'''
Longest consecutive subsequence

Given an array of positive integers.
Find the length of the longest sub-sequence such that elements in the subsequence are consecutive integers,
the consecutive numbers can be in any order.

Input:
N = 7
a[] = {2,6,1,9,4,5,3}
Output:
6
Explanation:
The consecutive numbers here
are 1, 2, 3, 4, 5, 6. These 6
numbers form the longest consecutive
subsquence.
'''

class LongestConsecutiveSubSequence:
    def findLongestConsecutiveSubSequence(self, arr):
        arr.sort()
        print(arr)
        maxSubSequence = 1
        currSubSequence = 1
        for i in range(1,len(arr)):
            if(arr[i] == arr[i-1]):
                continue;
            elif(arr[i]-1 == arr[i-1]):
                currSubSequence += 1
            else:
                currSubSequence = 1
            if(maxSubSequence < currSubSequence):
                maxSubSequence = currSubSequence
        return maxSubSequence

if __name__ == '__main__':
    # arr = [2,6,1,9,4,5,3]
    arr = [6, 6, 2, 3, 1, 4, 1, 5, 6, 2, 8, 7, 4, 2, 1, 3, 4, 5, 9, 6]
    obj = LongestConsecutiveSubSequence()
    print(obj.findLongestConsecutiveSubSequence(arr))


import torch
import torch.nn.functional as F
import sys
from torch_geometric.nn import MessagePassing
import matplotlib.pyplot as plt
from torch_geometric.explain import Explainer, Explanation
from torch_geometric.explain.algorithm.utils import clear_masks
from torch_geometric.explain.config import MaskType, ModelMode, ModelReturnType
from torch_geometric.explain.metric import unfaithfulness, fidelity

def evaluate_unfaithfulness(explainer: Explainer, explainer_name: str, evaluation_dataset):
    combined_unf = []
    benchmark = []
    for graph in evaluation_dataset:
        batch = torch.zeros(len(graph.x), dtype=torch.int64)
        explanation = explainer(graph.x, graph.edge_index, batch=batch)
        unf, benchmark_unf = my_unfaithfulness(explainer, explanation)
        print(f"unfaithfulness: {unf}\n")
        combined_unf.append(unf)
        benchmark.append(benchmark_unf)
    plt.boxplot([combined_unf, benchmark],labels=[explainer_name, "Random"])
    plt.title(f"Unfaitfulness by {explainer_name}")
    plt.show()

def evaluate_fidelity(explainers: list[Explainer], explainer_names: list[str], evaluation_dataset, dataset_name):

    fid_pluses = [[] for _ in range(len(explainers) + 1)]
    fid_minuses = [[] for _ in range(len(explainers) + 1)]

    explainer_names.append("Random")

    for graph in evaluation_dataset:
        for i, explainer in enumerate(explainers):
            batch = torch.zeros(len(graph.x), dtype=torch.int64)
            explanation = explainer(graph.x, graph.edge_index, batch=batch, target=graph.y)
            fid_plus, fid_minus = fidelity(explainer, explanation)
            fid_pluses[i].append(fid_plus)
            fid_minuses[i].append(fid_minus)

        random_edge_mask = torch.rand(explanation.get("edge_mask").shape)
        explanation.edge_mask = random_edge_mask
        fid_plus, fid_minus = fidelity(explainer, explanation)
        fid_pluses[len(explainers)].append(fid_plus)
        fid_minuses[len(explainers)].append(fid_minus)

    fig, ax = plt.subplots(1, 3, figsize=(15, 5))
    fig.suptitle(f"Fidelity evaluation of {dataset_name}")
    ax[0].boxplot(fid_pluses, labels=explainer_names)
    ax[0].set_title(f"Fidelity+")
    ax[1].boxplot(fid_minuses, labels=explainer_names)
    ax[1].set_title(f"Fidelity-")

    average_fid_plus = [sum(fidp)/len(fidp) for fidp in fid_pluses]
    one_minus_average_fid_minus = [1 - sum(fidm)/len(fidm) for fidm in fid_minuses]

    for i, explainer_name in enumerate(explainer_names):
        ax[2].scatter(average_fid_plus[i], one_minus_average_fid_minus[i], marker="X", label=f"{explainer_name}: ({average_fid_plus[i]}, {one_minus_average_fid_minus[i]})")
    ax[2].set_title(f"Correlation")
    ax[2].set_ylabel("1 - Fidelity-")
    ax[2].set_xlabel("Fidelity+")
    ax[2].set_ylim(bottom=0)
    ax[2].legend()
    plt.savefig(f"../explanation_visualizations/fidelity_{dataset_name}")
    plt.show()

def my_unfaithfulness(explainer: Explainer, explanation, top_k = 0.2, multi_label=False):
    if explainer.model_config.mode == ModelMode.regression:
        raise ValueError("Fidelity not defined for 'regression' models")

    if top_k is not None and explainer.node_mask_type == MaskType.object:
        raise ValueError("Cannot apply top-k feature selection based on a "
                         "node mask of type 'object'")

    node_mask = explanation.get('node_mask')
    edge_mask = explanation.get('edge_mask')
    edge_mask = convert_to_thresholded_edge_mask(edge_mask, top_k)
    print(f"Edge mask: {edge_mask}")
    x, edge_index = explanation.x, explanation.edge_index
    print(explanation._model_args)
    kwargs = {key: explanation[key] for key in explanation._model_args}

    y = explanation.get('prediction')
    if y is None:  # == ExplanationType.phenomenon
        y = explainer.get_prediction(x, edge_index, **kwargs)

    if node_mask is not None and False:
        feat_importance = node_mask.sum(dim=0)
        _, top_k_index = feat_importance.topk(top_k)
        node_mask = torch.zeros_like(node_mask)
        node_mask[:, top_k_index] = 1.0

    y_hat = explainer.get_masked_prediction(x, edge_index, node_mask, edge_mask, **kwargs)

    #random explanation
    random_node_mask = torch.rand(node_mask.shape) if node_mask is not None else None
    random_edge_mask = torch.rand(edge_mask.shape)
    random_edge_mask = convert_to_thresholded_edge_mask(random_edge_mask, top_k)
    print(f"Random mask: {random_edge_mask}")
    benchmark_y_hat = explainer.get_masked_prediction(x, edge_index, random_node_mask, random_edge_mask, **kwargs)

    if explanation.get('index') is not None:
        y, y_hat = y[explanation.index], y_hat[explanation.index]

    if explainer.model_config.return_type == ModelReturnType.raw and not multi_label:
        y, y_hat = y.sigmoid(), y_hat.sigmoid()
        benchmark_y_hat = benchmark_y_hat.sigmoid()
        print(f"y: {y} | y_hat: {y_hat} | benchmark_y_hat: {benchmark_y_hat}")
    elif explainer.model_config.return_type == ModelReturnType.raw and multi_label:
        y, y_hat = y.sigmoid(dim=-1), y_hat.softmax(dim=-1)
    elif explainer.model_config.return_type == ModelReturnType.log_probs:
        y, y_hat = y.exp(), y_hat.exp()

    # kl_div = F.kl_div(y_hat.log(), y, reduction="batchmean")
    loss = F.mse_loss(y_hat, y)
    benchmark_loss = F.mse_loss(benchmark_y_hat, y)
    print(f"loss: {loss} vs benchmark: {benchmark_loss}")
    return loss, benchmark_loss

def convert_to_thresholded_edge_mask(mask, topk):
    topk_num = int(len(mask) * topk)
    threshold = torch.sort(mask)[0][-topk_num]
    edge_mask = (mask>threshold).float()
    return edge_mask