import copy
import pickle
import random
from time import time
from gensim import matutils
from math import exp, log, inf, sqrt
import numpy as np
from sklearn.cluster import estimate_bandwidth, MeanShift

from utilitarian import Progresser
from scipy.stats import gaussian_kde
from sklearn.neighbors.kde import KernelDensity

class SuperPredictor:
    """Super Predictor

    Parameters
    ----------
    super_words : dict
        super_words[year][cat][context] is the frequency of that category-context pair in that year.
        Only the new context words should be in this dictionary.

    path : string
        The path to save the pickle file of predictions.

    start : int
        The year to start considering the words.

    threshold: int
        The year to start predicting the words.

    end : int
        The ending year.

    word_vectors : dict
        dictionary of words to their vector representations

    word_vector_yby: dict
        dictionary of years to word vectors in case of LDA and PCA dimension reduction in each year.

    swc : dict
        super-words-complete. a version of the super-words that all the context words and their frequencies are present,
        not just the new ones.

    new_words : dict
        A dictionary of years and the newly appearing words in that year.
    """

    def __init__(self, super_words, path, start, threshold, end, step, hbc, word_vectors, word_vector_yby, swc=None):
        self.word_vectors = word_vectors
        self.super_words = super_words
        self.hbc = hbc
        self.path = path
        self.kernel_width = None
        self.categories = super_words[threshold].keys()
        self.threshold = threshold
        self.start = start
        self.end = end
        self.step = step
        self.new_words = None
        self.predictions = None
        self.prior_dist = None
        self.cat_sim_method = None
        self.vec_sim_method = None
        self.super_words_so_far = None  # super words until current year for each category
        self.word_freq_so_far = None
        self.prg = None
        self.word_vectors_yby = word_vector_yby
        self.kde_functions = dict()
        self.super_words_c = swc
        self.prototypes = None

        # initiate frequency of each category in each year
        self.frequency = dict()
        for yr in range(start, end, step):
            self.frequency[yr] = dict()
            for cat in self.categories:
                self.frequency[yr][cat] = sum(self.super_words_c[yr][cat].values())

        # initialise new word list for each year
        self.new_words = dict()
        for yr in range(start, end, step):
            self.new_words[yr] = set()
            for cat, wrds in self.super_words[yr].items():
                self.new_words[yr] = self.new_words[yr] | wrds.keys()

    def predict_them_all(self, prior_dist, cat_sim_method, vec_sim_method, kw=1.0):
        """
        Predicts for new words of all years from the threshold until the end.

        Parameters
        ----------
        prior_dist : string
            valid prior distributions are
            ['uniform'|'frequency'|'items']

        cat_sim_method : string
            valid category similarity methods are
            ['one'|'gkde'|'nn'|'avg'|'wavg'|'eavg'|
            'tnn'|'tnn-avg'|'*nn'|'*tnn'|
            'pt0-avg'|'pt-avg'|'pt0-mode'|'pt-mode']

        vec_sim_method: string
            valid vector similarity methods are
            ['cos'|'euc'|'exp_euc'|'exp_euc_sq'|'hbc']

        kw: float or dict, (default 1.0)
            kernel width for exp_euc and exp_euc_sq similarity methods
        """

        t = time()
        if type(kw) == float or type(kw) == int or type(kw) == str:
            kw = float(kw)
            self.kernel_width = dict()
            for year in range(self.start, self.end, self.step):
                self.kernel_width[year] = kw
            kw_name = str(kw)
        else:
            self.kernel_width = kw
            kw_name = 'adj'
        self.prior_dist = prior_dist
        self.cat_sim_method = cat_sim_method
        self.vec_sim_method = vec_sim_method

        # initialise super words until current year for each category
        if 'tnn' not in cat_sim_method:
            self.__init_super_words_so_far()

        # iterate on years and predict for each one
        self.prg = Progresser(sum([len(self.new_words[yr]) for yr in range(self.threshold, self.end, self.step)]),
                              msg='predicting for ' + prior_dist + '-' + cat_sim_method + '-' +
                                  vec_sim_method + '_' + kw_name)

        # do the predictions for each year
        self.predictions = dict()
        for year in range(self.threshold, self.end, self.step):
            self.__predict_one_step(year)

        # write the results to file
        with open('./predictions/' + self.path + '/' + prior_dist + '-' + cat_sim_method + '-' + vec_sim_method +
                  '_' + kw_name + '.pkl', 'wb') as prediction_file:
            pickle.dump(self.predictions, prediction_file)

        load_time = time() - t
        print('\t prediction for ' + prior_dist + '-' + cat_sim_method + '-' + vec_sim_method + '_' + kw_name +
              ' done in', int(round(load_time / 60, 0)), ':', int(round(load_time % 60, 0)), 'sec')

    def __init_super_words_so_far(self):
        self.super_words_so_far = dict()
        for cat in self.categories:
            self.super_words_so_far[cat] = set()
        for yr in range(self.start, self.threshold, self.step):
            for cat, wrds in self.super_words[yr].items():
                self.super_words_so_far[cat] = self.super_words_so_far[cat] | wrds.keys()

        if 'pt-wavg' in self.cat_sim_method:
            self.word_freq_so_far = dict()
            for cat in self.categories:
                self.word_freq_so_far[cat] = dict()
            for yr in range(self.start, self.threshold, self.step):
                for cat, wrds in self.super_words[yr].items():
                    for w, f in wrds.items():
                        try:
                            self.word_freq_so_far[cat][w] += f
                        except:
                            self.word_freq_so_far[cat][w] = f

    def __update_prototypes(self, yr):
        self.prototypes = dict()
        for cat in self.categories:
            X = []
            if 'wavg' in self.cat_sim_method:
                ws = []
            for word in self.super_words_so_far[cat]:
                X.append(self.word_vectors[word])
                try:
                    ws.append(self.super_words[yr][cat][word])
                except KeyError:
                    ws.append(0)
                except:
                    continue

            pt_vec = np.nan
            if len(X) == 0:
                pass
            elif 'wavg' in self.cat_sim_method:
                if sum(ws) == 0:
                    pt_vec = np.mean(X, axis=0)
                else:
                    pt_vec = np.average(X, axis=0, weights=ws)
            elif 'avg' in self.cat_sim_method:
                pt_vec = np.mean(X, axis=0)
            elif 'mode' in self.cat_sim_method:
                try:
                    bandwidth = estimate_bandwidth(X, quantile=0.2, n_samples=10000)
                    ms = MeanShift(bandwidth=bandwidth, bin_seeding=True)
                    ms.fit(X)
                    labels_unique, l_counts = np.unique(ms.labels_, return_counts=True)
                    best_label = labels_unique[np.where(l_counts == max(l_counts))]
                    pt_vec = ms.cluster_centers_[best_label]
                except:
                    # if there are any errors just use mean instead
                    pt_vec = np.mean(X, axis=0)

            # print(pt_vec)
            self.prototypes[cat] = pt_vec

    def __predict_one_step(self, year):
        """
        update the super_words_so_far with new words if needed, prepare the word vector of this year if it is year by
        year and calculate the prior for each category and the similarities for each word.

        Parameters
        ----------
        year: (int)
            predict for this year
        """
        # prepare w2v list for this year if the vectors are Year by Year
        if self.word_vectors_yby is not None:
            self.word_vectors = self.word_vectors_yby[year]

        # update super words until now if the method is not based on only the immediate previous year
        if 'tnn' not in self.cat_sim_method:
            for cat, wrds in self.super_words[year - self.step].items():
                self.super_words_so_far[cat] = self.super_words_so_far[cat] | wrds.keys()
            if 'pt-wavg' in self.cat_sim_method:
                for cat, wrds in self.super_words[year - self.step].items():
                    for w, f in wrds.items():
                        try:
                            self.word_freq_so_far[cat][w] += f
                        except:
                            self.word_freq_so_far[cat][w] = f

        # initialise prototypes for each category in this year
        if 'pt' in self.cat_sim_method:
            if '0' not in self.cat_sim_method:
                self.__update_prototypes(yr=year - self.step)
            elif self.prototypes is None:
                self.__update_prototypes(yr=year - self.step)

        # prepare kde functions
        if self.cat_sim_method == 'gkde':
            ne = len(self.new_words[year])
            ol = sum(len(ws) for ws in self.super_words_so_far.values())
            print(year, '\t', '%' + str(int(100 * ne / (ol + ne))), ne, '\t', '%' + str(int(100 * ol / (ol + ne))), ol)
            for cat in self.categories:

                vectors = [self.word_vectors[w] for w in self.super_words_so_far[cat]]
                if len(vectors) > 1:
                    self.kde_functions[cat] = KernelDensity(kernel='gaussian', bandwidth=0.2).fit(vectors)
                else:
                    self.kde_functions[cat] = None

        # p_c = self.__calculate_prior(year=year - self.step,log_res=True)
        p_c = self.__calculate_prior(year=year - self.step, log_res=False)

        p_c_i = dict()  # P(category | item) ~ P(category) * P(item | category)
        # initialise the 2D dict
        for category in self.categories:
            p_c_i[category] = dict()

        for item in self.new_words[year]:
            self.prg.count()
            # standard knn methods:
            if self.cat_sim_method.startswith('s') and self.cat_sim_method.endswith('nn'):
                k = int(self.cat_sim_method[1:-2])
                max_sim = [(0, '') for _ in range(k)]
                for cat in self.categories:
                    for word in self.super_words_so_far[cat]:
                        try:
                            sim = self.__vector_similarity(word, item, year=year)
                            if sim > max_sim[0][0]:
                                max_sim[0] = (sim, cat)
                            max_sim = sorted(max_sim, key=lambda t: t[0])
                        except Exception as e:
                            print(10, e)
                nn_count = {cat: 0 for cat in self.categories}
                max_n = 0
                for x in max_sim:
                    nn_count[x[1]] += 1
                    max_n = max(max_n, nn_count[x[1]])
                best_cats = [cat for cat in self.categories if nn_count[cat] == max_n]
                if self.prior_dist == 'uniform':
                    best_cat = random.choice(best_cats)
                elif self.prior_dist == 'items' or self.prior_dist == 'frequency':
                    best_cats = {bc: p_c[bc] for bc in best_cats}
                    best_cat = sorted(best_cats.items(), key=lambda kv: kv[1])[-1][0]

                for category in self.categories:
                    p_c_i[category][item] = 0
                p_c_i[best_cat][item] = 1
                # TODO: check this
            else:  # other methods
                # for category in self.categories:
                #     # don't predict for categories that had it before
                #     if item in self.super_words_so_far[category]:
                #         p_c_i[category][item] = - inf
                #     else:
                #         # the values are in logarithm so the multiplication would be the sum of the two values
                #         p_c_i[category][item] = p_c[category] + self.__get_similarity(item, year - self.step, category)
                # TODO: the next lines or the previous lines
                sum_over_categories = 0.0
                for category in self.categories:
                    if len(self.super_words_so_far[category]) == 0:
                        p_c_i[category][item] = 0
                    # don't predict for categories that had it before
                    if item in self.super_words_so_far[category]:
                        p_c_i[category][item] = 0
                    else:
                        # the values are in logarithm so the multiplication would be the sum of the two values
                        p_c_i[category][item] = p_c[category] * self.__get_similarity(item, year - self.step, category,
                                                                                False)
                        sum_over_categories += p_c_i[category][item]

                for category in self.categories:
                    if len(self.super_words_so_far[category]) == 0 or item in self.super_words_so_far[category]:
                        p_c_i[category][item] = -inf
                    else:
                        # p_c_i[category][item] = log(p_c_i[category][item]) - log(sum_over_categories)

                        try:
                            p_c_i[category][item] = log(p_c_i[category][item]) - log(sum_over_categories)
                        except:
                            p_c_i[category][item] = -inf
                            # a=self.__get_similarity(item, year - self.step, category, False)
                            # a=self.__get_similarity(item, year - self.step, category, False)
                            # p_c_i[item][category] = -1000000000.0

        self.predictions[year] = p_c_i

    def set_params(self, prior_dist, cat_sim_method, vec_sim_method):
        self.prior_dist = prior_dist
        self.cat_sim_method = cat_sim_method
        self.vec_sim_method = vec_sim_method

    def build_super_words_until(self):
        self.super_words_until = {self.threshold - 2 * self.step: {}}
        for cat in self.categories:
            self.super_words_until[self.threshold - 2 * self.step][cat] = set()
        for yr in range(self.start, self.threshold - self.step):
            for cat, wrds in self.super_words[yr].items():
                self.super_words_until[self.threshold - 2 * self.step][cat] = \
                    self.super_words_until[self.threshold - 2 * self.step][cat] | wrds.keys()
        for yr in range(self.threshold - self.step, self.end, self.step):
            self.super_words_until[yr] = copy.deepcopy(self.super_words_until[yr - self.step])
            for cat, wrds in self.super_words[yr].items():
                self.super_words_until[yr][cat] = self.super_words_until[yr][cat] | wrds.keys()

    def log_likelihood_of_posterior(self, year, kw):
        """
        to find the best kernel width ...
        """
        kw = float(kw)
        self.kernel_width = dict()
        for y in range(self.start, self.end, self.step):
            self.kernel_width[y] = kw
        # self.prg = Progresser(len(self.new_words[year]),
        #                       msg='llp for ' + self.prior_dist + '-' + self.cat_sim_method + '-' + self.vec_sim_method
        #                           + '_' + str(self.kernel_width[year]) + 'y:' + str(year))

        # prepare w2v list for this year if the vectors are Year by Year
        if self.word_vectors_yby is not None:
            self.word_vectors = self.word_vectors_yby[year]

        # update super words until now if the method is not based on only the immediate previous year
        try:
            self.super_words_so_far = self.super_words_until[year - self.step]
        except:
            self.build_super_words_until()
            self.super_words_so_far = self.super_words_until[year - self.step]

        p_c = self.__calculate_prior(year=year - self.step, log_res=False)

        p_c_i = dict()  # P(category | item) ~ P(category) * P(item | category)
        posterior_list = []

        # For each new noun
        for item in self.new_words[year]:
            p_c_i[item] = dict()
            true_cats = []
            sum_over_categories = 0.0
            for category in self.categories:
                if len(self.super_words_so_far[category]) == 0:
                    continue
                # don't count categories that had it before
                # if item in self.super_words_so_far[category]:
                #     p_c_i[item][category] = 0
                # else:
                # the values are in logarithm so the multiplication would be the sum of the two values
                p_c_i[item][category] = p_c[category] * self.__get_similarity(item, year - self.step, category, False)
                sum_over_categories += p_c_i[item][category]
                if item in self.super_words[year][category]:
                    true_cats.append(p_c_i[item][category])

            for x in true_cats:
                try:
                    posterior_list.append(log(x) - log(sum_over_categories))
                except:
                    print('EXCEPTION IN LOG POSTERIOR!')

            # self.prg.count()

        # log_of_posteriors = sum(posterior_list)
        # print('length of the list:', len(posterior_list))
        log_of_posteriors = np.mean(posterior_list)

        return log_of_posteriors

    def __calculate_prior(self, year=None, log_res=True):
        """
        Parameters
        ----------
        year : (int)
            For the case of frequency based prior.

        Returns
        -------
        p_c : (dict)
            A dictionary of the categories and the logarithm of their prior.

        """
        p_c = dict()  # prior: P(category)

        # first methods - probability of each category with uniform distribution
        if self.prior_dist == 'uniform':
            for category in self.categories:
                p_c[category] = 1

        # second method - probability of each category based on frequency
        elif self.prior_dist == 'frequency':
            for category in self.categories:
                p_c[category] = 0
                for yr in range(self.start, year + self.step, self.step):
                    p_c[category] += sum(self.super_words[yr][category].values())

        elif self.prior_dist == 'frequency_sqr':
            for category in self.categories:
                p_c[category] = 0
                for yr in range(self.start, year + self.step, self.step):
                    p_c[category] += sum([sqrt(a) for a in self.super_words[yr][category].values()])

        elif self.prior_dist == 'frequency_log':
            for category in self.categories:
                p_c[category] = 0
                for yr in range(self.start, year + self.step, self.step):
                    p_c[category] += sum([log(a) for a in self.super_words[yr][category].values()])

        # third method - based on longer list of items
        elif self.prior_dist == 'items':
            for category in self.categories:
                p_c[category] = len(self.super_words_so_far[category])
        else:
            raise Exception('Prior calculation method not defined!')

        if log_res:
            # convert to log(p_c)
            for key in p_c.keys():
                if p_c[key] == 0:
                    p_c[key] = - inf
                else:
                    p_c[key] = log(p_c[key])
        return p_c

    def __get_similarity(self, item, year, category, log_res=True):
        """
        Parameters
        ----------
        item : (string)
            The word which its similarity is needed.

        category : (string)
            The category which the item is going to be compared for getting the similarity.

        year : (int)
            For the case of frequency based prior and kernel widths.

        Returns
        -------
        similarity : (float)
            The logarithm of the similarity between the word and the category.

        """
        similarity = 0
        if self.cat_sim_method == 'one':
            similarity = 1
        elif self.cat_sim_method == 'gkde':
            if self.kde_functions[category] is None:
                a = 0
            else:
                # a = self.kde_functions[category].evaluate(self.word_vectors[item])[0]
                a = self.kde_functions[category].score_samples([self.word_vectors[item]])[0]
            # print('--', a)
            # if a == -inf or a == np.nan or a == inf:
            #     a = 0
            return a
        elif self.cat_sim_method == 'nn':
            # nearest neighbour method
            max_similarity = 0
            for word in self.super_words_so_far[category]:
                try:
                    sim = self.__vector_similarity(word, item, year=year)
                    max_similarity = max(sim, max_similarity)
                except Exception as e:
                    print('1', e)
            similarity = max_similarity

        elif self.cat_sim_method == 'avg':
            # average similarity method
            similarity_sum = 0
            words_count = 0
            for word in self.super_words_so_far[category]:
                try:
                    similarity_sum += self.__vector_similarity(word, item, year=year)
                    words_count += 1
                except Exception as e:
                    print('2', e)
            similarity = similarity_sum / words_count if words_count != 0 else 0

        elif self.cat_sim_method == 'wavg':
            # weighted average similarity method
            similarity_sum = 0
            for word in self.super_words_c[year][category].keys():
                try:
                    similarity_sum += self.__vector_similarity(word, item, year=year) * \
                                      self.super_words_c[year][category][word]
                except Exception as e:
                    print('3', e)
            similarity = similarity_sum / self.frequency[year][category] if self.frequency[year][
                                                                                category] != 0 else 0
        elif self.cat_sim_method == 'wavg_sqr':
            # weighted average similarity method
            similarity_sum = 0
            for word in self.super_words_c[year][category].keys():
                try:
                    similarity_sum += self.__vector_similarity(word, item, year=year) * \
                                      sqrt(self.super_words_c[year][category][word])
                except Exception as e:
                    print('4', e)
            total_freq = sum([sqrt(a) for a in self.super_words[year][category].values()])
            similarity = similarity_sum / total_freq if total_freq != 0 else 0

        elif self.cat_sim_method == 'wavg_log':
            # weighted average similarity method
            similarity_sum = 0
            for word in self.super_words_c[year][category].keys():
                try:
                    similarity_sum += self.__vector_similarity(word, item, year=year) * \
                                      log(self.super_words_c[year][category][word])
                except Exception as e:
                    print('5', e)
            total_freq = sum([log(a) for a in self.super_words[year][category].values()])
            similarity = similarity_sum / total_freq if total_freq != 0 else 0

        elif self.cat_sim_method == 'eavg':
            # average similarity method except the 0 frequencies (not use hitherto)
            similarity_sum = 0
            words_count = 0
            for word in self.super_words_c[year][category].keys():
                try:
                    similarity_sum += self.__vector_similarity(word, item, year=year)
                    words_count += 1
                except Exception as e:
                    print('6', e)
            similarity = similarity_sum / words_count if words_count != 0 else 0

        elif self.cat_sim_method == 'tnn':
            # nearest neighbour method for the last decade
            max_similarity = 0
            for word in self.super_words[year][category]:
                try:
                    if word == item:
                        continue
                    sim = self.__vector_similarity(word, item, year=year)
                    max_similarity = max(sim, max_similarity)
                except Exception as e:
                    print('7', e)
            similarity = max_similarity

        elif self.cat_sim_method == 'tnn_avg':
            # average similarity method for the last decade
            similarity_sum = 0
            words_count = 0
            for word in self.super_words[year][category]:
                try:
                    if word == item:
                        continue
                    similarity_sum += self.__vector_similarity(word, item, year=year)
                    words_count += 1
                except Exception as e:
                    print('8', e)
            similarity = similarity_sum / words_count if words_count != 0 else 0

        elif 'tnn' in self.cat_sim_method:
            # avg of K nearest neighbours in each category in the last time point
            max_sim = [0 for _ in range(int(self.cat_sim_method[:-3]))]
            for word in self.super_words[year][category]:
                try:
                    if word == item:
                        continue
                    sim = self.__vector_similarity(word, item, year=year)

                    max_sim[0] = max(sim, max_sim[0])
                    max_sim = sorted(max_sim)
                except Exception as e:
                    print('9', e)
            similarity = np.mean(max_sim)

        elif 'nn' in self.cat_sim_method:
            # avg of K nearest neighbours in each category
            max_sim = [0 for _ in range(int(self.cat_sim_method[:-2]))]
            for word in self.super_words_so_far[category]:
                try:
                    sim = self.__vector_similarity(word, item, year=year)

                    max_sim[0] = max(sim, max_sim[0])
                    max_sim = sorted(max_sim)
                except Exception as e:
                    print(10, e)
            similarity = np.mean(max_sim)
        elif 'pt' in self.cat_sim_method:
            similarity = self.__vector_similarity(w1=item, v2=self.prototypes[category], year=year)
        else:
            raise Exception('Category similarity method not defined!')

        if log_res:
            if similarity == 0:
                return - inf
            return log(similarity) - log(self.kernel_width[year + self.step])  # TODO: I changed this
        else:
            return similarity / self.kernel_width[year + self.step]  # TODO: I changed this

    def __vector_similarity(self, w1=None, w2=None, v1=None, v2=None, year=None):
        """
        Parameters
        ----------
        w1 : (string)
            The first word to get the similarity.

        w2 : (string)
            The Second word to get the similarity.

        year : (int)
            The year parameter for kernel width value

        Returns
        -------
        similarity : (float)
            The similarity between the vectors of the two words.

        """
        if w1 is not None:
            vec1 = self.word_vectors[w1]
        elif v1 is not None:
            vec1 = v1
        if w2 is not None:
            vec2 = self.word_vectors[w2]
        elif v2 is not None:
            vec2 = v2

        if vec2 is np.nan:
            vec2 = np.zeros(len(vec1))

        if self.vec_sim_method == 'cos':
            return (np.dot(matutils.unitvec(vec1), matutils.unitvec(vec2)) + 1) / 2
        elif self.vec_sim_method == 'euc':
            return 1 / np.sqrt(np.sum((vec1 - vec2) ** 2))
        elif self.vec_sim_method == 'exp_euc':
            return exp(
                - np.sqrt(np.sum((vec1 - vec2) ** 2)) / self.kernel_width[year + self.step])
        elif self.vec_sim_method == 'exp_euc_sq':
            return exp(- np.sum((vec1 - vec2) ** 2) / self.kernel_width[year + self.step])
        elif self.vec_sim_method == 'hbc':
            try:
                return self.hbc[w1][w2]
            except IndexError:
                return 0.1
        else:
            raise Exception('Vector similarity method not defined!')

    def find_neighbouring_words(self, word, category, year, starting_year=1988):
        print(word, category, year)

        self.vec_sim_method = 'exp_euc_sq'
        self.kernel_width = dict()
        for year in range(self.threshold, self.end, self.step):
            self.kernel_width[year] = 1.0
        kw_name = str(1.0)
        self.word_vectors = self.word_vectors_yby[year]

        # find nearest neighbour in the specified category
        distances = dict()
        for yr in range(starting_year, year, self.step):
            for cat, wrds in self.super_words[yr].items():
                if cat == category:
                    for w in wrds.keys():
                        if w == word:
                            continue
                        distances[w] = self.__vector_similarity(word, w)
        sorted_words = sorted(distances, key=distances.get, reverse=True)
        print('--', sorted_words[0], distances[sorted_words[0]], '\t', category)
        # words = []
        # for i in range(5):
        #     try:
        #         words.append(sorted_words[i])
        #         print('+', sorted_words[i], distances[sorted_words[i]])
        #     except:
        #         pass
        # words.append('...')
        # for i in range(3):
        #     ind = len(sorted_words) - i - 1
        #     words.append(sorted_words[ind])
        #     print('-', sorted_words[ind], distances[sorted_words[ind]])
        # print(words)

        # find nearest neighbour in all categories
        distances = dict()
        for yr in range(starting_year, year, self.step):
            for cat, wrds in self.super_words[yr].items():
                for w in wrds.keys():
                    if w == word:
                        continue
                    distances[w] = self.__vector_similarity(word, w)
        sorted_words = sorted(distances, key=distances.get, reverse=True)
        for i in range(10):
            # find its real categories
            real_cats = []
            for yr in range(self.start, year, self.step):
                for cat, wrds in self.super_words[yr].items():
                    if sorted_words[i] in wrds:
                        real_cats.append((cat, yr))
            print(i + 1, ')', sorted_words[i], distances[sorted_words[i]], '\t', real_cats)


import os.path
import re

import textract

import bibtexparser as bibtex
from bibtexparser.bwriter import BibTexWriter

class Paper:
    __pdf_file_path = None
    __bib_file_path = None
    __bib = None
    __cached_text = None
    note = ''
    tags = []
    def __init__(self, pdf_file_path, bib_file_path=None):
        self.__pdf_file_path = pdf_file_path
        self.__bib_file_path = bib_file_path
        if self.__bib_file_path and os.path.isfile(self.__bib_file_path):
            with open(self.__bib_file_path) as f:
                self.__bib = bibtex.load(f)
                if 'note' in self.__bib.entries[0].keys():
                    self.note = self.__bib.entries[0]['note']
                if 'tags' in self.__bib.entries[0].keys():
                    tags = re.split('[;,\n]+',self.__bib.entries[0]['tags'])
                    self.tags = [t.strip() for t in tags]
    def text(self):
        if not self.__cached_text:
            self.__cached_text = str(textract.process(self.__pdf_file_path, encoding='utf-8')).replace('\\n',' ')
        return self.__cached_text

    def __str__(self):
        #return self.text()
        #return '\t"{}"\n\t{} ({})'.format(str(self.title()),str(self.authors()),str(self.year()))
        return '{}:\n\t"{}"\n\t{} ({})'.format(str(self.label()),str(self.title()),str(self.authors()),str(self.year()))

    def label(self, label=None):
        if label:
           self.__bib.entries[0]['ID'] = label
        return self.__bib.entries[0]['ID']

    def title(self):
        if self.__bib:
            return self.__bib.entries[0]['title']
        return None

    def year(self):
        if self.__bib:
            return self.__bib.entries[0]['year']
        return None

    def authors(self):
        if self.__bib:
            return ' '.join(self.__bib.entries[0]['author'].split())
        return None

    def bibtex_str(self, bibtex_text=None):
        if bibtex_text:
           self.__bib = bibtex.loads(bibtex_text)
        if 'note' in self.__bib.entries[0].keys():
           self.note = self.__bib.entries[0]['note']
        if 'tags' in self.__bib.entries[0].keys():
           tags = re.split('[;,\n]+',self.__bib.entries[0]['tags'])
           self.tags = [t.strip() for t in tags]
        return self.bibtex()

    def bibtexFile(self):
        return self.__bib_file_path

    def bibtex(self,simplified=False):
        if not self.__bib:
            return None
        from copy import deepcopy
        bib = deepcopy(self.__bib)
        if 'note' in bib.entries[0].keys():
            del bib.entries[0]['note']
        if 'tags' in bib.entries[0].keys():
            del bib.entries[0]['tags']
        if simplified:
            for k in ['doi','acmid','isbn', 'url','link']:
                if k in bib.entries[0].keys():
                    del bib.entries[0][k]
        writer = BibTexWriter()
        return writer.write(bib).strip()

    def persist(self):
        if self.__bib:
            self.__bib.entries[0]['note'] = self.note
            self.__bib.entries[0]['tags'] = ';'.join(self.tags)
            writer = BibTexWriter()
            with open(self.__bib_file_path, 'w') as bibfile:
                bibtex = writer.write(self.__bib)
                bibfile.write(bibtex)


def is_prime(n):
	'''check a number is a prime'''
	if n == 0:
		return False
	if n == 1:
		return False
	if n == 2:
		return True

	for i in range(2, n-1):
		if n % i == 0:
			return False
	return True

def find_primes(n):
	'''find primes from 2 to n using Sieve of Eratosthenes'''
	primes = [True] * (n+1)
	primes[0] = False
	primes[1] = False

	result = []
	for i in range(2, n+1):
		if primes[i]:
			result.append(i)
			for j in range(2*i, n+1, i):
				primes[j] == False

	return result

print(find_primes(10000))


import numpy as np
import struct
import math
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d

# Units convertion

GRA = 6.672e-8
#GRV = 1.0
BOL = 1.3806e-16
#BOL = 1.0
PROTON = 1.6726e-24
#PROTON = 1.6726e-24/(2.176e-5)

UL = 3.085678e21
#UL = 1.0
UM = 1.989e43
#UM = 1.0
UV = 1.0e5
#UV = 1.0

UT = UL/UV
UD = UM/UL**3
UP = UM/UL/UT**2
UE = UM*UL**2/UT**2

G = GRA*UM*UT**2/UL**3

# Meaningful items in the head for galaxy cluster simulation

head = ['npart[6]', 'mass[6]', 'time', 'redshift', 'Nall[6]', 'num_files', 'BoxSize', 'Omega0', 'OmegaLambda', 'HoubleParam']

name = ['Gas', 'Halo', 'Disk', 'Bulge', 'Stars', 'Bndry']

# Calculate the temperature based on internal energy u, metalicity X and degree of ionization Y for an adiabatic gas.

IONIZATION = 0.0

def temp(u, Y, X = 0.76, gamma = 5.0/3):
	M = PROTON*4.0/(3*X+1+4*X*Y)
	U = u*UE/UM
	tem = M*U*(gamma-1)/BOL
	return tem

# Read snapshot/IC files

def read(basename = 'snapshot', sn = 1, fn = 1, IC = 0, st = 0, so = -1, tem = 0):
	output = []
	para = []
	for i in range(st, fn):
		if IC==0:
			l = load(basename+'_'+str(i).zfill(3),sn,IC,so,tem)
		else:
			l = load(basename,sn,IC,so,tem)
		output.append(l[1])
		para.append(l[0])
	d = {}
	d['head'] = [head, para]
	d['l'] = output
	return d
# The outputs is a dictionary with 'head' leads to head items and their values at each snapshot (in the list d['head'][1][0] even if there is only one snapshot),
# while 'l' leads to the list of the results at each snapshot.

# Transfer binary data blocks to numpy arrays

def load(fname, fn = 1, IC = 0, so = -1, tem = 1, ed = '<', dummy = 4):
	inte = ed+'i'
	real = ed+'f'
	linte = ed+'q'
	dreal = ed+'d'
	lpara, lp, lv, lID, lm, lU, lrho, lNe = [], [], [], [], [], [], [], []
	not_withmass = 0
	for fi in range(fn):
		if fn!=1:
			fname0 = fname+'.'+str(fi)
		else:
			fname0 = fname
		lp.append([])
		lv.append([])
		lID.append([])
		lm.append([])
		lNe.append([])
		lU.append([])
		lrho.append([])
		with open(fname0, 'rb') as f:
			# Read the head
			a = f.read(dummy)
			npart = [struct.unpack(inte, f.read(4))[0] for x in range(6)]
			mass = [struct.unpack(dreal, f.read(8))[0] for x in range(6)]
			time = struct.unpack(dreal, f.read(8))[0]
			Redshift = struct.unpack(dreal, f.read(8))[0]
			print ('Redshift, time = ', Redshift, time)
			flag_sfr = struct.unpack(inte, f.read(4))[0]
			flag_feedback = struct.unpack(inte, f.read(4))[0]
			Nall = [struct.unpack(inte, f.read(4))[0] for x in range(6)]
			flag_cooling = struct.unpack(inte, f.read(4))[0]
			print ('flag_cooling', flag_cooling)
			num_files = struct.unpack(inte, f.read(4))[0]
			#if num_files!= fn:
			#	print('The number of files at each snapshot should be ',fn)
			#	return [[], []]
			BoxSize = struct.unpack(dreal, f.read(8))[0]
			Omega0 = struct.unpack(dreal, f.read(8))[0]
			OmegaLambda = struct.unpack(dreal, f.read(8))[0]
			HoubleParam = struct.unpack(dreal, f.read(8))[0]
			fill = 256-4*6-8*6-8*2-4*6-4-8*4-4*3
			a = f.read(fill)
			a = f.read(dummy)
			para = [npart, mass, time, Redshift, Nall, num_files, BoxSize, Omega0, OmegaLambda, HoubleParam]
			lpara.append(para)
			# Data structure for individual files
			index = []
			for i in range(6):
				if npart[i]!=0:
					index.append(i)
					lp[fi].append([])
					lv[fi].append([])
					lID[fi].append([])
					lm[fi].append([])
					if i==0:
						lU[fi].append([])
						lrho[fi].append([])
						lNe[fi].append([])
					if mass[i]==0: not_withmass += 1
			# Read positions
			a = f.read(dummy)
			for k in range(len(index)):
				lp[fi][k] = [np.zeros(npart[index[k]], dtype='float') for x in range(3)]
				for l in range(npart[index[k]]):
					lp[fi][k][0][l] = struct.unpack(real, f.read(4))[0]
					lp[fi][k][1][l] = struct.unpack(real, f.read(4))[0]
					lp[fi][k][2][l] = struct.unpack(real, f.read(4))[0]
			a = f.read(dummy)
			# Read velocities
			a = f.read(dummy)
			for k in range(len(index)):
				lv[fi][k] = [np.zeros(npart[index[k]], dtype='float') for x in range(3)]
				for l in range(npart[index[k]]):
					lv[fi][k][0][l] = struct.unpack(real, f.read(4))[0]
					lv[fi][k][1][l] = struct.unpack(real, f.read(4))[0]
					lv[fi][k][2][l] = struct.unpack(real, f.read(4))[0]
			a = f.read(dummy)
			# Read particle IDs
			a = f.read(dummy)
			for k in range(len(index)):
				lID[fi][k] = np.zeros(npart[index[k]], dtype = 'uint')
				for l in range(npart[index[k]]):
					lID[fi][k][l] = struct.unpack(inte, f.read(4))[0]
			a = f.read(dummy)
			# Read masses
			if not_withmass>0:
				a = f.read(dummy)
				for k in range(len(index)):
					if mass[index[k]]!=0:
						lm[fi][k] = np.ones(npart[index[k]], dtype = 'float')*mass[index[k]]
					else:
						lm[fi][k] = np.zeros(npart[index[k]], dtype = 'float')
						for l in range(npart[index[k]]):
							lm[fi][k][l] = struct.unpack(real, f.read(4))[0]
#						print('What?')
				a = f.read(dummy)
			else:
				for k in range(len(index)):
					lm[fi][k] = np.ones(npart[index[k]], dtype = 'float')*mass[index[k]]
			# Read gas properties
			if npart[0]!=0:
				lU[fi][0] = np.zeros(npart[0], dtype='float')
				lrho[fi][0] = np.zeros(npart[0], dtype='float')
				a = f.read(dummy)
				for l in range(npart[0]):
					lU[fi][0][l] =  struct.unpack(real, f.read(4))[0]
				a = f.read(dummy)
				if IC==0:
					a = f.read(dummy)
					for l in range(npart[0]):
						lrho[fi][0][l] = struct.unpack(real, f.read(4))[0]
					a = f.read(dummy)
				if flag_cooling!=1:
					lNe[fi][0] = np.ones(npart[0], dtype='float')*IONIZATION
				elif IC==0:
					lNe[fi][0] = np.zeros(npart[0], dtype='float')
					a = f.read(dummy)
					for l in range(npart[0]):
						lNe[fi][0][l] = struct.unpack(real, f.read(4))[0]
					a = f.read(dummy)
	# Generate the output
	output = []
	for k in range(len(index)):
		if index[k]==0:
			if IC==0:
				if tem==1:
					output.append([[] for x in range(12)])
				else:
					output.append([[] for x in range(11)])
			else:
				if tem==1:
					output.append([[] for x in range(11)])
				else:
					output.append([[] for x in range(10)])
		else:
			output.append([[] for x in range(8)])
		output[k][0] = np.hstack([lp[fi][k][0] for fi in range(fn)])
		output[k][1] = np.hstack([lp[fi][k][1] for fi in range(fn)])
		output[k][2] = np.hstack([lp[fi][k][2] for fi in range(fn)])
		output[k][3] = np.hstack([lv[fi][k][0] for fi in range(fn)])
		output[k][4] = np.hstack([lv[fi][k][1] for fi in range(fn)])
		output[k][5] = np.hstack([lv[fi][k][2] for fi in range(fn)])
		output[k][6] = np.hstack([lID[fi][k] for fi in range(fn)])
		output[k][7] = np.hstack([lm[fi][k] for fi in range(fn)])
		if index[k]==0:
			output[0][8] = np.hstack([lU[fi][0] for fi in range(fn)])
			if IC==0:
				output[0][9] = np.hstack([lrho[fi][0] for fi in range(fn)])
				output[0][10] = np.hstack([lNe[fi][0] for fi in range(fn)])
				if tem==1:
					output[0][11] = np.zeros(Nall[0], dtype='float')
					for l in range(Nall[0]):
						output[k][11][l] = temp(output[k][8][l], output[k][10][l])
			else:
				output[0][9] = np.hstack([lNe[fi][0] for fi in range(fn)])
				if tem==1:
					output[0][10] = np.zeros(Nall[0], dtype='float')
					for l in range(Nall[0]):
						output[k][10][l] = temp(output[k][8][l], output[k][9][l])
		if so>=0:
			output[k] = sort(output[k],so)
		#output[k] = sort(output[k], 6)
	return [lpara, output]

def write(d, name = 'test_ics', ed = '<', dummy = 4):
	inte = ed+'i'
	real = ed+'f'
	linte = ed+'q'
	dreal = ed+'d'
	not_withmass = 0
	with open(name, 'wb') as f:
		a = f.write(struct.pack(inte, 256))
		a = [f.write(struct.pack(inte, x)) for x in d['head'][1][0][0][4]]
		a = [f.write(struct.pack(dreal, x)) for x in d['head'][1][0][0][1]]
		a = f.write(struct.pack(dreal, d['head'][1][0][0][2]))
		a = f.write(struct.pack(dreal, d['head'][1][0][0][3]))
		a = f.write(struct.pack(inte, 0))
		a = f.write(struct.pack(inte, 0))
		a = [f.write(struct.pack(inte, x)) for x in d['head'][1][0][0][4]]
		a = f.write(struct.pack(inte, 0))
		a = f.write(struct.pack(inte, 1))
		a = f.write(struct.pack(dreal, d['head'][1][0][0][6]))
		a = f.write(struct.pack(dreal, d['head'][1][0][0][7]))
		a = f.write(struct.pack(dreal, d['head'][1][0][0][8]))
		a = f.write(struct.pack(dreal, d['head'][1][0][0][9]))
		fill = 256-4*6-8*6-8*2-4*6-4-8*4-4*3
		a = [f.write(struct.pack(inte, 0)) for x in range(int(fill/4))]
		a = f.write(struct.pack(inte, 256))
		index = []
		for i in range(6):
			if d['head'][1][0][0][4][i]!=0:
				index.append(i)
				if d['head'][1][0][0][1][i]==0: not_withmass += 1
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])*3))
		for i in range(len(index)):
			for k in range(d['head'][1][0][0][4][index[i]]):
				a = [f.write(struct.pack(real, d['l'][0][i][x][k])) for x in range(3)]
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])*3))
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])*3))
		for i in range(len(index)):
			for k in range(d['head'][1][0][0][4][index[i]]):
				a = [f.write(struct.pack(real, d['l'][0][i][3+x][k])) for x in range(3)]
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])*3))
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])))
		for i in range(len(index)):
			for k in range(d['head'][1][0][0][4][index[i]]):
				a = f.write(struct.pack(inte, d['l'][0][i][6][k]))
		a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])))
		if not_withmass>0:
			a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])))
			for k in range(len(index)):
				if d['head'][1][0][0][1][index[i]]!=0:
					a = [f.write(struct.pack(real, d['head'][1][0][0][1][index[i]])) for x in range(d['head'][1][0][0][4][i])]
				else:
					a = [f.write(struct.pack(real, d['l'][0][i][7][x])) for x in range(d['head'][1][0][0][4][i])]
			a = f.write(struct.pack(inte, sum(d['head'][1][0][0][4])))

		if d['head'][1][0][0][4][0]!=0:
			a = f.write(struct.pack(inte, d['head'][1][0][0][4][0]))
			a = [f.write(struct.pack(real, d['l'][0][0][8][x])) for x in range(d['head'][1][0][0][4][0])]
			a = f.write(struct.pack(inte, d['head'][1][0][0][4][0]))

def cut(d, box = (100.0, 100.0, 100.0), obs = (200.0,200.0,200.0)):
	head = d['head'][0]
	out = []
	lpara = []
	for i in range(len(d['l'])):
		out.append([])
		for k in range(len(d['l'][i])):
			out[i].append([])
			index = []
			for j in range(len(d['l'][i][k][0])):
				t = 0
				if (d['l'][i][k][0][j]>obs[0])and(d['l'][i][k][0][j]<obs[0]+box[0]):
					t += 1
				if (d['l'][i][k][1][j]>obs[1])and(d['l'][i][k][1][j]<obs[1]+box[1]):
					t += 1
				if (d['l'][i][k][2][j]>obs[2])and(d['l'][i][k][2][j]<obs[2]+box[2]):
					t += 1
				if t==3:
					index.append(j)
			for m in range(len(d['l'][i][k])):
				out[i][k].append(np.zeros(len(index), dtype=d['l'][i][k][m].dtype))
			for n in range(len(index)):
				for s in range(len(d['l'][i][k])):
					out[i][k][s][n] = d['l'][i][k][s][index[n]]
		lpara.append([])
		lpara[i].append([])
		lpara[i][0]=d['head'][1][i][0]
		count = 0
		for r in range(6):
			if d['head'][1][i][0][4][r]!=0:
				lpara[i][0][0][r]=len(out[i][count][0])
				lpara[i][0][4][r]=len(out[i][count][0])
				count += 1
		lpara[i][0][6]=box[0]
	dout = {}
	dout['head'] = [head, lpara]
	dout['l'] = out
	return dout

def sort(l, index = 7):
	print('Begin sorting')
	dim = len(l)
	if index>dim:
		print ('Error')
		return l
	a = np.matrix(l)
	b = np.array(a.transpose())
	b = sorted(b, key=lambda b:b[index])
	c = np.array(np.matrix(b).transpose())
	print('Done')
	return c
	lis[k].append(l[dim][k])


from models.cycleGAN import CycleGAN

from torch.utils.data.dataloader import DataLoader
from util.util import compute_error
import torch
import numpy as np
import sys

class Validator:
    """
    The Validator can validate a given cycleGAN model end to end by running a pretrained random forest on the generated fakes and computing the average relative error.
    """
    def __init__(self, opt):
        self.opt = opt

    def get_validation_score(self, model: CycleGAN, dataset: DataLoader, num_batches=sys.maxsize):
        """
        Computes various validation metrics for the given model.

        Parameters
        ----------
            - model (CycleGAN): The current CycleGAN model
            - dataset (DataLoader): The dataset the validation samples should be taken from. If none is given, the configured validation set will be used. Default=None

        Returns
        -------
            - The Mean Absolute Error (L1) per metabolite. (M) with M=number of metabolites, N=number of test samples
            - The relative error per metabolite per fake. (MxN) with M=number of metabolites, N=number of test samples
            - The Average Relative Error per metabolite. (M) with M=number of metabolites
            - The Coefficient of Determination (R^2) pre metabolite. (M) with M=number of metabolites
        """
        predictions = []
        labels = []
        for i, data in enumerate(dataset):
            if i>num_batches:
                break
            model.set_input(data)  # unpack data from data loader
            labels.append(data['label_A'])
            model.test()           # run inference
            prediction = model.get_prediction()
            predictions.append(prediction)
        predictions = np.concatenate(predictions)
        labels = torch.cat(labels).numpy()
        avg_abs_err, err_rel, avg_err_rel, r2 = compute_error(predictions, labels)

        return avg_abs_err, err_rel, avg_err_rel, r2


# Uses python3
import sys
import itertools

def partition3(A):
    sum_ = sum(A)
    if sum_ % 3:
        return 0
    n = sum_ // 3
    dp = [[0] * (n + 1) for _ in range(n + 1)]
    dp[0][0] = 1
    for a in A:
        for i in range(n, -1, -1):
            for j in range(n, -1, -1):
                if dp[i][j]:
                    if i + a <= n:
                        dp[i + a][j] = 1
                    if j + a <= n:
                        dp[i][j + a] = 1
        if dp[n][n] > 0:
            return 1
    return 0

if __name__ == '__main__':
    input = sys.stdin.read()
    n, *A = list(map(int, input.split()))
    print(partition3(A))


#BlackJAck#
import random
import time
biltoplam = 0
usertoplam = 0
sayac=0
sayec=0
kartlar = [1,2,3,4,5,6,7,8,9,10,11]
bilikincikart = [0,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11]
########################################################################################
while sayac<2:
    sayac+=1
    bilsecim = random.choice(kartlar)
    biltoplam += bilsecim
while True:
    if biltoplam<21:
        bilikin = random.choice(bilikincikart)
        biltoplam += bilikin
        if biltoplam<21:
            bilikin = random.choice(kartlar)
            biltoplam += bilikin
        if biltoplam == 21:

            break
    if biltoplam>21:

        break
    if biltoplam==21:

        break
###########################################################
while sayec<2:
    sayec +=1
    usersecim = random.choice(kartlar)
    usertoplam += usersecim
    time.sleep(1)
    print("Kart seçiliyor...")
    time.sleep(1)
    print(f"Elinizdeki kart ------------>{usertoplam}")
    while True:
        giris=input("Kart çekmek ister misin ------------> e/h :")
        if giris == "h":
            if biltoplam>21:
                print("Bilgisayar Battı")
                break
            if biltoplam<21:
                if usertoplam<biltoplam:
                    print(f"Bilgisayar sizi {biltoplam} ile yendi :) ")
                    break
                if usertoplam>biltoplam:
                    print(f"Bilgisayarı {usertoplam} ile yendiniz !!!")
                    break
                if usertoplam>21:
                    print("Batttınız!!!")
        if giris == "e":
            usersecim = random.choice(kartlar)
            usertoplam += usersecim
            time.sleep(1)
            print("Kart seçiliyor...")
            time.sleep(1)
            print(f"Çekilen kart ------------> {usersecim}")
            time.sleep(0.5)
            print(f"Kart toplamınız ------------> {usertoplam}")
            if usertoplam>21:
                print("Battınız!!")
                break
    break


"""
.. _linear_cells_example:

Linear Cells
~~~~~~~~~~~~

This example extends the :ref:`create_unstructured_example` example by
including an explanation of linear VTK cell types and how you can create them in
PyVista.

Linear cells are cells where points only occur at the edges of each
cell. Non-linear cells contain additional points along the edges of the cell.

For more details regarding what a :class:`pyvista.UnstructuredGrid` is, please
see :ref:`point_sets_api`.

"""
import numpy as np

import pyvista as pv
from pyvista.examples import cells as example_cells, plot_cell

# random generator for examples
rng = np.random.default_rng(2)

###############################################################################
# Plot an example cell
# ~~~~~~~~~~~~~~~~~~~~
# PyVista contains a simple utility to plot a single cell, which is the
# fundamental unit of each :class:`pyvista.UnstructuredGrid`. For example,
# let's plot a simple :func:`Wedge <pyvista.examples.cells.Wedge>`.
#
grid = example_cells.Wedge()
example_cells.plot_cell(grid)

###############################################################################
# This linear cell is composed of 6 points.

grid.points

###############################################################################
# The UnstructuredGrid is also composed of a single cell and the point indices
# of that cell are defined in :attr:`cells <pyvista.UnstructuredGrid.cells>`.
#
# .. note::
#    The leading ``6`` is the number of points in the cell.

grid.cells

###############################################################################
# Combine two UnstructuredGrids
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# We can combine two unstructured grids to create a single unstructured grid
# using the ``+`` operator.
#
# .. note::
#    This is an inefficient way of creating :class:`pyvista.UnstructuredGrid`
#    objects. To see a more efficient implementation see
#    :ref:`create_unstructured_example`.

grid_a = example_cells.Hexahedron()
grid_a.points += [0, 2.5, 0]

grid_b = example_cells.HexagonalPrism()

combined = grid_b + grid_a

plot_cell(combined, cpos='iso')

###############################################################################
# This example helps to illustrate meaning behind the :attr:`cells
# <pyvista.UnstructuredGrid.cells>` attribute. The first cell, a hexahedron
# contains 8 points and the hexagonal prism contains 12 points. The ``cells``
# attribute shows this along with indices composing each cell.

combined.cells

###############################################################################
# Cell Types
# ~~~~~~~~~~
# PyVista contains the :class:`pyvista.CellType` enumerator, which contains all the
# available VTK cell types mapped to a Python enumerator. These cell types are
# used when creating cells and also can be used when checking the
# :attr:`celltypes <pyvista.UnstructuredGrid.celltypes>` attribute. For example
# ``combined.celltypes`` contains both the ``pv.CellType.HEXAHEDRON`` and
# ``pv.CellType.HEXAGONAL_PRISM`` cell types.

print(pv.CellType.HEXAHEDRON, pv.CellType.HEXAGONAL_PRISM)
(pv.CellType.HEXAHEDRON, pv.CellType.HEXAGONAL_PRISM) == combined.celltypes

###############################################################################
# Create an UnstructuredGrid with a single linear cell
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Now that you know the three main inputs of an
# :class:`pyvista.UnstructuredGrid`, it's quite straightforward to create an
# unstructured grid with a one or more cells. If you need to reference point
# ordering or additional, you can either read the source of `cells.py
# <https://github.com/pyvista/pyvista/blob/main/pyvista/examples/cells.py>`_ or
# simply create a cell from the ``pyvista.core.cells`` module and inspect its attributes.

points = [
    [1.0, 1.0, 0.0],
    [-1.0, 1.0, 0.0],
    [-1.0, -1.0, 0.0],
    [1.0, -1.0, 0.0],
    [0.0, 0.0, 1.60803807],
]
cells = [len(points)] + list(range(len(points)))
pyrmaid = pv.UnstructuredGrid(cells, [pv.CellType.PYRAMID], points)
example_cells.plot_cell(pyrmaid)

###############################################################################
# Plot all linear cell Types
# ~~~~~~~~~~~~~~~~~~~~~~~~~~
# Let's create a ``(4, 4)`` :class:`pyvista.Plotter` and plot all 16 linear
# cells in a single plot.

def add_cell_helper(pl, text, grid, subplot, cpos=None):
    """Add a single cell to a plotter with fancy plotting."""
    pl.subplot(*subplot)
    pl.add_text(text, 'lower_edge', color='k', font_size=8)
    pl.add_mesh(grid, opacity=0.5, color='lightblue', line_width=5)
    edges = grid.extract_all_edges()
    if edges.n_cells:
        pl.add_mesh(grid.extract_all_edges(), line_width=5, color='k')
    pl.add_points(grid, render_points_as_spheres=True, point_size=20, color='r')
    pl.add_point_labels(
        grid.points,
        range(grid.n_points),
        always_visible=True,
        fill_shape=False,
        margin=0,
        shape_opacity=0.0,
        font_size=20,
        text_color='k',
    )
    if cpos is None:
        pl.camera.azimuth = 20
        pl.camera.elevation = -20
    else:
        pl.camera_position = cpos
    pl.camera.zoom(0.8)

pl = pv.Plotter(shape=(4, 4))
add_cell_helper(pl, f'VERTEX ({pv.CellType.VERTEX})', example_cells.Vertex(), (0, 0))
add_cell_helper(pl, f'POLY_VERTEX ({pv.CellType.POLY_VERTEX})', example_cells.PolyVertex(), (0, 1))
add_cell_helper(pl, f'LINE ({pv.CellType.LINE})', example_cells.Line(), (0, 2))
add_cell_helper(pl, f'POLY_LINE ({pv.CellType.POLY_LINE})', example_cells.PolyLine(), (0, 3))

add_cell_helper(
    pl, f'TRIANGLE ({pv.CellType.TRIANGLE})', example_cells.Triangle(), (1, 0), cpos='xy'
)
add_cell_helper(
    pl,
    f'TRIANGLE_STRIP ({pv.CellType.TRIANGLE_STRIP})',
    example_cells.TriangleStrip().rotate_z(90, inplace=False),
    (1, 1),
    cpos='xy',
)
add_cell_helper(pl, f'POLYGON ({pv.CellType.POLYGON})', example_cells.Polygon(), (1, 2), cpos='xy')
add_cell_helper(pl, f'PIXEL ({pv.CellType.PIXEL})', example_cells.Pixel(), (1, 3), cpos='xy')

# make irregular
quad_grid = example_cells.Quadrilateral()
quad_grid.points += rng.random((4, 3)) * 0.5

add_cell_helper(pl, f'QUAD ({pv.CellType.QUAD})', quad_grid, (2, 0))
add_cell_helper(pl, f'TETRA ({pv.CellType.TETRA})', example_cells.Tetrahedron(), (2, 1))
add_cell_helper(pl, f'VOXEL ({pv.CellType.VOXEL})', example_cells.Voxel(), (2, 2))

# make irregular
hex_grid = example_cells.Hexahedron()
hex_grid.points += rng.random((8, 3)) * 0.4
add_cell_helper(pl, f'HEXAHEDRON ({pv.CellType.HEXAHEDRON})', hex_grid, (2, 3))

add_cell_helper(pl, f'WEDGE ({pv.CellType.WEDGE})', example_cells.Wedge(), (3, 0))
add_cell_helper(pl, f'PYRAMID ({pv.CellType.PYRAMID})', example_cells.Pyramid(), (3, 1))
add_cell_helper(
    pl,
    f'PENTAGONAL_PRISM ({pv.CellType.PENTAGONAL_PRISM})',
    example_cells.PentagonalPrism(),
    (3, 2),
)
add_cell_helper(
    pl, f'HEXAGONAL_PRISM ({pv.CellType.HEXAGONAL_PRISM})', example_cells.HexagonalPrism(), (3, 3)
)

pl.background_color = 'w'
pl.show()


import re
import subprocess
import os
import signal

import xurl

VALID_URL = r'4gtv'

def getSource(url, fmt, ref):
    result = None
    cmd = 'google-chrome-stable --headless --disable-gpu --enable-logging --v=1 --virtual-time-budget=90000 \'%s\'' %(url)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
    for line in p.stdout:
        l = line.decode('utf8')
        m = re.search(r'https://4gtvfree-cds.cdn.hinet.net/.*', l)
        if m:
            result = m.group(0)
            break
        m = re.search(r'https://4gtvfreepc-mozai.4gtv.tv/.*', l)
        if m:
            result = m.group(0)
            break
    p.terminate()
    return result


# LeetCode. Yandex Task List. 20. Valid Parentheses

'''

20. Valid Parentheses
Easy

Given a string s containing just the characters '(', ')', '{', '}', '[' and ']', determine if the input string is valid.

An input string is valid if:

    Open brackets must be closed by the same type of brackets.
    Open brackets must be closed in the correct order.

Example 1:

Input: s = "()"
Output: true

Example 2:

Input: s = "()[]{}"
Output: true

Example 3:

Input: s = "(]"
Output: false

Example 4:

Input: s = "([)]"
Output: false

Example 5:

Input: s = "{[]}"
Output: true

Constraints:

    1 <= s.length <= 104
    s consists of parentheses only '()[]{}'.

Accepted
1,741,221
Submissions
4,299,585

Related Topics
String, Stack

Similar Questions
Generate Parentheses
Medium
Longest Valid Parentheses
Hard
Remove Invalid Parentheses
Hard
Check If Word Is Valid After Substitutions
Medium

Hide Hint 1
An interesting property about a valid parenthesis expression is that a sub-expression of a valid expression should also be a valid expression. (Not every sub-expression) e.g.

{ { } [ ] [ [ [ ] ] ] } is VALID expression
          [ [ [ ] ] ]    is VALID sub-expression
  { } [ ]                is VALID sub-expression

Can we exploit this recursive structure somehow?

Hide Hint 2
What if whenever we encounter a matching pair of parenthesis in the expression, we simply remove it from the expression? This would keep on shortening the expression. e.g.

{ { ( { } ) } }
      |_|

{ { (      ) } }
    |______|

{ {          } }
  |__________|

{                }
|________________|

VALID EXPRESSION!

Hide Hint 3
The stack data structure can come in handy here in representing this recursive structure of the problem. We can't really process this from the inside out because we don't have an idea about the overall structure. But, the stack can help us process this recursively i.e. from outside to inwards.

'''

import math
import time
import collections
from typing import List
import numpy as np
import random as rnd
import itertools as it
from collections import defaultdict, Counter
import re
from functools import reduce
from functools import lru_cache
from bisect import bisect, bisect_left

class Solution:
    def stackPop(self, stack: list, expectedStackTop: str) -> bool:
        if len(stack) > 0:
            if stack[-1] == expectedStackTop:
                stack.pop()
                return True
        return False

    def isValid(self, s: str) -> bool:
        # optimization for inputs that don't have an even number of elements
        if len(s) % 2:
            return False
        symbolsList = list(s)
        roundBracketsStack = []
        bracesStack = []
        squareBrackets = []
        generalStack = []
        for s in symbolsList:
            if s == "(":
                roundBracketsStack.append(s)
                generalStack.append(s)
            if s == "{":
                bracesStack.append(s)
                generalStack.append(s)
            if s == "[":
                squareBrackets.append(s)
                generalStack.append(s)
            if s == ")":
                self.stackPop(roundBracketsStack, "(")
                if not self.stackPop(generalStack, "("):
                    generalStack.append(s)
            if s == "}":
                self.stackPop(bracesStack, "{")
                if not self.stackPop(generalStack, "{"):
                    generalStack.append(s)
            if s == "]":
                self.stackPop(squareBrackets, "[")
                if not self.stackPop(generalStack, "["):
                    generalStack.append(s)
        return not len(roundBracketsStack) and not len(bracesStack) and not len(squareBrackets) and not len(generalStack)

if __name__ == "__main__":
    #
    start = time.perf_counter()
    solution = Solution()
    print(solution.twoSum(nums = [2 ,7 ,11 ,15], target = 9))
    end = time.perf_counter()
    print(f"test 1: {end - start:10.6f} sec")
    #
    start2 = time.perf_counter()
    print(solution.twoSum(nums = [3 ,2 ,4], target = 6))
    end2 = time.perf_counter()
    print(f"test 2: {end2 - start2:10.6f} sec")
    #
    start3 = time.perf_counter()
    print(solution.twoSum(nums = [3 ,3], target = 6))
    end3 = time.perf_counter()
    print(f"test 3: {end3 - start3:10.6f} sec")
    #
    start4 = time.perf_counter()
    print(solution.isIsomorphic("badc", "baba"))
    end4 = time.perf_counter()
    print(f"test 4: {end4 - start4:10.6f} sec")
    #
    start5 = time.perf_counter()
    print(solution.isIsomorphic("abab", "baba"))
    end5 = time.perf_counter()
    print(f"test 5: {end5 - start5:10.6f} sec")
    #
    start6 = time.perf_counter()
    print(solution.firstMissingPositive([1, 1]))
    end6 = time.perf_counter()
    print(f"test 6: {end6 - start6:10.6f} sec")


# MIV 2018-2023

# http://www.maths.surrey.ac.uk/explore/michaelspages/documentation/Simple
# http://hyperphysics.phy-astr.gsu.edu/hbase/pendl.html#c2
# https://www.grc.nasa.gov/www/k-12/airplane/dragsphere.html
# https://en.wikipedia.org/wiki/Density_of_air

import numpy as np
import pygame

# colors
black = (0, 0, 0)
white = (255, 255, 255)
red = (255, 0, 0)
blue = (0, 255, 0)
green = (0, 0, 255)

fontsize = 22

# window sizes
width = 600
height = 400

# position  coordinates of the pivot
pivotx = width / 2
pivoty = height / 4

fps = 60
clock = pygame.time.Clock()

endSim = False

g = 9.81  # m/s^2

# the mass, in kg
m = 1
mass_scale = 10  # to scale the circle for better visualization (10 for kg, 100 for gr...)
radius = m * mass_scale  # radius of the sphere

"""
# air drag (D = Cd * .5 * rho * V^2 * A)
drag = 0
cd = 0.47  # drag coefficient of a sphere
rho = 1.2754  # At IUPAC standard temperature and pressure (0 °C and 100 kPa), dry air has a density of 1.2754 kg/m^3
mass_area = 4 * np.pi * radius ** 2  # area of the spheric mass
"""

# position coordinates of the mass
x = 0
y = 0

# theta in rad
angle = np.pi / 2

# velocity (dtheta/dt) in m/s
angle_v = 0

# acceleration (d^2theta/dt^2) in m/s^2
angle_a = 0

# string length in m
r = 1
scale = 100  # scale r for better visualization (100 for m, 1000 for cm...)

# change in time
delta_t = 1 / fps

# data to display
period = (2 * np.pi * np.sqrt(r / g) * (1 + ((1/16) * angle ** 2) + ((11 / 3072) * angle ** 4)))  # s
total_force = 0  # N
angular_velocity = 0
linear_velocity = 0

pygame.init()

canvas = pygame.display.set_mode((width, height))
pygame.display.set_caption("Simple pendulum simulation")

def printOnScreen(msg, color, size, xcoord, ycoord):
    font = pygame.font.SysFont(None, size)
    text = font.render(msg, True, color)
    canvas.blit(text, [xcoord, ycoord])

while not endSim:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            endSim = True

    canvas.fill(white)

    x = r * np.sin(angle)
    y = r * np.cos(angle)

    pygame.draw.line(canvas, black, (pivotx, pivoty), (pivotx + x * scale, pivoty + y * scale), 1)
    pygame.draw.circle(canvas, black, (int(pivotx + x * scale), int(pivoty + y * scale)), int(radius))

    angle_a = - (g / r) * np.sin(angle)
    angle_v = angle_v + (delta_t * angle_a)

    angle = angle + (delta_t * angle_v)

    # drag = cd * .5 * rho * linear_velocity ** 2 * mass_area
    # very simple dampening
    angle_v *= 0.999

    total_force = - m * g * np.sin(angle)
    angular_velocity = - (g / r) * np.sin(angle)
    linear_velocity = r * angular_velocity

    printOnScreen("Period (T) = {} s".format(period), black, fontsize, 0, 0)
    printOnScreen("Total force = {} N".format(total_force), black, fontsize, 0, fontsize)
    printOnScreen("Angular velocity = {} rad/s".format(angular_velocity), black, fontsize, 0, fontsize * 2)
    printOnScreen("Linear velocity = {} m/s".format(linear_velocity), black, fontsize, 0, fontsize * 3)
    # printOnScreen("Air drag = {}".format(drag), black, 0, 100)

    pygame.display.update()
    clock.tick(fps)


#!/usr/bin/env python3
"""
ffdb
This module and the associated set of utility scripts
allow the creation, maintenance and usage of a database and document storage
which employs a single container file.

by Giuseppe Insana, 2020
"""
import os
import re
import sys
import time
import zlib
from math import ceil
from heapq import merge #alternative mergesort
from Cryptodome.Cipher import AES
from Cryptodome.Protocol import KDF
from Cryptodome import Random

# pylint: disable=C0103,R0912,R0915,W0603
VERSION = '2.5.5'

##CUSTOMIZATIONS:

#personalize this program by modifying the
#following SALT used by the PBKDF2 routine
SALT = b'5ed3a4284d6a9c1e4e4f6b4729b254be'

#change the following if you prefer space or other delimiter for the index
FIELDSEPARATOR = "\t"

#define temporary directory in case no TMPDIR is specified
if 'TMPDIR' in os.environ and os.environ['TMPDIR'] != '':
    TEMPDIR = os.environ['TMPDIR']
else:
    TEMPDIR = "/tmp"

#customize the progress bar (for multithread verbose progress)
#PROGRESSBARCHARS = False #use this for standard unicode smooth blocks
PROGRESSBARCHARS = ' ·•'

#only relevant for extraction via gztool
#from local or remote compressed flatfiles;
#what is the full path of gztool:
GZTOOL_EXE = None
#e.g.:
#GZTOOL_EXE = "/usr/local/bin/gztool"
#if not hardcoded here, be sure it is found in your PATH

#buffer size used by io operations
#BUFFERSIZE = 8192
BUFFERSIZE = 1048576

##CONSTANTS:

REIV = re.compile(r'^([0-9a-zA-Z{}]+)\.([0-9a-zA-Z{}]+)\|([A-Z])(.+)$') #only encrypted
REIVD = re.compile(r'^([0-9a-zA-Z{}]+)\+([0-9a-zA-Z{}]+)\|([A-Z])(.+)$') #deflated and encrypted
REESIV = re.compile(r'^([0-9a-zA-Z{}]+)(?:\.|\+)([0-9a-zA-Z{}]+)\|[A-Z](.+)$') #entrysize+iv stored
REESIVN = re.compile(r'^([0-9a-zA-Z{}]+)(?:\.|\+)([0-9a-zA-Z{}]+\|[A-Z].+)$') #don't capture iv
REES = re.compile(r'^([0-9a-zA-Z{}]+)(?::|-)([0-9a-zA-Z{}]+)$') #entrysize but no iv stored
RETERMINATOR = re.compile(r'^-$')

B64_CHARS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ{}'
B64_INDEX = dict((char, pos) for pos, char in enumerate(tuple(B64_CHARS)))

def calculate_chunknum(filename, blocksize):
    """
    calculate number of chunks from global defined blocksize
    and file's size
    """
    filesize = os.path.getsize(filename)
    number_of_chunks = filesize // blocksize + 1
    #eprint("filename {} filesize {} blocksize {}".format(filename, filesize, blocksize)) #debug
    return filesize, number_of_chunks

def calculate_blocksize(filename, number_of_chunks):
    """
    calculate size of a single block
    from number of chunk/splits desired and a file's filesize
    """
    filesize = os.path.getsize(filename)
    if filesize == 0:
        raise RuntimeError("File '{}' is empty".format(filename))
    blocksize = int(ceil(filesize / number_of_chunks))
    number_of_chunks = int(ceil(filesize / blocksize))
    return blocksize

def find_next_delimiter(fp, filesize, startpos, delimiter, buffersize=None):
    """
    return the position of first delimiter found after given starting position
    """
    if startpos > filesize:
        return filesize
    if buffersize is None:
        buffersize = 16384
    delimiterlen = len(delimiter)
    fp.seek(startpos)
    data = fp.read(buffersize)
    while data:
        delimiterpos = data.find(delimiter)
        #eprint("{}-{} checking for {}: {}".format(startpos, fp.tell(), delimiter, delimiterpos))
        if delimiterpos != -1:
            return startpos + delimiterpos + delimiterlen
        startpos += buffersize
        data = fp.read(buffersize)
    return filesize #could not find

def find_next_newline(fp, filesize, startpos):
    """
    return the position of first newline found after given starting position
    """
    if startpos > filesize:
        return filesize
    if startpos <= 0:
        return 0
    fp.seek(startpos - 1)
    fp.readline() # to avoid truncatedlines
    return fp.tell()

def chunk_startpos_onnewline(fp, filesize, blocksize, block):
    """
    return the initial byte of a chunk without breaking newlines
    """
    ini = block * blocksize
    end = ini + blocksize
    if ini > filesize:
        return None
    if end > filesize:
        end = filesize
    if ini <= 0:
        fp.seek(0)
    else:
        fp.seek(ini - 1)
        fp.readline() # to avoid truncatedlines
    return fp.tell()

def chunk_of_lines(fp, filesize, blocksize, block):
    """
    return a generator of the lines coming from a single chunk of the file
    """
    ini = block * blocksize
    end = ini + blocksize
    if ini > filesize:
        return
    if end > filesize:
        end = filesize
    if ini <= 0:
        fp.seek(0)
    else:
        fp.seek(ini - 1)
        fp.readline() # to avoid truncatedlines
    while fp.tell() < end:
        yield fp.readline()

def int_to_b64(number):
    """
    convert a decimal integer to a base64encoded number
    """
    if number == 0:
        return '0'
    string = ''
    while number:
        string = B64_CHARS[number % 64] + string
        number //= 64
        #~13% slower:
        #number, remainder = divmod(number, 64)
        #string = B64_CHARS[remainder] + string
    return string

def b64_to_int(string):
    """
    convert a base64encoded number to decimal integer
    """
    strlen = len(string)
    number = 0
    index = 0
    for char in string:
        index += 1
        power = strlen - index
        number += B64_INDEX[char] * (64 ** power)
        #~20% slower:
        # number += B64_CHARS.index(char) * (64 ** power)
    return number

def format_indexes(entry, index_type, cipher_type, checksums=False):
    """
    return array of formatted indexes
    """
    fieldseparator = FIELDSEPARATOR
    indexes = list()
    #if len(entry['ids']) == 0:
    #   eprint("processing entry '{}'".format(entry['full']))
    if index_type == "":
        for identifier in entry['ids']:
            indexes.append("{}\n".format(identifier))
        return indexes

    length = entry['length_b64']
    if index_type in (".", "+"): #encryption: the iv is part of the index
        entryposition = "{}{}{}|{}{}".format(entry['position'],
                                             index_type,
                                             length,
                                             cipher_type,
                                             entry['iv'].hex())
    else: #no iv
        entryposition = "{}{}{}".format(entry['position'],
                                        index_type,
                                        length)
    for identifier in entry['ids']:
        if checksums:
            indexes.append("{1}{0}{2}{0}{3}\n".format(fieldseparator,
                                                      identifier,
                                                      entryposition,
                                                      int_to_b64(entry['checksum'])))
        else:
            indexes.append("{}{}{}\n".format(identifier, fieldseparator, entryposition))
    return indexes

def deflate(data, compresslevel=9):
    """
    compress a byte object
    """
    compress = zlib.compressobj(
        compresslevel,        # level: 0-9
        zlib.DEFLATED,        # method: must be DEFLATED
        -zlib.MAX_WBITS,      # window size in bits:
                              #   -15..-8: negate, suppress header
                              #   8..15: normal
                              #   16..30: subtract 16, gzip header
        zlib.DEF_MEM_LEVEL,   # mem level: 1..8/9
        0                     # strategy:
                              #   0 = Z_DEFAULT_STRATEGY
                              #   1 = Z_FILTERED
                              #   2 = Z_HUFFMAN_ONLY
                              #   3 = Z_RLE
                              #   4 = Z_FIXED
    )
    deflated = compress.compress(data)
    deflated += compress.flush()
    return deflated

def inflate(data):
    """
    decompress a byte object
    """
    decompress = zlib.decompressobj(
        -zlib.MAX_WBITS
    )
    inflated = decompress.decompress(data)
    inflated += decompress.flush()
    return inflated

def get_cipher_type(cipher):
    """
    return cipher_type letter from cipher
    """
    cipher2letter = {'aes128': 'A', 'aes192': 'B', 'aes256': 'C'}
    if cipher not in cipher2letter:
        eprint("    => ERROR: cipher '{}' not recognised!".format(cipher))
        sys.exit(1)
    return cipher2letter[cipher]

def find_keysize_cipher(cipher_type):
    """
    infer cipher and keysize from cipher_type letter
    return keysize and cipher
    """
    if cipher_type == "A":
        cipher_name = "aes128"
        keysize = 16
    elif cipher_type == "B":
        cipher_name = "aes192"
        keysize = 24
    elif cipher_type == "C":
        cipher_name = "aes256"
        keysize = 32
    else:
        eprint("    => ERROR: cipher_type '{}' not recognised!".format(cipher_type))
        sys.exit(1)
    return keysize, cipher_name

def merge_indexes(index1_filename, index2_filename, pos_offset,
                  index_type, outfile, tempfile=None):
    """
    merge two index files into one outfile, shifting the second one of given offset
    returns number of identifiers from the second index
    """
    if tempfile is None: #read second index into array, then merge
        new_identifiers = shift_index(index2_filename,
                                      pos_offset,
                                      index_type)
        new_identifiers_count = len(new_identifiers)
        #faster than heap_mergesort if len(new_identifiers) is small
        mergesort_file_array(index1_filename, new_identifiers, outfile)
        #slower but with good performance no matter which size:
        #heap_mergesort_file_array(index1_filename, new_identifiers, outfile) #ALT
    else: #use a temporary file to shift the second index, then merge files
        new_identifiers_count = shift_index_file(index2_filename,
                                                 pos_offset,
                                                 tempfile,
                                                 index_type)
        mergesort_files([index1_filename, tempfile], outfile)
        os.remove(tempfile)
    return new_identifiers_count

def mergesort_file_array(filename, insertions, outfilename):
    """
    merge sort contents of a file and an array
    writing to outputfile
    """
    newline = insertions.pop(0)
    with open(outfilename, 'w') as outfh:
        with open(filename, 'r', 1) as sortedfh: #using line buffering
            for line in sortedfh:
                while insertions:
                    if newline <= line:
                        outfh.write(newline)
                        newline = insertions.pop(0) #prepare for next
                    else:
                        break
                if newline <= line:
                    outfh.write(newline)
                    newline = ""
                outfh.write(line)
            if newline != "":
                outfh.write(newline)
            if insertions: #if some left in the end, append
                for newline in insertions:
                    outfh.write(newline)

def heap_mergesort_file_array(filename, insertions, outfilename):
    """
    use heapq.merge to merge sort contents of a file and an array
    writing to outputfile
    """
    with open(outfilename, 'w') as outfh:
        with open(filename, 'r', 1) as sortedfh: #using line buffering
            outfh.writelines(merge(sortedfh, insertions))

def mergesort_files(filenames, outfilename):
    """
    use heapq.merge to merge sort contents of files
    writing to outputfile
    """
    filehandles = list()
    for filename in filenames:
        filehandles.append(open(filename, 'r', 1))
    with open(outfilename, 'w') as outfh:
        outfh.writelines(merge(*filehandles))
    for filehandle in filehandles:
        filehandle.close()

def file_string_binary_search(fh, string, mode=None):
    """
    perform a binary_search for a string in a sorted text file
    return all matches, unless mode specified (can be 'first' or 'last')
    """
    found = None
    foundpos = None
    lo = 0
    fh.seek(0, 2)
    hi = fh.tell() - 1 #filesize
    while lo < hi:
        mid = (hi + lo) >> 1
        if mid > 0:
            fh.seek(mid - 1)
            fh.readline() #to avoid truncated line we skip this one
            midf = fh.tell()
        else:
            midf = 0
            fh.seek(midf)
        line = fh.readline() #read next line
        #eprint("lo {} hi {} mid {} midf {}".format(lo, hi, mid, midf)) #debug
        if not line or string <= line:
            hi = mid
        else:
            lo = mid + 1
    if mid == lo: #range fully scanned
        foundpos = midf
    elif lo <= 0: #very beginning
        foundpos = 0
    if foundpos is None:
        line = fh.readline()
        #foundpos = fh.tell()
    #eprint("got fist match at pos {}".format(foundpos))
    if mode == 'first': # return earliest match
        if line.startswith(string):
            found = line
    elif mode == 'last': # return latest match
        while line.startswith(string): #continue skipping lines while they match
            found = line
            line = fh.readline()
    else: #multi: return all matches
        found = list()
        while line.startswith(string): #continue adding lines while they match
            found.append(line)
            line = fh.readline()
    return found

def _file_string_binary_search_any(fh, string):
    """
    perform a binary_search for a string in a sorted text file
    it will quickly return any match, not necessarily the first or the last in the file
    """
    found = None
    #start checking the very first line:
    fh.seek(0)
    line = fh.readline()
    if line.startswith(string):
        found = line
    else:
        lo = 0 #beginning of file
        fh.seek(0, 2)
        hi = fh.tell() #end of file
        while hi - lo > 1:
            mid = (hi + lo) // 2
            fh.seek(mid, 0)
            fh.readline() #to avoid truncated line we skip this one
            line = fh.readline() #read next line
            if line.startswith(string):
                found = line
                break
            if string > line:
                lo = mid
            else:
                hi = mid
        if found is None:
            fh.seek(hi)
            line = fh.readline()
            if line.startswith(string):
                found = line

    if found is not None:
        return found
    return None

def get_position_first(indexfh, identifier):
    """
    find from the index the position of the entry
    corresponding to specified identifier
    """
    identifier += FIELDSEPARATOR #e.g. "ABCDE\t"
    line = file_string_binary_search(indexfh, identifier, mode='first')
    if line:
        columns = line.rstrip().split(FIELDSEPARATOR)
        return columns[1]
    return None #if could not be found

def get_position_last(indexfh, identifier):
    """
    find from the index the position of the entry
    corresponding to specified identifier
    """
    identifier += FIELDSEPARATOR #e.g. "ABCDE\t"
    line = file_string_binary_search(indexfh, identifier, mode='last')
    if line:
        columns = line.rstrip().split(FIELDSEPARATOR)
        return columns[1]
    return None #if could not be found

def get_positions(indexfh, identifier):
    """
    find from the index all the positions
    corresponding to (possibly duplicated) specified identifier
    """
    positions = list()
    identifier += FIELDSEPARATOR #e.g. "ABCDE "
    lines = file_string_binary_search(indexfh, identifier)
    if lines:
        for line in lines:
            columns = line.rstrip().split(FIELDSEPARATOR)
            positions.append(columns[1]) #2nd column of index file is position
    return positions

def get_position_checksum_first(indexfh, identifier):
    """
    find from the index the position of the entry
    corresponding to specified identifier
    additionally return checksum
    """
    identifier += FIELDSEPARATOR #e.g. "ABCDE\t"
    line = file_string_binary_search(indexfh, identifier, mode='first')
    if line:
        columns = line.rstrip().split(FIELDSEPARATOR)
        return columns[1], columns[2]
    return None #if could not be found

def get_position_checksum_last(indexfh, identifier):
    """
    find from the index the position of the entry
    corresponding to specified identifier
    additionally return checksum
    """
    identifier += FIELDSEPARATOR #e.g. "ABCDE\t"
    line = file_string_binary_search(indexfh, identifier, mode='last')
    if line:
        columns = line.rstrip().split(FIELDSEPARATOR)
        return columns[1], columns[2]
    return None #if could not be found

def get_positions_checksums(indexfh, identifier):
    """
    find from the index all the positions
    corresponding to (possibly duplicated) specified identifier
    """
    positions = list()
    checksums = list()
    identifier += FIELDSEPARATOR #e.g. "ABCDE "
    lines = file_string_binary_search(indexfh, identifier)
    if lines:
        for line in lines:
            columns = line.rstrip().split(FIELDSEPARATOR)
            positions.append(columns[1]) #2nd column of index file is position
            checksums.append(columns[2]) #3rd column of index file is checksum
    return positions, checksums

def init_cipher(key, iv):
    """
    initialize and return a new cipher
    """
    return AES.new(key, AES.MODE_CFB, iv)

def generate_iv():
    """
    generate and return a new iv
    """
    return Random.new().read(AES.block_size)

def derive_key(passphrase, keysize):
    """
    derive key of desired bytesize from passphrase
    """
    key = KDF.PBKDF2(passphrase, SALT, keysize)
    if len(key) != keysize:
        msg = "Attention! Something has gone wrong when deriving key from passphrase!"
        raise RuntimeError(msg)
    ks2cipher = {16: 'aes128', 24: 'aes192', 32: 'aes256'}
    cipher_name = ks2cipher[keysize]
    return cipher_name, key

def check_index(line):
    """
    read first record of a positional index and gather needed information
    return index_type, cipher_name, keysize, has_checksums
    """
    cipher_name = None
    keysize = None
    has_checksums = False

    if line.find(FIELDSEPARATOR) == -1:
        eprint("    => ERROR: index does not have positional column. Wrong file??")
        sys.exit(1)

    columns = line.split(FIELDSEPARATOR)
    if len(columns) > 2: #3rd column is crc
        has_checksums = True
    position = columns[1] #2nd column of index file
    iv = None
    if position.find("+") != -1: #both compressed and encrypted
        index_type = "+"
        iv = REIVD.match(position)
    elif position.find(".") != -1: #encrypted but not compressed
        index_type = "."
        iv = REIV.match(position)
    elif position.find(":") != -1: #compressed but not encrypted
        index_type = ":"
    elif position.find("-") != -1: #not encrypted nor compressed but entry sizes stored
        index_type = "-"
    else:
        eprint("    => ERROR: index does not have index_type identifier. Wrong file??")
        sys.exit(1)

    if index_type in (".", "+"): #encrypted entries
        if iv is None:
            eprint("    => ERROR: missing iv, cannot infer cipher type and keysize!")
            sys.exit(1)
        cipher_type = iv.group(3) #A, B, C..
        keysize, cipher_name = find_keysize_cipher(cipher_type)
    return index_type, cipher_name, keysize, has_checksums

def read_from_size(fh, begin, size):
    """
    reads from filehandle a specified amount of bytes
    """
    fh.seek(begin)
    return fh.read(size)

def delete_files(filenames, path=None):
    """
    delete the temporary sub-files
    optionally specify a path where to look for the files
    """
    for filename in filenames:
        if path is not None:
            filename = os.path.join(path, filename)
        if os.path.isfile(filename):
            os.remove(filename)

def delete_file_if_exists(filename, path=None):
    """
    delete a file after confirming file exists
    optionally specify a path where to look for the files
    """
    if path is not None:
        filename = os.path.join(path, filename)
    if os.path.exists(filename):
        try: #just in case file disappears after the path.exists check..
            os.remove(filename)
        except FileNotFoundError:
            pass

def close_subfiles(fhlist):
    """
    close the filehandles for the temporary sub-files
    """
    for fh in fhlist:
        fh.close()

def print_subfiles(filenames, fh=None, offset=0):
    """
    read and print a series of temporary sub-files writing them sequentially
    to a given output filehandle or print them to stdout
    """
    mode = 'rb'
    if fh is None:
        fh = sys.stdout #print to stdout if no fh given
        mode = 'r'
    for filename in filenames:
        with open(filename, mode) as infile:
            ##straight, unbuffered, will read whole file in memory
            #fh.write(infile.read())
            ##buffered:
            filesize = os.path.getsize(filename)
            if offset > 0:
                infile.seek(offset)
                filesize -= offset
            blockcount, remainder = divmod(filesize, BUFFERSIZE)
            for _ in range(blockcount):
                buffered = infile.read(BUFFERSIZE)
                fh.write(buffered)
            fh.write(infile.read(remainder))

def shift_index(index_filename, offset, index_type):
    """
    shift positions in index file by given offset
    return list with identifiers/positions
    """
    identifiers = list()
    with open(index_filename, 'r', 1) as indexfh: #using line buffering
        for line in indexfh:
            columns = line.split(FIELDSEPARATOR)
            position = columns[1] #2nd column of index file is position
            if index_type in ("+", "."): #entry sizes and iv stored
                posmatch = REESIVN.match(position)
            elif index_type in (":", "-"): #entry sizes stored, no iv
                posmatch = REES.match(position)
            position, remaining = posmatch.groups()
            position = b64_to_int(position)
            columns[1] = "{}{}{}".format(int_to_b64(position + offset),
                                         index_type,
                                         remaining)
            identifiers.append(FIELDSEPARATOR.join(columns) + "\n")
    return identifiers

def shift_index_file(index_filename, offset, outfile, index_type):
    """
    shift positions in index file by given offset creating outfile
    """
    buffered = ""
    shifted_count = 0
    buflines = 100 #write every 100 lines
    bufcounter = 0
    with open(index_filename, 'r', 1) as indexfh: #using line buffering
        with open(outfile, 'w') as outfh:
            for line in indexfh:
                line = line.rstrip("\n")
                columns = line.split(FIELDSEPARATOR)
                position = columns[1] #2nd column of index file is position
                if index_type in ("+", "."): #entry sizes and iv stored
                    posmatch = REESIVN.match(position)
                elif index_type in (":", "-"): #entry sizes stored, no iv
                    posmatch = REES.match(position)
                position, remaining = posmatch.groups()
                position = b64_to_int(position)
                if bufcounter > buflines: #write to outfile and reset buffer
                    outfh.write(buffered)
                    shifted_count += bufcounter
                    bufcounter = 0
                    buffered = ""
                columns[1] = "{}{}{}".format(int_to_b64(position + offset),
                                             index_type,
                                             remaining)
                buffered += FIELDSEPARATOR.join(columns) + "\n"
                bufcounter += 1
            shifted_count += bufcounter
            outfh.write(buffered) #last block
    return shifted_count

def encode_delimiter_re(delimiter):
    """
    encode a delimiter regexp, replacing any ^ and $ with newlines
    """
    if delimiter[0] == "^" or delimiter[-1] == "$":
        delimiter = list(delimiter)
        if delimiter[0] == "^":
            delimiter[0] = "\n"
        if delimiter[-1] == "$":
            delimiter[-1] = "\n"
        delimiter = "".join(delimiter)
    return "{}".format(delimiter).encode('UTF-8')

def postprocess_entry(entry, mode):
    """
    perform any compression, encryption, checksum calculation as needed
    """
    if mode['xsanity']:
        entry['checksum'] = zlib.crc32(entry['full'])
    if mode['encrypt']:
        entry['iv'] = generate_iv() #generate and store new iv
        cipher = init_cipher(mode['key'], entry['iv'])
        if mode['compress']:
            entry['full'] = cipher.encrypt(deflate(entry['full'],
                                                   compresslevel=mode['compresslevel']))
            entry['length'] = len(entry['full']) #probably smaller, need to recompute
        else:
            entry['full'] = cipher.encrypt(entry['full'])
        mode['outffh'].write(entry['full'])
    elif mode['compress']: #compressed but not encrypted
        entry['full'] = deflate(entry['full'], compresslevel=mode['compresslevel'])
        entry['length'] = len(entry['full']) #probably smaller, need to recompute
        mode['outffh'].write(entry['full'])
    entry['length_b64'] = int_to_b64(entry['length'])

def entry_generator(path, delimiter, buffersize=None):
    """
    splits a file according to a delimiter, returning a generator of entries
    terminating in the delimiter
    """
    delimiter = encode_delimiter_re(delimiter)
    splen = len(delimiter)
    buf = b''
    with open(path, 'rb') as fh:
        data = fh.read(buffersize)
        while data:
            #buf += fh.read(buffersize)
            buf += data
            start = end = 0
            while end >= 0:# and len(buf)>=splen:
                start, end = end, buf.find(delimiter, end)
                if end != -1:
                    end += splen
                    yield buf[start:end], end - start
                else:
                    buf = buf[start:]
                    break
            data = fh.read(buffersize)
        #yield buf #if final content after terminator is desired

def compute_split_positions(filename, splitsize, delimiter=None):
    """
    compute and return start positions and sizes of chunks of splitsize
    without breaking lines or without breaking entries (if delimiter provided)
    """
    split_file_sizes = [0] #init with 0 size (this will be removed later)
    input_filesize = os.path.getsize(filename)

    split_file_startpos = [0] #first split start position is 0 (start of file)
    split_file_sizes = list() #init
    if delimiter is None:
        with open(filename, 'r', 1) as inputfh:
            split_file_end = find_next_newline(inputfh, input_filesize,
                                               splitsize)
            split_file_sizes.append(split_file_end)
            while split_file_end != input_filesize:
                split_file_start = split_file_end #of previous block
                split_file_startpos.append(split_file_start)
                split_file_end = find_next_newline(inputfh, input_filesize,
                                                   split_file_start + splitsize)
                split_file_sizes.append(split_file_end - split_file_start)
    else:
        delimiter = encode_delimiter_re(delimiter)
        bufsize = max(1024, min(1024*splitsize//65536, 1024*1024))
        with open(filename, 'rb') as inputfh:
            split_file_end = find_next_delimiter(inputfh, input_filesize,
                                                 splitsize, delimiter, bufsize)
            split_file_sizes.append(split_file_end)
            while split_file_end != input_filesize:
                split_file_start = split_file_end #of previous block
                split_file_startpos.append(split_file_start)
                split_file_end = find_next_delimiter(inputfh, input_filesize,
                                                     split_file_start + splitsize,
                                                     delimiter, bufsize)
                split_file_sizes.append(split_file_end - split_file_start)

    return split_file_startpos, split_file_sizes

def split_file(filename, splitsize, outprefix, delimiter=None):
    """
    split a file into chunks of splitsize without breaking lines
    returns list of chunk filenames and list of the chunk filesizes
    """
    split_files = list()
    split_file_startpos, split_file_sizes = compute_split_positions(filename, splitsize, delimiter)

    split_file_count = len(split_file_sizes)
    suffixlength = len(str(split_file_count))

    with open(filename, 'rb') as inputfh:
        for chunknum in range(split_file_count):
            chunk_suffix = str(chunknum).zfill(suffixlength)
            splitfile = outprefix + chunk_suffix
            split_files.append(splitfile)
            with open(splitfile, 'wb') as outfh:
                inputfh.seek(split_file_startpos[chunknum])
                ##straight, unbuffered, will read whole file in memory:
                #outfh.write(inputfh.read(split_file_sizes[chunknum]))
                ##buffered:
                blockcount, remainder = divmod(split_file_sizes[chunknum], BUFFERSIZE)
                for _ in range(blockcount):
                    buffered = inputfh.read(BUFFERSIZE)
                    outfh.write(buffered)
                outfh.write(inputfh.read(remainder))
    return split_files, split_file_sizes

def buffered_readwrite(inputfile, outputfile, startpos, size):
    """
    extract a piece of inputfile of given size from given position
    and write it to outputfile
    """
    with open(inputfile, 'rb') as inputfh:
        with open(outputfile, 'wb') as outfh:
            inputfh.seek(startpos)
            blockcount, remainder = divmod(size, BUFFERSIZE)
            for _ in range(blockcount):
                buffered = inputfh.read(BUFFERSIZE)
                outfh.write(buffered)
            outfh.write(inputfh.read(remainder))
            outfh.flush()

def check_iofiles(read_filenames, write_filenames):
    """
    check for ability to open input/output filenames
    """
    if read_filenames is not None:
        for filename in read_filenames:
            try:
                inputfh = open(filename, 'r')
                inputfh.close()
            except FileNotFoundError:
                eprint("    => ERROR: Cannot open file '{}' for reading".format(filename))
                sys.exit(2)

    if write_filenames is not None:
        for filename in write_filenames:
            if os.path.isfile(filename):
                eprint("    => ERROR: file '{}' exists".format(filename))
                eprint("       please remove it as we refuse to overwrite it!")
                sys.exit(1)
            try:
                myoutputfh = open(filename, 'w')
                myoutputfh.close()
            except PermissionError:
                eprint("    => ERROR: Cannot open file '{}' for writing".format(filename))
                sys.exit(1)
            #eprint("deleting {}".format(filename)) #DEBUG
            delete_files([filename])

def siprefix2num(numberstring):
    """
    Returns bytes from a metrix suffixed string
    """
    if numberstring == '0':
        return(0)

    prefix = {"k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4, "p": 1024**5}
    numberstring = numberstring.lower()
    try:
        return int(numberstring[:-1]) * prefix[numberstring[-1]]
    except KeyError:
        # no or unknown meter-prefix
        return int(numberstring)

def elapsed_time(start_time, work_done=None):
    """
    compute elapsed time from given start_time (in seconds) and return
    well formatted string. If work_done argument given, compute speed of the process
    """
    process_time = time.time() - start_time
    minutes, seconds = divmod(process_time, 60)
    hours, minutes = divmod(minutes, 60)
    if work_done is None:
        return("{:02.0f}h {:02.0f}m {:02.0f}s".format(
            hours, minutes, seconds))
    process_speed = str(round(work_done / process_time, 2))
    return "{:02.0f}h {:02.0f}m {:02.0f}s".format(hours, minutes, seconds), process_speed

def eprint(*myargs, **kwargs):
    """
    print to stderr, useful for error messages and to not clobber stdout
    """
    print(*myargs, file=sys.stderr, **kwargs)