__authors__ = ["T. Vincent"]
__license__ = "MIT"
__date__ = "16/05/2018"

import logging
import sys

import numpy
import pytest

from silx.utils.testutils import ParametricTestCase
from silx.math import colormap

_logger = logging.getLogger(__name__)

class TestNormalization(ParametricTestCase):
    """Test silx.math.colormap.Normalization sub classes"""

    def _testCodec(self, normalization, rtol=1e-5):
        """Test apply/revert for normalizations"""
        test_data = (numpy.arange(1, 10, dtype=numpy.int32),
                     numpy.linspace(1., 100., 1000, dtype=numpy.float32),
                     numpy.linspace(-1., 1., 100, dtype=numpy.float32),
                     1.,
                     1)

        for index in range(len(test_data)):
            with self.subTest(normalization=normalization, data_index=index):
                data = test_data[index]
                normalized = normalization.apply(data, 1., 100.)
                result = normalization.revert(normalized, 1., 100.)

                self.assertTrue(numpy.array_equal(
                    numpy.isnan(normalized), numpy.isnan(result)))

                if isinstance(data, numpy.ndarray):
                    notNaN = numpy.logical_not(numpy.isnan(result))
                    data = data[notNaN]
                    result = result[notNaN]
                self.assertTrue(numpy.allclose(data, result, rtol=rtol))

    def testLinearNormalization(self):
        """Test for LinearNormalization"""
        normalization = colormap.LinearNormalization()
        self._testCodec(normalization)

    def testLogarithmicNormalization(self):
        """Test for LogarithmicNormalization"""
        normalization = colormap.LogarithmicNormalization()
        # relative tolerance is higher because of the log approximation
        self._testCodec(normalization, rtol=1e-3)

        # Specific extra tests
        self.assertTrue(numpy.isnan(normalization.apply(-1., 1., 100.)))
        self.assertTrue(numpy.isnan(normalization.apply(numpy.nan, 1., 100.)))
        self.assertEqual(normalization.apply(numpy.inf, 1., 100.), numpy.inf)
        self.assertEqual(normalization.apply(0, 1., 100.), - numpy.inf)

    def testArcsinhNormalization(self):
        """Test for ArcsinhNormalization"""
        self._testCodec(colormap.ArcsinhNormalization())

    def testSqrtNormalization(self):
        """Test for SqrtNormalization"""
        normalization = colormap.SqrtNormalization()
        self._testCodec(normalization)

        # Specific extra tests
        self.assertTrue(numpy.isnan(normalization.apply(-1., 0., 100.)))
        self.assertTrue(numpy.isnan(normalization.apply(numpy.nan, 0., 100.)))
        self.assertEqual(normalization.apply(numpy.inf, 0., 100.), numpy.inf)
        self.assertEqual(normalization.apply(0, 0., 100.), 0.)

class TestColormap(ParametricTestCase):
    """Test silx.math.colormap.cmap"""

    NORMALIZATIONS = (
        'linear',
        'log',
        'arcsinh',
        'sqrt',
        colormap.LinearNormalization(),
        colormap.LogarithmicNormalization(),
        colormap.GammaNormalization(2.),
        colormap.GammaNormalization(0.5))

    @staticmethod
    def ref_colormap(data, colors, vmin, vmax, normalization, nan_color):
        """Reference implementation of colormap

        :param numpy.ndarray data: Data to convert
        :param numpy.ndarray colors: Color look-up-table
        :param float vmin: Lower bound of the colormap range
        :param float vmax: Upper bound of the colormap range
        :param str normalization: Normalization to use
        :param Union[numpy.ndarray, None] nan_color: Color to use for NaN
        """
        norm_functions = {'linear': lambda v: v,
                          'log': numpy.log10,
                          'arcsinh': numpy.arcsinh,
                          'sqrt': numpy.sqrt}

        if isinstance(normalization, str):
            norm_function = norm_functions[normalization]
        else:
            def norm_function(value):
                return normalization.apply(value, vmin, vmax)

        with numpy.errstate(divide='ignore', invalid='ignore'):
            # Ignore divide by zero and invalid value encountered in log10, sqrt
            norm_data, vmin, vmax = map(norm_function, (data, vmin, vmax))

        if normalization == 'arcsinh' and sys.platform == 'win32':
            # There is a difference of behavior of numpy.arcsinh
            # between Windows and other OS for results of infinite values
            # This makes Windows behaves as Linux and MacOS
            norm_data[data == numpy.inf] = numpy.inf
            norm_data[data == -numpy.inf] = -numpy.inf

        nb_colors = len(colors)
        scale = nb_colors / (vmax - vmin)

        # Substraction must be done in float to avoid overflow with uint
        indices = numpy.clip(scale * (norm_data - float(vmin)),
                             0, nb_colors - 1)
        indices[numpy.isnan(indices)] = nb_colors  # Use an extra index for NaN
        indices = indices.astype('uint')

        # Add NaN color to array
        if nan_color is None:
            nan_color = (0,) * colors.shape[-1]
        colors = numpy.append(colors, numpy.atleast_2d(nan_color), axis=0)

        return colors[indices]

    def _test(self, data, colors, vmin, vmax, normalization, nan_color):
        """Run test of colormap against alternative implementation

        :param numpy.ndarray data: Data to convert
        :param numpy.ndarray colors: Color look-up-table
        :param float vmin: Lower bound of the colormap range
        :param float vmax: Upper bound of the colormap range
        :param str normalization: Normalization to use
        :param Union[numpy.ndarray, None] nan_color: Color to use for NaN
        """
        image = colormap.cmap(
            data, colors, vmin, vmax, normalization, nan_color)

        ref_image = self.ref_colormap(
            data, colors, vmin, vmax, normalization, nan_color)

        self.assertTrue(numpy.allclose(ref_image, image))
        self.assertEqual(image.dtype, colors.dtype)
        self.assertEqual(image.shape, data.shape + (colors.shape[-1],))

    def test(self):
        """Test all dtypes with finite data

        Test all supported types and endianness
        """
        colors = numpy.zeros((256, 4), dtype=numpy.uint8)
        colors[:, 0] = numpy.arange(len(colors))
        colors[:, 3] = 255

        # Generates (u)int and floats types
        dtypes = [e + k + i for e in '<>' for k in 'uif' for i in '1248'
                  if k != 'f' or i != '1']
        dtypes.append(numpy.dtype(numpy.longdouble).name)  # Add long double

        for normalization in self.NORMALIZATIONS:
            for dtype in dtypes:
                with self.subTest(dtype=dtype, normalization=normalization):
                    _logger.info('normalization: %s, dtype: %s',
                                 normalization, dtype)
                    data = numpy.arange(-5, 15).astype(dtype).reshape(4, 5)

                    self._test(data, colors, 1, 10, normalization, None)

    def test_not_finite(self):
        """Test float data with not finite values"""
        colors = numpy.zeros((256, 4), dtype=numpy.uint8)
        colors[:, 0] = numpy.arange(len(colors))
        colors[:, 3] = 255

        test_data = {  # message: data
            'no finite values': (float('inf'), float('-inf'), float('nan')),
            'only NaN': (float('nan'), float('nan'), float('nan')),
            'mix finite/not finite': (float('inf'), float('-inf'), 1., float('nan')),
        }

        for normalization in self.NORMALIZATIONS:
            for msg, data in test_data.items():
                with self.subTest(msg, normalization=normalization):
                    _logger.info('normalization: %s, %s', normalization, msg)
                    data = numpy.array(data, dtype=numpy.float64)
                    self._test(data, colors, 1, 10, normalization, (0, 0, 0, 0))

    def test_errors(self):
        """Test raising exception for bad vmin, vmax, normalization parameters
        """
        colors = numpy.zeros((256, 4), dtype=numpy.uint8)
        colors[:, 0] = numpy.arange(len(colors))
        colors[:, 3] = 255

        data = numpy.arange(10, dtype=numpy.float64)

        test_params = [  # (vmin, vmax, normalization)
            (-1., 2., 'log'),
            (0., 1., 'log'),
            (1., 0., 'log'),
            (-1., 1., 'sqrt'),
            (1., -1., 'sqrt'),
        ]

        for vmin, vmax, normalization in test_params:
            with self.subTest(
                    vmin=vmin, vmax=vmax, normalization=normalization):
                _logger.info('normalization: %s, range: [%f, %f]',
                             normalization, vmin, vmax)
                with self.assertRaises(ValueError):
                    self._test(data, colors, vmin, vmax, normalization, None)

def test_apply_colormap():
    """Basic test of silx.math.colormap.apply_colormap"""
    data = numpy.arange(256)
    expected_colors = numpy.empty((256, 4), dtype=numpy.uint8)
    expected_colors[:, :3] = numpy.arange(256, dtype=numpy.uint8).reshape(256, 1)
    expected_colors[:, 3] = 255
    colors = colormap.apply_colormap(
        data,
        colormap="gray",
        norm="linear",
        autoscale="minmax",
        vmin=None,
        vmax=None,
        gamma=1.0)
    assert numpy.array_equal(colors, expected_colors)

testdata_normalize = [
    (numpy.arange(512), numpy.arange(512) // 2, 0, 511),
    ((numpy.nan, numpy.inf, -numpy.inf), (0, 255, 0), 0, 1),
    ((numpy.nan, numpy.inf, -numpy.inf, 1), (0, 255, 0, 0), 1, 1),
]

@pytest.mark.parametrize(
    "data,expected_data,expected_vmin,expected_vmax",
    testdata_normalize,
)
def test_normalize(data, expected_data, expected_vmin, expected_vmax):
    """Basic test of silx.math.colormap.normalize"""
    result = colormap.normalize(
        numpy.asarray(data),
        norm="linear",
        autoscale="minmax",
        vmin=None,
        vmax=None,
        gamma=1.0,
    )
    assert result.vmin == expected_vmin
    assert result.vmax == expected_vmax
    assert numpy.array_equal(
        result.data,
        numpy.asarray(expected_data, dtype=numpy.uint8),
    )


import numpy as np
import pandas as pd
import scipy
from sklearn.linear_model import LinearRegression as linreg
from sklearn.linear_model import LogisticRegression as logreg
from sklearn.cross_validation import KFold
from sklearn.cross_validation import *
from sklearn import  cross_validation

titanic=pd.read_csv("train.csv")

#print(titanic.describe())
#print(titanic.head(5))

# ------------------- DATA CORRECTION --------------------------------

# 1) Fill missing Age data with median
titanic["Age"]=titanic["Age"].fillna(titanic["Age"].median())

# 2) Convert Sex string with 0 or 1
titanic.loc[titanic["Sex"] == "male", "Sex"] = 0 #convert 0 for men
titanic.loc[titanic["Sex"] =="female", "Sex"]=1 #convert 1 for women

# 3) Fill missing Embarked data with most common char
print(pd.value_counts(titanic["Embarked"].values, sort=False))
     # "S" is most common char -> chosen as default for missing values
titanic["Embarked"]=titanic["Embarked"].fillna("S")

#4) Replace Embarked char with numeric code
#titanic.loc[titanic["Embarked"]=="S", "Embarked"]=0 # 'S' -> 0
#titanic.loc[titanic["Embarked"]=="C", "Embarked"]=1 # 'C' -> 1
titanic.loc[titanic["Embarked"]=="S", "Embarked"]=0
titanic.loc[titanic["Embarked"]=="C", "Embarked"]=1
titanic.loc[titanic["Embarked"]=="Q", "Embarked"]=2 # 'Q' -> 2

# input column used for predictions :
predictors=["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare", "Embarked"]

# Initialize the algorithm
algo_linreg = linreg()

# Generate cross-validation folds with random splits
# return rows indices for corresponding train and set
kf =KFold(titanic.shape[0], n_folds=3, random_state=1)

# Make the predictions
predictions =[]
for train, test in kf:
    # Which predictors used on train fold
    train_predictors = (titanic[predictors].iloc[train,:])
    # Target/goal used to train the algo
    train_target= titanic["Survived"].iloc[train]

    # Train the algo with the predictors and target
    # .fit(x input, y output)
    algo_linreg.fit(train_predictors, train_target)
    # Make predictions with the trained algo on test fold
    test_predictions = algo_linreg.predict(titanic[predictors].iloc[test,:])
    predictions.append(test_predictions)

# The predictions are in 3 Numpy arrays
# So we concatenate the arrays on axis 0 (bc only 1 axis)
predictions=np.concatenate(predictions, axis=0)

predictions[predictions> .5]=1
predictions[predictions<= .5]=0

print(predictions)

print(sum(predictions==titanic["Survived"]))
accuracy= sum(predictions==titanic["Survived"])/len(predictions)
print(accuracy) # = 0.783

#------------------- Logistic Regression method ---------------------

# Initialize the algo
algo_logreg = logreg(random_state=1)

# Compute accuracy score for all cross-V folds;
# cross_val_score(algo, predictors, target, cross-validation fold)
scores = cross_validation.cross_val_score(algo_logreg, titanic[predictors], titanic["Survived"], cv=3)
# Mean of the scores for each folds (3 folds)
print(scores.mean())

#----------------------------------- Log Reg. with test set ---------------------

titanic_test = pd.read_csv("test.csv")

# I) Clean data
titanic_test["Age"] = titanic_test["Age"].fillna(titanic["Age"].median())
titanic_test["Fare"] = titanic_test["Fare"].fillna(titanic_test["Fare"].median())
titanic_test.loc[titanic_test["Sex"] == "male", "Sex"] = 0
titanic_test.loc[titanic_test["Sex"] == "female", "Sex"] = 1
titanic_test["Embarked"] = titanic_test["Embarked"].fillna("S")

titanic_test.loc[titanic_test["Embarked"] == "S", "Embarked"] = 0
titanic_test.loc[titanic_test["Embarked"] == "C", "Embarked"] = 1
titanic_test.loc[titanic_test["Embarked"] == "Q", "Embarked"] = 2

# II) Test algo on data

# Initialize the algo
algo_logreg_test=logreg(random_state=1)

# Train algo on using all training data
algo_logreg_test.fit(titanic[predictors], titanic["Survived"])

# Make predictions with algo on data
predictions=algo_logreg_test.predict(titanic_test[predictors])

# Generate new dataset for kaggle submission
submission= pd.DataFrame({
    "PassengerId" : titanic_test["PassengerId"],
    "Survived": predictions
})

submission.to_csv("kaggle.csv", index=False)


"""
Contains classes Quandle, Biquandle, Identity_Quandle,
Alexander_Quandle, and Singquandle.

FIXME:
    - Nothing for now.

TODO:
    - If X is a rack with operation a*b, then it is a birack if we
      define a**b as the identity a**b == a. Thus biquandle matrix2
      should be optional.
    - Does the above apply to singquandles/singracks?
    - homomorphism methods.
"""

import numpy as np
from pyknots.modules.magmas import Magma
from pyknots.modules.groups import Group
from pyknots.modules.utils import issquare, applymorphism
import json
import os

__all__ = ['Quandle', 'Biquandle', 'Singquandle',
'Alexander_Quandle', 'Conj_Quandle', 'Trivial_Quandle']

class Quandle(Magma):
    """Instantiate a quandle object by passing Quandle() a matrix, a
       string representation of the RIG index, or a numpy array.
    """
    def __init__(self, matrix):
        if type(matrix) is str:
            dirname = os.path.dirname(os.path.realpath(__file__))
            path = os.path.join(os.path.dirname(dirname), 'data', 'RIG_quandles.json')
            try:
                with open(path, 'r') as fp:
                    matrix = json.load(fp)[matrix]
                    self.__init__(matrix)
            except KeyError:
                raise TypeError('Input %s not a RIG quandle.' % (matrix))
        else:
            super().__init__(matrix)

    def __str__(self):
        return str(self.array)

    def as_biquandle(self):
        """ Generate identity matrix and return biquandle class object."""
        M1 = self.array
        M2 = Trivial_Quandle(self.order, self.index).array
        B = Biquandle(M1, M2)
        return B

    def inverse_quandle(self):
        M = self.array
        n = self.order
        new_M = np.zeros((n, n), dtype=int)
        for i in range(n):
            for j in range(n):
                k = M[i,j]
                new_M[k,j] = i
        return Quandle(new_M)

    def is_rack(self):
        """ A rack is a set with 2 axioms: for a, b, c in X,
            1) a*b is a bijection.
            2) (a*b)*c == (a*c)*(b*c).
        """
        M = self.array
        ind = self.index
        for i in range(self.order):
            col = []
            for j in range(self.order):
                for k in range(self.order):
                    if M[M[i,j]-ind,k] != M[M[i,k]-ind,M[j,k]-ind]:
                        return False
                col.append(M[j,i])
            for c in range(len(col)):
                if not c+ind in col:
                    return False
        return True

    def is_quandle(self):
        """ A quandle is a rack that satisfies the third axiom that for
            all a in X, a*a == a.
        """
        M = self.array
        ind = self.index
        if not self.is_rack():
            return False
        for i in range(self.order):
            if M[i,i] != i+ind:
                return False
        return True

    def is_biquandle(self):
        return False

    def is_singquandle(self):
        return False

    def is_kei(self):
        if self.is_quandle() and self.is_involutory():
            return True
        return False

    def is_trivial(self):
        """ A quandle is trivial if for all a, b in X, a*b == a."""
        M = self.array
        for i in range(self.order):
            for j in range(self.order):
                if M[i,j] != i:
                    return False
        return True

    def is_involutory(self):
        """ A quandle is involutory if for all a, b in X, a*(a*b) == b."""
        M = self.array
        for i in range(self.order):
            for j in range(self.order):
                if M[i,M[i,j]] != j:
                    return False
        return True

    def is_dihedral(self):
        """ If for all a, b in X, a*b == 2*b-a then it is dihedral.
            Equivalent to isomorphism to Alexander_Quandle(p, -1)
        """
        M = self.array
        ind = self.index
        p = self.order
        for i in range(self.order):
            for j in range(self.order):
                if M[i,j] != ((2*(j) - (i)) % p)+ind:
                    return False
        return True

    def is_medial(self):
        """ Equivalent to abelian quandle. If X satisfies the property that for
            any a, b, c, d in Q, (a*b)*(c*d) == (a*c)*(b*d) it is medial.
        """
        M = self.array
        ind = self.index
        for i in range(self.order):
            for j in range(self.order):
                for m in range(self.order):
                    for n in range(self.order):
                        if M[M[i,j]-ind,M[m,n]-ind] != M[M[i,m]-ind,M[j,n]-ind]:
                            return False
        return True

class Biquandle(object):
    """ Instantiate a biquandle object by passing Biquandle() a pair of
        matrices, a string representation of the RIG index, or a
        numpy array.
    """
    def __init__(self, matrix1, matrix2=None):
        if matrix2 is None:
            M1 = Quandle(matrix1)
            M2 = Identity_Quandle(M1.order, M1.index)
            self.__init__(M1.array, M2.array)
        else:
            self.quandle1, self.quandle2 = Quandle(matrix1), Quandle(matrix2)
            self.array1, self.array2 = np.array(matrix1), np.array(matrix2)
            self.order = len(matrix1[0])
            self.index = self._index()

    def __str__(self):
        return str(self.array1)+str(self.array2)

    def _index(self):
        """ Verify that indices of input match."""
        ind1, ind2 = np.amin(self.array1), np.amin(self.array2)
        if ind1 != ind2:
            raise IndexError('%s, %s have non-matching indices.' % (self.array1, self.array2))
        return ind1

    def is_birack(self):
        """ A birack is a set with 4 axioms: for a, b, c in X,
            1) a*b, a**b is a bijection.
            2) (a**b)**(c**b) == (a**c)**(b*c).
            3) (a*b)*(c*b) == (a*c)*(b**c)
            4) (a*b)**(c*b) == (a**c)*(b**c)
        """
        M1, M2 = self.array1, self.array2
        ind = self.index
        if not self.is_invertible():
            return False
        for a in range(self.order):
            for b in range(self.order):
                for c in range(self.order):
                    if M2[M2[a,b]-ind,M2[c,b]-ind] != M2[M2[a,c]-ind,M1[b,c]-ind]:
                        return False
                    if M1[M1[a,b]-ind,M1[c,b]-ind] != M1[M1[a,c]-ind,M2[b,c]-ind]:
                        return False
                    if M2[M1[a,b]-ind,M1[c,b]-ind] != M1[M2[a,c]-ind,M2[b,c]-ind]:
                        return False
        return True

    def is_biquandle(self):
        """ A biquandle is a birack such that for all a in X, there
            exists x such that: x*a == x <==> a**x == a"""
        M1, M2 = self.array1, self.array2
        if not self.is_birack():
            return False
        for i in range(self.order):
            for j in range(self.order):
                if M1[i,j] == i and M2[j,i] != j:
                    return False
        return True

    def is_singquandle(self):
        return False

    def is_invertible(self):
        if self.quandle1.is_left_invertible():
            if self.quandle2.is_left_invertible():
                return True
        return False

    def check_wada(self):
        M1, M2 = self.array1, self.array2

        for a in range(self.order):
            for b in range(self.order):
                for c in range(self.order):
                    if M1[M1[a,b],M1[M2[a,b],c]] != M1[a,M1[b,c]]:
                        return False
                    if M2[M1[a,b],M1[M2[a,b],c]] != M1[M2[a,M1[b,c]],M2[b,c]]:
                        return False
                    if M2[M2[a,b],c] != M2[M2[a,M1[b,c]],M2[b,c]]:
                        return False
        return True

class Singquandle(object):
    """ Instantiate a singquandle object by passing Singquandle() three
        matrices (denoting Cayley tables for *, R1, and R2), a string
        representation of the RIG index, or a numpy array.
        (Only matrices supported)
    """
    def __init__(self, matrix1, matrix2, matrix3):
        self.quandle1, self.quandle2 = Quandle(matrix1), Quandle(matrix2)
        self.quandle3 = Quandle(matrix3)
        self.array1, self.array2 = np.array(matrix1), np.array(matrix2)
        self.array3 = np.array(matrix3)
        self.order = len(matrix1[0])
        self.index = self._index()

    def __str__(self):
        return str(self.array1)+str(self.array2)+str(self.array3)

    def _index(self):
        """ Verify that indices of input match."""
        ind1, ind2, ind3 = np.amin(self.array1), np.amin(self.array2), np.amin(self.array3)
        if ind1 != ind2 != ind3:
            raise IndexError('%s, %s, %s have non-matching indices.' % (self.array1, self.array2, self.array3))
        return ind1

    def is_invertible(self):
        """ Check whether * is an invertible operation."""
        if not self.quandle1.is_left_invertible():
            return False
        return True

    """
    def check_identity(self):
         R1(x,y) = R2(y,x)*x, R2(x,y) = R1(y,x)*y.
        M1, M2, M3 = self.array1, self.array2, self.array3
        ind = self.index
        for a in range(self.order):
            for b in range(self.order):
                if M2[a,b] != M1[M3[b,a]-ind,a]:
                    return False
                if M3[a,b] != M1[M2[b,a]-ind,b]:
                    return False
        return True
    """

    def is_singquandle(self):
        """ Check if the object is a singquandle."""
        if self.is_nonoriented_singquandle() or self.is_oriented_singquandle():
            return True
        return False

    def is_nonoriented_singquandle(self):
        """ Check if the singquandle satisfies the axioms of a nonoriented
            singquandle.
        """
        M1, M2, M3 = self.array1, self.array2, self.array3
        ind = self.index
        if not self.is_invertible():
            return False
        for a in range(self.order):
            for b in range(self.order):
                if M3[a,b] != M2[b,M1[a,b]-ind]:
                    return False
                if M3[b,M1[a,b]-ind] != M1[M2[a,b]-ind,M3[a,b]-ind]:
                    return False
                if M2[a,b] != M3[M1[b,a]-ind,a]:
                    return False
                if M2[M1[b,a]-ind,a] != M1[M3[a,b]-ind,M2[a,b]-ind]:
                    return False
                for c in range(self.order):
                    if M1[M1[b,c]-ind,M3[a,c]-ind] != M1[M1[b,a]-ind,M2[a,c]-ind]:
                        return False
                    if M1[M2[a,b]-ind,c] != M2[M1[a,c]-ind,M1[b,c]-ind]:
                        return False
                    if M1[M3[a,b]-ind,c] != M3[M1[a,c]-ind,M1[b,c]-ind]:
                        return False
        return True

    def is_oriented_singquandle(self):
        """ Check if the singquandle satisfies the axioms of an oriented
            singquandle.
        """
        M1, M2, M3 = self.array1, self.array2, self.array3
        n, ind = self.order, self.index
        inv = self.quandle1.inverse_quandle().array
        if not self.is_invertible():
            return False
        for x in range(n):
            for y in range(n):
                for z in range(n):
                    if M1[M2[inv[x,y],z],y] != M2[x,M1[z,y]]:
                        return False
                    if M3[inv[x,y],z] != inv[M3[x,M1[z,y]], y]:
                        return False
                    if M1[inv[y,M2[x,z]],x] != inv[M1[y,M3[x,z]], z]:
                        return False
                    if M3[x,y] != M2[y,M1[x,y]]:
                        return False
                    if M1[M2[x,y], M3[x,y]] != M3[y, M1[x,y]]:
                        return False
        return True

class Alexander_Quandle(Quandle):
    """ Returns quandle generated by Alexander module Z_p/((t**b)-a).
        Setting exponent b not supported.
    """
    def __init__(self, p, a=-1, b=None):
        M = np.zeros((p, p), dtype=int)
        for i in range(p):
            for j in range(p):
                M[i,j] = (a*i + (1 - a)*j) % p
        super().__init__(M.tolist())

class Conj_Quandle(Quandle):
    """ Returns quandle generated by the group cayley table with automorphism f. Pass
        f as a permutation in the form (1, 2, 3, ...). Then f maps index(i) to i.
        Quandle is given by conjugation operation x*y = f(y)^{-1}f(x)f(y).
    """
    def __init__(self, matrix, *args):
        n = len(matrix[0])
        M = np.zeros((n, n), dtype=int)
        for arg in args:
            if isinstance(arg, tuple):
                matrix = applymorphism(matrix, arg)
        G = Group(matrix)
        m = G.array
        for i in range(n):
            for j in range(n):
                M[i,j] = m[m[G.inverse(j), i], j]
        super().__init__(M)

class Trivial_Quandle(Quandle):
    """ Returns a trivial quandle such that for all a, b in X,
        a*b == a. Optional index for non-0-indexed quandles.
    """
    def __init__(self, dim, index=0, flip=False):
        ind = index
        M = []
        if not flip:
            for i in range(dim):
                row = [i+ind for j in range(ind, dim+ind)]
                M.append(row)
        else:
            for i in range(dim):
                row = [j+ind for j in range(ind, dim+ind)]
                M.append(row)
        super().__init__(M)


import numpy as np
import torch

from affogato.affinities import compute_affinities
from torchvision.utils import make_grid
from inferno.extensions.criteria import SorensenDiceLoss

class ConcatDataset(torch.utils.data.Dataset):
    def __init__(self, *datasets):
        self.datasets = datasets
        self.lens = [len(ds) for ds in self.datasets]

        self.start_idx = np.cumsum(self.lens)
        self.start_idx[-1] = 0
        self.start_idx = np.roll(self.start_idx, 1)

    def __len__(self):
        return sum(self.lens)

    def __getitem__(self, index):
        ds_index = np.where(index - self.start_idx >= 0)[0][-1]
        item_index = index - self.start_idx[ds_index]
        return self.datasets[ds_index][item_index]

class DefaultDataset(torch.utils.data.Dataset):
    """ Simple default dataset for generating affinities
    from segmentation and mask.
    """
    patch_shape = [512, 512]  # TODO expose this and other parameters

    def to_affinities(self, seg, mask):
        seg[~mask] = 0
        affs, aff_mask = compute_affinities(seg, self.offsets, have_ignore_label=True)
        aff_mask = aff_mask.astype('bool')
        affs = 1. - affs

        mask_transition, aff_mask2 = compute_affinities(mask, self.offsets)
        mask_transition[~aff_mask2.astype('bool')] = 1
        aff_mask[~mask_transition.astype('bool')] = True
        return affs, aff_mask

    @staticmethod
    def estimate_n_samples(shape, patch_shape):
        # we estimate the number of samples by tiling shape with patch_shape
        crops_per_dim = [sh / float(cs) for sh, cs in zip(shape, patch_shape)]
        return int(np.prod(crops_per_dim))

    def __init__(self, raw, seg, mask_ids, offsets, transforms=None):
        self.raw = raw
        self.seg = seg
        self.mask_ids = mask_ids
        self.offsets = offsets
        self.transforms = transforms
        self.n_samples = self.estimate_n_samples(self.raw.shape, self.patch_shape)

    def __getitem__(self, index):

        # TODO sample so that we are biased towards the mask
        def sample_raw_seg_mask():
            offset = [np.random.randint(0, sh - csh) if sh > csh else 0
                      for sh, csh in zip(self.raw.shape, self.patch_shape)]
            bb = tuple(slice(off, off + csh) for off, csh in zip(offset, self.patch_shape))

            raw = self.raw[bb]
            seg = self.seg[bb]

            if self.transforms is not None:
                raw, seg = self.transforms(raw, seg)

            raw, seg = raw.copy(), seg.copy()
            mask = np.isin(seg, self.mask_ids)

            return raw, seg, mask

        raw, seg, mask = sample_raw_seg_mask()

        # TODO ensure that we have some in-mask area
        # # some arbitrary but very small pixel threshold
        # while mask.sum() < 25:
        #     raw, seg, mask = sample_raw_seg_mask()

        # add channel dim
        raw = raw[None]

        # make affs and aff_mask
        affs, aff_mask = self.to_affinities(seg, mask)

        return raw, affs, aff_mask

    def __len__(self):
        return self.n_samples

class MaskedLoss(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.criterion = SorensenDiceLoss()

    def forward(self, pred, y, mask):
        mask.requires_grad = False
        masked_prediction = pred * mask
        loss = self.criterion(masked_prediction, y)
        return loss

def default_training(proc_id, net, ds,
                     pipe, device, step):
    loader = torch.utils.data.DataLoader(ds, batch_size=1, num_workers=2)

    p_out, p_in = pipe

    optimizer = torch.optim.Adam(net.parameters(), lr=1e-4)
    loss = MaskedLoss()
    loss = loss.to(device)

    logger = torch.utils.tensorboard.SummaryWriter('./runs/imws')

    add_gradients = True
    log_frequency = 10

    net.train()
    while True:
        if p_out.poll():
            if not p_out.recv():
                p_in.send(step)
                break

        for x, y, mask in loader:
            x = x.to(device)
            y, mask = y.to(device), mask.to(device)

            optimizer.zero_grad()

            pred = net(x)
            pred.retain_grad()

            loss_val = loss(pred, y, mask)
            loss_val.backward()
            optimizer.step()

            logger.add_scalar("loss", loss_val.item(), step)

            step += 1
            if step % log_frequency == 0:
                print("Background training process iteration", step)
                x = x[0].detach().cpu()
                logger.add_image('input', x, step)
                y = y[0].detach().cpu()

                if add_gradients:
                    grads = pred.grad[0].detach().cpu()
                    grads -= grads.min()
                    grads /= grads.max()

                pred = torch.clamp(pred[0].detach().cpu(), 0.001, 0.999)
                tandp = [target.unsqueeze(0) for target in y]
                nrow = len(tandp)
                tandp.extend([p.unsqueeze(0) for p in pred])

                if add_gradients:
                    tandp.extend([grad.unsqueeze(0) for grad in grads])
                tandp = make_grid(tandp, nrow=nrow)
                logger.add_image('target_and_prediction', tandp, step)

                # for debugging
                # return x, y, pred, grads


import argparse
from dateutil import tz
from datetime import datetime

from spotipy import Spotify
import spotipy.util

from models import Play, Track, Album, Artist, PostgreSQLConnection

import settings

def set_timezone_to_datetime(datetime_to_set, timezone):
    return datetime_to_set.replace(tzinfo=tz.gettz(timezone))

def convert_played_at_from_response_to_datetime(played_at):
    try:
        return datetime.strptime(played_at, '%Y-%m-%dT%H:%M:%S.%fZ')
    except:
        # For the single moment where the played at time hits a full second
        return datetime.strptime(played_at, '%Y-%m-%dT%H:%M:%SZ')

def convert_datetime_from_timezone_to_timezone(datetime_to_convert, from_tz_code, to_tz_code):
    from_tz = tz.gettz(from_tz_code)
    to_tz = tz.gettz(to_tz_code)

    datetime_to_convert = datetime_to_convert.replace(tzinfo=from_tz)
    converted_datetime = datetime_to_convert.astimezone(to_tz)
    return converted_datetime

class SpotifyConnection(object):

    def __init__(self, user_data):
        self.user_name = user_data['user_name']
        token = spotipy.util.prompt_for_user_token(self.user_name,
                                                   scope='user-read-recently-played',
                                                   client_id=user_data['client_id'],
                                                   client_secret=user_data['client_secret'],
                                                   redirect_uri=user_data['redirect_uri'])
        self.client = Spotify(auth=token)
        self.db = self.init_db()

    def init_db(self):
        return PostgreSQLConnection()

    def get_artist(self, artist_id):
        artist = self.db.session.query(Artist).get(artist_id)
        if artist:
            return artist
        else:
            artist_response = self.client.artist(artist_id)
            artist = Artist()
            artist.artist_id = artist_id
            artist.artist_data = artist_response
            self.db.save_instance(artist)
            print("> Artist {} was not in database.".format(artist.artist_data['name']))
            return self.db.session.query(Artist).get(artist_id)

    def get_album(self, album_id):
        album = self.db.session.query(Album).get(album_id)
        if album:
            return album
        else:
            album_response = self.client.album(album_id)
            album = Album()
            album.album_data = album_response
            album.album_id = album_response['id']
            # Artists
            for album_artist_response in album_response['artists']:
                album.artists.append(self.get_artist(album_artist_response['id']))
            self.db.save_instance(album)
            print("> Album {} was not in database.".format(album.album_data['name']))
            return self.db.session.query(Album).get(album_id)

    def get_track(self, track_id):
        track = self.db.session.query(Track).get(track_id)
        if track:
            return track
        else:
            response = self.client.track(track_id)

            track = Track()
            track.track_id = track_id
            track.track_data = response
            # Album
            track.album = self.get_album(response['album']['id'])
            # Artists
            for artist_response in response['artists']:
                track.artists.append(self.get_artist(artist_response['id']))
            # Audio feature
            audio_feature_response = self.client.audio_features(track_id)[0]
            if audio_feature_response:  # Some tracks do not have audio features
                track.audio_feature_data = audio_feature_response
            print("> Track {} was not in database.".format(track.track_data['name']))
            self.db.save_instance(track)
            return self.db.session.query(Track).get(track_id)

    def get_play_from_played_at_utc_and_track_id(self, played_at_utc, track_id):
        played_at_utc = convert_played_at_from_response_to_datetime(played_at_utc)
        played_at_utc = set_timezone_to_datetime(played_at_utc, timezone='UTC')
        played_at_cet = convert_datetime_from_timezone_to_timezone(played_at_utc,
                                                                   from_tz_code='UTC',
                                                                   to_tz_code='CET')
        # Play
        play = Play()
        play.user_name = self.user_name
        play.played_at_utc_timestamp = played_at_utc.timestamp() * 1000
        play.played_at_utc = played_at_utc
        play.played_at_cet = played_at_cet
        play.day = played_at_cet.day
        play.month = played_at_cet.month
        play.year = played_at_cet.year
        play.hour = played_at_cet.hour
        play.minute = played_at_cet.minute
        play.second = played_at_cet.second
        play.day_of_week = played_at_cet.weekday()
        play.week_of_year = played_at_cet.date().isocalendar()[1]
        # Track
        track = self.get_track(track_id)
        play.track = track
        play.track_id = track_id
        return play

    def _get_play_tuples_from_response(self, response):
        plays = []
        for item in response['items']:
            play_tuple = (item['played_at'], item['track']['id'])
            plays.append(play_tuple)
        return plays

    def _get_play_tuples(self, limit=50, after=None):
        play_tuples = []
        response = self.client._get('me/player/recently-played', after=after, limit=limit)
        play_tuples.extend(self._get_play_tuples_from_response(response))

        while response and 'next' in response:
            response = self.client.next(response)
            if response:
                play_tuples.extend(self._get_play_tuples_from_response(response))

        return play_tuples

    def extract_plays(self):
        print("* Extracting latest plays of {}.".format(self.user_name))
        play_tuples = self._get_play_tuples()

        for played_at, track_id in play_tuples:
            play = self.get_play_from_played_at_utc_and_track_id(played_at, track_id)
            self.db.save_play(play)

class HoergewohnheitenManager(object):

    def __init__(self, spotify_user_data):
        self.spotify = SpotifyConnection(user_data=spotify_user_data)

    def process_hoergewohnheiten(self):
        self.spotify.extract_plays()

def process_hoergewohnheiten(user_name):
    print("***", user_name, "***")
    user_data = settings.SPOTIFY_USERS[user_name]
    mgr = HoergewohnheitenManager(user_data)
    mgr.process_hoergewohnheiten()

if __name__ == '__main__':
    print('''
 _     ___   ____  ___   __    ____  _       ___   _     _      _     ____  _  _____  ____  _
| |_| / / \ | |_  | |_) / /`_ | |_  \ \    // / \ | |_| | |\ | | |_| | |_  | |  | |  | |_  | |\ |
|_| | \_\_/ |_|__ |_| \ \_\_/ |_|__  \_\/\/ \_\_/ |_| | |_| \| |_| | |_|__ |_|  |_|  |_|__ |_| \|
''')

    print("Started at {}.".format(datetime.now()))

    # Argparse
    parser = argparse.ArgumentParser(description='Hoergewohnheiten')
    parser.add_argument('-u', dest='user_name')
    args = parser.parse_args()

    if args.user_name:
        process_hoergewohnheiten(args.user_name)
    else:
        for user_name in settings.SPOTIFY_USERS:
            process_hoergewohnheiten(user_name)

    print("Finished at {}.".format(datetime.now()))


import collections

class Rectangle():
    def __init__(self, w, h, placed=None, free_wh=None):
        self.wh = w, h
        self.placed = placed or []
        self.free_wh = free_wh or (0,0)

    @property
    def w(self):
        return self.wh[0]

    @property
    def h(self):
        return self.wh[1]

    def transposed(self):
        return Rectangle(self.h, self.w,
                         [r.transposed() for r in self.placed],
                        (self.free_wh[1], self.free_wh[0]),
                        )

    @staticmethod
    def placed_build(W, H, rect):
        # build rectangle of size (W, H) with placed rectangle rect
        if not W or not H:
            return

        w, h = rect.wh

        if (W, H) == (w, h):
            return Rectangle(W, H, [rect])

        elif (W, H) == (h, w):
            return Rectangle(W, H, [rect.transposed()])

        H_h = H - h, W >= w
        W_w = W - w, H >= h
        H_w = H - w, W >= h
        W_h = W - h, H >= w
        cases = [H_h, W_w, H_w, W_h]
        residue = [c[0] for c in cases if c[1] and c[0] >= 0]
        if not residue:
            return None

        min_size = min(residue)

        if H_h[0] == min_size and H_h[1]:
            placed_r = Rectangle(w, H,
                                [rect],
                                (w, H - h))
            free_wh = (W - w, H)
        elif W_w[0] == min_size and W_w[1]:
            placed_r = Rectangle(W, h,
                                [rect],
                                (W - w, h))
            free_wh = (W, H - h)
        elif H_w[0] == min_size and H_w[1]:
            placed_r = Rectangle(h, H,
                                 [rect.transposed()],
                                (h, H - w))
            free_wh = (W - h, H)
        elif W_h[0] == min_size and W_h[1]:
            placed_r = Rectangle(W, w,
                                [rect.transposed()],
                                (W - h, w))
            free_wh = (W, H - w)
        else:
            assert False, 'impossible'

        out = Rectangle(W, H, [placed_r], free_wh)
        return out

    def place(self, rect):
        W, H = self.free_wh
        r = Rectangle.placed_build(W, H, rect)
        if not r:
            return False
        self.placed.append(r.placed[0])
        self.free_wh = r.free_wh
        #print(f'place {rect.wh}: free {W,H} -> {self.free_wh}')
        return True

    @staticmethod
    def concat(rects, by='w'):
        w = list(map(lambda r : r.w, rects))
        h = list(map(lambda r : r.h, rects))

        if 'w' == by:
            max_w = max(w)
            placed_r = [
                Rectangle.placed_build(max_w, r.h, r)
                for r in rects
            ]
            out = Rectangle(max_w, sum(h), placed_r)
        else:
            max_h = max(h)
            placed_r = [
                Rectangle.placed_build(r.w, max_h, r)
                for r in rects
            ]
            out = Rectangle(sum(w), max_h, placed_r)

        return out

    @staticmethod
    def min_concat(W, H, rect1, rect2):
        rect2T = Rectangle(rect2.h, rect2.w, rect2.placed, rect2.free_wh)
        concat_cases = [
            Rectangle.concat([rect1, rect2], by='w'),
            Rectangle.concat([rect1, rect2], by='h'),
            Rectangle.concat([rect1, rect2T], by='w'),
            Rectangle.concat([rect1, rect2T], by='h'),
        ]
        if W < H:
            W, H = H, W
        concat_cases = [r for r in concat_cases if max(r.wh) <= W and min(r.wh) <= H]
        if not concat_cases:
            return
        return min(concat_cases, key=lambda r : r.free_square)

    @property
    def square(self):
        return self.w * self.h

    @property
    def free_square(self):
        out = self.free_wh[0] * self.free_wh[1]
        for r in self.placed:
            out += r.free_square
        return out

    def free_print(self):
        if self.free_wh[0] and self.free_wh[1]:
            print(self.free_wh)
        for r in self.placed:
            r.free_print()

    @property
    def fullness(self):
        return (1 - self.free_square / self.square) * 100

    def __repr__(self):
        return f'Rectangle(w={self.w}, h={self.h}, childs={len(self.placed)}, free_wh={self.free_wh}, fullness={self.fullness}%)'

def equal_side_concat_step(rects, W, H, order='descending'):
    w = list(map(lambda r : r.w, rects))
    h = list(map(lambda r : r.h, rects))
    side_cnt = collections.Counter([s for s in w + h if s <= max(W, H)])
    side_repeats = [ side
                     for side, cnt in side_cnt.most_common()
                     if cnt > 1
                    ]
    side_repeats.sort(reverse=('descending' == order))

    single_rects = list(rects)
    side_to_rects = {}
    for side in side_repeats:
        side_to_rects[side] = [r for r in single_rects
                               if side in r.wh]
        single_rects = [r for r in single_rects
                        if side not in r.wh]

    # TODO: Если прямоугольник совпадает каждой стороной с другими -> варианты по какой стороне объединять
    concat_rects = []
    for side, side_rects in side_to_rects.items():
        if 1 == len(side_rects):
            single_rects.append(side_rects[0])
            continue
        for r in side_rects:
            if side != r.w:
                r.wh = r.h, r.w
        # TODO: 1d упаковка вдоль H или W
        # далее идет упаковка вдоль максимальной возможной стороны
        side_rects.sort(key=lambda r : r.h, reverse=True)
        concat_side = max(H, W) if side <= min(H, W) else min(H, W)
        while side_rects:
            rects_for_concat = []
            rest_rects = []
            sum_h = 0
            for r in side_rects:
                if sum_h + r.h <= concat_side:
                    rects_for_concat.append(r)
                    sum_h += r.h
                else:
                    rest_rects.append(r)
            if len(rects_for_concat) == 1:
                single_rects.append(rects_for_concat[0])
            elif len(rects_for_concat) > 1:
                concat_rects.append(Rectangle.concat(rects_for_concat, by='w'))
            else:
                single_rects.extend(rest_rects)
                break
                #assert False, f'side_rects={side_rects}, rest_rects={rest_rects}, max_h={max_h}'
            side_rects = rest_rects

    return single_rects, concat_rects

def exact_concat(rects, W, H):
    merge_rects = list(rects)
    while True:
        single_rects, concat_rects = equal_side_concat_step(merge_rects, W, H, order='descending')
        #print(f'single_rects={single_rects} \n concat_rects={concat_rects} \n')
        merge_rects = single_rects + concat_rects
        if not concat_rects:
            break

    while True:
        single_rects, concat_rects = equal_side_concat_step(merge_rects, W, H, order='ascending')
        merge_rects = single_rects + concat_rects
        if not concat_rects:
            break
    return merge_rects

def pallet_exact_side_placement(rects, pallet, side='max'):
    W, H = pallet.free_wh
    side = max(W, H) if 'max' == side else min (W, H)

    rest_rects = []
    for r in rects:
        if side in r.wh:
            if pallet.place(r):
                continue
        rest_rects.append(r)
    return rest_rects

def exact_placement(rects, pallet):
    rest_rects = list(rects)
    while rest_rects:
        rest_rects1 = pallet_exact_side_placement(rest_rects, pallet, side='max')
        rest_rects2 = pallet_exact_side_placement(rest_rects1, pallet, side='min')
        if len(rest_rects) == len(rest_rects2):
            break
        rest_rects = rest_rects2
    return rest_rects

def rects_flatten(rects):
    out = []
    for r in rects:
        if r.placed:
            out.extend(rects_flatten(r.placed))
        else:
            out.append(r)
    return out

def min_residue(WH, rects):
    W, H = WH
    min_r = None
    min_residue_wh = WH
    for r in rects:
        placed_r = Rectangle.placed_build(W, H, r)
        if placed_r:
            residue_wh = placed_r.placed[0].free_wh
            if residue_wh[0] * residue_wh[1] < min_residue_wh[0] * min_residue_wh[1]:
                min_residue_wh = residue_wh
                min_r = r
    return min_r, min_residue_wh

def min_residue_placement(pallet, rects):
    max_r, min_residue_wh = min_residue(pallet.free_wh, rects)
    if not max_r or min_residue_wh[0] * min_residue_wh[1] > max_r.square:
        return rects

    rest_rects = [r for r in rects if r is not max_r]

    r, _ = min_residue(min_residue_wh, rest_rects)
    if not r:
        pallet.place(max_r)
        return rest_rects

    return rects

def find_concat_pair(W, H, rects):
    min_loss = 1
    min_values = None
    for i in range(len(rects)):
        for j in range(i + 1, len(rects)):
            cur_concat = Rectangle.min_concat(W, H, rects[i], rects[j])
            if not cur_concat:
                continue
            cur_loss = cur_concat.free_square / min(rects[i].square, rects[j].square)
            if cur_loss < min_loss:
                min_loss = cur_loss
                min_values = (i, j, cur_concat)
    return min_values

def free_placement(pallet, rects):
    rest_rects = list(rects)
    while rest_rects:
        W, H = pallet.free_wh
        if not W * H:
            break
        concat_rects = exact_concat(rest_rects, W, H)
        #print(f'concat_rects={concat_rects}')
        rest_rects = exact_placement(concat_rects, pallet)
        #print(f'exact_placement: rest_rects={rest_rects}, concat_rects={concat_rects}, pallet={pallet}')
        if len(rest_rects) == len(concat_rects):
            rest_rects2 = min_residue_placement(pallet, rest_rects)
            if len(rest_rects2) == len(rest_rects):
                find_values = find_concat_pair(W, H, rest_rects)
                if not find_values:
                    #print(f'not find_concat_pair for rest_rects={rest_rects}')
                    break
                i, j, concat_r = find_values
                assert pallet.place(concat_r)
                del rest_rects[j]
                del rest_rects[i]
            else:
                rest_rects = rest_rects2
        rest_rects = rects_flatten(rest_rects)

    return rest_rects

def pallet_placement(pallet, rects):
    rest_rects = free_placement(pallet, rects)

#     placed = len(rects) - len(rest_rects)
#     if placed:
#         print(f'pallet: {pallet}, placed: {placed} \n')

    for r in pallet.placed:
        rest_rects = pallet_placement(r, rest_rects)

    return rest_rects

def assign_coordinates(x, y, W, H, rects):
    out_xywh = []
    for r in rects:
        if W == r.h or H == r.w:
            r = r.transposed()
        if not r.placed:
            out_xywh.append((x, y, r.w, r.h))
            #print(f'append {x,y,r.w,r.h}')
        else:
            out_xywh.extend(assign_coordinates(x, y, r.w, r.h, r.placed))
        if W == r.w:
            y += r.h
            H -= r.h
        elif H == r.h:
            x += r.w
            W -= r.w
        else:
            assert False, f'WH={W,H}, r={r}'

    return out_xywh


from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from google.oauth2.credentials import Credentials
from google.auth.transport.requests import Request
import os

# All scopes together
ALL_SCOPES = [
    'https://www.googleapis.com/auth/contacts.readonly',
    'https://www.googleapis.com/auth/calendar.readonly',
    'https://www.googleapis.com/auth/gmail.readonly'
]

CREDENTIALS_FILE = 'env/token.json'

def get_credentials(scopes):
    creds = None

    # Load credentials from the file if it exists
    if os.path.exists(CREDENTIALS_FILE):
        creds = Credentials.from_authorized_user_file(CREDENTIALS_FILE, scopes)

    # Refresh or obtain new credentials
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())  # Use `Request()` as the required argument for `refresh()`
        else:
            flow = InstalledAppFlow.from_client_secrets_file('env/oauth2_credentials.json', scopes)
            creds = flow.run_local_server(port=0)

        # Save the credentials for the next run
        with open(CREDENTIALS_FILE, 'w') as token:
            token.write(creds.to_json())

    return creds

# Build the service using credentials
def get_authenticated_service(api, version, creds):
    return build(api, version, credentials=creds)

# Fetch contacts
def get_contacts(creds):
    service = get_authenticated_service('people', 'v1', creds)
    results = service.people().connections().list(
        resourceName='people/me',
        personFields='names,emailAddresses'
    ).execute()
    print("Data type of get_contacts results: ", type(results))
    # print("First 500 characters of results: ", json.dumps(results, indent=4)[:500])
    return results  # Add this line

# Fetch calendar events
def get_calendar_events(creds):
    service = get_authenticated_service('calendar', 'v3', creds)
    results = service.events().list(calendarId='primary').execute()
    print("Data type of get_calendar_events results: ", type(results))
    # print("First 500 characters of results: ", json.dumps(results, indent=4)[:500])
    return results  # Add this line

# Fetch emails
def get_emails(creds):
    service = get_authenticated_service('gmail', 'v1', creds)
    results = service.users().messages().list(userId='me', maxResults=10).execute()
    messages = results.get('messages', [])
    full_messages = []  # List to hold the full message details
    for message in messages:
        msg = service.users().messages().get(userId='me', id=message['id']).execute()
        full_messages.append(msg)
    return full_messages

if __name__ == "__main__":
    creds = get_credentials(ALL_SCOPES)
    get_contacts(creds)
    get_calendar_events(creds)
    get_emails(creds)


"""
This helps in finding the means and standards of the images to normalize before training.

  To run
python3 calculate_means_std.py -i path/to/image/folder/
"""

import argparse
import subprocess
import yaml
import os
import sys
sys.path.remove("/opt/ros/kinetic/lib/python2.7/dist-packages")
import cv2
import numpy as np

def is_image(filename):
  return any(filename.endswith(ext) for ext in ['.jpg', '.png'])

if __name__ == '__main__':
  parser = argparse.ArgumentParser()

  parser.add_argument(
      '--image', '-i',
      type=str,
      required=True,
      default=None,
      help='Directory to get the images from. If not passed, do from scratch!'
  )
  FLAGS, unparsed = parser.parse_known_args()

  # print summary of what we will do
  print("----------")
  print("INTERFACE:")
  #
  print("image dir", FLAGS.image)
  print("----------\n")

  print("----------\n")

  #

  # create list of images and examine their pixel values
  filenames = [os.path.join(dp, f) for dp, dn, fn in os.walk(
      os.path.expanduser(FLAGS.image)) for f in fn if is_image(f)]

  # examine individually pixel values
  counter = 0.0
  pix_val = np.zeros(3, dtype=np.float)
  for filename in filenames:
    # analize
    print("Accumulating mean", filename)

    # open as rgb
    cv_img = cv2.imread(filename, cv2.IMREAD_COLOR)
    cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)

    # normalize to 1
    cv_img = cv_img.astype(np.float) / 255.0

    # count pixels and add them to counter
    h, w, d = cv_img.shape
    counter += h * w

    # sum to moving pix value counter in each channel
    pix_val += np.sum(cv_img, (0, 1))

  # calculate means
  means = (pix_val / counter)

  # means
  print("means(rgb): ", means)

  # pass again and calculate variance
  pix_var = np.zeros(3, dtype=np.float)
  for filename in filenames:
    # analizel
    print("Accumulating variance", filename)

    # open as rgb
    cv_img = cv2.imread(filename, cv2.IMREAD_COLOR)
    cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)

    # normalize to 1
    cv_img = cv_img.astype(np.float) / 255.0

    # sum to moving pix value counter in each channel
    pix_var += np.sum(np.square(cv_img - means), (0, 1))

  # calculate the standard deviations
  stds = np.sqrt(pix_var / counter)
  print("stds(rgb): ", stds)

  # finalize by printing both
  print("*" * 80)
  print("means(rgb): ", means)
  print("stds(rgb): ", stds)
  print("*" * 80)


# Importancia de la característica de permutación (PFI) para la clasificación de latidos utilizando un perceptrón multicapa (MLP)
#
#
# - Código 'PFI.py'
# - Trabajo Fin de Máster.
# - Néstor Bolaños Bolaños. (nestorbolanos@correo.ugr.es)

import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pickle
import glob
import matplotlib.pyplot as plt
import pandas as pd
from scipy import *
import os
import seaborn as sns
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn import *
from sklearn.metrics import *
from sklearn.model_selection import StratifiedKFold
sns.set()

# Cargamos los datos y codificamos las clases de cada latido mediante One Hot

# Cargamos los datos de entrenamiento y de test:
tamaño = 277
valores_train = np.empty(shape=[0, tamaño])
valores_test = np.empty(shape=[0, tamaño])

latidos_entrenamiento = glob.glob('train_beats.csv')
latidos_test = glob.glob('test_beats.csv')

for j in latidos_entrenamiento:
    filas = np.loadtxt(j, delimiter=',')
    valores_train = np.append(valores_train, filas, axis=0)

for j in latidos_test:
    filas = np.loadtxt(j, delimiter=',')
    valores_test = np.append(valores_test, filas, axis=0)

print(valores_train.shape)
print(valores_test.shape)

# Separamos los datos de entrenamiento y de test, y aplicamos la codificación One Hot a Y:
X_train = valores_train[:,:-2]
X_test = valores_test[:,:-2]
y_train = valores_train[:,-2]
y_test = valores_test[:,-2]

# Combinamos todo nuevamente:
X = np.concatenate((X_train, X_test), axis = 0)
Y = np.concatenate((y_train, y_test), axis = 0)

# codificación One Hot de Y:
Y = to_categorical(Y)

# Construimos el perceptrón multicapa

# Construimos el modelo MLP
def getModel():
    model_mlp = Sequential()
    model_mlp.add(Dense(100, activation = 'relu'))
    model_mlp.add(Dense(9, activation = 'softmax'))
    return model_mlp

model_mlp.summary()

# Implementamos y aplicamos PFI para el perceptrón multicapa

# Métodos de perturbación:
# Hay diferentes tipos de perturbación para la importancia de la característica de permutación, como la perturbación media, la perturbación cero y la perturbación aleatoria. En la implementación que hemos realizado en este cuaderno, los datos dentro de cada corte se han barajado aleatoriamente.

fig, ax = plt.subplots(1, 4, figsize = (20, 4), sharex = True, sharey=True)

# Sin perturbación: señal original.
ax[0].set_title('Sin perturbación')
ax[0].plot(np.arange(len( X[20, :])), X[20, :])

# perturbación 0: se establecen los valores de cada corte a 0.
ax[1].set_title('Perturbación 0')
X_zero_perturbed = X[20, :].copy()
X_zero_perturbed[5 * 25 : 6 * 25] = 0.0
ax[1].plot(np.arange(len(X[20, :])), X_zero_perturbed)

# Perturbación aleatoria: los valores de cada corte se reemplazan con valores aleatorios.
ax[2].set_title('Perturbación aleatoria')
X_random_perturbed = X[20, :].copy()
X_random_perturbed[5 * 25 : 6 * 25] = np.std(X[20, :]) * np.random.randn(25) + np.mean(X[20, :])
ax[2].plot(np.arange(len(X[20, :])), X_random_perturbed)

# Perturbación media: se promedian los valores del corte actual.
ax[3].set_title('Perturbación Media')
X_mean_perturbed = X[20, :].copy()
X_mean_perturbed[5 * 25 : 6 * 25] = np.mean(X[20, 5 * 25 : 6 * 25])
ax[3].plot(np.arange(len(X[20, :])), X_mean_perturbed)

for i in range(4):
    ax[i].set_xlabel('Tiempo')
    ax[i].axvspan(5 * 25, 6 * 25, color = 'green', alpha = 0.25)

# Importancia de la característica de permutación:
kf = StratifiedKFold(n_splits = 5, shuffle = True)
contador_pliegues = 0
M = np.zeros((X.shape[0], 11))
for indice_train, indice_test in kf.split(X, np.argmax(Y, axis = 1)):
    print('Fold ', contador_pliegues)

    # Separamos los datos en cada pliegue:
    X_train, X_test = X[indice_train], X[indice_test]
    y_train, y_test = Y[indice_train], Y[indice_test]

    # Construimos el modelo de aprendizaje con los datos de entrenamiento:
    model_mlp = getModel()
    model_mlp.compile(optimizer = 'adam', loss = tf.keras.losses.CategoricalCrossentropy())
    model_mlp.fit(X_train, y_train, epochs = 100, verbose = 0)

    # Realizamos predicciones con los datos de test sin permutaciones:
    predicciones = model_mlp.predict(X_test)

    # Para cada característica:
    for corte in range(0, 275, 25):
        # Permutamos y realizamos predicciones:
        x_permutacion = np.copy(X_test)
        x_corte = X_test[:, corte:corte+25]
        x_corte_permutacion = np.random.permutation(x_corte)
        x_permutacion[:, corte:corte + 25] = x_corte_permutacion
        pred_perm = model_mlp.predict(x_permutacion)

        # Obtenemos la importancia:
        importancia = ((np.argmax(y_test, axis = 1) - np.argmax(pred_perm, axis = 1))**2
                      - (np.argmax(y_test, axis = 1) - np.argmax(predicciones, axis = 1))**2)
        M[indice_test, corte // 25] = importancia
    contador_pliegues += 1
importancia_media = np.mean(M, axis = 0)

indices_ordenados = np.argsort(-1 * importancia_media)
cortes = np.arange(1, 12)
colores = ['forestgreen', 'limegreen', 'royalblue', 'blue', 'darkorange', 'cyan', 'purple', 'red', 'pink', 'yellow', 'coral']

fig, ax = plt.subplots(1, 2, figsize = (15, 4))
ax[0].bar(range(11), importancia_media[indices_ordenados], color = np.array(colores)[indices_ordenados])
ax[0].set_title('Importancia de cada característica del modelo MLP')
ax[0].set_xticks(np.arange(11))
ax[0].set_xticklabels(cortes[indices_ordenados].astype(int))
ax[0].set_xlabel('Corte')
ax[0].set_ylabel('Importancia de cada característica')

ecg_normalizado = (X[20, :] - X[20, :].min()) / (X[20, :].max() - X[20, :].min())
Importancia_caraceristica_normalizada = (importancia_media - importancia_media.min()) / (importancia_media.max() - importancia_media.min())
ax[1].plot(np.arange(len(ecg_normalizado)), ecg_normalizado, label='Datos ECG')
ax[1].plot(np.repeat(Importancia_caraceristica_normalizada, 25), label = 'Importancia de cada característica')
ax[1].set_title('Importancia de cada característica \npara el modelo MLP en una muestra de ECG')
ax[1].set_xlabel('Tiempo')
ax[1].set_ylabel('Señal ECG / Importancia de cada característica')
ax[1].legend()


import os
from google.cloud import storage

class GoogleStorageLoader():
    def __init__(self) -> None:
        """Start Google Cloud clint - could be used for uploading to storage
        """
        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "../content/key-bucket.json"
        self.client = storage.Client()

    def upload_to_bucket(self, bucket_name, source_file, destination):
        """uploads file to the bucket

        Args:
            bucket_name (str): _description_
            source_file (str): _description_
            destination (str): _description_
        """
        bucket = self.client.bucket(bucket_name)
        blob = bucket.blob(destination)
        blob.upload_from_filename(source_file)


"""
words list가 링크만 있고, 글자 데이터가 없음. 글자 리스트를 받아오기 위한 파일
"""

from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd

class Get_chndic_data:

    def __init__(self, link):
        self.link = link
        self.get_data = []

    def beautiful_soup(self, link):
        """
        Beautiful Soup Type의 객체를 return
        return값에 다음과 같은 메소드 가능
        data = soup.find('div', id='container').find('div', class_='section_hsk')
        :param link: https://zh.dict.naver.com/ 뒤에 들어갈 {letter_link}  주소
        :return: row 데이터
        """
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('headless')
        chrome_options.add_argument('window-size=1920x1080')
        chrome_options.add_argument("disable-gpu")
        # https://beomi.github.io/2017/09/28/HowToMakeWebCrawler-Headless-Chrome/

        # driver = webdriver.Chrome("D:/dev/chromedriver.exe", chrome_options=chrome_options)  # 집에서 chromedriver 경로
        driver = webdriver.Chrome("C:/Users/user/Downloads/chromedriver.exe", chrome_options=chrome_options)
        # 학원에서 chromedriver 경로
        url = f'https://zh.dict.naver.com/{link}'
        driver.get(url)
        driver.minimize_window()
        content = driver.page_source.encode('utf-8').strip()
        soup = BeautifulSoup(content, "html.parser")
        driver.close()
        return soup

    def find_letter_inf(self):
        """
        글자 link를 이용해 글자와 글자 뜻, 병음, 단어일 경우 구성 글자와 구성 글자 링크를 받아옴
        :return:  위 자료로 구성된 list
        """
        # 병음 추가해야 함.
        soup = self.beautiful_soup(self.link)
        letter = soup.find('div', id='container').find('div', class_="section section_entry _section_entry") \
            .find('div', class_="entry_title _guide_lang").find('strong', class_='word').text
        return letter

temp_df = pd.read_csv('../csv/hsk_words_listed.csv', encoding='UTF-8')
hsk_words_link = temp_df.iloc[:, 1]
index = 500
get_data_list = []

########################################## 파일명 변환 ##################################
for i in range(3000, 3100):
    while True:
        try:
            get_data = Get_chndic_data(hsk_words_link[i])
            get_data_list.append(get_data.find_letter_inf())
            df = pd.DataFrame(get_data_list)
            print(df.tail())
            df.to_csv(f'../csv/letters_list{3000}.csv')
            break
        except AttributeError:
            print('try again')
            continue


#!/usr/bin/env python
# coding: utf-8

# In[36]:

import requests
from bs4 import BeautifulSoup
import pandas

list1=[]

for page in range(0,30,10):

    r = requests.get("http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s="+str(page)+".html", headers={'User-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'})
    c= r.content

    soup=BeautifulSoup(c,"html.parser")
    all=soup.find_all("div",{"class":"propertyRow"})
    x=all[0].find("h4",{"class":"propPrice"}).text

    for item in all:
        d={}

        d["Address"]=item.find_all("span",{"class":"propAddressCollapse"})[0].text

        try:
            d["Locality"]=item.find_all("span",{"class":"propAddressCollapse"})[1].text
        except:
            d["Locality"]=None

        d["Price"]=item.find("h4",{"class":"propPrice"}).text.replace("\n","").strip()
        try:
            d["Beds"]=item.find("span",{"class":"infoBed"}).find("b").text
        except:
            d["Beds"]=None

        try:
            d["Area"]=item.find("span",{"class":"infoSqFt"}).find("b").text
        except:
            d["Area"]=None

        try:
            d["Full Baths"]=item.find("span",{"class":"infoValueFullBath"}).find("b").text
        except:
            d["Full Baths"]=None

        try:
            d["Half Baths"]=item.find("span",{"class":"infoValueHalfBath"}).find("b").text
        except:
            d["Half Baths"]=None

        for column_group in item.find_all("div",{"class":"columnGroup"}):
            for fg , fn in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})):
                if "Lot Size" in fg.text :
                    d["Lot Size"]=fn.text

        list1.append(d)
df=pandas.DataFrame(list1)
df.to_csv("output.csv")


import logging
import os
import typing
from collections import defaultdict
from typing import Dict

import dpath.util
from voluptuous import Any

from dvc.exceptions import DvcException
from dvc.utils.serialize import ParseError, load_path
from dvc_data.hashfile.hash_info import HashInfo

from .base import Dependency

logger = logging.getLogger(__name__)

class MissingParamsError(DvcException):
    pass

class MissingParamsFile(DvcException):
    pass

class ParamsIsADirectoryError(DvcException):
    pass

class BadParamFileError(DvcException):
    pass

class ParamsDependency(Dependency):
    PARAM_PARAMS = "params"
    PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
    DEFAULT_PARAMS_FILE = "params.yaml"

    def __init__(self, stage, path, params=None, repo=None):
        self.params = list(params) if params else []
        info = (
            {self.PARAM_PARAMS: params} if isinstance(params, dict) else None
        )
        repo = repo or stage.repo
        path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
        super().__init__(stage, path, info=info, repo=repo)

    def dumpd(self):
        ret = super().dumpd()
        if not self.hash_info:
            ret[self.PARAM_PARAMS] = self.params or {}
        return ret

    def fill_values(self, values=None):
        """Load params values dynamically."""
        if values is None:
            return

        info = {}
        if not self.params:
            info.update(values)
        for param in self.params:
            if param in values:
                info[param] = values[param]
        self.hash_info = HashInfo(self.PARAM_PARAMS, info)

    def read_params(
        self, flatten: bool = True, **kwargs: typing.Any
    ) -> Dict[str, typing.Any]:
        try:
            config = self.read_file()
        except MissingParamsFile:
            config = {}

        if not self.params:
            return config

        ret = {}
        if flatten:
            for param in self.params:
                try:
                    ret[param] = dpath.util.get(config, param, separator=".")
                except KeyError:
                    continue
            return ret

        from dpath.util import merge

        for param in self.params:
            merge(
                ret,
                dpath.util.search(config, param, separator="."),
                separator=".",
            )
        return ret

    def workspace_status(self):
        if not self.exists:
            return {str(self): "deleted"}
        if self.hash_info.value is None:
            return {str(self): "new"}

        from funcy import ldistinct

        status = defaultdict(dict)
        info = self.hash_info.value if self.hash_info else {}
        actual = self.read_params()

        # NOTE: we want to preserve the order of params as specified in the
        # status. In case of tracking the whole file, the order is top-level
        # keys in the file and then the keys in the `info` from `dvc.lock`
        # (which are alphabetically sorted).
        params = self.params or ldistinct([*actual.keys(), *info.keys()])
        for param in params:
            if param not in actual:
                st = "deleted"
            elif param not in info:
                st = "new"
            elif actual[param] != info[param]:
                st = "modified"
            else:
                assert actual[param] == info[param]
                continue

            status[str(self)][param] = st

        return status

    def status(self):
        return self.workspace_status()

    def validate_filepath(self):
        if not self.exists:
            raise MissingParamsFile(f"Parameters file '{self}' does not exist")
        if self.isdir():
            raise ParamsIsADirectoryError(
                f"'{self}' is a directory, expected a parameters file"
            )

    def read_file(self):
        self.validate_filepath()
        try:
            return load_path(self.fs_path, self.repo.fs)
        except ParseError as exc:
            raise BadParamFileError(
                f"Unable to read parameters from '{self}'"
            ) from exc

    def get_hash(self):
        info = self.read_params()

        missing_params = set(self.params) - set(info.keys())
        if missing_params:
            raise MissingParamsError(
                "Parameters '{}' are missing from '{}'.".format(
                    ", ".join(missing_params), self
                )
            )

        return HashInfo(self.PARAM_PARAMS, info)

    def save(self):
        if not self.exists:
            raise self.DoesNotExistError(self)

        if not self.isfile and not self.isdir:
            raise self.IsNotFileOrDirError(self)

        self.ignore()
        self.hash_info = self.get_hash()


import torch
import torch.nn as nn
from torch import cat, exp
import torch.nn.functional as F
from torch.nn.functional import pad
from torch.nn.modules.batchnorm import _BatchNorm

class my_AFF(nn.Module):
    '''
    Point-wise Convolution based Attention module (PWAtt)
    '''

    def __init__(self, channels=64, r=2):
        super(my_AFF, self).__init__()
        inter_channels = int(channels // r)

        self.local_att = nn.Sequential(
            nn.Conv1d(in_channels=channels, out_channels=inter_channels, kernel_size=1, stride=1, padding=0),
            nn.BatchNorm1d(inter_channels),
            nn.ReLU(inplace=True),
            nn.Conv1d(in_channels=inter_channels, out_channels=channels, kernel_size=1, stride=1, padding=0),
            nn.BatchNorm1d(channels),
        )

        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        xa = self.local_att(x)
        wei = self.sigmoid(xa)

        xo = 2 * x * wei
        return xo, wei

# Root Mean Squared Logarithmic Error (RMSLE) loss
class RMSLELoss(nn.Module):
    def __init__(self, eps=1e-6):
        super(RMSLELoss, self).__init__()
        self.squared_error = nn.MSELoss(reduction='none')
        self.eps = eps

    def forward(self, y_hat, y, mask, seq_length, sum_losses=False):
        # the log(predictions) corresponding to no data should be set to 0
        # log_y_hat = y_hat.log().where(mask, torch.zeros_like(y))
        log_y_hat = torch.log(y_hat + 1).where(mask, torch.zeros_like(y))
        # the we set the log(labels) that correspond to no data to be 0 as well
        # log_y = y.log().where(mask, torch.zeros_like(y))
        log_y = torch.log(y + 1).where(mask, torch.zeros_like(y))
        # where there is no data log_y_hat = log_y = 0, so the squared error will be 0 in these places
        loss = self.squared_error(log_y_hat, log_y)
        rmsle_loss = torch.sqrt(loss + self.eps)
        loss = torch.sum(rmsle_loss, dim=1)
        if not sum_losses:
            loss = loss / seq_length.clamp(min=1)
        return loss.mean()

# Root Mean Squared Error (MSE) loss
class RMSELoss(nn.Module):
    def __init__(self, eps=1e-6):
        super(RMSELoss, self).__init__()
        self.squared_error = nn.MSELoss(reduction='none')
        self.eps = eps

    def forward(self, y_hat, y, mask, seq_length, sum_losses=False):
        # the predictions corresponding to no data should be set to 0
        y_hat = y_hat.where(mask, torch.zeros_like(y))
        # the we set the labels that correspond to no data to be 0 as well
        y = y.where(mask, torch.zeros_like(y))
        # where there is no data log_y_hat = log_y = 0, so the squared error will be 0 in these places
        loss = self.squared_error(y_hat, y)
        rmse_loss = torch.sqrt(loss + self.eps)
        loss = torch.sum(rmse_loss, dim=1)
        if not sum_losses:
            loss = loss / seq_length.clamp(min=1)
        return loss.mean()

# Mean Squared Logarithmic Error (MSLE) loss
class MSLELoss(nn.Module):
    def __init__(self):
        super(MSLELoss, self).__init__()
        self.squared_error = nn.MSELoss(reduction='none')

    def forward(self, y_hat, y, mask, seq_length, sum_losses=False):
        # the log(predictions) corresponding to no data should be set to 0
        log_y_hat = y_hat.log().where(mask, torch.zeros_like(y))
        # the we set the log(labels) that correspond to no data to be 0 as well
        log_y = y.log().where(mask, torch.zeros_like(y))
        # where there is no data log_y_hat = log_y = 0, so the squared error will be 0 in these places
        loss = self.squared_error(log_y_hat, log_y)
        loss = torch.sum(loss, dim=1)
        if not sum_losses:
            loss = loss / seq_length.clamp(min=1)
        return loss.mean()

# Mean Squared Error (MSE) loss
class MSELoss(nn.Module):
    def __init__(self):
        super(MSELoss, self).__init__()
        self.squared_error = nn.MSELoss(reduction='none')

    def forward(self, y_hat, y, mask, seq_length, sum_losses=False):
        # the predictions corresponding to no data should be set to 0
        y_hat = y_hat.where(mask, torch.zeros_like(y))
        # the we set the labels that correspond to no data to be 0 as well
        y = y.where(mask, torch.zeros_like(y))
        # where there is no data log_y_hat = log_y = 0, so the squared error will be 0 in these places
        loss = self.squared_error(y_hat, y)
        loss = torch.sum(loss, dim=1)
        if not sum_losses:
            loss = loss / seq_length.clamp(min=1)
        return loss.mean()

class MyBatchNorm(_BatchNorm):
    def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True,
                 track_running_stats=True):
        super(MyBatchNorm, self).__init__(
            num_features, eps, momentum, affine, track_running_stats)

    def forward(self, input):
        self._check_input_dim(input)

        # hack to work around model.eval() issue
        if not self.training:
            self.eval_momentum = 0  # set the momentum to zero when the model is validating

        if self.momentum is None:
            exponential_average_factor = 0.0
        else:
            exponential_average_factor = self.momentum if self.training else self.eval_momentum

        if self.track_running_stats:
            if self.num_batches_tracked is not None:
                self.num_batches_tracked = self.num_batches_tracked + 1
                if self.momentum is None:  # use cumulative moving average
                    exponential_average_factor = 1.0 / float(self.num_batches_tracked)
                else:  # use exponential moving average
                    exponential_average_factor = self.momentum if self.training else self.eval_momentum

        return F.batch_norm(
            input, self.running_mean, self.running_var, self.weight, self.bias,
            training=True, momentum=exponential_average_factor, eps=self.eps)  # set training to True so it calculates the norm of the batch

class MyBatchNorm1d(MyBatchNorm):
    def _check_input_dim(self, input):
        if input.dim() != 2 and input.dim() != 3:
            raise ValueError('expected 2D or 3D input (got {}D input)'.format(input.dim()))

class EmptyModule(nn.Module):
    def forward(self, X):
        return X

class TempSepConv_CAFF(nn.Module):
    def __init__(self, config, no_ts_features=None, no_daig_features=None, no_flat_features=None):

        super(TempSepConv_CAFF, self).__init__()
        self.task = config['task']
        self.n_layers = config['n_layers']
        self.diagnosis_size = config['diagnosis_size']
        self.main_dropout_rate = config['main_dropout_rate']
        self.temp_dropout_rate = config['temp_dropout_rate']
        self.kernel_size = config['kernel_size']
        self.temp_kernels = config['temp_kernels']
        self.last_linear_size = config['last_linear_size']
        self.no_ts_features = no_ts_features
        self.no_daig_features = no_daig_features
        self.no_flat_features = no_flat_features

        self.no_diag = config['no_diag']

        self.alpha = 100

        self.keep_prob = 1-config['main_dropout_rate'] #0.5

        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()
        self.hardtanh = nn.Hardtanh(min_val=1/48, max_val=100)  # keep the end predictions between half an hour and 100 days
        self.rmsle_loss = RMSLELoss()
        self.msle_loss = MSLELoss()
        self.mse_loss = MSELoss()
        self.bce_loss = nn.BCELoss()

        self.main_dropout = nn.Dropout(p=self.main_dropout_rate)
        self.temp_dropout = nn.Dropout(p=self.temp_dropout_rate)

        self.remove_none = lambda x: tuple(xi for xi in x if xi is not None)  # removes None items from a tuple
        self.empty_module = EmptyModule()

        self.batchnormclass = MyBatchNorm1d
        # self.batchnormclass = nn.BatchNorm1d

        self.diagnosis_encoder = nn.Linear(in_features=self.no_daig_features, out_features=self.diagnosis_size)

        self.diagnosis_encoder1 = nn.Linear(in_features=self.no_daig_features, out_features=self.temp_kernels[0]+1)
        self.flat_encoder = nn.Linear(in_features=self.no_flat_features, out_features=self.temp_kernels[0]+1)

        self.bn_diagnosis_encoder = self.batchnormclass(num_features=self.diagnosis_size, momentum=0.1)  # input shape: B * diagnosis_size
        self.bn_point_last_los = self.batchnormclass(num_features=self.last_linear_size, momentum=0.1)  # input shape: (B * T) * last_linear_size
        self.bn_point_last_mort = self.batchnormclass(num_features=self.last_linear_size, momentum=0.1)  # input shape: (B * T) * last_linear_size

        # self.bn_diagnosis_encoder = self.empty_module
        # self.bn_point_last_los = self.empty_module
        # self.bn_point_last_mort = self.empty_module

        # input shape: (B * T) * last_linear_size
        # output shape: (B * T) * 1
        self.final_los = nn.Linear(in_features=self.last_linear_size, out_features=1)
        self.final_mort = nn.Linear(in_features=self.last_linear_size, out_features=1)

        # TDSC layers settings
        self.layers = []
        for i in range(self.n_layers):
            dilation = i * (self.kernel_size - 1) if i > 0 else 1  # dilation = 1 for the first layer, after that it captures all the information gathered by previous layers
            temp_k = self.temp_kernels[i]

            self.layers.append({})
            if temp_k is not None:
                padding = [(self.kernel_size - 1) * dilation, 0]  # [padding_left, padding_right]
                self.layers[i]['temp_kernels'] = temp_k
                self.layers[i]['dilation'] = dilation
                self.layers[i]['padding'] = padding
                self.layers[i]['stride'] = 1

        self.layer_modules = nn.ModuleDict()

        self.Y = 0  # Y is the number of channels in the previous temporal layer (could be 0 if this is the first layer)
        self.n = 0  # n is the layer number

        for i in range(self.n_layers):

            temp_in_channels = (self.no_ts_features + self.n) * (1 + self.Y) if i > 0 else 2 * self.no_ts_features  # (F + n) * (Y + 1)
            temp_out_channels = (self.no_ts_features + self.n) * self.layers[i]['temp_kernels']  # (F + n) * temp_kernels
            out_channels_caff = (self.no_ts_features+self.n+1)*(self.layers[i]['temp_kernels']+1)
            if self.n == 0:
                linear_input_dim = (self.no_ts_features + self.n - 1) * self.Y + 2 * self.no_ts_features + 2 + self.no_flat_features
            else:
                linear_input_dim = (self.no_ts_features + self.n - 1) * self.Y + (self.layers[i]['temp_kernels']+1) + 2 * self.no_ts_features + 2 + self.no_flat_features  # (F + n-1) * Y + Z + 2F + 2 + no_flat_features
            linear_output_dim = (self.layers[i]['temp_kernels']+1)

            temp = nn.Conv1d(in_channels=temp_in_channels,  # (F + n) * (Y + 1)
                             out_channels=temp_out_channels,  # (F + n) * Y
                             kernel_size=self.kernel_size,
                             stride=self.layers[i]['stride'],
                             dilation=self.layers[i]['dilation'],
                             groups=self.no_ts_features + self.n)

            caff_fc = nn.Linear(in_features=linear_input_dim, out_features=linear_output_dim)

            bn_temp = self.batchnormclass(num_features=temp_out_channels, momentum=0.1)
            bn_caff = self.batchnormclass(num_features=linear_output_dim, momentum=0.1)

            # bn_temp = bn_point = self.empty_module  # linear module; does nothing

            A_layer = my_AFF(out_channels_caff)
            FFA_layer = my_AFF(linear_input_dim)

            self.layer_modules[str(i)] = nn.ModuleDict({
                'temp': temp,
                'bn_temp': bn_temp,
                'caff_fc': caff_fc,
                'bn_caff': bn_caff,
                'A_layer': A_layer,
                'FFA_layer': FFA_layer})

            self.Y = self.layers[i]['temp_kernels']
            self.n += 1

        # input shape: (B * T) * ((F + n) * (1 + Y) + diagnosis_size + no_flat_features)
        # output shape: (B * T) * last_linear_size
        # input_size = (self.no_ts_features + self.n) * (1 + self.Y) + self.diagnosis_size + self.no_flat_features
        #input_size = (self.no_ts_features + self.n) * (1 + self.Y) + self.diagnosis_size + self.no_flat_features

        input_size = (self.no_ts_features + self.n) * (1 + self.Y) + (self.n_layers * (1 + self.Y)) + self.diagnosis_size + self.no_flat_features
        if self.no_diag:
            # input_size = input_size - self.diagnosis_size
            input_size = input_size - self.diagnosis_size #input_size - self.diagnosis_size

        self.last_los_fc = nn.Linear(in_features=input_size, out_features=self.last_linear_size)
        self.last_mort_fc = nn.Linear(in_features=input_size, out_features=self.last_linear_size)

        return

    def tdsc_caff(self, B=None, T=None, X=None, repeat_flat=None, X_orig=None, temp=None, bn_temp=None, caff_fc=None,
                       bn_caff=None, A_layer=None, FFA_layer=None, temp_kernels=None, padding=None, prev_temp=None, prev_caff=None, m_scale_output=None,
                       caff_skip=None):

        X_padded = pad(X, padding, 'constant', 0)  # B * ((F + n) * (Y + 1)) * (T + padding)
        X_temp = self.temp_dropout(bn_temp(temp(X_padded)))  # B * ((F + n) * temp_kernels) * T

        #### Context Aware Attentive Feature Fusion (CAFF) #####
        if prev_caff is None:
            X_concat = cat(self.remove_none((prev_temp,  # (B * T) * ((F + n-1) * Y)
                                            prev_caff,  # (B * T) * 1
                                            X_orig,  # (B * T) * (2F + 2)
                                            repeat_flat)),  # (B * T) * no_flat_features
                        dim=1)  # (B * T) * (((F + n-1) * Y) + 1 + 2F + 2 + no_flat_features)
        else:
            X_concat = cat(self.remove_none((prev_temp.view(B*T,-1),  # (B * T) * ((F + n-1) * Y)
                                            prev_caff.permute(0,3,1,2).view(B*T,-1),  # (B * T) * 1
                                            X_orig,  # (B * T) * (2F + 2)
                                            repeat_flat)),  # (B * T) * no_flat_features
                        dim=1)  # (B * T) * (((F + n-1) * Y) + 1 + 2F + 2 + no_flat_features)

        X_concat, wei_1 = FFA_layer(X_concat.view(B,T,-1).permute(0,2,1)) # Step 2 Attention
        X_concat = X_concat.permute(0,2,1).view(B*T,-1)
        caff_output = self.main_dropout(bn_caff(caff_fc(X_concat)))  # (B * T) * 1
        caff_output = caff_output.view(B, T, -1).unsqueeze(2).permute(0,2,3,1)

        # Accumulate multi-scale features
        m_scale_output = cat((m_scale_output,caff_output), dim=1) if m_scale_output is not None else caff_output

        caff_skip = cat((caff_skip, prev_caff[:,:,-1,:].unsqueeze(2)), dim=1) if prev_caff is not None else caff_skip

        temp_skip = cat((caff_skip,  # B * (F + n) * 1 * T
                         X_temp.view(B, caff_skip.shape[1], temp_kernels, T)),  # B * (F + n) * temp_kernels * T
                        dim=2)  # B * (F + n) * (1 + temp_kernels) * T

        X_combined = self.relu(cat((temp_skip, caff_output), dim=1))  # B * (F + n) * (1 + temp_kernels) * T
        next_X = X_combined.view(B, (caff_skip.shape[1] + 1) * (1 + temp_kernels), T)  # B * ((F + n + 1) * (1 + temp_kernels)) * T

        next_X, wei_2 = A_layer(next_X.view(B,-1,T)) # step 4 attention
        next_X =  next_X.view(B, (caff_skip.shape[1] + 1) * (1 + temp_kernels), T)

        temp_output = X_temp.permute(0, 2, 1).contiguous().view(B * T, caff_skip.shape[1] * temp_kernels)  # (B * T) * ((F + n) * temp_kernels)

        return (temp_output,  # (B * T) * ((F + n) * temp_kernels)
                caff_output,  # (B * T) * 1
                next_X,  # B * ((F + n) * (1 + temp_kernels)) * T
                caff_skip, # caff features of the prevous layer
                m_scale_output, #  keeping track of the caff multi scale features from all layers; B * (F + n) * T
                wei_1, wei_2)  #  PWatt Attention weights

    def forward(self, X, diagnoses, flat, time_before_pred=5):

        # flat is B * no_flat_features
        # diagnoses is B * no_daig_features
        # X is B * no_daig_features * T

        # split into features and indicator variables
        X_separated = torch.split(X[:, 1:-1, :], self.no_ts_features, dim=1)  # tuple ((B * F * T), (B * F * T))

        # prepare repeat arguments and initialise layer loop
        B, _, T = X_separated[0].shape

        repeat_flat = flat.repeat_interleave(T, dim=0)  # (B * T) * no_flat_features
        X_orig = X.permute(0, 2, 1).contiguous().view(B * T, 2 * self.no_ts_features + 2)  # (B * T) * (2F + 2)
        repeat_args = {'repeat_flat': repeat_flat,
                        'X_orig': X_orig,
                        'B': B,
                        'T': T}

        next_X = torch.stack(X_separated, dim=2).reshape(B, 2 * self.no_ts_features, T)
        caff_skip = X_separated[0].unsqueeze(2)  # ts features without indicators, keeps track of caff skip connections generated from caff module;
        temp_output = None
        caff_output = None
        m_scale_output = None
        wei_step2 = []
        wei_step4 = []
        for i in range(self.n_layers):
            kwargs = dict(self.layer_modules[str(i)], **repeat_args)

            temp_output, caff_output, next_X, caff_skip, m_scale_output, wei_1, wei_2 = self.tdsc_caff(X=next_X, caff_skip=caff_skip,
                                                                prev_temp=temp_output, prev_caff=caff_output,
                                                                temp_kernels=self.layers[i]['temp_kernels'],
                                                                padding=self.layers[i]['padding'],
                                                                m_scale_output= m_scale_output,
                                                                **kwargs)

            wei_step2.append(wei_1.detach().cpu())
            wei_step4.append(wei_2.detach().cpu())

        m_scale_output = m_scale_output.view(B,-1,T)
        if self.no_diag:
            combined_features = cat((flat.repeat_interleave(T - time_before_pred, dim=0),  # (B * (T - time_before_pred)) * no_flat_features
                                     next_X[:, :, time_before_pred:].permute(0, 2, 1).contiguous().view(B * (T - time_before_pred), -1),
                                     m_scale_output[:, :, time_before_pred:].permute(0, 2, 1).contiguous().view(B * (T - time_before_pred), -1)), dim=1)  # (B * (T - time_before_pred)) * (((F + n) * (1 + Y)) + no_flat_features) for tpc
        else:
            diagnoses_enc = self.relu(self.main_dropout(self.bn_diagnosis_encoder(self.diagnosis_encoder(diagnoses))))  # B * diagnosis_size
            combined_features = cat((flat.repeat_interleave(T - time_before_pred, dim=0),  # (B * (T - time_before_pred)) * no_flat_features
                                     diagnoses_enc.repeat_interleave(T - time_before_pred, dim=0),  # (B * (T - time_before_pred)) * diagnosis_size
                                     next_X[:, :, time_before_pred:].permute(0, 2, 1).contiguous().view(B * (T - time_before_pred), -1),
                                     m_scale_output[:, :, time_before_pred:].permute(0, 2, 1).contiguous().view(B * (T - time_before_pred), -1)), dim=1)  # (B * (T - time_before_pred)) * (((F + n) * (1 + Y)) + diagnosis_size + no_flat_features) for tpc

        last_los = self.relu(self.main_dropout(self.bn_point_last_los(self.last_los_fc(combined_features))))
        last_mort = self.relu(self.main_dropout(self.bn_point_last_mort(self.last_mort_fc(combined_features))))

        los_predictions = self.hardtanh(exp(self.final_los(last_los).view(B, T - time_before_pred)))  # B * (T - time_before_pred)
        mort_predictions = self.sigmoid(self.final_mort(last_mort).view(B, T - time_before_pred))  # B * (T - time_before_pred)

        return los_predictions, mort_predictions, wei_step2, wei_step4

    def loss(self, y_hat_los, y_hat_mort, y_los, y_mort, mask, seq_lengths, device, sum_losses, loss_type):
        # mortality loss
        if self.task == 'mortality':
            loss = self.bce_loss(y_hat_mort, y_mort) * self.alpha
        # LoS loss
        else:
            bool_type = torch.cuda.BoolTensor if device == torch.device('cuda:3') else torch.BoolTensor
            if loss_type == 'rmsle':
                los_loss = self.rmsle_loss(y_hat_los, y_los, mask.type(bool_type), seq_lengths, sum_losses)
            if loss_type == 'msle':
                los_loss = self.msle_loss(y_hat_los, y_los, mask.type(bool_type), seq_lengths, sum_losses)
            elif loss_type == 'mse':
                los_loss = self.mse_loss(y_hat_los, y_los, mask.type(bool_type), seq_lengths, sum_losses)

            loss = los_loss
        return loss


# -*- coding: utf-8 -*-
"""

@author: Fatih Kemal Terzi
"""

import cv2
import numpy as np

# Image reading
img = cv2.imread('pools.png')
count=0
# Image converting to HSV color space
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

# Adjusting range of blue color for detecting pools
lower_blue = np.array([80,50,50])
upper_blue = np.array([115,255,255])

# To spesifiying blue region creating binary mask
mask = cv2.inRange(hsv, lower_blue, upper_blue)

# Perform morphological operations to reduce noise
kernel = np.ones((5,5),np.uint8)
mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

# Find contours of blue regions
contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

# Draw bounding boxes around the blue regions
for cnt in contours:
    x, y, w, h = cv2.boundingRect(cnt)
    cv2.rectangle(img, (x, y), (x + w, y + h), (0, 0, 255), 2)
    count+=1
# Display the result
cv2.imshow('Detected_pools', img)
print('Number of pools : ',count)
cv2.waitKey(0)
cv2.destroyAllWindows()


import requests

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}

"传入参数的形式"
params = {"wd":"haha"}

"最后面的问号可加可不加，不加的话，程序会自动帮你加上"
url_temp = "https://www.baidu.com/?"

"注意用requests调用post和get时的函数"
r = requests.get(url_temp, headers=headers, params=params)
print(r.status_code)       # 获取请求得到的网页的状态
print(r.request.url)       # 获取我们请求得到的网页网址

"下面这个更加简洁"
".format用起来和%s效果是一样的"
url_2 = "https://www.baidu.com/?wd={}".format("haha")
r = requests.get(url_2, headers=headers)
print(r.status_code)       # 获取请求得到的网页的状态
print(r.request.url)       # 获取我们请求得到的网页网址