# coding: utf-8

import werkzeug.datastructures

from functools import wraps
from jinja2 import Markup, escape
from flask import (request, session, current_app, redirect, render_template,
                    url_for)
from wtforms.fields import HiddenField
from wtforms.widgets import HiddenInput
from wtforms.validators import ValidationError
from wtforms.ext.csrf.form import SecureForm
from ._compat import text_type, string_types
from .csrf import generate_csrf, validate_csrf

try:
    from .i18n import translations
except ImportError:
    translations = None  # babel not installed

class _Auto():
    '''Placeholder for unspecified variables that should be set to defaults.

    Used when None is a valid option and should not be replaced by a default.
    '''
    pass

def _is_hidden(field):
    """Detect if the field is hidden."""
    if isinstance(field, HiddenField):
        return True
    if isinstance(field.widget, HiddenInput):
        return True
    return False

class Form(SecureForm):
    """
    Flask-specific subclass of WTForms **SecureForm** class.

    If formdata is not specified, this will use flask.request.form.
    Explicitly pass formdata = None to prevent this.

    :param csrf_context: a session or dict-like object to use when making
                         CSRF tokens. Default: flask.session.

    :param secret_key: a secret key for building CSRF tokens. If this isn't
                       specified, the form will take the first of these
                       that is defined:

                       * SECRET_KEY attribute on this class
                       * WTF_CSRF_SECRET_KEY config of flask app
                       * SECRET_KEY config of flask app
                       * session secret key

    :param csrf_enabled: whether to use CSRF protection. If False, all
                         csrf behavior is suppressed.
                         Default: WTF_CSRF_ENABLED config value
    """

    SECRET_KEY = None
    TIME_LIMIT = None

    def __init__(self, formdata=_Auto, obj=None, prefix='', csrf_context=None,
                 secret_key=None, csrf_enabled=None, *args, **kwargs):

        if csrf_enabled is None:
            csrf_enabled = current_app.config.get('WTF_CSRF_ENABLED', True)

        self.csrf_enabled = csrf_enabled

        if formdata is _Auto:
            if self.is_submitted():
                formdata = request.form
                if request.files:
                    formdata = formdata.copy()
                    formdata.update(request.files)
                elif request.json:
                    formdata = werkzeug.datastructures.MultiDict(request.json)
            else:
                formdata = None

        if self.csrf_enabled:
            if csrf_context is None:
                csrf_context = session
            if secret_key is None:
                # It wasn't passed in, check if the class has a SECRET_KEY
                secret_key = getattr(self, "SECRET_KEY", None)

            self.SECRET_KEY = secret_key
        else:
            csrf_context = {}
            self.SECRET_KEY = ''
        super(Form, self).__init__(formdata, obj, prefix,
                                   csrf_context=csrf_context,
                                   *args, **kwargs)

    def generate_csrf_token(self, csrf_context=None):
        if not self.csrf_enabled:
            return None
        return generate_csrf(self.SECRET_KEY, self.TIME_LIMIT)

    def validate_csrf_token(self, field):
        if not self.csrf_enabled:
            return True
        if hasattr(request, 'csrf_valid') and request.csrf_valid:
            # this is validated by CsrfProtect
            return True
        if not validate_csrf(field.data, self.SECRET_KEY, self.TIME_LIMIT):
            raise ValidationError(field.gettext('CSRF token missing'))

    def validate_csrf_data(self, data):
        """Check if the csrf data is valid.

        .. versionadded: 0.9.0

        :param data: the csrf string to be validated.
        """
        return validate_csrf(data, self.SECRET_KEY, self.TIME_LIMIT)

    def is_submitted(self):
        """
        Checks if form has been submitted. The default case is if the HTTP
        method is **PUT** or **POST**.
        """

        return request and request.method in ("PUT", "POST")

    def hidden_tag(self, *fields):
        """
        Wraps hidden fields in a hidden DIV tag, in order to keep XHTML
        compliance.

        .. versionadded:: 0.3

        :param fields: list of hidden field names. If not provided will render
                       all hidden fields, including the CSRF field.
        """

        if not fields:
            fields = [f for f in self if _is_hidden(f)]

        name = current_app.config.get('WTF_HIDDEN_TAG', 'div')
        attrs = current_app.config.get(
            'WTF_HIDDEN_TAG_ATTRS', {'style': 'display:none;'})

        tag_attrs = u' '.join(
            u'%s="%s"' % (escape(k), escape(v)) for k, v in attrs.items())
        tag_start = u'<%s %s>' % (escape(name), tag_attrs)
        tag_end = u'</%s>' % escape(name)

        rv = [tag_start]
        for field in fields:
            if isinstance(field, string_types):
                field = getattr(self, field)
            rv.append(text_type(field))
        rv.append(tag_end)

        return Markup(u"".join(rv))

    def validate_on_submit(self):
        """
        Checks if form has been submitted and if so runs validate. This is
        a shortcut, equivalent to ``form.is_submitted() and form.validate()``
        """
        return self.is_submitted() and self.validate()

    def _get_translations(self):
        if not current_app.config.get('WTF_I18N_ENABLED', True):
            return None
        return translations

def form_page(template, login_route, **parameters_to_render_template):
    """
    If you decorate a view with this, this will ensure that your HTML form
    has no fields blank, and allows the developer don't worry with the type
    of request is "POST" or "GET". For example::

        @app.route('/register', methods=["GET", "POST"])
        @form_page("register.html", ".register", title="Register user")
        def register():
            pass

    This feature is useful when you prefer using HTML to forms. It also allows
    less polluting your view, since it does not need to check which type
    of request or make validation of fields sent by HTML forms.

    :param template: Indicates the template page which contains the form.
        It is used if the request is of type "GET".
    :type template: string
    :param login_route: Indicates the route belonging to the form page.
        It is used if the HTML form is not valid.
    :type login_route: string
    :param parameters_to_render_template: Indicates the parameters to be
        passed to the template at the time of the request type "GET".
        For example, if we need to spend a title for our template, we use
        these parameters. Defaults to "None".
    :type parameters_to_render_template: any type.
    """
    def decorated_function(func):
        """
        :param func: The view function to decorate.
        :type func: function
        """
        @wraps(func)
        def decorated_view(*args, **kwargs):
            if request.method in ("POST", "PUT"):
                fields = request.form.to_dict()
                if _valid_form(fields):
                    return redirect(url_for(login_route))
                return func(*args, **kwargs)
            return render_template(template, **parameters_to_render_template)
        return decorated_view
    return decorated_function

def _valid_form(fields):
    """
    Check if form have empty fields.

    :param fields: Indicates a past dictionary with the parameters of the
        request "POST".
    :type fields: dict
    """
    fields_errors = {}
    for field in fields:
        field_value = fields[field]
        if field_value.isspace() or not len(field_value):
            fields_errors[field] = field_value
    return bool(fields_errors)


import matplotlib.pyplot as plt
import numpy as np

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def periodogram6(ax, x, y, vlines, plot_min_t=1, max_f=1, spp=100):

	from scipy.signal import find_peaks
	from astropy.timeseries import LombScargle

	time_span = (max(x) - min(x))
	min_f   = 1/time_span

	frequency, power = LombScargle(x, y).autopower(minimum_frequency=min_f,
												   maximum_frequency=max_f,
												   samples_per_peak=spp)

	plot_x = 1/frequency
	idxx = (plot_x>plot_min_t) & (plot_x<100)
	height = max(power[idxx])*0.9
	ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.7)
	peaks, _ = find_peaks(power[idxx], height=height)
	ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

	for n in range(len(plot_x[idxx][peaks])):
		ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

	for xc in vlines:
		ax.axvline(x=xc, color='r', linestyle='-', lw=2, alpha=0.3)

	ax.set_xlim([plot_min_t,100])
	ax.set_ylim([0,1.7*height])
	ax.set_xscale('log')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_all(k_mode, t, rv, erv, ind, eind, ts_xlabel, rv_xlabel, pe_xlabel, ind_yalbel, file_name, height_ratio=0.7, vlines=[], HARPS=True):

	'''
	e.g.
		k_mode 		= 11
		t 			= bjd_daily
		rv 			= rv_daily
		erv 		= erv_daily
		ind 		= ΔRV_k
		eind 	 	= eRV_FT_k
		ts_xlabel 	= 'BJD - 2400000 [d]'
		rv_xlabel 	= '$RV_{HARPS}$'
		pe_xlabel 	= 'Period [days]'
		ind_yalbel	= 'A'
		file_name 	= 'time-series_and_shift_correlation.pdf'

	'''

	def new_periodogram(x, y, dy, vlines, height_ratio=height_ratio, plot_min_t=2, max_f=1, spp=100):

		from scipy.signal import find_peaks
		from astropy.timeseries import LombScargle

		time_span = (max(x) - min(x))
		min_f   = 1/time_span

		frequency, power = LombScargle(x, y, dy).autopower(minimum_frequency=min_f,
													   maximum_frequency=max_f,
													   samples_per_peak=spp)

		plot_x = 1/frequency
		idxx = (plot_x>plot_min_t) & (plot_x<time_span/2)
		height = max(power[idxx])*height_ratio
		ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.5)
		peaks, _ = find_peaks(power[idxx], height=height)
		ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

		for n in range(len(plot_x[idxx][peaks])):
			ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

		ax.set_xlim([plot_min_t,time_span/2])
		ax.set_ylim([0, 1.25*max(power[idxx])])

		if vlines!=[]:
			for xc in vlines:
				ax.axvline(x=xc, color='r', linestyle='-', lw=2, alpha=0.2)
		if HARPS == True:
			ax.axvspan(200, 220, facecolor='r', alpha=0.2)

		ax.set_xscale('log')

	from sklearn.linear_model import LinearRegression

	# set up the plotting configureations
	alpha1, alpha2 = [0.5,0.2]
	widths 	= [7,1,7]
	heights = [1]*(k_mode+1)
	gs_kw 	= dict(width_ratios=widths, height_ratios=heights)
	plt.rcParams.update({'font.size': 12})
	fig6, f6_axes = plt.subplots(figsize=(16, k_mode+1), ncols=3, nrows=k_mode+1, constrained_layout=True,
	                             gridspec_kw=gs_kw)

	# plots
	for r, row in enumerate(f6_axes):
		for c, ax in enumerate(row):

			# time-series
			if c==0:
				if r==0:
					ax.errorbar(t, rv-np.mean(rv), erv, marker='.', ms=5, color='black', ls='none', alpha=alpha1)
					ax.set_title('Time-series')
					ax.set_ylabel(rv_xlabel)
				else:
					ax.errorbar(t, ind[r-1,:], eind[r-1,:],  marker='.', ms=5, color='black', ls='none', alpha=alpha1)
					ax.set_ylabel(ind_yalbel + '$_{' + str(r) + '}$')
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				else:
					ax.set_xlabel(ts_xlabel)

			if c==1:
				if r==0:
					reg = LinearRegression().fit(rv.reshape(-1, 1), rv.reshape(-1, 1))
					score = reg.score(rv.reshape(-1, 1), rv.reshape(-1, 1))
					adjust_R2 = 1-(1-score)*(len(t)-1)/(len(t)-1-1)
					title = r'$\bar{R}$' + r'$^2$'
					ax.set_title(title + ' = {:.2f}'.format(adjust_R2))
					ax.plot(rv-np.mean(rv), rv-np.mean(rv), 'k.', alpha = alpha2)
				if r>0:
					reg = LinearRegression().fit(rv.reshape(-1, 1), ind[r-1,:].reshape(-1, 1))
					score = reg.score(rv.reshape(-1, 1), ind[r-1,:].reshape(-1, 1))
					adjust_R2 = 1-(1-score)*(len(t)-1)/(len(t)-1-1)
					title = r'$\bar{R}$' + r'$^2$'
					ax.set_title(title + ' = {:.2f}'.format(adjust_R2))
					ax.plot(rv-np.mean(rv), ind[r-1,:], 'k.', alpha = alpha2)
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				else:
					ax.set_xlabel(rv_xlabel)
				ax.yaxis.tick_right()

			if c==2:
				if r==0:
					new_periodogram(t, rv, erv, vlines)
					ax.set_title('Periodogram')
				if r>0:
					new_periodogram(t, ind[r-1,:], eind[r-1,:], vlines)
				if r!=k_mode:
					ax.tick_params(labelbottom=False)
				if r==k_mode:
					ax.set_xlabel(pe_xlabel)

	fig6.align_ylabels(f6_axes[:, 0])
	plt.savefig(file_name)
	plt.show()
	plt.close('all')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_all_but_corr(k_mode, t, ind, eind, height_ratio, ts_xlabel, pe_xlabel, ind_yalbel, file_name):

	'''
	e.g.
		k_mode 		= 11
		t 			= bjd_daily
		ind 		= shift_function
		eind 	 	= err_shift_spectrum
		ts_xlabel 	= 'BJD - 2400000 [d]'
		pe_xlabel 	= 'Period [days]'
		ind_yalbel	= 'A'
		file_name 	= 'time-series_and_shift_correlation.png'

	'''
	def new_periodogram(x, y, dy, height_ratio=height_ratio, plot_min_t=2, max_f=1, spp=100):

		from scipy.signal import find_peaks
		from astropy.timeseries import LombScargle

		time_span = (max(x) - min(x))
		min_f   = 1/time_span

		frequency, power = LombScargle(x, y, dy).autopower(minimum_frequency=min_f,
													   maximum_frequency=max_f,
													   samples_per_peak=spp)

		plot_x = 1/frequency
		idxx = (plot_x>plot_min_t) & (plot_x<time_span/2)
		height = max(power[idxx])*height_ratio
		ax.plot(plot_x[idxx], power[idxx], 'k-', alpha=0.5)
		peaks, _ = find_peaks(power[idxx], height=height)
		ax.plot(plot_x[idxx][peaks], power[idxx][peaks], "ro")

		for n in range(len(plot_x[idxx][peaks])):
			ax.text(plot_x[idxx][peaks][n], power[idxx][peaks][n], '%.1f' % plot_x[idxx][peaks][n], fontsize=10)

		ax.set_xlim([plot_min_t,time_span/2])
		ax.set_ylim([0, 1.25*max(power[idxx])])

		ax.set_xscale('log')

	from sklearn.linear_model import LinearRegression

	# set up the plotting configureations
	alpha1, alpha2 = [0.5,0.2]
	widths 	= [7,7]
	heights = [1]*k_mode
	gs_kw 	= dict(width_ratios=widths, height_ratios=heights)
	plt.rcParams.update({'font.size': 12})
	fig6, f6_axes = plt.subplots(figsize=(16, k_mode), ncols=2, nrows=k_mode, constrained_layout=True,
	                             gridspec_kw=gs_kw)

	# plots
	for r, row in enumerate(f6_axes):
		for c, ax in enumerate(row):

			# time-series
			if c==0:
				ax.errorbar(t, ind[r,:], eind[r,:],  marker='.', ms=5, color='black', ls='none', alpha=alpha1)
				if len(ind_yalbel)==1:
					ax.set_ylabel(ind_yalbel[0] + '$_{' + str(r+1) + '}$')
				else:
					ax.set_ylabel(ind_yalbel[r])
				if r==0:
					ax.set_title('Time-series')
				if r!=(k_mode-1):
					ax.set_xticks([])
				if r==(k_mode-1):
					ax.set_xlabel(ts_xlabel)

			if c==1:
				new_periodogram(t, ind[r,:], eind[r,:])
				if r==0:
					ax.set_title('Periodogram')
				if r!=(k_mode-1):
					ax.set_xticks([])
				if r==(k_mode-1):
					ax.set_xlabel(pe_xlabel)

	fig6.align_ylabels(f6_axes[:, 0])
	plt.savefig(file_name)
	plt.show()
	plt.close('all')

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def weighted_pca(X, X_err, n_pca=None, nor=False):
	'''
	X.shape 	= (n_samples, n_features)
	X_err.shape = (n_samples, n_features)
	nor: normalization = True / False
	'''
	from wpca import WPCA

	weights = 1/X_err
	kwds 	= {'weights': weights}

	# subtract the weighted mean for each measurement type (dimension) of X
	X_new 	= np.zeros(X.shape)
	mean 	= np.zeros(X.shape[1])
	std 	= np.zeros(X.shape[1])

	for i in range(X.shape[1]):
		mean[i] = np.average(X[:,i], weights=weights[:,i])
		std[i] 	= np.average((X[:,i]-mean[i])**2, weights=weights[:,i])**0.5

	for i in range(X.shape[1]):
		if nor == True:
			X_new[:,i] 		= (X[:,i] - mean[i]) / std[i]
			weights[:,i]	= weights[:,i] * std[i] 	# may need to include the Fourier power later
		else:
			X_new[:,i] = X[:,i] - mean[i]

	if n_pca==None:
		n_pca = X.shape[1]

	# Compute the PCA vectors & variance
	pca 		= WPCA(n_components=n_pca).fit(X_new, **kwds)
	pca_score 	= pca.transform(X_new, **kwds)
	P 			= pca.components_

	# The following computes the errorbars
	# The transpose is only intended to be consistent with the matrix format from Ludovic's paper
	X_wpca 	= X_new.T
	C 		= np.zeros(X_wpca.shape)
	err_C 	= np.zeros(X_wpca.shape)

	W 		= weights.T
	P 		= P.T

	for i in range(X_wpca.shape[1]):
		w = np.diag(W[:, i]) ** 2
		C[:, i] = np.linalg.inv(P.T @ w @ P) @ (P.T @ w @ X_wpca[:, i])

		# the error is described by a covariance matrix of C
		Cov_C = np.linalg.inv(P.T @ w @ P)
		diag_C = np.diag(Cov_C)
		err_C[:, i] = diag_C ** 0.5

	P = pca.components_
	err_pca_score = err_C.T

	#---------#
	# results #
	#---------#
	cumulative_variance_explained = np.cumsum(
		pca.explained_variance_ratio_) * 100  # look again the difference calculated from the other way
	print('Cumulative variance explained vs PCA components')
	for i in range(len(cumulative_variance_explained)):
		print(i+1, '\t', '{:.3f}'.format(cumulative_variance_explained[i]))

	for i in range(len(cumulative_variance_explained)):
		if cumulative_variance_explained[i] < 89.5:
			n_pca = i
	n_pca += 2
	if cumulative_variance_explained[0] > 89.5:
		n_pca = 1

	print('{:d} pca scores account for {:.2f}% variance explained'.format(n_pca,
			cumulative_variance_explained[n_pca - 1]))

	print('Standard deviations of each component and the midean uncertainty are\n',
		  np.around(np.std(C[0:n_pca, :], axis=1), decimals=1), '\n',
		  np.around(np.median(err_C[0:n_pca, :], axis=1), decimals=1))

	return P, pca_score, err_pca_score, n_pca

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def long_short_divide(x, y, yerr, r):
	'''
	x 		= bjd_daily
	y 		= shift_function[i,:]
	yerr 	= err_shift_spectrum[i,:]
	'''

	import george
	from george import kernels

	kernel 	= np.var(y) * kernels.Matern52Kernel(r**2)
	gp 		= george.GP(kernel)
	gp.compute(x, yerr)

	y_pred, _ 	= gp.predict(y, x, return_var=True)
	long_term 	= y_pred
	short_term 	= y - y_pred

	return gp, short_term, long_term

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def lasso_lambda(X, Y, Weight, alphas, n_folds=5, N=100, title='', file_name=None):

	from sklearn.linear_model import Lasso
	from sklearn.model_selection import GridSearchCV
	from sklearn.utils import shuffle
	from datetime import datetime
	from alive_progress import alive_bar

	start_time  = datetime.now()

	alpha_size 	= len(alphas)
	array_size 	= len(Y)

	alpna_n 		= np.zeros(N)
	# coef 			= np.zeros((36,N))
	scores_test 	= np.zeros((alpha_size, N))
	scores_train 	= np.zeros((alpha_size, N))
	scores_test_std = np.zeros((alpha_size, N))
	scores_train_std= np.zeros((alpha_size, N))

	tuned_parameters 	= [{"alpha": alphas}]

	with alive_bar(N) as bar:

		for n in range(N):
			idx_random 	= shuffle(np.arange(array_size))
			X 			= X[idx_random,:]
			Y 			= Y[idx_random]
			sw 			= Weight[idx_random]

			lasso 		= Lasso()
			clf 		= GridSearchCV(lasso, tuned_parameters, cv=n_folds, scoring='neg_root_mean_squared_error', refit=True, return_train_score=True)
			clf.fit(X, Y, sample_weight=sw)
			# coef[:,n] 	= clf.best_estimator_.coef_
			alpna_n[n] 	= clf.best_estimator_.alpha

			scores_test[:,n] 		= -clf.cv_results_["mean_test_score"]
			scores_train[:,n] 		= -clf.cv_results_["mean_train_score"]
			scores_test_std[:,n]	= clf.cv_results_["std_test_score"] / np.sqrt(n_folds)
			scores_train_std[:,n] 	= clf.cv_results_["std_train_score"] / np.sqrt(n_folds)

			bar()

	scores_test_mean 		= np.mean(scores_test, axis=1)
	scores_train_mean 		= np.mean(scores_train, axis=1)
	scores_test_std_mean 	= np.mean(scores_test_std, axis=1)
	scores_train_std_mean 	= np.mean(scores_train_std, axis=1)

	end_time = datetime.now()
	print('Duration: {}'.format(end_time - start_time))

	# plots
	if file_name != None:
		plt.figure().set_size_inches(8, 6)
		plt.rcParams['font.size'] = '16'
		plt.gcf().subplots_adjust(left=0.15)

		# plot error lines showing +/- std. errors of the scores
		plt.title(title)
		plt.semilogx(alphas, scores_test_mean, 'r', label='testing')
		plt.semilogx(alphas, scores_train_mean, 'b', label='training')
		plt.semilogx(alphas, scores_test_mean + scores_test_std_mean, "r--")
		plt.semilogx(alphas, scores_test_mean - scores_test_std_mean, "r--")
		plt.semilogx(alphas, scores_train_mean + scores_train_std_mean, "b--")
		plt.semilogx(alphas, scores_train_mean - scores_train_std_mean, "b--")
		plt.fill_between(alphas, scores_test_mean + scores_test_std_mean, scores_test_mean - scores_test_std_mean, color='r', alpha=0.2)
		plt.fill_between(alphas, scores_train_mean + scores_train_std_mean, scores_train_mean - scores_train_std_mean, color='b', alpha=0.2)

		xx 	= np.median(alpna_n)
		plt.axvline(xx, color='k', alpha=0.5)
		plt.text(xx, scores_test_mean[alphas==xx], r'$\hat\lambda_1$ = {:.3f}'.format(xx))

		# determine a more robust lambda using the 1-sigma rule
		scores_1sigma = (scores_test_mean+scores_test_std_mean)[alphas==xx][0]
		for i in range(alpha_size-1):
			if (scores_test_mean[i] < scores_1sigma) & (scores_test_mean[i+1] > scores_1sigma):
				alpha_1sigma = alphas[i]
		plt.axvline(alpha_1sigma, color='k', alpha=0.5)
		plt.axhline(scores_1sigma, linestyle="--", color=".5")
		plt.text(alpha_1sigma, scores_1sigma, r'$\hat\lambda_2$ = {:.3f}'.format(alpha_1sigma))
		plt.ylabel("Residual WRMS [m/s]")
		plt.xlabel(r"$\lambda$")
		plt.axhline(np.min(scores_test_mean), linestyle="--", color=".5")
		plt.xlim([alphas[0], alphas[-1]])
		plt.legend()

		plt.savefig(file_name + '.pdf')
		plt.show()
		plt.close()

	return scores_test_mean, scores_train_mean, scores_test_std_mean, scores_train_std_mean, alpna_n

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def mlr(feature_matrix, target_vector, etarget_vector, alpha, lag='True', day=5, feature_matrix2=None):

	'''
		Multiple linear regression
		no lag involved
	'''

	from sklearn import linear_model
	from sklearn.linear_model import Lasso

	regr 	= linear_model.LinearRegression()
	lasso 	= Lasso(alpha=alpha, max_iter=10000).fit(feature_matrix, target_vector, sample_weight=1/etarget_vector**2)

	y_hat 			= lasso.predict(feature_matrix)
	_, w_std_all 	= weighted_avg_and_std(target_vector, 1/etarget_vector**2)
	_, res_wrms 	= weighted_avg_and_std((y_hat - target_vector), weights=1/etarget_vector**2)
	_, model_wrms 	= weighted_avg_and_std(y_hat, weights=1/etarget_vector**2)
	score 			= lasso.score(feature_matrix, target_vector, sample_weight=1/etarget_vector**2)
	n, p 			= len(target_vector), feature_matrix.shape[1]
	adjust_R2 		= 1-(1-score)*(n-1)/(n-p-1)
	score 			= adjust_R2

	if lag=='False':
		k_feature2 	= feature_matrix.shape[1]
	else:
		k_feature2 	= int(feature_matrix.shape[1]/(day+1))
		k_feature 	= int(k_feature2/2)

	w_std = np.zeros(k_feature2)
	for i in range(len(w_std)):
		if lag=='True':
			_, w_std[i] = weighted_avg_and_std(feature_matrix2[:,i], 1/etarget_vector**2)
		else:
			_, w_std[i] = weighted_avg_and_std(feature_matrix[:,i], 1/etarget_vector**2)
	print('Weighted rms is reduced from {:.2f} to {:.2f} (-{:.0f}%); \n\
		Modelled RV weigthed rms = {:.2f};\n\
		Adjusted R squared = {:.3f}.'
			.format(w_std_all, res_wrms, (1-res_wrms/w_std_all)*100, model_wrms, score))

	if lag=='False':

		import pandas as pd
		df = pd.DataFrame({	'coefficients'		: lasso.coef_,
							'std'				: w_std,
							'variance'			: (lasso.coef_*w_std)**2,
							'variance percentage': (lasso.coef_*w_std)**2 / sum((lasso.coef_*w_std)**2) * 100
							})
		print(df.round({'coefficients'	: 2,
						'std'		: 2,
						'variance'		: 2,
						'variance percentage': 1
						}))

		return y_hat, w_std_all, res_wrms, score, df

	else:

		coeff_matrix = np.zeros((day*2+1, k_feature2))
		for i in range(k_feature):
			coeff_matrix[:,i] = lasso.coef_[i:k_feature*(2*day+1):k_feature]
		coeff_matrix[day,k_feature:]=lasso.coef_[k_feature*(2*day+1):]

		variance_matrix = np.zeros(coeff_matrix.shape)
		for i in range(day*2+1):
			variance_matrix[i,:] = coeff_matrix[i,:]*w_std
		variance_matrix = variance_matrix**2
		variance_matrix = variance_matrix / variance_matrix.sum() * 100

		return y_hat, w_std_all, res_wrms, score, variance_matrix

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def weighted_avg_and_std(values, weights):
    """
    Return the weighted average and standard deviation.

    values, weights -- Numpy ndarrays with the same shape.
    """
    average = np.average(values, weights=weights)
    variance = np.average((values-average)**2, weights=weights)

    return (average, np.sqrt(variance))

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def imshow_matrix(coeff_array, score, res_wrms, alpha, k_max, file_name):

	from matplotlib import colors

	'''
		file_name = fwhm_bis_coef
		file_name = fwhm_bis_multi_coef
		file_name = pca_coef
	'''

	day = int((coeff_array.shape[0]-1)/2)
	x = np.arange(day * 2 + 1) - day
	y = np.arange(coeff_array.shape[1]) + 1

	from mpl_toolkits.axes_grid1 import make_axes_locatable
	fig = plt.figure(figsize=(len(x)/1.5+2, len(y)/1.5+1), frameon=False)
	title = r'$\bar{R}$' + r'$^2$'
	plt.title(title + ' = {:.3f}'.format(score)
			  + '   Residual WRMS = {:.2f} m/s'.format(res_wrms))
	plt.xlabel('Lag [days]')
	ax = plt.gca()

	if file_name == 'lasso_coef':
		im2 = plt.imshow(np.transpose(coeff_array),  vmin=-5, vmax=5, cmap=plt.cm.bwr, alpha=.9)
	else:
		im2 = plt.imshow(np.transpose(coeff_array),  cmap=plt.cm.get_cmap('Reds'), alpha=.9, norm=colors.LogNorm(vmin=0.1, vmax=100))

	# Loop over data dimensions and create text annotations.
	for i in range(coeff_array.shape[1]):
		for j in range(day*2+1):
			if coeff_array[j, i] != 0:
				text = ax.text(j, i, '{:.1f}'.format(coeff_array[j, i]),
							   ha="center", va="center", color="k")

	ax.set_xticks(np.arange(len(x)))
	ax.set_yticks(np.arange(len(y)))
	ax.set_xticklabels(x)
	if file_name == 'fwhm_bis_coef':
		ax.set_yticklabels(['FWHM', 'BIS'])
	if file_name == 'Figure/fwhm_bis_multi_coef':
		ax.set_yticklabels(['S-FWHM', 'S-BIS', 'L-FWHM', 'L-BIS'])
	if file_name == 'pca_coef':
		ax.set_yticklabels([r'PC$_1$', r'PC$_2$', r'PC$_3$'])
	if file_name == 'Figure/fiesta_multi_coef':
		ax.set_yticklabels([r'S-PC$_1$', r'S-PC$_2$', r'S-PC$_3$', r'L-PC$_1$', r'L-PC$_2$', r'L-PC$_3$'])

	divider = make_axes_locatable(ax)
	cax = divider.append_axes("right", size="5%", pad=0.05)

	plt.colorbar(im2, cax=cax, label='Variance percentage')
	plt.savefig(file_name + '_{:.2f}_{:d}_{:d}'.format(alpha, day, k_max) +'.pdf')
	plt.show()
	plt.close()

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def scatter_hist(x, y, xerr, yerr, ax, ax_histx, ax_histy):

    ax_histx.tick_params(axis="x", labelbottom=False)
    ax_histy.tick_params(axis="y", labelleft=False)

    # the scatter plot:
    ax.errorbar(x, y, xerr, yerr, c='black', marker='o', ls='none', alpha=0.2)
    ax.set_xlabel('Model 3 residual [m/s]')
    ax.set_ylabel('Model 6 residual [m/s]')

    # now determine nice limits by hand:
    binwidth = 0.25
    xymax = max(np.max(np.abs(x)), np.max(np.abs(y)))
    lim = (int(xymax/binwidth) + 1) * binwidth

    bins = np.arange(-lim, lim + binwidth, binwidth)
    ax_histx.hist(x, bins=bins, color='black', alpha=0.5)
    ax_histy.hist(y, bins=bins, orientation='horizontal', color='black', alpha=0.5)

#---------------------------------------------------------------------------
#---------------------------------------------------------------------------

def plot_soap_config(FEATURE):

    N_sample  = 8
    lat       = np.linspace(10, 80, num=N_sample)
    RV_gauss_all = np.zeros((N_sample, 100))
    ΔA_k_all = np.zeros((N_sample, 5, 100))
    Δv_k_all = np.zeros((N_sample, 5, 100))

    for i in range(N_sample):
        print('\nlatitude = ' + str(lat[i]))
        folder_name = 'CCF_PROT=25.05_i=90.00_lon=(180.0,0.0,0.0,0.0)_lat=(' + '%.1f' % lat[i] +  ',0.0,0.0,0.0)_size=(0.1000,0.0000,0.0000,0.0000)'
        DIR     = 'Data/' + FEATURE + '_lat/' + folder_name
        FILE    = sorted(glob.glob(DIR + '/fits/*fits'))
        N       = len(FILE)

        for n in range(N):
            hdulist     = fits.open(FILE[n])
            CCF[:, n]   = hdulist[0].data[v_idx]

        v_k, A_k, RV_gauss = FIESTA(V_grid_SOAP, CCF, eCCF=CCF*0, template=CCF[:,0], k_max=5)
        v_k *= 1000
        RV_gauss = (RV_gauss - RV_gauss[0]) * 1000
        RV_gauss_all[i,:] = RV_gauss

        ΔA_k_all[i,:,:] = np.array([A_k[:,kk] - A_k[:,0] for kk in range(100)]).T

        for k in range(v_k.shape[0]):
            v_k[k,:] -= RV_gauss
        Δv_k_all[i,:,:] = v_k

    #
    colors = cm.seismic(np.linspace(0, 1, RV_gauss_all.shape[0]))

    plt.rcParams.update({'font.size': 12})
    fig, axes = plt.subplots(5,2,figsize=(8, 6))
    plt.subplots_adjust(left = 0.15, hspace=0.1, wspace=0.35, top = 0.95, right = 0.95)

    phase = np.arange(100)/100
    idx = (phase<=0.8) & (phase>=0.2)

    for i in range(5):

        for j in range(N_sample):
            axes[i,0].plot(phase[idx], ΔA_k_all[j,i,idx], alpha=0.8, color=colors[7-j], label=r'$%d\degree$' %lat[j])
            axes[i,1].plot(phase[idx], Δv_k_all[j,i,idx], alpha=0.8, color=colors[7-j])
        axes[i,0].set_ylabel(r'$k$=%d' %(i+1))
        axes[i,1].set_ylabel('[m/s]')
        axes[0,0].set_title(FEATURE + r' $\Delta A_k$')
        axes[0,1].set_title(FEATURE + r' $\Delta RV_k$')

        if i == 1:
            axes[i,0].legend(fontsize=6, loc=3)

        for l in range(2):
            if i!=4:
                axes[i,l].set_xticks([])
            else:
                axes[i,l].set_xlabel('Rotation phase')

    fig.align_ylabels(axes[:,0])
    fig.align_ylabels(axes[:,1])
    plt.savefig('Figure/FIESTA-' + FEATURE + '_latitude.pdf')
    plt.show()

    RV_gauss = RV_gauss.reshape(-1, 1)
    plt.rcParams.update({'font.size': 16})
    fig, axes = plt.subplots(figsize=(20, 4))
    fig.tight_layout()
    plt.subplots_adjust(left=0.06, bottom=0.16, right=0.98, top=0.9, wspace=0.4, hspace=0.4)

    for j in range(5):
        ax = plt.subplot(1, 5, j+1)
        for i in range(RV_gauss_all.shape[0]):
            ax.plot(RV_gauss_all[i,:], Δv_k_all[i,j,:], '.-', alpha=0.8, color=colors[7-i], label=r'$%d\degree$' %lat[i])
            ax.set_xlabel(r'$RV_{Gaussian}$ [m/s]')
            ax.set_ylabel(r'$\Delta RV_{}$ [m/s]'.format(j+1))
            if FEATURE == 'spot':
                if j==0:
                    ax.legend(prop={'size': 10})
            if j==2:
                ax.set_title('Solar '+ FEATURE)
    plt.savefig('Figure/FIESTA-' + FEATURE + '_latitude_correlation.pdf')
    plt.show()


#!/bin/python3
# author: Jan Hybs

import sys
import os
import re
import subprocess
import re, sys
import time
import progressbar
import argparse
from dateutil.relativedelta import relativedelta
import utils.config as cfg

__dir__ = os.path.abspath(os.path.dirname(__file__))
__root__ = os.path.dirname(__dir__)
__cfg__ = os.path.join(__root__, 'cfg')
__src__ = os.path.join(__root__, 'src')

config_path = os.path.join(__cfg__, 'variables.yaml')
config = cfg.load_config(config_path)
print(cfg.yaml_dump(config))

cwd = '/home/jan-hybs/projects/fenics-project/hpc-ci'
cwd = '/lustre/home/ec036/janhybs/projects/hpc-ci'
cwd = '/home/jan-hybs/mnt/cirrus/projects/hpc-ci'
cwd = config['hpc-ci-home']

# attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
# human_readable = lambda delta: [
#     '%d %s' % (getattr(delta, attr), getattr(delta, attr) > 1 and attr or attr[:-1])
#         for attr in attrs if getattr(delta, attr)
# ]

# git log -n 10 --graph '--pretty=format:%h:%ar:%ae:%d:%s'
class GitHash(object):
    def __init__(self,relation, git_hash, rel_date, email, branches, message):
        self.relation = relation
        self.hash = git_hash
        self.rel_date = rel_date
        self.email = email
        self.branches = branches
        self.message = message

        self.branch = 'master'
        self.message_peek = ''
        if message:
            self.message_peek = message[:64]
            if len(self.message_peek) != len(message):
                self.message_peek = message[:60] + ' ...'

def read_process_file(git_file):
    hash_match = re.compile(r'([ |*\\/]+) ([a-f0-9]{6,8})\| ?(.*)')
    result = list()
    with open(git_file, 'r') as fp:
        for line in fp.read().splitlines():
            if line.startswith('#'):
                continue

            match = hash_match.match(line)
            if match:
                relation = match.group(1)
                git_hash = match.group(2)
                rel_date, email, branches, message = [x.strip() for x in match.group(3).split('|', 4)]
                result.append(GitHash(
                    relation, git_hash, rel_date,
                    email, branches, message
                ))
            else:
                result.append(line)
    return result

def trigger_install(git):
    path = os.path.join(cwd, 'tmp-install-%s.log' % git.hash)
    # return subprocess.Popen(['sleep', '1']), None, path
    fp = open(path, 'w')
    return subprocess.Popen([
        'python3', 'src/main.py', 'install',
        '--commit=dolfin:%s' % git.hash,
        '--branch=dolfin:%s' % git.branch,
    ], cwd=cwd, stdout=fp, stderr=subprocess.STDOUT), fp, path

def trigger_tests(git):
    path = os.path.join(cwd, 'tmp-tests-%s.log' % git.hash)
    # return subprocess.Popen(['sleep', '1']), None, path
    fp = open(path, 'w')
    return subprocess.Popen([
        '/bin/bash', 'entrypoint', 'test',
    ], cwd=cwd, stdout=fp, stderr=subprocess.STDOUT), fp, path

def read_last_line(path):
    try:
        p = subprocess.Popen(['tail', '-1', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        o, e = p.communicate()
        return str(o.decode().strip())
    except Exception:
        return ''

def git_debug_info(common_prefix):
    print(common_prefix, '$> git rev-parse HEAD')
    o, e = subprocess.Popen(
        'git rev-parse HEAD'.split(),
        cwd='/lustre/home/ec036/janhybs/projects/hpc-ci/projects/fenics/dolfin',
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    ).communicate()
    for l in o.decode().strip().splitlines():
        print(common_prefix, l)
    print(common_prefix)

    print(common_prefix, '$> git log -n 1 --format=fuller --date=relative')
    o, e = subprocess.Popen(
        'git log -n 1 --format=fuller --date=relative'.split(),
        cwd='/lustre/home/ec036/janhybs/projects/hpc-ci/projects/fenics/dolfin',
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    ).communicate()
    for l in o.decode().strip().splitlines():
        print(common_prefix, l)
    print(common_prefix)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('git_file')
    parser.add_argument('--preview', '-p', action='store_true', default=False)
    args = parser.parse_args()

    no_preview = not args.preview
    result = read_process_file(args.git_file)
    i = 0

    space = ' '
    total = len([x for x in result if type(x) is GitHash])
    len_total = len(str(total))
    for git in result:
        prefix_fmt = '%{len_total}d/%{len_total}d)'.format(len_total=len_total)
        prefix_len = prefix_fmt % (0, 0)
        prefix_fill = ' ' * len(prefix_len)

        if type(git) is GitHash:
            i += 1
            prefix = prefix_fmt % (i, total)
            info = '{prefix} {git.relation:12s} [{git.hash:7s}] {git.message_peek:64} [{git.rel_date}]'
            common_prefix = '{prefix_fill} {git.relation:12s} {space:9s}'.format(**locals())
            # line = '%s %-12s [%6s] %s'% (prefix, git.relation, git.hash, git.message_peek)

            print(info.format(**locals()))
            print(common_prefix)

            # ----------------------------------------------------------------------

            def update_progress(common_prefix, start_time, path):
                duration = time.strftime('%H:%M:%S', time.gmtime(int(time.time() - start_time)))
                print('\r{common_prefix} reading last line ...{space:100s}'.format(space=' ', **locals()), end='')
                last_line = read_last_line(path)[:64]
                print('\r{common_prefix} {last_line:64} [{duration}]'.format(**locals()), end='')
                sys.stdout.flush()

            print(common_prefix, 'Installing ...')
            if no_preview:
                proc, fp, path = trigger_install(git)
                start_time = time.time()
                while proc.poll() is None:
                    update_progress(common_prefix, start_time, path)
                    time.sleep(5)
                if fp: fp.close()
                update_progress(common_prefix, start_time, path)
                print('\n' + common_prefix)

            # ----------------------------------------------------------------------

            if no_preview:
                git_debug_info(common_prefix)

            # ----------------------------------------------------------------------

            print(common_prefix, 'Testing ...')
            if no_preview:
                proc, fp, path = trigger_tests(git)
                start_time = time.time()
                while proc.poll() is None:
                    update_progress(common_prefix, start_time, path)
                    time.sleep(5)
                if fp: fp.close()
                update_progress(common_prefix, start_time, path)
                print('\n' + common_prefix)

            # label = progressbar.FormatCustomText('%(value)s', dict(value='<last-line>'))
            # bar = progressbar.ProgressBar(
            #     term_width=100,
            #     max_value=progressbar.UnknownLength,
            #     widgets=[
            #         line, ' | ', progressbar.Timer(), ' | ', label
            #     ]
            # )
            # while proc.poll() is None:
            #     if path:
            #         try:
            #             p = subprocess.Popen(['tail', '-1', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            #             o, e = p.communicate()
            #             o = str(o.decode().strip())[:64]
            #             o = o + ' ' * (64-len(o))
            #             label.update_mapping(**dict(value=o))
            #         except Exception as e:
            #             pass
            #
            #     bar.update(0)
            #     time.sleep(5)
            # if fp:
            #     fp.close()
            # bar.finish()
            #
            # # ----------------------------------------------------------------------

            # print(common_prefix)
            # proc = trigger_tests(git)
            # print(proc.wait())

        else:
            print('{prefix_fill} {git}'.format(**locals()))


from game import *
from round_robin import *

class Tournament(object):
    """Reproduce Prof. Axelrod's tournament"""

    def __init__(self, players, game=None, turns=200, repetitions=10):
        """Initiate a tournmanent of players"""
        self.players = players
        self.nplayers = len(players)
        self.plist = list(range(self.nplayers))
        if game is None:
            self.game = Game()
        else:
            self.game = game
        self.turns = turns
        self.replist = list(range(repetitions))
        self.results = self.initialise_results()
        self.deterministic_cache = {}

    def initialise_results(self):
        """
        Build the initial results containing just zeros. This is an embedded
        that could be made more efficient using a NumPy array.
        """
        results = [[[0 for irep in self.replist] for j in self.plist]
                   for i in self.plist]
        return results

    def play(self):
        """Play the tournament with repetitions of round robin"""
        round_robin = RoundRobin(self.players, self.game, self.turns, self.deterministic_cache)
        for irep in self.replist:
            payoffs = round_robin.play()
            self.deterministic_cache = round_robin.deterministic_cache
            for i in self.plist:
                for j in self.plist:
                    self.results[i][j][irep] = payoffs[i][j]
        return self.results


#!/usr/bin/env python
""" """

# Standard library modules.
import unittest
import logging

# Third party modules.

# Local modules.
from pymontecarlo.testcase import TestCase

from pymontecarlo.util.electron_range import kanaya_okayama

# Globals and constants variables.

class TestModule(TestCase):

    def setUp(self):
        super().setUp()

        self.comp1 = {29: 1.0}
        self.comp2 = {29: 0.5, 30: 0.5}

    def testkanaya_okayama(self):
        self.assertAlmostEqual(1.45504, kanaya_okayama(self.comp1, 20e3) * 1e6, 4)
        self.assertAlmostEqual(1.61829, kanaya_okayama(self.comp2, 20e3) * 1e6, 4)

if __name__ == '__main__': #pragma: no cover
    logging.getLogger().setLevel(logging.DEBUG)
    unittest.main()


import torch
import torch.nn as nn
import torch.nn.functional as F

from modules import get_positional_encoding, MultiHeadAttention, FeedForward, label_smoothing

PAD = 0
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def _get_key_and_query_masking(x, y):
    """Get key pad mask and query pad mask.

    Arguments:
        x: Source sentence. Tensor of shape=(batch_size, T_k)
        y: Target sentence. Tensor of shape=(batch_size, T_q)
    """
    _, T_k = x.size()
    _, T_q = y.size()
    key_pad_masks = x.eq(PAD).unsqueeze(1).repeat(1, T_q, 1)  # (N, T_q, T_k)
    query_pad_masks = y.eq(PAD).unsqueeze(2).repeat(1, 1, T_k)  # (N, T_q, T_k)
    return key_pad_masks.to(device), query_pad_masks.to(device)

class Transformer(nn.Module):
    def __init__(self, embedding_size, enc_voc_size, dec_voc_size, num_blocks=6, is_label_smooth=True):
        super(Transformer, self).__init__()
        self.enc_voc_size = enc_voc_size
        self.dec_voc_size = dec_voc_size
        self.num_blocks = num_blocks
        self.is_label_smooth = is_label_smooth

        # encoder
        self.enc_emb = nn.Embedding(enc_voc_size, embedding_size, padding_idx=0)
        self.enc_dropout = nn.Dropout(p=0.1)
        for i in range(num_blocks):
            self.__setattr__("enc_self_attention_{}".format(i), MultiHeadAttention(embedding_size, num_heads=8, causality=False))
            self.__setattr__("enc_feed_forward_{}".format(i), FeedForward(embedding_size, hiddens=[2048, embedding_size]))

        # decoder
        self.dec_emb = nn.Embedding(dec_voc_size, embedding_size, padding_idx=0)
        self.dec_dropout = nn.Dropout(p=0.1)
        for i in range(num_blocks):
            self.__setattr__("dec_self_attention_{}".format(i), MultiHeadAttention(embedding_size, num_heads=8, causality=True))
            self.__setattr__("dec_vanilla_attention_{}".format(i), MultiHeadAttention(embedding_size, num_heads=8, causality=False))
            self.__setattr__("dec_feed_forward_{}".format(i), FeedForward(embedding_size, hiddens=[2048, embedding_size]))

        # output
        self.fc = nn.Linear(embedding_size, dec_voc_size)

    def forward(self, x, y):
        """Transformer feed-forward function.

        Arguments:
            x: Tensor of shape=(batch_size, T_x)
            y: Tensor of shape=(batch_size, T_y)
            Both x and y are sentences with <EOS> as the last vocab!
            So we need to add <BOS> and remove <EOS> to y for the decoder input.
        Returns:
            mean_loss: A scalar tensor.
            preds: Tensor of shape=(batch_size, T_y)
            acc: A scalar tensor.
        """
        # No pad mask
        istarget = (1 - y.eq(PAD)).float().view(-1)  # (N*T, )
        # Backup labels for cal loss and acc
        labels = y.view(-1)  # (N*T, )

        # add <BOS> and remove <EOS>
        y = torch.cat((torch.ones(y.shape[0], 1, dtype=torch.int64).to(device) * 2, y[:, :-1]), dim=-1)  # (N, T_y)

        # encoder prepare
        enc_key_pad_masks, enc_query_pad_masks = _get_key_and_query_masking(x, x)  # (N, T_x, T_x)

        # decoder prepare
        dec_self_key_pad_masks, dec_self_query_pad_masks = _get_key_and_query_masking(y, y)  # (N, T_y, T_y)
        dec_key_pad_masks, dec_query_pad_masks = _get_key_and_query_masking(x, y)  # (N, T_y, T_x)

        # encoder
        x = self.enc_emb(x)  # (N, T_x, C)
        x = x + get_positional_encoding(x.shape[1], x.shape[2], padding_idx=None)
        x = self.enc_dropout(x)
        for i in range(self.num_blocks):
            x = self.__getattr__("enc_self_attention_{}".format(i))(
                x, x, x, key_pad_masks=enc_key_pad_masks, query_pad_masks=enc_query_pad_masks)
            x = self.__getattr__("enc_feed_forward_{}".format(i))(x)

        # decoder
        y = self.dec_emb(y)  # (N, T_y, C)
        y = y + get_positional_encoding(y.shape[1], y.shape[2], padding_idx=None)
        y = self.dec_dropout(y)
        for i in range(self.num_blocks):
            y = self.__getattr__("dec_self_attention_{}".format(i))(
                y, y, y, key_pad_masks=dec_self_key_pad_masks, query_pad_masks=dec_self_query_pad_masks)
            y = self.__getattr__("dec_vanilla_attention_{}".format(i))(
                y, x, x, key_pad_masks=dec_key_pad_masks, query_pad_masks=dec_query_pad_masks)
            y = self.__getattr__("dec_feed_forward_{}".format(i))(y)  # (N, T_y, C)

        logits = self.fc(y)  # (N, T_y, dec_voc_size)

        probs = F.softmax(logits, dim=-1).view(-1, self.dec_voc_size)  # (N*T, dec_voc_size)

        preds = probs.argmax(dim=-1)  # (N*T, )
        acc = (preds.eq(labels).float() * istarget).sum() / istarget.sum()  # (1, )

        y_onehot = torch.zeros(labels.shape[0], self.dec_voc_size).to(device).scatter_(1, labels.view(-1, 1), 1)  # (N*T, dev_voc_size)
        if self.is_label_smooth:
            y_onehot = label_smoothing(y_onehot)
        loss = -torch.sum(y_onehot * torch.log(probs), dim=-1)  # (N*T, )
        mean_loss = (loss * istarget).sum() / istarget.sum()  # (1, )

        return mean_loss, preds.view(logits.shape[0], -1), acc


from sqlalchemy import create_engine
from pandas.io import sql
import pandas as pd

data = pd.read_csv ('../csv_data/input.csv')
print('data:')
print (data)

# create the database engine
engine = create_engine ('sqlite:///:memory:')

# Store the dataframe as a table
data.to_sql ('data_table', engine)

# Query 1: Delete row from data_table
sql.execute ('DELETE FROM data_table WHERE name = (?)',
engine, params=[('Gary')])
res1 = pd.read_sql_query ('SELECT ID,Dept,Name,Salary,Start_date FROM data_table',
engine)
print ('\nres1: SELECT ID,Dept,Name,Salary,Start_date FROM data_table')
print(res1)

# Query 2: Delete row from data_table
sql.execute ('DELETE FROM data_table WHERE name = (?)',
engine, params=[('Guru')])
res2 = pd.read_sql_query ('SELECT ID,Dept,Name,Salary,Start_date FROM data_table',
engine)
print ('\nres2: SELECT ID,Dept,Name,Salary,Start_date FROM data_table')
print(res2)


# Function to sort an array with bubble sorting algorithm
# @@Param: list of numbers
def bubble_sort(array):

    for i in range(len(array)):
        for j in range(i,len(array)):
            if array[i] > array[j]:         # if first element is greater then second element
                array[i], array[j] = array[j], array[i]     # Swap

def main():
    array = [100,90,80,70]
    bubble_sort(array)
    print(array)

main()


# Exercise 2: Write a program to look for lines of the form:
# New Revision: 39772
# Extract the number from each of the lines using a regular expression and the findall() method. Compute the average of the numbers and print out the average as an integer.

import re

file = open("mbox.txt")
lst = []
for line in file:
    num = re.findall("^New Revision: ([0-9]+)", line)
    if len(num) != 1:
        continue
    for n in num:
        n = float(n)
    lst.append(n)

print("Average:", sum(lst)/len(lst))


# uncompyle6 version 3.7.4
# Python bytecode 3.8 (3413)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/takumakanari/.pyenv/versions/japanese-numbers-py3/lib/python3.8/site-packages/japanese_numbers/parser/to_arabic_parser.py
# Compiled at: 2018-07-01 03:23:12
# Size of source mod 2**32: 2309 bytes
from __future__ import absolute_import, unicode_literals
from past.builtins import xrange
from japanese_numbers.result import ParsedResult
from japanese_numbers.token import Tokenized, NUMERICS
from japanese_numbers.kind import UNIT_KIND, NUMBERS_KIND, MULTIPLES_KIND, NUMERIC_KIND
import sys
PY3 = sys.version_info.major == 3
if PY3:
    unicode = str

def _collect_numerics(val, pos):
    stack = []
    for c in val[pos:]:
        if c not in NUMERICS:
            break
        stack.append(c)
    else:
        return (
         int(''.join(stack)), len(stack))

def to_arabic(val, encode='utf8'):
    stacks, numbers, texts, analyzing, index = ([], [], [], False, -1)
    results = []

    def _append_result():
        results.append(ParsedResult(text=(''.join(texts)), number=(sum(stacks) + sum(numbers)),
          index=index))

    decoded_val = val if isinstance(val, unicode) else val.decode(encode)
    token = Tokenized(decoded_val)
    while token.has_next():
        kind, num = token.kind, token.num_of_kind
        if kind == UNIT_KIND:
            if token.last_kind != UNIT_KIND:
                ret = (numbers[(-1)] if numbers else 1) * num
                if numbers:
                    numbers[-1] = ret
            else:
                numbers.append(ret)
        else:
            if kind in (NUMBERS_KIND, UNIT_KIND):
                numbers.append(num)
            else:
                if kind == MULTIPLES_KIND:
                    stacks.append(sum(numbers) * num)
                    numbers = []
                else:
                    if kind == NUMERIC_KIND:
                        n, s = _collect_numerics(token.val, token.pos)
                        numbers.append(n)
                        index = token.pos if index < 0 else index
                        texts.append(''.join((token.origin_char_at(x) for x in xrange(token.pos, token.pos + s))))
                        token.next(incr=s)
                    else:
                        if analyzing:
                            _append_result()
                            stacks, numbers, texts, analyzing, index = ([], [], [], False, -1)
        analyzing = kind is not None
        if analyzing:
            if kind != NUMERIC_KIND:
                texts.append(token.origin_char)
            if index < 0:
                if token.last_kind is None:
                    index = token.pos
        if kind != NUMERIC_KIND:
            token.next()

    if stacks or numbers:
        _append_result()
    return results

def to_arabic_numbers(val, encode='utf8'):
    return tuple((x.number for x in to_arabic(val, encode=encode)))


import numpy as np
import cv2 as cv

def imshow(winname,img):

    cv.imshow(winname,img)
    cv.waitKey(0)
    cv.destroyAllWindows()

def kemeans_roi(img):

    # choose the midfreespace manullay
    img1 = img[285:320, 0:1279]

    # print(img1.shape)

    # gaussian blur it's a spatial operation does't effect color
    img_gs = cv.GaussianBlur(img1, (15, 15), 0)

    # first change the color space from BGR to HSV
    img_cvt = cv.cvtColor(img_gs, cv.COLOR_BGR2HSV)

    # change img_cvt in shape(?,3)
    Z = img_cvt[:, :, 0].reshape((-1, 1))

    # convert to np.float32 because kmeans in opencv use data type of float
    Z = np.float32(Z)

    # define criteria, number of clusters(K) and apply kmeans()
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 10, 1.0)

    # define cluster number
    K = 2

    # use opencv kmeans
    ret, label, center = cv.kmeans(Z, K, None, criteria, 10, cv.KMEANS_RANDOM_CENTERS)

    # choose the color belongs to freespace
    index = []
    for i in range(K):

        # set the threshold for centern
        if (center[i, 0] <= 50):
            index.append(i)

    # initialize mask
    mask_out = np.zeros((img_cvt.shape[0] * img_cvt.shape[1], 1), dtype=np.uint8)

    # get mask from index
    for i in range(len(index)):
        mask = np.zeros((img_cvt.shape[0] * img_cvt.shape[1], 1), dtype=np.uint8)
        loc = np.where(label.flatten() == index[i])
        mask[loc] = 255
        mask_out += mask

    mask_out = np.reshape(mask_out, (img_cvt.shape[0], img_cvt.shape[1]))

    # inverse mask
    img_inv = 255 - mask_out

    # erode
    kernel = np.ones((3, 3), np.uint8)
    mask_out = cv.dilate(img_inv, kernel, iterations=5)

    # use canny to detect edges
    canny = cv.Canny(mask_out, 30, 200)

    # choose one line from canny ,use this line to judge ,whether it is rightline or leftline
    sample = canny[15, :]
    mask_out_sample = mask_out[15, :]
    index = np.where(sample > 128)
    x_1 = np.array(index[0])

    if(len(x_1)>0):
        dif = np.diff(x_1)

        # remove the line close to each other
        dif = np.array(dif)
        deletidx = []
        for i in range(len(dif)):
            if (dif[i] < 10):
                deletidx.append(i)
        x_1 = np.delete(x_1, deletidx)

        car_rightline = []
        car_leftline = []

        for i in range(len(x_1)):

            if (mask_out_sample[x_1[i] - 1] > 128):
                car_rightline.append(x_1[i])
            else:
                car_leftline.append((x_1[i]))
        print("leftline ", car_leftline)
        print("rightline ", car_rightline)

        freespacerange = []

        if (car_leftline[0] < car_rightline[0]):
            freespacerange.append([0, car_leftline[0]])
            car_leftline = np.delete(car_leftline, 0)
        if (car_rightline[-1] > car_leftline[-1]):
            freespacerange.append([car_rightline[-1], len(sample)])
            car_rightline = np.delete(car_rightline, len(car_rightline) - 1)
        for i in range(len(car_rightline)):
            loc = np.where(car_leftline > car_rightline[i])[0]
            freespacerange.append([car_rightline[i], car_leftline[loc[0]]])

    return freespacerange


# Import modules
import enum
import itertools
import logging
import numpy as np
import regions
from rig import machine

# Import classes
from collections import defaultdict
from pacman.model.graphs.machine.impl.machine_vertex \
    import MachineVertex
from pacman.model.resources.resource_container import ResourceContainer
from pacman.model.resources.sdram_resource import SDRAMResource
from spinn_front_end_common.abstract_models.abstract_has_associated_binary \
    import AbstractHasAssociatedBinary
from spinn_front_end_common.abstract_models.abstract_provides_n_keys_for_partition \
    import AbstractProvidesNKeysForPartition
from spinn_front_end_common.abstract_models.abstract_starts_synchronized \
    import AbstractStartsSynchronized
from utils import Args

# Import functions
from six import iteritems
from utils import (calc_bitfield_words, calc_slice_bitfield_words,
                   get_model_executable_filename, load_regions, split_slice)

logger = logging.getLogger("pynn_spinnaker")

# ----------------------------------------------------------------------------
# Regions
# ----------------------------------------------------------------------------
class Regions(enum.IntEnum):
    """Region names, corresponding to those defined in `ensemble.h`"""
    system = 0
    neuron = 1
    synapse = 2
    input_buffer = 3
    back_prop_output = 4
    flush = 5
    intrinsic_plasticity = 6
    spike_recording = 7
    analogue_recording_0 = 8
    analogue_recording_1 = 9
    analogue_recording_2 = 10
    analogue_recording_3 = 11
    analogue_recording_start = analogue_recording_0
    analogue_recording_end = analogue_recording_3 + 1
    profiler = analogue_recording_end
    statistics = analogue_recording_end + 1

# ----------------------------------------------------------------------------
# Vertex
# ----------------------------------------------------------------------------
class Vertex(MachineVertex, AbstractHasAssociatedBinary,
             AbstractProvidesNKeysForPartition, AbstractStartsSynchronized):
    def __init__(self, neuron_slice, vert_index, sdram, app_name):
        self.neuron_slice = neuron_slice

        self.input_verts = []
        self.back_prop_out_buffers = None
        self.region_memory = None
        self.app_name = app_name
        self.vert_index = vert_index

        # Superclass
        # **NOTE** as vertex partitioning is already done,
        # only SDRAM is required for subsequent placing decisions
        MachineVertex.__init__(
            self, label="<neuron slice:%s>" % str(self.neuron_slice),
            resources_required=ResourceContainer(sdram=SDRAMResource(sdram)))

    # ------------------------------------------------------------------------
    # AbstractProvidesNKeysForPartition methods
    # ------------------------------------------------------------------------
    def get_n_keys_for_partition(self, partition, graph_mapper):
        # Each neuron requires two keys
        # **HACK** allocate fixed keyspace for each neuron cluster:
        # means a fixed bit can always be used to signify flushing
        return 2 * 1024

    # ------------------------------------------------------------------------
    # AbstractHasAssociatedBinary methods
    # ------------------------------------------------------------------------
    def get_binary_file_name(self):
        return self.app_name

    # ------------------------------------------------------------------------
    # Public methods
    # ------------------------------------------------------------------------
    def get_back_prop_in_buffer(self, post_slice):
        # Check the slices involved overlap and that this
        # neuron vertex actually has back propagation buffers
        assert post_slice.overlaps(self.neuron_slice)
        assert self.back_prop_out_buffers is not None

        # Calculate start and end bit in neuron id-space
        neuron_start_bit = max(post_slice.start, self.neuron_slice.start)
        neuron_end_bit = min(post_slice.stop, self.neuron_slice.stop)
        logger.debug("\t\t\tNeuron start bit:%u, Neuron end bit:%u",
                     neuron_start_bit, neuron_end_bit)

        # Calculate where in the buffer post_slice starts
        buffer_start_bit = neuron_start_bit - self.neuron_slice.start
        assert buffer_start_bit >= 0

        # Seperate where the buffer starts in words and bits
        buffer_start_word = buffer_start_bit // 32
        buffer_start_bit -= (buffer_start_word * 32)
        buffer_end_bit = (neuron_end_bit - neuron_start_bit) + buffer_start_bit
        buffer_num_words = calc_bitfield_words(buffer_end_bit)
        logger.debug("\t\t\tBuffer start word:%u, Buffer start bit:%u, Buffer end bit:%u, Buffer num words:%u",
                     buffer_start_word, buffer_start_word,
                     buffer_end_bit, buffer_num_words)

        # Return offset pointers into out buffers
        return (
            [b + (buffer_start_word * 4) for b in self.back_prop_out_buffers],
            buffer_num_words, buffer_start_bit, buffer_end_bit)

    def get_spike_tx_key(self, routing_info):
        # Routing key is used for spike transmission
        return self.get_routing_key(routing_info)

    def get_flush_tx_key(self, routing_info):
        # **YUCK** flush events are transmitted with fixed bit set
        return self.get_routing_key(routing_info) | (1 << 10)

    def get_routing_key(self, routing_info):
        vert_routing = routing_info.get_routing_info_from_pre_vertex(self, 0)
        return 0xFFFFFFFF if vert_routing is None else vert_routing.first_key

    def get_routing_mask(self, routing_info):
        vert_routing = routing_info.get_routing_info_from_pre_vertex(self, 0)
        return vert_routing.first_mask

# -----------------------------------------------------------------------------
# NeuralCluster
# -----------------------------------------------------------------------------
class NeuralCluster(object):
    # Tag names, corresponding to those defined in neuron_processor.h
    profiler_tag_names = {
        0:  "Synapse shape",
        1:  "Update neurons",
        2:  "Apply buffer",
    }

    # Names of statistics
    statistic_names = (
        "task_queue_full",
        "timer_event_overflows",
    )

    def __init__(self, pop_id, cell_type, parameters, initial_values,
                 sim_timestep_ms, timer_period_us, sim_ticks,
                 record_sample_interval, indices_to_record, config,
                 frontend, post_synaptic_width, requires_back_prop, pop_size):
        # Create standard regions
        self.regions = {}
        self.regions[Regions.system] = regions.System(
            timer_period_us, sim_ticks)
        self.regions[Regions.neuron] = cell_type._neuron_region_class(
            cell_type, parameters, initial_values, sim_timestep_ms, pop_size)
        self.regions[Regions.back_prop_output] = regions.SDRAMBackPropOutput(
            requires_back_prop)
        self.regions[Regions.flush] = regions.Flush(config.flush_time,
                                                    sim_timestep_ms)
        self.regions[Regions.spike_recording] = regions.SpikeRecording(
            indices_to_record, sim_timestep_ms, sim_ticks)
        self.regions[Regions.statistics] = regions.Statistics(len(self.statistic_names))

        # If cell type has any receptors i.e. any need for synaptic input
        if len(cell_type.receptor_types) > 0:
            # Add a synapse region and an input buffer
            self.regions[Regions.synapse] = regions.ParameterSpace(
                cell_type._synapse_mutable_param_map,
                cell_type._synapse_immutable_param_map,
                parameters, initial_values, pop_size,
                sim_timestep_ms=sim_timestep_ms)

            self.regions[Regions.input_buffer] = regions.InputBuffer()

         # If cell type has an intrinsic plasticity parameter map
        if hasattr(cell_type, "intrinsic_plasticity_param_map"):
            self.regions[Regions.intrinsic_plasticity] =\
                regions.HomogeneousParameterSpace(
                    cell_type._intrinsic_plasticity_param_map,
                    parameters,
                    sim_timestep_ms)

        # Assert that there are sufficient analogue
        # recording regions for this celltype's needs
        num_analogue_rec_regions = Regions.analogue_recording_end -\
            Regions.analogue_recording_start
        assert num_analogue_rec_regions >= (len(cell_type.recordable) - 1)

        # Loop through cell's non-spike recordables
        # and create analogue recording regions
        # **HACK** this assumes the first entry is spike
        for i, v in enumerate(cell_type.recordable[1:]):
            self.regions[Regions(Regions.analogue_recording_start + i)] =\
                regions.AnalogueRecording(indices_to_record, v,
                                          record_sample_interval,
                                          sim_timestep_ms, sim_ticks)

        # Add profiler region if required
        if config.num_profile_samples is not None:
            self.regions[Regions.profiler] =\
                regions.Profiler(config.num_profile_samples)

        # Split population slice
        neuron_slices = split_slice(pop_size, post_synaptic_width)

        # Get neuron executable name
        neuron_app = get_model_executable_filename(
            "neuron_", cell_type, config.num_profile_samples is not None)

        logger.debug("\t\tNeuron application:%s", neuron_app)
        logger.debug("\t\t%u neuron vertices", len(neuron_slices))

        # Build neuron vertices for each slice
        self.verts = []
        for vert_id, neuron_slice in enumerate(neuron_slices):
            # Create vertex
            vert = Vertex(neuron_slice, vert_id,
                          self._estimate_sdram(neuron_slice), neuron_app)

            # Add to frontend and verts list
            frontend.add_machine_vertex(vert)
            self.verts.append(vert)

    # --------------------------------------------------------------------------
    # Public methods
    # --------------------------------------------------------------------------
    def allocate_out_buffers(self, placements, transceiver, app_id):
        # Loop through vertices
        for v in self.verts:
            # Get placement
            placement = placements.get_placement_of_vertex(v)

            logger.debug("\t\tVertex %s (%u, %u, %u)",
                         v, placement.x, placement.y, placement.p)

            # **TODO** zero memory - waiting on https://github.com/SpiNNakerManchester/SpiNNMan/issues/59
            # If back propagation is enabled, allocate two back
            # propagation out buffers for this neuron vertex
            if self.regions[Regions.back_prop_output].enabled:
                back_prop_buffer_bytes =\
                    calc_slice_bitfield_words(v.neuron_slice) * 4
                v.back_prop_out_buffers = [
                    transceiver.malloc_sdram(placement.x, placement.y,
                                             back_prop_buffer_bytes,
                                             app_id=app_id)
                    for _ in range(2)]

    def load(self, routing_info, placements, transceiver, app_id):
        # Loop through vertices
        for v in self.verts:
            # Get placement and allocation
            placement = placements.get_placement_of_vertex(v)

            # Use routing info to get spike and flush TX keys
            spike_tx_key = v.get_spike_tx_key(routing_info)
            flush_tx_key = v.get_flush_tx_key(routing_info)

            logger.debug("\t\t\tVertex %s (%u, %u, %u): Spike key:%08x, Flush key:%08x",
                            v, placement.x, placement.y, placement.p,
                            spike_tx_key, flush_tx_key)

            # Get the input buffers from each synapse vertex
            in_buffers = [
                s.get_in_buffer(v.neuron_slice)
                for s in v.input_verts]

            # Get regiona arguments
            region_arguments = self._get_region_arguments(
                spike_tx_key, flush_tx_key, v.neuron_slice,
                in_buffers, v.back_prop_out_buffers)

            # Load regions
            v.region_memory = load_regions(self.regions, region_arguments,
                                           placement, transceiver, app_id)

    def read_recorded_spikes(self):
        # Loop through all neuron vertices and read spike times into dictionary
        spike_times = {}
        region = self.regions[Regions.spike_recording]
        for v in self.verts:
            region_mem = v.region_memory[Regions.spike_recording]
            spike_times.update(region.read_spike_times(v.neuron_slice,
                                                       region_mem))
        return spike_times

    def read_recorded_signal(self, channel):
        # Get index of channelread_profile
        region_index = Regions(Regions.analogue_recording_start + channel)
        region = self.regions[region_index]

        # Loop through all neuron vertices and read signal
        signal = {}
        for v in self.verts:
            region_mem = v.region_memory[region_index]
            signal.update(region.read_signal(v.neuron_slice, region_mem))

        return signal

    def read_profile(self):
        # Get the profile recording region and
        region = self.regions[Regions.profiler]

        # Return profile data for each vertex that makes up population
        return [(v.neuron_slice.python_slice,
                 region.read_profile(v.region_memory[Regions.profiler],
                                     self.profiler_tag_names))
                for v in self.verts]

    def read_statistics(self):
        # Get the statistics recording region
        region = self.regions[Regions.statistics]

        # Read stats from all vertices
        return region.read_stats(
            [v.region_memory[Regions.statistics] for v in self.verts],
            self.statistic_names)

    # --------------------------------------------------------------------------
    # Private methods
    # --------------------------------------------------------------------------
    def _estimate_sdram(self, vertex_slice):
        # Begin with size of spike recording region
        sdram = self.regions[Regions.spike_recording].sizeof(vertex_slice);

        # Add on size of neuron region
        sdram += self.regions[Regions.neuron].sizeof(vertex_slice)

        # If profiler region exists, add its size
        if Regions.profiler in self.regions:
            sdram += self.regions[Regions.profiler].sizeof()

        # Loop through possible analogue recording regions
        for t in range(Regions.analogue_recording_start,
                       Regions.analogue_recording_end):
            # If region exists, add its size to total
            if Regions(t) in self.regions:
                sdram += self.regions[Regions(t)].sizeof(vertex_slice)

        return sdram

    def _get_region_arguments(self, spike_tx_key, flush_tx_key, vertex_slice,
                              in_buffers, back_prop_out_buffers):
        region_arguments = defaultdict(Args)

        analogue_recording_regions = range(Regions.analogue_recording_start,
                                           Regions.analogue_recording_end)
        # Add vertex slice to regions that require it
        for r in itertools.chain((Regions.neuron,
                                  Regions.synapse,
                                  Regions.spike_recording),
                                 analogue_recording_regions):
            region_arguments[r] = Args(vertex_slice)

        # Add kwargs for regions that require them
        region_arguments[Regions.system].kwargs["application_words"] =\
            [spike_tx_key, flush_tx_key, len(vertex_slice)]
        region_arguments[Regions.input_buffer].kwargs["in_buffers"] =\
            in_buffers
        region_arguments[Regions.back_prop_output].kwargs["out_buffers"] =\
            back_prop_out_buffers
        return region_arguments


from twisted.internet import defer, threads, error
from twisted.trial import unittest

from lbrynet import conf
from lbrynet import lbry_file
from lbrynet import reflector
from lbrynet.core import BlobManager
from lbrynet.core import PeerManager
from lbrynet.core import Session
from lbrynet.core import StreamDescriptor
from lbrynet.lbry_file import EncryptedFileMetadataManager
from lbrynet.lbry_file.client import EncryptedFileOptions
from lbrynet.file_manager import EncryptedFileCreator
from lbrynet.file_manager import EncryptedFileManager

from lbrynet.tests import mocks
from lbrynet.tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir

class TestReflector(unittest.TestCase):
    def setUp(self):
        mocks.mock_conf_settings(self)
        self.session = None
        self.stream_info_manager = None
        self.lbry_file_manager = None
        self.server_blob_manager = None
        self.reflector_port = None
        self.port = None
        self.addCleanup(self.take_down_env)
        wallet = mocks.Wallet()
        peer_manager = PeerManager.PeerManager()
        peer_finder = mocks.PeerFinder(5553, peer_manager, 2)
        hash_announcer = mocks.Announcer()
        sd_identifier = StreamDescriptor.StreamDescriptorIdentifier()

        self.expected_blobs = [
            (
                'dc4708f76a5e7af0f1cae0ee96b824e2ed9250c9346c093b'
                '441f0a20d3607c17948b6fcfb4bc62020fe5286693d08586',
                2097152
            ),
            (
                'f4067522c1b49432a2a679512e3917144317caa1abba0c04'
                '1e0cd2cf9f635d4cf127ce1824fa04189b63916174951f70',
                2097152
            ),
            (
                '305486c434260484fcb2968ce0e963b72f81ba56c11b08b1'
                'af0789b55b44d78422600f9a38e3cf4f2e9569897e5646a9',
                1015056
            ),
        ]
        ## Setup reflector client classes ##
        self.db_dir, self.blob_dir = mk_db_and_blob_dir()
        self.session = Session.Session(
            conf.settings['data_rate'],
            db_dir=self.db_dir,
            node_id="abcd",
            peer_finder=peer_finder,
            hash_announcer=hash_announcer,
            blob_dir=self.blob_dir,
            peer_port=5553,
            use_upnp=False,
            wallet=wallet,
            blob_tracker_class=mocks.BlobAvailabilityTracker,
            external_ip="127.0.0.1"
        )

        self.stream_info_manager = EncryptedFileMetadataManager.DBEncryptedFileMetadataManager(
                                    self.db_dir)

        self.lbry_file_manager = EncryptedFileManager.EncryptedFileManager(
            self.session, self.stream_info_manager, sd_identifier)

        ## Setup reflector server classes ##
        self.server_db_dir, self.server_blob_dir = mk_db_and_blob_dir()
        self.server_session = Session.Session(
            conf.settings['data_rate'],
            db_dir=self.server_db_dir,
            node_id="abcd",
            peer_finder=peer_finder,
            hash_announcer=hash_announcer,
            blob_dir=self.server_blob_dir,
            peer_port=5553,
            use_upnp=False,
            wallet=wallet,
            blob_tracker_class=mocks.BlobAvailabilityTracker,
            external_ip="127.0.0.1"
        )

        self.server_blob_manager = BlobManager.DiskBlobManager(
                                    hash_announcer, self.server_blob_dir, self.server_db_dir)
        self.server_stream_info_manager = \
            EncryptedFileMetadataManager.DBEncryptedFileMetadataManager(self.server_db_dir)

        self.server_lbry_file_manager = EncryptedFileManager.EncryptedFileManager(
                                    self.server_session, self.server_stream_info_manager,
                                    sd_identifier)

        d = self.session.setup()
        d.addCallback(lambda _: self.stream_info_manager.setup())
        d.addCallback(lambda _: EncryptedFileOptions.add_lbry_file_to_sd_identifier(sd_identifier))
        d.addCallback(lambda _: self.lbry_file_manager.setup())
        d.addCallback(lambda _: self.server_session.setup())
        d.addCallback(lambda _: self.server_blob_manager.setup())
        d.addCallback(lambda _: self.server_stream_info_manager.setup())
        d.addCallback(lambda _: self.server_lbry_file_manager.setup())

        def verify_equal(sd_info):
            self.assertEqual(mocks.create_stream_sd_file, sd_info)

        def save_sd_blob_hash(sd_hash):
            self.sd_hash = sd_hash
            self.expected_blobs.append((sd_hash, 923))

        def verify_stream_descriptor_file(stream_hash):
            self.stream_hash = stream_hash
            d = lbry_file.get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True)
            d.addCallback(verify_equal)
            d.addCallback(
                lambda _: lbry_file.publish_sd_blob(
                    self.lbry_file_manager.stream_info_manager,
                    self.session.blob_manager, stream_hash
                )
            )
            d.addCallback(save_sd_blob_hash)
            return d

        def create_stream():
            test_file = mocks.GenFile(5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)]))
            d = EncryptedFileCreator.create_lbry_file(
                self.session,
                self.lbry_file_manager,
                "test_file",
                test_file,
                key="0123456701234567",
                iv_generator=iv_generator()
            )
            return d

        def start_server():
            server_factory = reflector.ServerFactory(
                peer_manager, self.server_blob_manager, self.server_stream_info_manager,
                self.server_lbry_file_manager)
            from twisted.internet import reactor
            port = 8943
            while self.reflector_port is None:
                try:
                    self.reflector_port = reactor.listenTCP(port, server_factory)
                    self.port = port
                except error.CannotListenError:
                    port += 1

        d.addCallback(lambda _: create_stream())
        d.addCallback(verify_stream_descriptor_file)
        d.addCallback(lambda _: start_server())
        return d

    def take_down_env(self):
        d = defer.succeed(True)
        ## Close client classes ##
        d.addCallback(lambda _: self.lbry_file_manager.stop())
        d.addCallback(lambda _: self.session.shut_down())
        d.addCallback(lambda _: self.stream_info_manager.stop())

        ## Close server classes ##
        d.addCallback(lambda _: self.server_blob_manager.stop())
        d.addCallback(lambda _: self.server_lbry_file_manager.stop())
        d.addCallback(lambda _: self.server_session.shut_down())
        d.addCallback(lambda _: self.server_stream_info_manager.stop())

        d.addCallback(lambda _: self.reflector_port.stopListening())

        def delete_test_env():
            try:
                rm_db_and_blob_dir(self.db_dir, self.blob_dir)
                rm_db_and_blob_dir(self.server_db_dir, self.server_blob_dir)
            except:
                raise unittest.SkipTest("TODO: fix this for windows")

        d.addCallback(lambda _: threads.deferToThread(delete_test_env))
        d.addErrback(lambda err: str(err))
        return d

    def test_stream_reflector(self):
        def verify_blob_on_reflector():
            check_blob_ds = []
            for blob_hash, blob_size in self.expected_blobs:
                check_blob_ds.append(verify_have_blob(blob_hash, blob_size))
            return defer.DeferredList(check_blob_ds)

        @defer.inlineCallbacks
        def verify_stream_on_reflector():
            # check stream_info_manager has all the right information
            streams = yield self.server_stream_info_manager.get_all_streams()
            self.assertEqual(1, len(streams))
            self.assertEqual(self.stream_hash, streams[0])

            blobs = yield self.server_stream_info_manager.get_blobs_for_stream(self.stream_hash)
            blob_hashes = [b[0] for b in blobs if b[0] is not None]
            expected_blob_hashes = [b[0] for b in self.expected_blobs[:-1] if b[0] is not None]
            self.assertEqual(expected_blob_hashes, blob_hashes)
            sd_hashes = yield self.server_stream_info_manager.get_sd_blob_hashes_for_stream(
                self.stream_hash)
            self.assertEqual(1, len(sd_hashes))
            expected_sd_hash = self.expected_blobs[-1][0]
            self.assertEqual(self.sd_hash, sd_hashes[0])

            # check lbry file manager has the file
            files = yield self.server_lbry_file_manager.lbry_files
            self.assertEqual(1, len(files))
            self.assertEqual(self.sd_hash, files[0].sd_hash)
            self.assertEqual('test_file', files[0].file_name)

            status = yield files[0].status()
            self.assertEqual('stopped', status.running_status)
            num_blobs = len(self.expected_blobs) -1 # subtract sd hash
            self.assertEqual(num_blobs, status.num_completed)
            self.assertEqual(num_blobs, status.num_known)

            # check should_announce blobs on blob_manager
            blob_hashes = yield self.server_blob_manager._get_all_should_announce_blob_hashes()
            self.assertEqual(2, len(blob_hashes))
            self.assertTrue(self.sd_hash in blob_hashes)
            self.assertTrue(expected_blob_hashes[0] in blob_hashes)

        def verify_have_blob(blob_hash, blob_size):
            d = self.server_blob_manager.get_blob(blob_hash)
            d.addCallback(lambda blob: verify_blob_completed(blob, blob_size))
            return d

        def send_to_server():
            fake_lbry_file = mocks.FakeLBRYFile(self.session.blob_manager,
                                                self.stream_info_manager,
                                                self.stream_hash)
            factory = reflector.ClientFactory(fake_lbry_file)

            from twisted.internet import reactor
            reactor.connectTCP('localhost', self.port, factory)
            return factory.finished_deferred

        def verify_blob_completed(blob, blob_size):
            self.assertTrue(blob.get_is_verified())
            self.assertEqual(blob_size, blob.length)
            return

        d = send_to_server()
        d.addCallback(lambda _: verify_blob_on_reflector())
        d.addCallback(lambda _: verify_stream_on_reflector())
        return d

    def test_blob_reflector(self):
        def verify_data_on_reflector():
            check_blob_ds = []
            for blob_hash, blob_size in self.expected_blobs:
                check_blob_ds.append(verify_have_blob(blob_hash, blob_size))
            return defer.DeferredList(check_blob_ds)

        def verify_have_blob(blob_hash, blob_size):
            d = self.server_blob_manager.get_blob(blob_hash)
            d.addCallback(lambda blob: verify_blob_completed(blob, blob_size))
            return d

        def send_to_server(blob_hashes_to_send):
            factory = reflector.BlobClientFactory(
                self.session.blob_manager,
                blob_hashes_to_send
            )

            from twisted.internet import reactor
            reactor.connectTCP('localhost', self.port, factory)
            return factory.finished_deferred

        def verify_blob_completed(blob, blob_size):
            self.assertTrue(blob.get_is_verified())
            self.assertEqual(blob_size, blob.length)

        d = send_to_server([x[0] for x in self.expected_blobs])
        d.addCallback(lambda _: verify_data_on_reflector())
        return d

    def test_blob_reflector_v1(self):
        @defer.inlineCallbacks
        def verify_stream_on_reflector():
            # this protocol should not have any impact on stream info manager
            streams = yield self.server_stream_info_manager.get_all_streams()
            self.assertEqual(0, len(streams))
            # there should be no should announce blobs here
            blob_hashes = yield self.server_blob_manager._get_all_should_announce_blob_hashes()
            self.assertEqual(0, len(blob_hashes))

        def verify_data_on_reflector():
            check_blob_ds = []
            for blob_hash, blob_size in self.expected_blobs:
                check_blob_ds.append(verify_have_blob(blob_hash, blob_size))
            return defer.DeferredList(check_blob_ds)

        def verify_have_blob(blob_hash, blob_size):
            d = self.server_blob_manager.get_blob(blob_hash)
            d.addCallback(lambda blob: verify_blob_completed(blob, blob_size))
            return d

        def send_to_server(blob_hashes_to_send):
            factory = reflector.BlobClientFactory(
                self.session.blob_manager,
                blob_hashes_to_send
            )
            factory.protocol_version = 0

            from twisted.internet import reactor
            reactor.connectTCP('localhost', self.port, factory)
            return factory.finished_deferred

        def verify_blob_completed(blob, blob_size):
            self.assertTrue(blob.get_is_verified())
            self.assertEqual(blob_size, blob.length)

        d = send_to_server([x[0] for x in self.expected_blobs])
        d.addCallback(lambda _: verify_data_on_reflector())
        return d

    # test case when we reflect blob, and than that same blob
    # is reflected as stream
    def test_blob_reflect_and_stream(self):

        def verify_blob_on_reflector():
            check_blob_ds = []
            for blob_hash, blob_size in self.expected_blobs:
                check_blob_ds.append(verify_have_blob(blob_hash, blob_size))
            return defer.DeferredList(check_blob_ds)

        @defer.inlineCallbacks
        def verify_stream_on_reflector():
            # check stream_info_manager has all the right information

            streams = yield self.server_stream_info_manager.get_all_streams()
            self.assertEqual(1, len(streams))
            self.assertEqual(self.stream_hash, streams[0])

            blobs = yield self.server_stream_info_manager.get_blobs_for_stream(self.stream_hash)
            blob_hashes = [b[0] for b in blobs if b[0] is not None]
            expected_blob_hashes = [b[0] for b in self.expected_blobs[:-1] if b[0] is not None]
            self.assertEqual(expected_blob_hashes, blob_hashes)
            sd_hashes = yield self.server_stream_info_manager.get_sd_blob_hashes_for_stream(
                self.stream_hash)
            self.assertEqual(1, len(sd_hashes))
            expected_sd_hash = self.expected_blobs[-1][0]
            self.assertEqual(self.sd_hash, sd_hashes[0])

            # check should_announce blobs on blob_manager
            blob_hashes = yield self.server_blob_manager._get_all_should_announce_blob_hashes()
            self.assertEqual(2, len(blob_hashes))
            self.assertTrue(self.sd_hash in blob_hashes)
            self.assertTrue(expected_blob_hashes[0] in blob_hashes)

        def verify_have_blob(blob_hash, blob_size):
            d = self.server_blob_manager.get_blob(blob_hash)
            d.addCallback(lambda blob: verify_blob_completed(blob, blob_size))
            return d

        def send_to_server_as_blobs(blob_hashes_to_send):
            factory = reflector.BlobClientFactory(
                self.session.blob_manager,
                blob_hashes_to_send
            )
            factory.protocol_version = 0

            from twisted.internet import reactor
            reactor.connectTCP('localhost', self.port, factory)
            return factory.finished_deferred

        def send_to_server_as_stream(result):
            fake_lbry_file = mocks.FakeLBRYFile(self.session.blob_manager,
                                                self.stream_info_manager,
                                                self.stream_hash)
            factory = reflector.ClientFactory(fake_lbry_file)

            from twisted.internet import reactor
            reactor.connectTCP('localhost', self.port, factory)
            return factory.finished_deferred

        def verify_blob_completed(blob, blob_size):
            self.assertTrue(blob.get_is_verified())
            self.assertEqual(blob_size, blob.length)

        # Modify this to change which blobs to send
        blobs_to_send = self.expected_blobs

        d = send_to_server_as_blobs([x[0] for x in self.expected_blobs])
        d.addCallback(send_to_server_as_stream)
        d.addCallback(lambda _: verify_blob_on_reflector())
        d.addCallback(lambda _: verify_stream_on_reflector())
        return d

def iv_generator():
    iv = 0
    while True:
        iv += 1
        yield "%016d" % iv


import json
import os
from typing import (
    Any,
)

def load_json(path: str) -> Any:
    with open(path) as file:
        return json.load(file)

def write_json(path, data: Any) -> None:
    with open(path, "w") as file:
        json.dump(data, file, indent=2, sort_keys=True)

def main() -> None:
    for project in sorted(os.listdir("projects")):
        print(project)
        project_path = os.path.join("projects", project)
        for version in sorted(os.listdir(project_path)):
            version_path = os.path.join(project_path, version)
            if os.path.isdir(version_path):
                py_common_path = os.path.join(version_path, "python3*.json")
                py_common = (
                    set(load_json(py_common_path).items())
                    if os.path.exists(py_common_path)
                    else set()
                )

                py = {
                    py_version: set(load_json(py_path).items()) | py_common
                    for py_version in ["36", "37", "38", "39"]
                    for py_path in [
                        os.path.join(
                            version_path,
                            f"python{py_version}.json",
                        )
                    ]
                    if os.path.exists(py_path)
                }

                if not py:
                    continue

                py_common = set.intersection(*py.values())
                py_common_ = dict(py_common)
                py_common_.pop(project, None)

                write_json(py_common_path, py_common_)

                for py_version in py:
                    py[py_version] = dict(py[py_version] - py_common)
                    py[py_version].pop(project, None)
                    write_json(
                        os.path.join(version_path, f"python{py_version}.json"),
                        py[py_version],
                    )

if __name__ == "__main__":
    main()


from .serializers import PostSerializer
from .models import Post
from rest_framework.views import APIView
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.response import Response
from rest_framework import status
import csv
import os
import boto3
from pandas import DataFrame, read_csv, ExcelWriter
import pandas as pd
import xlwt
# Create your views here.

class PostView(APIView):
    parser_classes = (MultiPartParser, FormParser)

    def get(self, request, *args, **kwargs):
        posts = Post.objects.all()
        serializer = PostSerializer(posts, many=True)
        return Response(serializer.data)

    def post(self, request, *args, **kwargs):
        posts_serializer = PostSerializer(data=request.data)
        QueryDict = request.data
        object_dict = QueryDict.dict()
        file_name = str(object_dict['file'])
        file_realname = file_name.replace(" ", "_")
        file_realnames = file_realname.replace("̀", "")
        if posts_serializer.is_valid():
            posts_serializer.save()

            print('working until here')
            df = pd.read_excel('./file/post_images/' + file_realnames, sheet_name='BAREME-MEASUREMENT CHART', header=1)
            df = df.drop(df.columns[[2]], axis=1)
            df = df.iloc[:, :-8]

            if file_name.startswith('01001-5934') == True:
                searchfor = ['Tour de poitrine', 'Tour de taille', 'Longueur de manche ML', '65cm']

            elif file_name.startswith('01804-8230') == True:
                searchfor = ['Tour de taille étirée', 'Tour de taille', 'Tour de bassin', '1/2 Tour de  cuisse à 2.5cm', 'Longueur d\'entrejambe']

            elif file_name.startswith('00400-0270') == True:
                searchfor = ['1/2 Tour de poitrine à 2,5 cms\n1/2 Chest', 'Tour de taille', 'Longueur de manche raglan dépliée', '65cm']

            elif file_name.startswith('01804-3921') == True:
                searchfor = ['( elastique à partir du 44)', '1/2 Tour de taille étirée\n1/2 Waistround', '1/2 Tour de bassin\n1/2 Hips round', '1/2 Tour de  cuisse à 2.5cm\n1/2 Thigh round', 'Longueur d\'entrejambe' + '\n' + 'Inleg length']

            elif file_name.startswith('01804-3922') == True:
                searchfor = ['( elastique à partir du 44)', '1/2 Tour de taille étirée\n1/2 Waistround', '1/2 Tour de bassin\n1/2 Hips round', '1/2 Tour de  cuisse à 2.5cm\n1/2 Thigh round', 'Longueur d\'entrejambe' + '\n' + 'Inleg length']

            df = df[df['TAILLES FRANCAISES '].str.contains('|'.join(searchfor)) ]
            writer = ExcelWriter('export' + file_realname)
            df.to_excel(writer)
            writer.save()

            s3 = boto3.client('s3')
            bucket_name = 'compartiment-thimothe'
            print('jusquala')
            s3.upload_file('export' + str(file_realname), bucket_name, 'export' + str(file_realname))
            print('Successfully uploaded')

            return Response(posts_serializer.data, status=status.HTTP_201_CREATED)
        else:
            print('error', posts_serializer.errors)
            return Response(posts_serializer.errors, status=status.HTTP_400_BAD_REQUEST)


from rizzanet.models import Content
from rizzanet.api.validators import required_fields, resource_exists
from rizzanet.api.responses import api_response, api_error
from rizzanet.api.auth import api_auth
from flask import Blueprint

content_controller = Blueprint('content', __name__, url_prefix='/content')

@content_controller.route('/<regex("\d+"):id>',methods=['GET','FETCH'])
@api_auth
@resource_exists(function=Content.get_by_id, name='id')
def get_by_id(id):
    from rizzanet.models import Content
    response = Content.get_by_id(id)
    return handle_get_content_response(response)

@content_controller.route('/path', defaults={'path': ''})
@content_controller.route('/path/<path:path>',methods=['GET','FETCH'])
@api_auth
@resource_exists(function=Content.get_by_path, name='path')
def get_by_path(path):
    from rizzanet.models import Content
    response = Content.get_by_path(path)
    return handle_get_content_response(response)

def handle_get_content_response(response):
    try:
        parts = get_parts()
    except Exception as error:
        return api_error('Error: invalid parts format.',400)
    response_data = response.as_dict()
    if parts != False:
        if any([ part in parts for part in ['all','content_data']]):
            content_data = response.get_content_data()
            response_data['content_data'] = content_data.as_dict()
        if any([ part in parts for part in ['all','content_type']]):
            from rizzanet.models import ContentType
            content_type = response.get_content_type()
            response_data['content_type'] = content_type.as_dict()
    return api_response(response_data)

def get_parts():
    '''gets parts for content objects'''
    from flask import request
    valid_parts = ['content', 'content_data', 'content_type', 'all']
    parts = request.values.get('parts')
    if parts == None:
        return False
    parts = parts.split(',')
    if len(parts) == 0 or not all([part in valid_parts for part in parts]):
        raise Exception('Invalid format for parts')
    else:
        return parts


#from django.conf.urls import patterns, include, url
#from django.conf.urls.defaults import *
from django.conf.urls  import *

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

from tastypie.api import Api
from restapi.api import EndpointResource, TeamResource, ProjectResource, EnvironmentResource, ReleaseResource, ReleaseTypeResource, StatusResource

v1_api = Api(api_name='v1')
v1_api.register(EndpointResource())
v1_api.register(TeamResource())
v1_api.register(ProjectResource())
v1_api.register(EnvironmentResource())
v1_api.register(ReleaseResource())
v1_api.register(StatusResource())
v1_api.register(ReleaseTypeResource())

urlpatterns = [
    # Examples:
    # url(r'^$', 'django_dashboard.views.home', name='home'),
    # url(r'^django_dashboard/', include('django_dashboard.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    url(r'^api/', include(v1_api.urls)),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', admin.site.urls),
]