#!/usr/bin/env python
'''
A basic mapper function/program that
takes whatever is passed on stdin and
outputs tuples of all the words formatted
as (word, 1)
'''
import sys

for line in sys.stdin:

    # create tuples of all words in line
    L = [ (word.strip().lower(), 1 ) for word in line.strip().split() ]

    for word in line.strip().split():
        # write the results to STDOUT (standard output);
        # what we output here will be the input for the
        # Reduce step, i.e. the input for freqRed1.py
        #
        # tab-delimited; the trivial word count is 1
        print( '%s\t1' % (word) )


from statsmodels.compat.platform import PLATFORM_LINUX32

import numpy as np
from numpy.testing import (
    assert_,
    assert_allclose,
    assert_array_equal,
    assert_equal,
)
import pandas as pd
import pytest

import statsmodels.api as sm

from .results.results_discrete import RandHIE
from .test_discrete import CheckModelMixin

class CheckGeneric(CheckModelMixin):
    def test_params(self):
        assert_allclose(self.res1.params, self.res2.params, atol=1e-5, rtol=1e-5)

    def test_llf(self):
        assert_allclose(self.res1.llf, self.res2.llf, atol=1e-5, rtol=1e-5)

    def test_conf_int(self):
        assert_allclose(self.res1.conf_int(), self.res2.conf_int, atol=1e-3, rtol=1e-5)

    def test_bse(self):
        assert_allclose(self.res1.bse, self.res2.bse, atol=1e-3, rtol=1e-3)

    def test_aic(self):
        assert_allclose(self.res1.aic, self.res2.aic, atol=1e-2, rtol=1e-2)

    def test_bic(self):
        assert_allclose(self.res1.aic, self.res2.aic, atol=1e-1, rtol=1e-1)

    def test_t(self):
        unit_matrix = np.identity(self.res1.params.size)
        t_test = self.res1.t_test(unit_matrix)
        assert_allclose(self.res1.tvalues, t_test.tvalue)

    def test_fit_regularized(self):
        model = self.res1.model

        alpha = np.ones(len(self.res1.params))
        alpha[-2:] = 0
        res_reg = model.fit_regularized(alpha=alpha*0.01, disp=False, maxiter=500)

        assert_allclose(res_reg.params[2:], self.res1.params[2:],
            atol=5e-2, rtol=5e-2)

    def test_init_keys(self):
        init_kwds = self.res1.model._get_init_kwds()
        assert_equal(set(init_kwds.keys()), set(self.init_keys))
        for key, value in self.init_kwds.items():
            assert_equal(init_kwds[key], value)

    def test_null(self):
        # call llnull, so null model is attached, side effect of cached attribute
        self.res1.llnull
        # check model instead of value
        exog_null = self.res1.res_null.model.exog
        exog_infl_null = self.res1.res_null.model.exog_infl
        assert_array_equal(exog_infl_null.shape,
                     (len(self.res1.model.exog), 1))
        assert_equal(np.ptp(exog_null), 0)
        assert_equal(np.ptp(exog_infl_null), 0)

    @pytest.mark.smoke
    def test_summary(self):
        summ = self.res1.summary()
        # GH 4581
        assert 'Covariance Type:' in str(summ)

class TestZeroInflatedModel_logit(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()
        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = sm.add_constant(data.exog[:,1:4], prepend=False)
        exog_infl = sm.add_constant(data.exog[:,0], prepend=False)
        cls.res1 = sm.ZeroInflatedPoisson(data.endog, exog,
            exog_infl=exog_infl, inflation='logit').fit(method='newton', maxiter=500,
                                                        disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset']
        cls.init_kwds = {'inflation': 'logit'}
        res2 = RandHIE.zero_inflated_poisson_logit
        cls.res2 = res2

class TestZeroInflatedModel_probit(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()
        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = sm.add_constant(data.exog[:,1:4], prepend=False)
        exog_infl = sm.add_constant(data.exog[:,0], prepend=False)
        cls.res1 = sm.ZeroInflatedPoisson(data.endog, exog,
            exog_infl=exog_infl, inflation='probit').fit(method='newton', maxiter=500,
                                                         disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset']
        cls.init_kwds = {'inflation': 'probit'}
        res2 = RandHIE.zero_inflated_poisson_probit
        cls.res2 = res2

    @pytest.mark.skipif(PLATFORM_LINUX32, reason="Fails on 32-bit Linux")
    def test_fit_regularized(self):
        super().test_fit_regularized()

class TestZeroInflatedModel_offset(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()
        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = sm.add_constant(data.exog[:,1:4], prepend=False)
        exog_infl = sm.add_constant(data.exog[:,0], prepend=False)
        cls.res1 = sm.ZeroInflatedPoisson(data.endog, exog,
            exog_infl=exog_infl, offset=data.exog[:,7]).fit(method='newton',
                                                            maxiter=500,
                                                            disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset']
        cls.init_kwds = {'inflation': 'logit'}
        res2 = RandHIE.zero_inflated_poisson_offset
        cls.res2 = res2

    def test_exposure(self):
        # This test mostly the equivalence of offset and exposure = exp(offset)
        # use data arrays from class model
        model1 = self.res1.model
        offset = model1.offset
        model3 = sm.ZeroInflatedPoisson(model1.endog, model1.exog,
            exog_infl=model1.exog_infl, exposure=np.exp(offset))
        res3 = model3.fit(start_params=self.res1.params,
                          method='newton', maxiter=500, disp=False)

        assert_allclose(res3.params, self.res1.params, atol=1e-6, rtol=1e-6)
        fitted1 = self.res1.predict()
        fitted3 = res3.predict()
        assert_allclose(fitted3, fitted1, atol=1e-6, rtol=1e-6)

        ex = model1.exog
        ex_infl = model1.exog_infl
        offset = model1.offset
        fitted1_0 = self.res1.predict(exog=ex, exog_infl=ex_infl,
                                      offset=offset.tolist())
        fitted3_0 = res3.predict(exog=ex, exog_infl=ex_infl,
                                 exposure=np.exp(offset))
        assert_allclose(fitted3_0, fitted1_0, atol=1e-6, rtol=1e-6)

        ex = model1.exog[:10:2]
        ex_infl = model1.exog_infl[:10:2]
        offset = offset[:10:2]
        # # TODO: this raises with shape mismatch,
        # # i.e. uses offset or exposure from model -> fix it or not?
        # GLM.predict to setting offset and exposure to zero
        # fitted1_1 = self.res1.predict(exog=ex, exog_infl=ex_infl)
        # fitted3_1 = res3.predict(exog=ex, exog_infl=ex_infl)
        # assert_allclose(fitted3_1, fitted1_1, atol=1e-6, rtol=1e-6)

        fitted1_2 = self.res1.predict(exog=ex, exog_infl=ex_infl,
                                      offset=offset)
        fitted3_2 = res3.predict(exog=ex, exog_infl=ex_infl,
                                 exposure=np.exp(offset))
        assert_allclose(fitted3_2, fitted1_2, atol=1e-6, rtol=1e-6)
        assert_allclose(fitted1_2, fitted1[:10:2], atol=1e-6, rtol=1e-6)
        assert_allclose(fitted3_2, fitted1[:10:2], atol=1e-6, rtol=1e-6)

        # without specifying offset and exposure
        fitted1_3 = self.res1.predict(exog=ex, exog_infl=ex_infl)
        fitted3_3 = res3.predict(exog=ex, exog_infl=ex_infl)
        assert_allclose(fitted3_3, fitted1_3, atol=1e-6, rtol=1e-6)

class TestZeroInflatedModelPandas(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load_pandas()
        cls.endog = data.endog
        cls.data = data
        exog = sm.add_constant(data.exog.iloc[:,1:4], prepend=False)
        exog_infl = sm.add_constant(data.exog.iloc[:,0], prepend=False)
        # we do not need to verify convergence here
        start_params = np.asarray([0.10337834587498942, -1.0459825102508549,
                                   -0.08219794475894268, 0.00856917434709146,
                                   -0.026795737379474334, 1.4823632430107334])
        model = sm.ZeroInflatedPoisson(data.endog, exog,
            exog_infl=exog_infl, inflation='logit')
        cls.res1 = model.fit(start_params=start_params, method='newton',
                             maxiter=500, disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset']
        cls.init_kwds = {'inflation': 'logit'}
        res2 = RandHIE.zero_inflated_poisson_logit
        cls.res2 = res2

    def test_names(self):
        param_names = ['inflate_lncoins', 'inflate_const', 'idp', 'lpi',
                       'fmde', 'const']
        assert_array_equal(self.res1.model.exog_names, param_names)
        assert_array_equal(self.res1.params.index.tolist(), param_names)
        assert_array_equal(self.res1.bse.index.tolist(), param_names)

        exog = sm.add_constant(self.data.exog.iloc[:,1:4], prepend=True)
        exog_infl = sm.add_constant(self.data.exog.iloc[:,0], prepend=True)
        param_names = ['inflate_const', 'inflate_lncoins', 'const', 'idp',
                       'lpi', 'fmde']
        model = sm.ZeroInflatedPoisson(self.data.endog, exog,
            exog_infl=exog_infl, inflation='logit')
        assert_array_equal(model.exog_names, param_names)

class TestZeroInflatedPoisson_predict:
    @classmethod
    def setup_class(cls):
        expected_params = [1, 0.5]
        np.random.seed(999)
        nobs = 2000
        exog = np.ones((nobs, 2))
        exog[:nobs//2, 1] = 2
        mu_true = exog.dot(expected_params)
        cls.endog = sm.distributions.zipoisson.rvs(mu_true, 0.05,
                                                   size=mu_true.shape)
        model = sm.ZeroInflatedPoisson(cls.endog, exog)
        cls.res = model.fit(method='bfgs', maxiter=5000, disp=False)

        cls.params_true = [mu_true,  0.05, nobs]

    def test_mean(self):
        def compute_conf_interval_95(mu, prob_infl, nobs):
            dispersion_factor = 1 + prob_infl * mu

            # scalar variance of the mixture of zip
            var = (dispersion_factor*(1-prob_infl)*mu).mean()
            var += (((1-prob_infl)*mu)**2).mean()
            var -= (((1-prob_infl)*mu).mean())**2
            std = np.sqrt(var)
            # Central limit theorem
            conf_intv_95 = 2 * std / np.sqrt(nobs)
            return conf_intv_95

        conf_interval_95 = compute_conf_interval_95(*self.params_true)
        assert_allclose(self.res.predict().mean(), self.endog.mean(),
                        atol=conf_interval_95, rtol=0)

    def test_var(self):
        def compute_mixture_var(dispersion_factor, prob_main, mu):
            # the variance of the mixture is the mixture of the variances plus
            # a non-negative term accounting for the (weighted)
            # dispersion of the means, see stats.stackexchange #16609 and
            #  Casella & Berger's Statistical Inference (Example 10.2.1)
            prob_infl = 1-prob_main
            var = (dispersion_factor*(1-prob_infl)*mu).mean()
            var += (((1-prob_infl)*mu)**2).mean()
            var -= (((1-prob_infl)*mu).mean())**2
            return var

        res = self.res
        var_fitted = compute_mixture_var(res._dispersion_factor,
                                         res.predict(which='prob-main'),
                                         res.predict(which='mean-main'))

        assert_allclose(var_fitted.mean(),
                        self.endog.var(), atol=5e-2, rtol=5e-2)

    def test_predict_prob(self):
        res = self.res

        pr = res.predict(which='prob')
        pr2 = sm.distributions.zipoisson.pmf(np.arange(pr.shape[1])[:, None],
                                             res.predict(), 0.05).T
        assert_allclose(pr, pr2, rtol=0.05, atol=0.05)

    def test_predict_options(self):
        # check default exog_infl, see #4757
        res = self.res
        n = 5
        pr1 = res.predict(which='prob')
        pr0 = res.predict(exog=res.model.exog[:n], which='prob')
        assert_allclose(pr0, pr1[:n], rtol=1e-10)

        fitted1 = res.predict()
        fitted0 = res.predict(exog=res.model.exog[:n])
        assert_allclose(fitted0, fitted1[:n], rtol=1e-10)

@pytest.mark.slow
class TestZeroInflatedGeneralizedPoisson(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()
        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = sm.add_constant(data.exog[:,1:4], prepend=False)
        exog_infl = sm.add_constant(data.exog[:,0], prepend=False)
        cls.res1 = sm.ZeroInflatedGeneralizedPoisson(data.endog, exog,
            exog_infl=exog_infl, p=1).fit(method='newton', maxiter=500, disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset', 'p']
        cls.init_kwds = {'inflation': 'logit', 'p': 1}
        res2 = RandHIE.zero_inflated_generalized_poisson
        cls.res2 = res2

    def test_bse(self):
        pass

    def test_conf_int(self):
        pass

    def test_bic(self):
        pass

    def test_t(self):
        unit_matrix = np.identity(self.res1.params.size)
        t_test = self.res1.t_test(unit_matrix)
        assert_allclose(self.res1.tvalues, t_test.tvalue)

    def test_minimize(self, reset_randomstate):
        # check additional optimizers using the `minimize` option
        model = self.res1.model
        # use the same start_params, but avoid recomputing
        start_params = self.res1.mle_settings['start_params']

        res_ncg = model.fit(start_params=start_params,
                            method='minimize', min_method="trust-ncg",
                            maxiter=500, disp=False)

        assert_allclose(res_ncg.params, self.res2.params,
                        atol=1e-3, rtol=0.04)
        assert_allclose(res_ncg.bse, self.res2.bse,
                        atol=1e-3, rtol=0.6)
        assert_(res_ncg.mle_retvals['converged'] is True)

        res_dog = model.fit(start_params=start_params,
                            method='minimize', min_method="dogleg",
                            maxiter=500, disp=False)

        assert_allclose(res_dog.params, self.res2.params,
                        atol=1e-3, rtol=3e-3)
        assert_allclose(res_dog.bse, self.res2.bse,
                        atol=1e-3, rtol=0.6)
        assert_(res_dog.mle_retvals['converged'] is True)

        # Ser random_state here to improve reproducibility
        random_state = np.random.RandomState(1)
        seed = {'seed': random_state}
        res_bh = model.fit(start_params=start_params,
                           method='basinhopping', niter=500, stepsize=0.1,
                           niter_success=None, disp=False, interval=1, **seed)

        assert_allclose(res_bh.params, self.res2.params,
                        atol=1e-4, rtol=1e-4)
        assert_allclose(res_bh.bse, self.res2.bse,
                        atol=1e-3, rtol=0.6)
        # skip, res_bh reports converged is false but params agree
        #assert_(res_bh.mle_retvals['converged'] is True)

class TestZeroInflatedGeneralizedPoisson_predict:
    @classmethod
    def setup_class(cls):
        expected_params = [1, 0.5, 0.5]
        np.random.seed(999)
        nobs = 2000
        exog = np.ones((nobs, 2))
        exog[:nobs//2, 1] = 2
        mu_true = exog.dot(expected_params[:-1])
        cls.endog = sm.distributions.zigenpoisson.rvs(mu_true, expected_params[-1],
                                                      2, 0.5, size=mu_true.shape)
        model = sm.ZeroInflatedGeneralizedPoisson(cls.endog, exog, p=2)
        cls.res = model.fit(method='bfgs', maxiter=5000, disp=False)

        cls.params_true = [mu_true, expected_params[-1], 2,  0.5, nobs]

    def test_mean(self):
        def compute_conf_interval_95(mu, alpha, p, prob_infl, nobs):
            p = p-1
            dispersion_factor = (1 + alpha * mu**p)**2 + prob_infl * mu

            # scalar variance of the mixture of zip
            var = (dispersion_factor*(1-prob_infl)*mu).mean()
            var += (((1-prob_infl)*mu)**2).mean()
            var -= (((1-prob_infl)*mu).mean())**2
            std = np.sqrt(var)
            # Central limit theorem
            conf_intv_95 = 2 * std / np.sqrt(nobs)
            return conf_intv_95

        conf_interval_95 = compute_conf_interval_95(*self.params_true)
        assert_allclose(self.res.predict().mean(), self.endog.mean(),
                        atol=conf_interval_95, rtol=0)

    def test_var(self):
        def compute_mixture_var(dispersion_factor, prob_main, mu):
            prob_infl = 1-prob_main
            var = (dispersion_factor*(1-prob_infl)*mu).mean()
            var += (((1-prob_infl)*mu)**2).mean()
            var -= (((1-prob_infl)*mu).mean())**2
            return var

        res = self.res
        var_fitted = compute_mixture_var(res._dispersion_factor,
                                         res.predict(which='prob-main'),
                                         res.predict(which='mean-main'))

        assert_allclose(var_fitted.mean(),
                        self.endog.var(), atol=0.05, rtol=0.1)

    def test_predict_prob(self):
        res = self.res

        pr = res.predict(which='prob')
        pr2 = sm.distributions.zinegbin.pmf(np.arange(pr.shape[1])[:, None],
                                            res.predict(), 0.5, 2, 0.5).T
        assert_allclose(pr, pr2, rtol=0.08, atol=0.05)

class TestZeroInflatedNegativeBinomialP(CheckGeneric):
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()
        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = sm.add_constant(data.exog[:,1], prepend=False)
        exog_infl = sm.add_constant(data.exog[:,0], prepend=False)
        # cheating for now, parameters are not well identified in this dataset
        # see https://github.com/statsmodels/statsmodels/pull/3928#issuecomment-331724022
        sp = np.array([1.88, -10.28, -0.20, 1.14, 1.34])
        cls.res1 = sm.ZeroInflatedNegativeBinomialP(data.endog, exog,
            exog_infl=exog_infl, p=2).fit(start_params=sp, method='nm',
                                          xtol=1e-6, maxiter=5000, disp=False)
        # for llnull test
        cls.res1._results._attach_nullmodel = True
        cls.init_keys = ['exog_infl', 'exposure', 'inflation', 'offset', 'p']
        cls.init_kwds = {'inflation': 'logit', 'p': 2}
        res2 = RandHIE.zero_inflated_negative_binomial
        cls.res2 = res2

    def test_params(self):
        assert_allclose(self.res1.params, self.res2.params,
                        atol=1e-3, rtol=1e-3)

    def test_conf_int(self):
        pass

    def test_bic(self):
        pass

    def test_fit_regularized(self):
        model = self.res1.model

        alpha = np.ones(len(self.res1.params))
        alpha[-2:] = 0
        res_reg = model.fit_regularized(alpha=alpha*0.01, disp=False, maxiter=500)

        assert_allclose(res_reg.params[2:], self.res1.params[2:],
            atol=1e-1, rtol=1e-1)

    # possibly slow, adds 25 seconds
    def test_minimize(self, reset_randomstate):
        # check additional optimizers using the `minimize` option
        model = self.res1.model
        # use the same start_params, but avoid recomputing
        start_params = self.res1.mle_settings['start_params']

        res_ncg = model.fit(start_params=start_params,
                            method='minimize', min_method="trust-ncg",
                            maxiter=500, disp=False)

        assert_allclose(res_ncg.params, self.res2.params,
                        atol=1e-3, rtol=0.03)
        assert_allclose(res_ncg.bse, self.res2.bse,
                        atol=1e-3, rtol=0.06)
        assert_(res_ncg.mle_retvals['converged'] is True)

        res_dog = model.fit(start_params=start_params,
                            method='minimize', min_method="dogleg",
                            maxiter=500, disp=False)

        assert_allclose(res_dog.params, self.res2.params,
                        atol=1e-3, rtol=3e-3)
        assert_allclose(res_dog.bse, self.res2.bse,
                        atol=1e-3, rtol=7e-3)
        assert_(res_dog.mle_retvals['converged'] is True)

        res_bh = model.fit(start_params=start_params,
                           method='basinhopping', maxiter=500,
                           niter_success=3, disp=False)

        assert_allclose(res_bh.params, self.res2.params,
                        atol=1e-4, rtol=3e-4)
        assert_allclose(res_bh.bse, self.res2.bse,
                        atol=1e-3, rtol=1e-3)
        # skip, res_bh reports converged is false but params agree
        #assert_(res_bh.mle_retvals['converged'] is True)

class TestZeroInflatedNegativeBinomialP_predict:
    @classmethod
    def setup_class(cls):

        expected_params = [1, 1, 0.5]
        np.random.seed(999)
        nobs = 5000
        exog = np.ones((nobs, 2))
        exog[:nobs//2, 1] = 0

        prob_infl = 0.15
        mu_true = np.exp(exog.dot(expected_params[:-1]))
        cls.endog = sm.distributions.zinegbin.rvs(mu_true,
                    expected_params[-1], 2, prob_infl, size=mu_true.shape)
        model = sm.ZeroInflatedNegativeBinomialP(cls.endog, exog, p=2)
        cls.res = model.fit(method='bfgs', maxiter=5000, disp=False)

        # attach others
        cls.prob_infl = prob_infl
        cls.params_true = [mu_true, expected_params[-1], 2,  prob_infl, nobs]

    def test_mean(self):
        def compute_conf_interval_95(mu, alpha, p, prob_infl, nobs):
            dispersion_factor = 1 + alpha * mu**(p-1) + prob_infl * mu

            # scalar variance of the mixture of zip
            var = (dispersion_factor*(1-prob_infl)*mu).mean()
            var += (((1-prob_infl)*mu)**2).mean()
            var -= (((1-prob_infl)*mu).mean())**2
            std = np.sqrt(var)
            # Central limit theorem
            conf_intv_95 = 2 * std / np.sqrt(nobs)
            return conf_intv_95

        conf_interval_95 = compute_conf_interval_95(*self.params_true)
        mean_true = ((1-self.prob_infl)*self.params_true[0]).mean()
        assert_allclose(self.res.predict().mean(),
                        mean_true, atol=conf_interval_95, rtol=0)

    def test_var(self):
        # todo check precision
        def compute_mixture_var(dispersion_factor, prob_main, mu):
            prob_infl = 1 - prob_main
            var = (dispersion_factor * (1 - prob_infl) * mu).mean()
            var += (((1 - prob_infl) * mu) ** 2).mean()
            var -= (((1 - prob_infl) * mu).mean()) ** 2
            return var

        res = self.res
        var_fitted = compute_mixture_var(res._dispersion_factor,
                                         res.predict(which='prob-main'),
                                         res.predict(which='mean-main'))

        assert_allclose(var_fitted.mean(),
                        self.endog.var(), rtol=0.2)

    def test_predict_prob(self):
        res = self.res
        endog = res.model.endog

        pr = res.predict(which='prob')
        pr2 = sm.distributions.zinegbin.pmf(np.arange(pr.shape[1])[:,None],
            res.predict(), 0.5, 2, self.prob_infl).T
        assert_allclose(pr, pr2, rtol=0.1, atol=0.1)
        prm = pr.mean(0)
        pr2m = pr2.mean(0)
        freq = np.bincount(endog.astype(int)) / len(endog)
        assert_allclose(((pr2m - prm)**2).mean(), 0, rtol=1e-10, atol=5e-4)
        assert_allclose(((prm - freq)**2).mean(), 0, rtol=1e-10, atol=1e-4)

    def test_predict_generic_zi(self):
        # These tests do not use numbers from other packages.
        # Tests are on closeness of estimated to true/DGP values
        # and theoretical relationship between quantities
        res = self.res
        endog = self.endog
        exog = self.res.model.exog
        prob_infl = self.prob_infl
        nobs = len(endog)

        freq = np.bincount(endog.astype(int)) / len(endog)
        probs = res.predict(which='prob')
        probsm = probs.mean(0)
        assert_allclose(freq, probsm, atol=0.02)

        probs_unique = res.predict(exog=[[1, 0], [1, 1]],
                                   exog_infl=np.asarray([[1], [1]]),
                                   which='prob')
        # no default for exog_infl yet
        #probs_unique = res.predict(exog=[[1, 0], [1, 1]], which='prob')

        probs_unique2 = probs[[1, nobs-1]]

        assert_allclose(probs_unique, probs_unique2, atol=1e-10)

        probs0_unique = res.predict(exog=[[1, 0], [1, 1]],
                                    exog_infl=np.asarray([[1], [1]]),
                                    which='prob-zero')
        assert_allclose(probs0_unique, probs_unique2[:, 0], rtol=1e-10)

        probs_main_unique = res.predict(exog=[[1, 0], [1, 1]],
                                        exog_infl=np.asarray([[1], [1]]),
                                        which='prob-main')
        probs_main = res.predict(which='prob-main')
        probs_main[[0,-1]]
        assert_allclose(probs_main_unique, probs_main[[0,-1]],  rtol=1e-10)
        assert_allclose(probs_main_unique, 1 - prob_infl, atol=0.01)

        pred = res.predict(exog=[[1, 0], [1, 1]],
                           exog_infl=np.asarray([[1], [1]]))
        pred1 = endog[exog[:, 1] == 0].mean(), endog[exog[:, 1] == 1].mean()
        assert_allclose(pred, pred1, rtol=0.05)

        pred_main_unique = res.predict(exog=[[1, 0], [1, 1]],
                                       exog_infl=np.asarray([[1], [1]]),
                                       which='mean-main')
        assert_allclose(pred_main_unique, np.exp(np.cumsum(res.params[1:3])),
                        rtol=1e-10)

        # TODO: why does the following fail, params are not close enough to DGP
        # but results are close statistics of simulated data
        # what is mu_true in DGP sm.distributions.zinegbin.rvs
        # assert_allclose(pred_main_unique, mu_true[[1, -1]] * (1 - prob_infl), rtol=0.01)

        # mean-nonzero
        mean_nz = (endog[(exog[:, 1] == 0) & (endog > 0)].mean(),
                   endog[(exog[:, 1] == 1) & (endog > 0)].mean())
        pred_nonzero_unique = res.predict(exog=[[1, 0], [1, 1]],
                                          exog_infl=np.asarray([[1], [1]]), which='mean-nonzero')
        assert_allclose(pred_nonzero_unique, mean_nz, rtol=0.05)

        pred_lin_unique = res.predict(exog=[[1, 0], [1, 1]],
                                      exog_infl=np.asarray([[1], [1]]),
                                      which='linear')
        assert_allclose(pred_lin_unique, np.cumsum(res.params[1:3]), rtol=1e-10)

class TestZeroInflatedNegativeBinomialP_predict2:
    @classmethod
    def setup_class(cls):
        data = sm.datasets.randhie.load()

        cls.endog = np.asarray(data.endog)
        data.exog = np.asarray(data.exog)
        exog = data.exog
        start_params = np.array([
            -2.83983767, -2.31595924, -3.9263248,  -4.01816431, -5.52251843,
            -2.4351714,  -4.61636366, -4.17959785, -0.12960256, -0.05653484,
            -0.21206673,  0.08782572, -0.02991995,  0.22901208,  0.0620983,
            0.06809681,  0.0841814,   0.185506,    1.36527888])
        mod = sm.ZeroInflatedNegativeBinomialP(
            cls.endog, exog, exog_infl=exog, p=2)
        res = mod.fit(start_params=start_params, method="bfgs",
                      maxiter=1000, disp=False)

        cls.res = res

    def test_mean(self):
        assert_allclose(self.res.predict().mean(), self.endog.mean(),
                        atol=0.02)

    def test_zero_nonzero_mean(self):
        mean1 = self.endog.mean()
        mean2 = ((1 - self.res.predict(which='prob-zero').mean()) *
                 self.res.predict(which='mean-nonzero').mean())
        assert_allclose(mean1, mean2, atol=0.2)

class TestPandasOffset:

    def test_pd_offset_exposure(self):
        endog = pd.DataFrame({'F': [0.0, 0.0, 0.0, 0.0, 1.0]})
        exog = pd.DataFrame({'I': [1.0, 1.0, 1.0, 1.0, 1.0],
                             'C': [0.0, 1.0, 0.0, 1.0, 0.0]})
        exposure = pd.Series([1., 1, 1, 2, 1])
        offset = pd.Series([1, 1, 1, 2, 1])
        sm.Poisson(endog=endog, exog=exog, offset=offset).fit()
        inflations = ['logit', 'probit']
        for inflation in inflations:
            sm.ZeroInflatedPoisson(endog=endog, exog=exog["I"],
                                   exposure=exposure,
                                   inflation=inflation).fit()


# -*- coding: utf-8 -*-
"""
Created on Wed Dec  1 09:10:59 2021

@author: anshu
"""

# Selenium
# pip install selenium
# pip install webdriver-manager

# =============================================================================
# 1. open chrom browser
# 2. launch google.com
# 3. perform a search
# 4. close the browser
#
# =============================================================================

import time
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager

# opening the chrome browser
driver = webdriver.Chrome(ChromeDriverManager().install())
# maximize the chrom browser
driver.maximize_window()

driver.get("https://www.google.com/")
time.sleep(3)
# open the second tab
driver.execute_script("window.open('about:blank','secondtab');")
# switch to second tab
driver.switch_to.window('secondtab')
driver.get("https://www.python.org/")
time.sleep(5)
driver.close()

#############################################################
#############################################################

from selenium.webdriver.common.keys import Keys

# opening the chrome browser
driver = webdriver.Chrome(ChromeDriverManager().install())
# maximize the chrom browser
driver.maximize_window()

driver.get("https://www.google.com/")
time.sleep(2)
# find the search box element by name - 'q'
# type Xceedance into it
driver.find_element_by_name("q").send_keys("Xceedance")
time.sleep(1)
# click on the search button
driver.find_element_by_name("btnK").send_keys(Keys.ENTER)
time.sleep(5)
driver.close()


from keras_preprocessing.text import Tokenizer
from keras_preprocessing.sequence import pad_sequences

def prepare_data(data_set, length=None):
    #tokenize the data set
    bodies_tokenizer, headlines_tokenizer = (Tokenizer(), Tokenizer())

    #find the max length of each dataset
    bodies_max_length = 0
    headlines_max_length = 0
    if not length:
        bodies_max_length = data_set['articleBody'].map(lambda x : len(x.split())).max()
        headlines_max_length = data_set['Headline'].map(lambda x : len(x.split())).max()
    else:
        bodies_max_length = length[0]
        headlines_max_length = length[1]

    #fit the tokenizer on the data set
    bodies_tokenizer.fit_on_texts(data_set['articleBody'])
    headlines_tokenizer.fit_on_texts(data_set['Headline'])

    #convert the texts to sequences
    bodies_sequences = bodies_tokenizer.texts_to_sequences(data_set['articleBody'])
    headlines_sequences = headlines_tokenizer.texts_to_sequences(data_set['Headline'])

    #pad the data to be the max length
    bodies_sequences = pad_sequences(bodies_sequences, maxlen=bodies_max_length, padding='post', truncating='post')
    headlines_sequences = pad_sequences(headlines_sequences, maxlen=headlines_max_length, padding='post', truncating='post')

    return bodies_sequences, headlines_sequences, bodies_tokenizer.word_index, headlines_tokenizer.word_index, data_set['Stance']


from cryptography.fernet import Fernet

file = open('key.key', 'rb')
key = file.read()
file.close()

print('input message to be encrypted:')

message = input()
encoded = message.encode()

f = Fernet(key)
encrypted = f.encrypt(encoded)
print('encrypted successfully')

file = open('message.msg', 'wb')
file.write(encrypted)
file.close()

var = input()
if var == '':
    quit()

#   / \__
#  (    @\___
#  /         O
# /   (_____/
#/_____/   U


#Find Minimum in Subarray
def indexOfMinimum(array, startIndex):
    minValue = array[startIndex]
    minIndex = startIndex
    for i in range(startIndex + 1,len(array)):
        if(array[i] < minValue):
            minValue = array[i]
            minIndex = i
    return minIndex

minIndex = indexOfMinimum([2,4,3], 0)
print(minIndex)


#Aditya Chunduru
#00335780
#Problem Set 6
#CIS 153 L8
#Program description: Dictionaries and tuples.

#Chapter 9 exercise 3 and 4
def email_addresses():
    #dictionary of all emails from the file
    Dict = {'gopal.ramasammycook@gmail.com': 1, 'louis@media.berkeley.edu': 3,
            'cwen@iupui.edu': 5, 'antranig@caret.cam.ac.uk': 1,
            'rjlowe@iupui.edu': 2, 'gsilver@umich.edu': 3,
            'david.horwitz@uct.ac.za': 4, 'wagnermr@iupui.edu': 1,
            'zqian@umich.edu': 4, 'stephen.marquard@uct.ac.za': 2,
            'ray@media.berkeley.edu': 1}
    largest = None
    for itervar in Dict:
        if largest is None or Dict[itervar] > Dict[largest]:
            largest = itervar

    print('Largest:', largest)

email_addresses()

#Chapter 9 exercise 5/ exercise 1 and 2 chapter 10 revision
hour_counts = {}

file_name = input("enter a file name:")

try:
    with open (file_name, 'r') as file:
        for line in file:
            if line.startswith('From'):
                parts = line.split()

                if len(parts) >= 6:

                    time = parts[1]

                    hour = time.split(':')[0]

                    hour_counts[hour] = hour_counts.get(hour, 0) + 1

    sorted_counts = sorted(hour_counts.items())

    for hour, count in sorted_counts:
        print(hour, count)

except FileNotFoundError:
    print(f"File '{file_name}' not found ")


from bs4 import BeautifulSoup
from pymongo import MongoClient
client = MongoClient()

import mechanize
import re
import urllib
import json
import pymongo

class ArchitectFinderScraper(object):
    def __init__(self):
        self.url = "http://timetable.gmit.ie"
        self.br = mechanize.Browser()
        self.br.addheaders = [('User-agent',
                               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.7')]

    def goto_programmes_page(self):
        self.br.open(self.url)
        s = BeautifulSoup(self.br.response().read())

        for link in s.findAll("a", text="Academic Year 2017/18"):
            if 'href' in link.attrs:
                self.url = link.attrs['href']

        self.goto_form_page()

    def goto_form_page(self):
        self.br.open(self.url)
        s = BeautifulSoup(self.br.response().read())

        saved_form = s.find('form', id='form1').prettify()

        view_state = s.find('input', id='__VIEWSTATE')
        view_state_generator = s.find('input', id='__VIEWSTATEGENERATOR')
        event_validation = s.find('input', id='__EVENTVALIDATION')

        self.br.select_form('form1')
        self.br.form.set_all_readonly(False)
        self.br.form['__EVENTTARGET'] = "LinkBtn_programmesofstudy"
        self.br.form['__VIEWSTATE'] = view_state['value']
        self.br.form['__VIEWSTATEGENERATOR'] = view_state_generator['value']
        self.br.form['__EVENTVALIDATION'] = event_validation['value']
        self.br.form['tLinkType'] = "information"
        self.br.form.fixup()

        self.br.submit()

        s = BeautifulSoup(self.br.response().read())
        self.br.select_form('form1')
        items = self.br.form.find_control('dlObject').get_items()

        for item in items:
            if len(item.name) < 1:
                continue

            print('Scraping timetable for course: \n([ %s ])' % item.name)
            self.scrape_course_timetables(item)

    def scrape_course_timetables(self, item):
        s = BeautifulSoup(self.br.response().read())
        saved_form = s.find('form', id='form1').prettify()

        view_state = s.find('input', id='__VIEWSTATE')
        view_state_generator = s.find('input', id='__VIEWSTATEGENERATOR')
        event_validation = s.find('input', id='__EVENTVALIDATION')
        list_view = s.find('option', text="List Timetable")

        self.br.select_form('form1')
        self.br.form.set_all_readonly(False)
        self.br.form['__VIEWSTATE'] = view_state['value']
        self.br.form['__VIEWSTATEGENERATOR'] = view_state_generator['value']
        self.br.form['__EVENTVALIDATION'] = event_validation['value']
        self.br.form['tLinkType'] = "programmesofstudy"
        self.br.form['dlObject'] = [ item.name ]
        self.br.form['dlType'] = ["TextSpreadsheet;swsurl;SWSCUST Object TextSpreadsheet", ]

        self.br.form.fixup()

        self.br.submit(name='bGetTimetable')

        resp = self.br.response().read()

        self.scrape_timetable(item)

        self.br.back()

    def scrape(self):
        self.goto_programmes_page()

    def scrape_timetable(self, item):
        db = client['timetables']

        soup = BeautifulSoup(self.br.response().read())

        mondays = []
        tuesdays = []
        wednesdays = []
        thursdays = []
        fridays = []

        days = soup.findAll('span', {'class': 'labelone'})

        for num, table in enumerate(soup.findAll('table', {'class', 'spreadsheet'})):
            day = days[num].get_text()
            for tr in table.findAll('tr')[1:]:
                cells = tr.findAll('td')
                index = 0
                for i in range(len(cells)):
                    if i == 0:
                        moduleName = cells[i].get_text()
                    if i == 1:
                        moduleType = cells[i].get_text()
                    if i == 2:
                        startTime = cells[i].get_text()
                    if i == 3:
                        endTime = cells[i].get_text()
                    if i == 4:
                        duration = cells[i].get_text()
                    if i == 5:
                        weeks = cells[i].get_text()
                    if i == 6:
                        room = cells[i].get_text()
                    if i == 7:
                        lecturer = cells[i].get_text()
                    index = index + 1

                module = {
                    "moduleName": moduleName,
                    "moduleType": moduleType,
                    "startTime": startTime,
                    "endTime": endTime,
                    "duration": duration,
                    "weeks": weeks,
                    "room": room,
                    "lecturer": lecturer
                }

                if day == 'Monday':
                    mondays.append(module)
                if day == 'Tuesday':
                    tuesdays.append(module)
                if day == 'Wednesday':
                    wednesdays.append(module)
                if day == 'Thursday':
                    thursdays.append(module)
                if day == 'Friday':
                    fridays.append(module)

        jsonObj = {
            "_id": item.name,
            "courseName": item.attrs['label'].split(' ', 1)[1],
            "Monday":     mondays,
            "Tuesday":    tuesdays,
            "Wednesday":  wednesdays,
            "Thursday":   thursdays,
            "Friday":     fridays
        }

        courses = db.courses

        result = courses.insert_one(jsonObj)

        print('One post: {0}'.format(result.inserted_id))

if __name__ == '__main__':
    scraper = ArchitectFinderScraper()
    scraper.scrape()


"""Module that implements the bytecode compiler that produces raw bytecode
from input files in various formats.
"""

import os

from pathlib import Path
from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union

from .ast import Node
from .errors import CompilerError, UnsupportedInputFormatError
from .formats import InputFormat, InputFormatLike, OutputFormat, OutputFormatLike
from .optimisation import create_optimiser_for_level
from .plan import Plan
from .stages import (
    ASTObjectToBytecodeCompilationStage,
    ASTObjectToJSONBytecodeCompilationStage,
    ASTObjectToLEDSourceCodeCompilationStage,
    ASTObjectToPickleCompilationStage,
    ASTObjectToRawBytesCompilationStage,
    ASTOptimisationStage,
    BytecodeToASTObjectCompilationStage,
    CompilationStageExecutionEnvironment,
    JSONBytecodeToASTObjectCompilationStage,
    LEDSourceCodeToASTObjectCompilationStage,
    PassthroughStage,
    RawBytesToASTObjectCompilationStage,
)

class BytecodeCompiler:
    """Bytecode compiler that produces raw bytecode from input files in
    various formats.
    """

    _input_format_to_ast_stage_factory: Dict[
        InputFormat, Type[RawBytesToASTObjectCompilationStage]
    ]
    _output_format_to_output_stage_factory: Dict[
        OutputFormat, Type[ASTObjectToRawBytesCompilationStage]
    ]
    _optimisation_level: int

    environment: CompilationStageExecutionEnvironment
    progress: bool
    verbose: bool

    def __init__(
        self,
        *,
        optimisation_level: int = 0,
        progress: bool = False,
        verbose: bool = False
    ):
        """Constructor.

        Parameters:
            optimisation_level: the optimisation level that the compiler
                will use. Defaults to not optimising the bytecode at all.
            progress: whether to print a progress bar showing the
                progress of the compilation
            verbose: whether to print additional messages about the compilation
                process above the progress bar
        """
        self._optimisation_level = 0

        self._input_format_to_ast_stage_factory = {
            InputFormat.LEDCTRL_BINARY: BytecodeToASTObjectCompilationStage,
            InputFormat.LEDCTRL_SOURCE: LEDSourceCodeToASTObjectCompilationStage,
            InputFormat.LEDCTRL_JSON: JSONBytecodeToASTObjectCompilationStage,
        }
        self._output_format_to_output_stage_factory = {
            OutputFormat.LEDCTRL_BINARY: ASTObjectToBytecodeCompilationStage,
            OutputFormat.LEDCTRL_SOURCE: ASTObjectToLEDSourceCodeCompilationStage,
            OutputFormat.LEDCTRL_JSON: ASTObjectToJSONBytecodeCompilationStage,
            OutputFormat.AST_PICKLE: ASTObjectToPickleCompilationStage,
        }

        self.optimisation_level = int(optimisation_level)
        self.progress = progress
        self.verbose = verbose

        self.environment = CompilationStageExecutionEnvironment()
        self.output = None

    def compile(
        self,
        input: Any,
        output_file: Optional[Union[str, Path]] = None,
        *,
        input_format: Optional[InputFormatLike] = None,
        output_format: Optional[OutputFormatLike] = None
    ) -> Tuple[Any]:
        """Runs the compiler.

        Parameters:
            input: the input to compile. When it is a string or a Path object, it is
                assumed to be the name of a file that contains the input. When it is
                a bytes object, it is assumed to contain the raw data to compile;
                in this case, the ``input_format`` parameter must be specified.
                When it is a dictionary, it is assumed to be the Python
                representation of a JSON object and the ``input_format`` will
                be assumed to be ``InputFormat.LEDCTRL_JSON``.
            output_file: the name of the output file that the compiler will
                produce. When the compiler is expected to produce multiple
                output objects, the output file is expected to contain a
                ``{}`` placeholder where the index of the output object will be
                substituted. The output file may also be ``None`` if the
                compiler should only return the output
            input_format: the preferred input format or ``None`` if it should
                be inferred from the extension of the input file
            output_format: the preferred output format or ``None`` if it should
                be inferred from the extension of the output file

        Returns:
            a tuple containing all the objects that the compiler returned.
            Typically the tuple will contain a single item only (if the
            compilation yields a single output).

        Raises:
            CompilerError: in case of a compilation error
        """
        if isinstance(input, Path):
            input = str(input)

        if isinstance(input, str):
            if input_format is None:
                input_format = InputFormat.detect_from_filename(input)

            description = os.path.basename(input)
            with open(input, "rb") as fp:
                input = fp.read()

        elif isinstance(input, bytes):
            description = "<<raw bytes>>"

        elif isinstance(input, dict):
            from json import dumps

            description = "<<JSON>>"
            input = dumps(input).encode("utf-8")
            input_format = InputFormat.LEDCTRL_JSON

        elif isinstance(input, Node):
            description = "<<AST>>"
            input_format = InputFormat.AST

        else:
            description = "<<unknown>>"

        if input_format is None:
            raise CompilerError("input format must be specified")

        if output_format is None:
            if output_file is not None:
                output_format = OutputFormat.detect_from_filename(str(output_file))
            else:
                output_format = OutputFormat.AST

        input_format = InputFormat(input_format)
        output_format = OutputFormat(output_format)

        plan = Plan()
        self._collect_stages(plan, input, input_format, output_format)
        self.output = plan.execute(
            self.environment,
            force=True,
            progress=self.progress,
            description=description,
            verbose=self.verbose,
        )
        if output_file:
            self._write_outputs_to_file(self.output, str(output_file))

        return self.output

    @property
    def optimisation_level(self) -> int:
        """The optimisation level that the compiler will use.

        Currently we have the following optimisation levels:

            - 0: don't optimise the AST at all

            - 1: perform only basic optimisations

            - 2: perform more aggressive optimisations to make the generated
            bytecode smaller (default)
        """
        return self._optimisation_level

    @optimisation_level.setter
    def optimisation_level(self, value: int):
        self._optimisation_level = max(0, int(value))

    def _collect_stages(
        self,
        plan: Plan,
        input_data: Union[bytes, Node],
        input_format: InputFormat,
        output_format: OutputFormat,
    ) -> None:
        """Collects the compilation stages that will turn the given input
        file into the given output file.

        Parameters:
            plan: compilation plan where the collected stages will be added to
            input_data: the input data to work on
            input_format: the format of the input data
            output_format: the preferred output format

        Raises:
            UnsupportedInputFormatError: when the format of the input file is
                not known to the compiler
        """
        if isinstance(input_data, Node) and input_format != InputFormat.AST:
            raise ValueError(
                "input_format must be InputFormat.AST when input_data is an AST"
            )

        # Add the stages required to produce an abstract syntax tree
        # representation of the LED program based on the extension of the
        # input file
        if isinstance(input_data, Node):
            ast_stage = PassthroughStage(input_data)
        else:
            create_ast_stage = self._input_format_to_ast_stage_factory.get(input_format)
            if create_ast_stage is None:
                raise UnsupportedInputFormatError(format=input_format.value)
            ast_stage = create_ast_stage(input_data)
        plan.add_step(ast_stage)

        # Create a list containing our only AST stage; this may be useful later
        # when one input file may produce multiple ASTs
        ast_stages = [ast_stage]

        # Create a function that adds an optimization stage for the AST stage
        # given as an input
        def create_optimisation_stage(ast_stage):
            optimiser = create_optimiser_for_level(self.optimisation_level)
            return ASTOptimisationStage(ast_stage, optimiser)

        # Determine which factory to use for the output stages
        create_output_stage = self._output_format_to_output_stage_factory.get(
            output_format
        )

        # Create the optimization stages and the output stages for each AST
        for _index, ast_stage in enumerate(ast_stages):
            optimisation_stage = create_optimisation_stage(ast_stage)
            plan.add_step(optimisation_stage)

            if create_output_stage:
                output_stage = create_output_stage(optimisation_stage)
                plan.add_step(output_stage)
            else:
                output_stage = optimisation_stage

            plan.mark_as_output(output_stage)

    def _write_outputs_to_file(self, outputs: Sequence[Any], output_file: str):
        if not outputs:
            return

        if len(outputs) > 1 and "{}" not in output_file:
            raise CompilerError(
                "output filename needs to include a {} placeholder if the "
                "compiler produces multiple outputs"
            )

        num_digits = len(str(len(outputs) - 1))
        id_format = "{0:0" + str(num_digits) + "}"

        for index, output in enumerate(outputs):
            id = id_format.format(index)
            with open(output_file.format(id), "wb") as fp:
                fp.write(output)

def compile(
    input: Any,
    output_file: Optional[str] = None,
    *,
    input_format: Optional[InputFormatLike] = None,
    output_format: Optional[OutputFormatLike] = None,
    **kwds
):
    """Runs the compiler.

    This function is a syntactic sugar for one-time throwaway compilations.
    For more sophisticated use-cases, use the BytecodeCompiler_ class.

    Keyword arguments not mentioned here are forwarded to the BytecodeCompiler_
    constructor.

    Parameters:
        input: the input to compile. When it is a string, it is assumed to
            be the name of a file that contains the input. When it is a
            bytes object, it is assumed to contain the raw data to compile;
            in this case, the ``input_format`` parameter must be specified.
            When it is a dictionary, it is assumed to be the Python
            representation of a JSON object and the ``input_format`` will
            be assumed to be ``InputFormat.LEDCTRL_JSON``.
        output_file: the name of the output file that the compiler will
            produce. When the compiler is expected to produce multiple
            output objects, the output file is expected to contain a
            ``{}`` placeholder where the index of the output object will be
            substituted. The output file may also be ``None`` if the
            compiler should only return the output
        input_format: the preferred input format or ``None`` if it should
            be inferred from the extension of the input file
        output_format: the preferred output format or ``None`` if it should
            be inferred from the extension of the output file

    Returns:
        None if the compiler returned nothing; the result of the compilation if
        the compiler returned a single object only, or a tuple containing the
        result of the compilation if the compiler returned multiple objects

    Raises:
        CompilerError: in case of a compilation error
    """
    compiler = BytecodeCompiler(**kwds)
    result = compiler.compile(
        input, output_file, input_format=input_format, output_format=output_format
    )

    if not result:
        return None

    if len(result) == 1:
        return result[0]

    return result


lib = dict()
lib["B4"] = "Before"
lib["TX"] = "Thanks"
lib["BBL"] = "Be Back Later"
lib["BCNU"] = "Be Seeing You"
lib["HAND"] = "Have A Nice Day"
st = input("번역할 문장을 입력하시오 : ")
st = st.split()
result = ""
for i in st:
    if i in lib: # 입력한 문장이 lib에 있을 때
        result += lib[i] + " " # 해당 딕셔너리 출력
    else: # 없으면
        result += i + " " # 그대로 출력
print(result)


from base64 import b64encode
from datetime import datetime

from aioresponses import aioresponses

from azure_monitor_client import AzureMonitorClient

async def test_send_log_message() -> None:
    """
    Ensure no exceptions are raised when sending an event.
    """
    client = AzureMonitorClient(
        workspace_id="test-workspace-id",
        shared_key=b64encode(b"test-shared-key").decode(),
    )

    with aioresponses() as mock:
        mock.post(client.endpoint, status=200)

        await client.send(
            "TestEvent",
            {
                "test": True,
                "date": datetime.now().isoformat(),
                "value": 65.02,
            },
        )


from MLNPY import ML
import numpy as np
import random
import copy

class GenAlgoLearning:
	def __init__(self, layers, populationSize = 100):
		#Makes a population
		self.layers = np.array(layers)
		self.populationSize = populationSize
		self.population = self.initialisePopulation(self.layers, self.populationSize)

	def sortBots(self):
		#Sorts the bots by fitness
		self.population = np.array(sorted(self.population, key = lambda element: element.fitness, reverse = True))

	def initialisePopulation(self,layers, populationSize):
		#Makes a random population of solutions
		return np.array([ML(layers) for i in range(0,populationSize)])

	def getFitnesses(self):
		fitnesses = np.vectorize(lambda chro: chro.fitness) #Way 2. Vectorise works like making a function with another.
		return fitnesses(self.population) #now when we run the new fintessES function it will take every input and return its fitness

	def printFitness(self):
		#print([chromosome.fitness for chromosome in self.population]) #1 Way
		print(self.getFitnesses())

	def sumFitness(self):
		return np.sum(self.getFitnesses())

	def updateFitness(self, newFitness):
		#self.getFitnesses() = newFitness # I was trying to find a way to update them all at onece but I don't think that is going to work
		for i in range(0,self.populationSize):
			self.population[i].fitness = newFitness[i]
		pass

	def makeNewGeneration(self):
		newGeneration = []
		self.sortBots()
		#print(self.population)
		#print(type(self.population))
		#print(round(self.populationSize/2.0))
		#For loop for half the number of rounds for kinds, because every round will yield 2 kids.
		for newChild in range(0, round(self.populationSize/2.0)):
			#Choose parents, Crossover, and mutate
			parent1 = self.chooseParent()
			parent2 = self.chooseParent()
			#print(parent1,parent2)
			#DEep copy, coppies and dissassociates objects
			child1, child2 = self.crossoverBreed(copy.deepcopy(parent1),copy.deepcopy(parent2))

			child1.fitness = 0
			child2.fitness = 0
			#Then add to the start of the next generation
			newGeneration = np.append(newGeneration, [self.mutate(copy.deepcopy(child1)), copy.deepcopy(self.mutate(child2))])
		#print(newGeneration)
		self.population = newGeneration

	def chooseParent(self):
		#ok so picture we hjave the the fitneesses as rectangles all pushed together with the length of the rectangles being relative to the fitness. If we choose a random number between 0 and the sum, this is a weighted selection.
		#https://en.wikipedia.org/wiki/Fitness_proportionate_selection
		sumFitness = self.sumFitness()
		randVal = random.random() * sumFitness
		#print(randVal)
		runningSum = 0
		fitness = self.getFitnesses()
		for i in range(0, self.populationSize):
			#We find which rectangle the random number seelcted by running and addin the rectangles till we pass the number. THen the rectange we just ran by is the one it picked.
			runningSum += fitness[i]
			if(runningSum > randVal):
				return self.population[i]
		return self.population[0]

	def crossoverBreed(self, parent1, parent2):
		# parent1 = np.copy(parent1)
		# parent2 = np.copy(parent2)
		#Inspiration Here
		#https://stackoverflow.com/questions/31708478/how-to-evolve-weights-of-a-neural-network-in-neuroevolution
		#Returns 2 children
		#High chance to swap all the weights comming out of a node, and low chance to swap an entire layer
		if(random.random() > 0.7):
			#print("Swap Layer")
			#print(F"p1: {parent1.weights } \n p2: {parent2.weights}")
			#print(f"layerSwap {int(random.random() * (len(self.layers) - 1))}")
			swapLayer = int(random.random() * (len(self.layers) - 1))
			#print(f"layerSwap {swapLayer}")
			temp = np.copy(parent1.weights[swapLayer])
			#print(f"temp {temp}")
			parent1.weights[swapLayer] = parent2.weights[swapLayer]
			parent2.weights[swapLayer] = temp
			#print(f"temp {temp}")
			#print(F"p1: {parent1.weights } \n p2: {parent2.weights}")
		else:
			#print("Swap Node")
			swapLayer = int(random.random() * (len(self.layers) - 1))
			swapNode  = int(random.random() * (self.layers[swapLayer]))
			#print(f"Swap {swapLayer,swapNode}")
			#print(parent1.weights[swapLayer,swapNode])
			#print(F"p1: {parent1.weights } \n p2: {parent2.weights}")
			temp = np.copy(parent1.weights[swapLayer,swapNode])
			parent1.weights[swapLayer,swapNode] = parent2.weights[swapLayer,swapNode]
			parent2.weights[swapLayer,swapNode] = temp
			#print(F"p1: {parent1.weights } \n p2: {parent2.weights}")
		return parent1, parent2

	def mutate(self, chromosome):
		randomChoice = random.random()
		#Again link
		#https://stackoverflow.com/questions/31708478/how-to-evolve-weights-of-a-neural-network-in-neuroevolution
		#4 different mutation options for a random weigth
		swapLayer  = int(random.random() * (len(self.layers) - 1))
		swapNodeL   = int(random.random() * (self.layers[swapLayer]))
		swapNodeR   = int(random.random() * (self.layers[swapLayer+1]))
		#print(swapLayer,swapNodeL,swapNodeR)
		#print(chromosome.weights)
		#print(chromosome.weights[swapLayer,swapNodeL,swapNodeR])
		if(randomChoice < 0.25):
			#print(f"1 {randomChoice}")
			#print("New Random Value")
			#New Value from -1,1
			chromosome.weights[swapLayer,swapNodeL,swapNodeR] = 2 * random.random() - 1
		elif(randomChoice < 0.5):
			#print(f"2 {randomChoice}")
			#print("Precentage Multiplication")
			#Scale between x0 to x2 aka 1/2 is x0.5 or 3/2 is x1.5
			chromosome.weights[swapLayer,swapNodeL,swapNodeR] *= random.random() * 2
		elif(randomChoice < 0.75):
			#print(f"3 {randomChoice}")
			#print("Random Addition or subtraction")
			#Add a random value to the weight. The second part is 1 and -1 half of the time so it is random either additnio or subtracton
			chromosome.weights[swapLayer,swapNodeL,swapNodeR] += random.random() * (2 * (random.random() > 0.5) - 1)
		else:
			#print(f"4 {randomChoice}")
			#print("Sign Switch")
			chromosome.weights[swapLayer,swapNodeL,swapNodeR] *= -1
		#print(chromosome.weights)
		#print(chromosome.weights[swapLayer,swapNodeL,swapNodeR])
		return chromosome

'''

myLayers = [8,10,4]
chromosomes = 10

AI = GenAlgoLearning(myLayers, chromosomes)
#print(AI.population)
newFit = np.arange(0,2 * chromosomes,2)
#print(newFit)
AI.updateFitness(newFit)
AI.sortBots()
AI.printFitness()
#print(AI.population[0].fitness)

print(AI.population)
AI.makeNewGeneration()
print(AI.population)

#'''

#print(2 * (0.4 > 0.5))
'''
bots = np.array([])

for i in range(0, 10):
	newLayers = [8,10,4]
	bot = ML(newLayers)
	bot.fitness = i + i % 4
	bots = np.append(bots, bot)
#'''


## Part B Task 5
import re
import os
import sys
import pandas as pd
import nltk
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfTransformer
import math
from numpy import dot
from numpy.linalg import norm
from nltk.stem.porter import *
from nltk.tokenize.treebank import TreebankWordDetokenizer
# nltk.download('stopwords')
from nltk.corpus import stopwords
stopWords = set(stopwords.words('english'))

# reuse the code from part b task 4 to get the list of documents containing all the keywords.

porterStemmer = PorterStemmer()

doc_IDs = pd.read_csv('partb1.csv', encoding = 'ISO-8859-1')

# reuse the function for preprocessing text, this time removing stop words as well
def preprocess(article_name):
    article_text = open(article_name, "r").read()
    preproc_text = re.sub(r'[^a-zA-Z]+', ' ', article_text)
    wordlist = nltk.word_tokenize(preproc_text)
    for i in range(len(wordlist)):
        wordlist[i] = porterStemmer.stem(wordlist[i])
    stopwords_removed = [word for word in wordlist if not word in stopWords]
    preproc_text = TreebankWordDetokenizer().detokenize(stopwords_removed)
    return preproc_text.lower()

# navigate to the correct directory.
fulldir = os.getcwd()
os.chdir(r'cricket')

# iterate through the articles searching for the keywords
article_list = os.listdir()
contain_keywords = []
total_wordlist = []
keyword_list=[]

for article_name in article_list:
    # make sure its only the text files being searched.
    if article_name[-3:] != 'txt':
        continue
    article_text = preprocess(article_name)
    contains_keywords = 1

# search for the stem of each keyword in the updated stem text files.
    for keyword in sys.argv[1:]:
        stem = porterStemmer.stem(keyword)
        keyword_list.append(stem)
        pattern = r'( |\n|^)'+stem.lower()+r'( |\n|\.)'
        if not re.search(pattern, article_text):
            contains_keywords = 0
            break
    if contains_keywords:
        doc_ID = list(doc_IDs.loc[doc_IDs['filename'] == article_name].documentID.values)[0]
        contain_keywords.append(doc_ID)
# create a list of all the words contained in articles that have all the keywords.
        for word in nltk.word_tokenize(article_text):
            total_wordlist.append(word)

# find the term count of the total words in each document.
# Each document from contain_keywords will be represented by a row in the term counts array.
total_words = set(total_wordlist)
term_counts = []

for docID in contain_keywords:
    article = list(doc_IDs.loc[doc_IDs['documentID'] == docID].filename.values)[0]
    article_text = preprocess(article)
    article_dic = defaultdict(int)
    for word in total_words:
        pattern = r'( |\n|^)'+word+r'( |\n|\.)'
        article_dic[word] = len(re.findall(pattern, article_text))
    term_counts.append(list(article_dic.values()))

# find the TF-IDF vector for each document.
transformer = TfidfTransformer()
tfidf = transformer.fit_transform(term_counts)
tfidf_array = tfidf.toarray()

# create the query vector of the stemmed keywords.
query = []
for i in range(len(total_words)):
    if list(total_words)[i] in keyword_list:
        query.append(1/math.sqrt(len(keyword_list)))
    else:
        query.append(0)

# calculate the cosine similarities between the query and the TF-IDF vector of each document.
cosine_sims = []

for i in range(len(contain_keywords)):
    tfidf_vec = tfidf_array[i]
    cosine_sim = dot(query, tfidf_vec)/(norm(query)*norm(tfidf_vec))
    cosine_sims.append(cosine_sim)

# present the results in a dataframe.
similarity_scores = pd.DataFrame(cosine_sims, index = contain_keywords, columns = ['score'])
similarity_scores.index.name = 'documentID'
sorted_similarity_scores = similarity_scores.sort_values( by = 'score', ascending = False)
print(sorted_similarity_scores, file = sys.stdout)

os.chdir(fulldir)


# all functions and constants to be used. organize hierarchicaly
from modules import *

############# CONSTANTS AND PARAMETERS #################################
SMALL = np.finfo(float).eps
rho_air = 1.2 # [kg/m3]
ni = 14.88*10**-6 # [m2/s] kinematic viscosity air
kappa = 0.4 # von Karman constant for atmospheric flows

############# PLOTTING CUSTOMIZATIONS ##################################
plt.rcParams["figure.figsize"] = np.array(plt.rcParams["figure.figsize"])*1
fsize = min(plt.rcParams['figure.figsize'])
#plt.rcParams["image.cmap"] = 'Spectral'
plt.rcParams["image.cmap"] = 'gray' # for images
plt.rcParams["font.size"] = 18
plt.rcParams["axes.grid"] = False
plt.rcParams["figure.dpi"] = 200 # 100
markerlist = [ k for k in Line2D.markers.keys() ]
lineslist = [ k for k in Line2D.lineStyles.keys() ]
plt.rcParams['lines.markersize'] = 10
plt.rcParams["legend.frameon"] = True
plt.rcParams["legend.framealpha"] = 1
plt.rcParams["legend.fancybox"] = False
plt.rcParams["xtick.minor.visible"] = True
plt.rcParams["ytick.minor.visible"] = True
ploting = True # disable if you don't want to see plots

############# TIER 1 ###################################################
def zero_to_small(A):
	"""
	take array A and values whose abs is smalleer than SMALL
	are converted to SMALL
	"""
	A[(A<SMALL) & (A>=0)] = SMALL
	A[(A>-SMALL) & (A<0)] = -SMALL
	return A

def poly_2d(x,y,K):
	return (K[0]+K[1]*x+K[2]*y + K[3]*x**2+K[4]*x*y+K[5]*y**2 +
		K[6]*x**3+K[7]*x**2*y+K[8]*x*y**2+K[9]*y**3)

def fname_dir(fname):
	"""
	returns the directory in which fname is located
	"""
	return os.path.abspath(os.path.join(fname,os.pardir))

def df_add(df,index,column,value):
	"""
	Add a new value at column and index. If either column or index
	do not exist they are created.
	Works for pandas dataframes.
	"""
	try:
		df[column]
	except:
		df[column]=np.nan
	try:
		df.loc[index]
	except:
		df.loc[index]=np.nan
	df.loc[index,column]=value
	return df

def gaussian(x,mu,sigma,A):
	return A/(sigma*np.sqrt(2*np.pi))*np.exp(
		-(x-mu)**2/(2*sigma**2))

def read_external_data(fname,sep='	',coma=False,bn=False,header=0):
	"""
	read a data file located in fname structured in lines and columns
	where each column is separated by sep. If the data uses comas
	instead of dots put replace=True to sort compatibility. In case \n
	shows up at the end of each line but bn=True. Header is
	the number of lines to be skipped at the start of the file.
	returns a matrix
	"""
	f = open(fname,"r")
	Lines = f.readlines()[header:]
	N = len(Lines)
	nVal = len(Lines[N-1].split(sep)) # using last line as reference for number of cloumns
	A = np.zeros((N,nVal))
	for line in range(N):
		if coma:
			Lines[line] = Lines[line].replace(',' , '.')
		if bn:
			Lines[line] = Lines[line].replace('\n' , '')
		A[line] = np.array(Lines[line].split(sep))
	f.close()
	return A.transpose()

def detect_circles(img,resolution=1,delta=10,canny=30,akku=15,rmin=10,rmax=-1):
	"""
	circles: x,y vecotor
	resolution: when multiplied by image size
	delta: min space between circles in px
	canny: threshold for canny edge. The higher the stricter.
	akku: threshold for hough circles accumulator. The higher the stricter.
	rmin: minimum radius of circles
	"""
	circles = cv2.HoughCircles(img,cv2.HOUGH_GRADIENT,resolution,delta,
              param1=canny,param2=akku,minRadius=rmin,maxRadius=rmax)[0,:,:-1]
	return circles

def nan_interp(A):
	"""
	Interpolate nans in a matrix
	"""
	ni,nj = np.shape(A)
	# extend edges of A by one
	A = np.concatenate((np.array([A[:,0]]).transpose(),A,np.array([A[:,-1]]).transpose()),axis=1)
	A = np.concatenate((np.array([A[0,:]]),A,np.array([A[-1,:]])),axis=0)

	#nit = 0
	#while np.sum(np.isnan(A)) != 0:
	#nit+=1
	nanp = np.isnan(A)
	for i in range(1,ni+1):
		for j in range(1,nj+1):
			if nanp[i,j]:
				#	# edges
				#	if (i==0) & (j!=0)& (j!=nj-1):
				#		b = np.array([A[i+1,j],A[i,j-1],A[i,j+1]])
				#	if (i==ni-1) & (j!=0)& (j!=nj-1):
				#		b = np.array([A[i-1,j],A[i,j-1],A[i,j+1]])
				#	if (j==0) & (i!=0)& (i!=ni-1):
				#		b = np.array([A[i-1,j],A[i+1,j],A[i,j+1]])
				#	if (j==nj-1) & (i!=0)& (i!=ni-1):
				#		b = np.array([A[i-1,j],A[i+1,j],A[i,j-1]])
				#	# corners
				#	if (i==0) & (j==0):
				#		b = np.array([A[i+1,j],A[i,j+1]])
				#	if (i==ni-1) & (j==0):
				#		b = np.array([A[i-1,j],A[i,j+1]])
				#	if (i==0) & (j==nj-1):
				#		b = np.array([A[i+1,j],A[i,j-1]])
				#	if (i==ni-1) & (j==nj-1):
				#		b = np.array([A[i-1,j],A[i,j-1]])
				#	# core
				#	else:
				b = np.array([A[i-1,j],A[i,j-1],A[i+1,j],A[i,j+1]])
				snan = np.sum(np.isnan(b))
				sb = np.nansum(b)
				A[i,j] = sb/(len(b)-snan)
				#print(i,j)
	# only the core matters
	A = A[1:ni+1,1:nj+1]
	return A

def new_directory(name):
	if not os.path.isdir(name):
		os.mkdir(name)
	return name

def read_image(fname):
	"""
	Return the loaded image as is (with alpha channel)
	As a numpy array
	"""
	img = cv2.imread(fname,cv2.IMREAD_GRAYSCALE)
	return img

def dir_ftype(directory,extension):
	"""
	find all files with given extension in directory
	order them alphabetically in a list and return them
	"""
	extension = extension.replace(".","") # make sure theres no "."
	fnames = directory+os.sep+"*"+"."+extension
	fnames = glob.glob(fnames)
	fnames = np.sort(fnames) # order files from 0 to last
	return fnames

def dir_fname(directory,nametype):
	"""
	find all files with given extension and string in directory
	order them alphabetically in a list and return them
	"""
	fnames = directory+os.sep+nametype
	fnames = glob.glob(fnames)
	fnames = np.sort(fnames) # order files from 0 to last
	return fnames

def crop_image(img,roi):
	"""
	roi: region of interest, numpy slice, 4 elements,
	[y_min:y_max,x_min:x_max]
	"""
	img = img[roi]
	return img

def canny_edge(img,threshold=15,hysteresis=5):
	img = cv2.Canny(img, threshold-hysteresis,threshold+hysteresis)
	return img

def blur_image(img,Ks,strength=1):
	"""
	Gaussian blur. Currently using the cv2 package.
	Ks is the kernel size and strength is the sigma
	"""
	Ks = int(Ks)
	if Ks%2 != 1:
		print("blur_image: Ks must be odd! Continuing with Ks = Ks-1")
		Ks = Ks-1
	img = cv2.GaussianBlur(img,(Ks,Ks),strength)#sigmaX=
	return img

def blurd_image(img,order=5,direction='horizontal',strength=0.25,speed='slow'):
	"""
	blur in a given direction with a
	slow: low pass butterworth filter
	fast: directional averaging convolution
	"""
	ny,nx = np.shape(img)
	if speed == 'slow':
		b, a = scig.butter(order, strength)
		if direction == 'horizontal':
			for i in range(ny):
				img[i,:] = scig.filtfilt(b, a, img[i,:])
		elif direction == 'vertical':
			for i in range(nx):
				img[:,i] = scig.filtfilt(b, a, img[:,i])
		return img
	elif speed=='fast':
		K = np.ones((int(strength),1))
		K = K/np.sum(K)
		if direction=='vertical':
			img=scig.convolve2d(img,K,mode='same')
		elif direction=='horizontal':
			img=scig.convolve2d(img,K.transpose(),mode='same')
		return img

def zernike_Vnm(rho,theta,n,m):
	"""
	from [2] a function necessary for the calculation of any Anm
	"""
	Rnm = 0
	fact = lambda x: np.math.factorial(x)
	am = abs(m)
	for s in range(0,(n-am)/2):
		Rnm+= (-1)**s*fact(n-s)*rho**(n-2*s)/(
			fact(s)*fact((n+am)/2-s)*fact((n-am)/2-s))
	Vnm = Rnm*np.exp(1j*m*theta)

def ddx(a):
	"""
	The first derivative of a vector assuming a constant spacing of
	1 unit between poitns. Avoid edge effects
	"""
	# avoid corner effects
	thick = 2
	a = np.concatenate((a[(thick-1)::-1],a,a[:-(thick+1):-1]))
	mode="same"
	mode = "valid"

	K = np.array([-0.5,0,0.5])
	da = scig.convolve(a,K,mode=mode)
	return da

def ghosal_edge(img,Ks,thr=1,thrmax=0.995,lmin = 0.5,phimin=1.4,thresholding=True, debug=False):
	"""
	implementation of the subpixel edge detection method of [2]. The
	pixels are projected into a new orthogonal domain where a parameter
	k defines the intesity of the edge. By filtering out edges of small
	k the relevant ones remain. The extracted parameters are enough to
	define a straight edge.
	img: the image to be treated.
	Ks: kernel size
	thr: threshold paramter limiting the minimum of k
	thrmax: threshold limiting the maximum of k. This is usefull to
		remove reflections which have excessively sharp edges
	lmin: is the minimum distance between the pixel center and the
		detected edge. This avoids that big kernels do errors, when
		close to multiple edges. It also avoids values of l which are
		nonesense, since they only make sense for real edges.
	phimin: allows the user to define a minimum angle in redians
		measured between the y-axis and the edge.
	"""
	totaltime = time.time()
	kerneltime = time.time()
	# Ks must be odd
	if Ks%2 != 1:
		print("Ks must be odd! Continuing with Ks = Ks-1")
		Ks = Ks-1
	# define the rectangular kernels
	#Vc00 = np.zeros((Ks,Ks),dtype=complex)
	Vc11 = np.zeros((Ks,Ks),dtype=complex)
	Vc20 = np.zeros((Ks,Ks),dtype=complex)
	ofs = 1 *(1-1/Ks) # offset for centering kernel around 0,0
	for i in range(Ks):
		for j in range(Ks):
			Kx = 2*j/Ks-ofs # limits of integration between -1 and 1
			Ky = 2*i/Ks-ofs
			if Kx**2+Ky**2 <= 1: # only a circle
				#Vc00[i,j] = 1 # the conjugate of V00
				Vc11[i,j] = Kx-Ky*1j # ...
				Vc20[i,j] = 2*Kx**2+2*Ky**2-1
	kerneltime = time.time() - kerneltime

	# Kernel Plots
	#	VCplot = Vc00
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real w K Vc00")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag w K Vc00")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = Vc11
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real w K Vc11")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag w K Vc11")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = Vc20
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real w K Vc20")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag w K Vc20")
	#	plt.colorbar()
	#	plt.show()

	# do the convolution with the images to get the zernike moments
	Anorm = lambda n : (n+1)/np.pi	# a normalization value
	convolvetime = time.time()
	#A00 = scig.convolve2d(img,Vc00,mode='same')
	#	A11 = Anorm(1)*scig.convolve2d(img,Vc11,mode='same')
	#	A20 = Anorm(2)*scig.convolve2d(img,Vc20,mode='same')
	A11 = Anorm(1)*scig.oaconvolve(img,Vc11,mode='same')
	A20 = Anorm(2)*scig.oaconvolve(img,Vc20,mode='same')
	convolvetime = time.time() - convolvetime
	# Plot Zernike moments
	#	VCplot = A00
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real A00")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag A00")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = A11
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real A11")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag A11")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = A20
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real A20")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag A20")
	#	plt.colorbar()
	#	plt.show()

	paramstime = time.time()
	# calculate the edge paramters
	#	tanphi = np.imag(A11)/np.real(A11)
	#	phi = np.arctan(tanphi)
	#	cosphi = np.cos(phi)
	#	sinphi = cosphi*tanphi
	#	Al11 = np.real(A11)*cosphi+np.imag(A11)*sinphi

	phi = np.arctan(np.imag(A11)/np.real(A11))
	Al11 = np.real(A11)*np.cos(phi)+np.imag(A11)*np.sin(phi)

	#	Al11 = A11*np.exp(-phi*1j)
	l = A20/Al11 # A20 has no imaginary component so A20 = A'20

	k = 3*Al11/(2*(1-l**2)**(3/2))
	paramstime = time.time() - paramstime

	# Plot edge paramters
	#	VCplot = phi
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real phi")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag phi")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = Al11
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real A\'11")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag A\'11")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = l
	#	plt.pcolormesh(np.real(VCplot))#,vmin=-5,vmax=5
	#	plt.title("real l")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot)) # ,vmin=-5,vmax=5
	#	plt.title("imag l")
	#	plt.colorbar()
	#	plt.show()
	#	VCplot = k
	#	plt.pcolormesh(np.real(VCplot))
	#	plt.title("real k")
	#	plt.colorbar()
	#	plt.show()
	#	plt.pcolormesh(np.imag(VCplot))
	#	plt.title("imag k")
	#	plt.colorbar()
	#	plt.show()

	treattime = time.time()
	if thresholding==True:
		# do the thresholding
		if (thrmax<0)&(thr>0):
			knorm = np.sort(k.flatten())[[int(thr*np.size(k)),int(thrmax*np.size(k))]]
			idx = (abs(l)<lmin)&(abs(phi)>phimin)&(abs(k)>knorm[0])
		elif thrmax>0:
			knorm = np.sort(k.flatten())[[int(thr*np.size(k)),int(thrmax*np.size(k))]]
			idx = (abs(l)<lmin)&(abs(phi)>phimin)&(abs(k)>knorm[0])&(abs(k)<knorm[1])
		elif thr<0:
			idx = (abs(l)<lmin)&(abs(phi)>phimin)
			knorm = np.sort(k[idx].flatten())[int(thr)]
			idx = idx&(abs(k)>abs(knorm))
		ne = np.sum(idx)
	elif thresholding==False:
		raise ValueError("this option is not still uncer development")
		# no thresholding
		idx = np.ones(np.shape(l),dtype=bool)
		ne =np.sum(idx)
	else:
		raise ValueError("thresholding should be boolean")

	# put all detected points in a vector of (x,y) values
	edg = np.zeros((ne,2))
	org = np.zeros((ne,2))
	nx,ny = np.shape(img)
	e = 0
	for i in range(nx):
		for j in range(ny):
			if idx[i,j]:
				edg[e]=np.array([i,j]) + l[i,j]*Ks/2*np.array(
					[np.sin(phi[i,j]),-np.cos(phi[i,j])])
				org[e]=np.array([i,j])
				e +=1
	treattime = time.time() - treattime
	totaltime = time.time() - totaltime
	print("total %0.5f	convolution %0.5f	thresholding %0.5f	paramters %0.5f	kernel %0.5f"%(totaltime,convolvetime,treattime,paramstime,kerneltime))

	if debug==True:
		return edg, org, k, l, phi
	else:
		return edg, org

def ghosal_edge_v2(img,Ks,kmin=0,kmax=1000,lmax=0.5,phimin=1,thresholding=True,debug=False,mirror=False):
	"""
	implementation of the subpixel edge detection method of [2]. The
	pixels are projected into a new orthogonal domain where a parameter
	k defines the intesity of the edge. By filtering out edges of small
	k the relevant ones remain. The extracted parameters are enough to
	define a straight edge.
	img: the image to be treated.
	Ks: kernel size
	thr: threshold paramter limiting the minimum of k
	kmax/min: threshold limiting k. This is usefull to define the edge
		intensity range that we are looking for. Neither too strong nor
		too weak
	lmax: is the maximum distance between the pixel center and the
		detected edge. This avoids that big kernels do errors, when
		close to multiple edges. It also avoids values of l which are
		nonesense, since they only make sense for real edges.
	phimin: allows the user to define a minimum angle in radians
		measured between the y-axis and the edge.
	thresholding: in case no thresholding is desired, might make sense
		for debugging or for post-processing raw data of edges
	debug: will output k, l and phi matrices
	mirror: mirror the limits of the image during convolution
		so that no aliasing happens during the convolution.
		Convolution time is doubled for some cases if this is activated.
	"""
	# gather image properties before its altered
	ni,nj = np.shape(img)
	# Ks must be odd
	if Ks%2 != 1:
		print("Ks must be odd! Continuing with Ks = Ks-1")
		Ks = Ks-1
	# define the rectangular kernels
	#Vc00 = np.zeros((Ks,Ks),dtype=complex) # not needed
	Vc11 = np.zeros((Ks,Ks),dtype=complex)
	Vc20 = np.zeros((Ks,Ks),dtype=complex)
	ofs = 1 *(1-1/Ks) # offset for centering kernel around 0,0
	for i in range(Ks):
		for j in range(Ks):
			Kx = 2*j/Ks-ofs # limits of integration between -1 and 1
			Ky = 2*i/Ks-ofs
			if Kx**2+Ky**2 <= 1: # only a circle
				#Vc00[i,j] = 1 # the conjugate of V00 # not needed
				Vc11[i,j] = Kx-Ky*1j # ...
				Vc20[i,j] = 2*Kx**2+2*Ky**2-1
	# mirror the edges to avoid edge effects from convolution
	if mirror:
		thick = int((Ks-1)/2)
		img = np.concatenate((img[:,(thick-1)::-1],img,img[:,:-(thick+1):-1]),1)
		img = np.concatenate((img[(thick-1)::-1,:],img,img[:-(thick+1):-1,:]),0)
		mode = "valid"
	else:
		mode = "same"

	# do the convolution with the images to get the zernike moments
	Anorm = lambda n : (n+1)/np.pi	# a normalization value
	#A00 = scig.convolve2d(img,Vc00,mode='same') # not needed
	A11 = Anorm(1)*scig.oaconvolve(img,Vc11,mode=mode)
	A20 = Anorm(2)*scig.oaconvolve(img,Vc20,mode=mode)

	phi = np.arctan(np.imag(A11)/zero_to_small(np.real(A11)))
	Al11 = np.real(A11)*np.cos(phi)+np.imag(A11)*np.sin(phi)
	l = np.real(A20)/Al11 # A20 has no imaginary component so A20 = A'20
	l = np.minimum(l,1-SMALL) # chop off those that go beyond the kernel boundaries
	l = np.maximum(l,-1+SMALL)
	k = abs(3*Al11/(2*(1-l**2)**(3/2)))

	if thresholding==True:
		# conditions
		phi_c = abs(phi)>phimin
		l_c = abs(l)<lmax
		k_c = (k<kmax) & (k>kmin)
		valid = phi_c & (k_c & l_c)
	elif thresholding==False:
		valid = np.ones_like(k)
	# define a grid of pixel positions
	i,j = np.meshgrid(np.arange(nj),np.arange(ni))

	# get a list of the valid relevant parameters
	i = i[valid]
	j = j[valid]
	#	k = k[valid] # not necessary
	l = l[valid]
	phi = phi[valid]

	# convert to the subpixel position
	i_s = i+l*Ks/2*np.cos(phi)
	j_s = j+l*Ks/2*np.sin(phi)

	# put all detected points in a vector of (x,y) values
	edg = np.squeeze((j_s,i_s)).transpose()
	org = np.squeeze((j,i)).transpose()
	if debug==True:
		return edg, org, k, l, phi
	else:
		return edg, org

def line_fit(x,y):
	"""
	Returns a straight line (two points) that is the best fit to a cloud
	of x,y values
	"""
	# clean
	x = np.squeeze(x)
	y = np.squeeze(y)
	# concatenate
	xy = np.concatenate((x[:,np.newaxis],y[:,np.newaxis]),1)
	# sort by x values
	xy = xy[xy[:,0].argsort()]
	#print(xy)
	f = lambda x,m,b : m*x+b
	pars,_ = opt.curve_fit(f,xy[:,0],xy[:,1])
	m = pars[0]
	b = pars[1]
	pts = np.zeros((2,2))
	pts[0,0] = xy[0,0]
	pts[1,0] = xy[-1,0]
	pts[:,1] = pts[:,0]*m+b
	sig = np.std((xy[:,1]-f(xy[:,0],m,b)))
	return pts, sig

def remove_dbledge(img):
	"""
	The image has to be binary. Get only the points closest to the
	bottom of the image.
	"""
	(ny,nx) = np.shape(img)
	for i in range(nx):
		idx = np.array(np.nonzero(img[:,i]))
		if np.size(idx) == 0:
			continue
		idx = idx[0][-1]
		img[idx-1::-1,i] = 0
	return img

def solve_L(centers_i,centers_r):
	"""
	From [1] as an intermediary step to obtain the reorientation of the
	calibration plate.
	L is a matrix which when multiplied with the rotation and
	translation parameters r'1, r'2, r'4, r'5 and T'x returns the vector
	of the x_r values at each calibration point. Solving it results in
	the	vector of rotation parameters. T'x and T'y require special
	treatment later on.
	"""
	# The first term is a column vector of size N. Each of its rows
	# multiplies with the respective row on the x_r with 2 columns and
	# N rows.
	Ly = centers_i[:,1][np.newaxis,:].transpose() * centers_r
	Lx = - centers_i[:,0][np.newaxis,:].transpose() * centers_r
	L = np.concatenate((Ly,centers_i[:,1][np.newaxis,:].transpose(),Lx),
		axis=1)
	b = centers_i[:,0]
	print("solving for the rotation and translation coefficients...")
	rl,resids,rank,svals = np.linalg.lstsq(L,b)
	print("residue:%0.4f	rank:%0.4f"%(np.sum(resids),rank))
	return rl

############# TIER 2 ###################################################
def select_points(fname,n=-1,prompt=""):
	"""
	Open an image and allow for the selection of points to be diplayed
	as an array later
	"""
	#def onclose(event):
	#	raise ValueError("Figure Closed by user. No input. Exiting.")
	if type(fname) is type(plt.figure()):
		fig = fname
		ax1 = fig.axes[0]
	else:
		fig,ax1 = plt.subplots(1)

	if type(fname) is str:
		img = read_image(fname)
	elif type(fname) is np.ndarray:
		img = fname
	#if n == 1:
	#	mouse = {"mouse_add":2, "mouse_pop":1,"mouse_stop":3}
	#	plt.xlabel("middle click: add point",fontsize="xx-small")
	#else:
	mouse = {"mouse_add":2, "mouse_pop":1,"mouse_stop":3}
	try:
		ax1.imshow(img)
	except:
		1 == 2
	fig.suptitle(prompt)
	ax1.set_xlabel("L click: remove previous - middle click: add point - R click: exit",fontsize="xx-small")
	#fig = plt.gcf() # get current figure
	#fig.canvas.mpl_connect('close_event', onclose) # raise value error if figure is closed
	pts = fig.ginput(n=n,timeout=0,**mouse)
	plt.close()
	return pts

def gradient_edge(img):
	def error(y,mu,sigma,A):
		return y-gaussian(np.arange(0,len(y)),mu,sigma,A)
	#	optimum,residu = opt.leastsq(\
	#		error, guess, args=(x,y))
	nx,ny = np.shape(img)

	for i in range(nx):
		I = img[:,i]
		dI = ddx(I)
		maxx = np.where(dI == max(dI))
		minx = np.where(dI == min(dI))
		plt.plot(I)
		plt.plot(dI)
		plt.show()
		#if i == 0:

def polyfit_2d(Xu,X):
	"""
	Xu is a vector of the positions of the points in pixels
	X is the vector of the positions of the points in meters
	The result is a set of coefficients k which allow the mapping of a
	point in the reference of Xu to the reference of X using poly_2d
	"""
	xu = Xu[:,0]
	yu = Xu[:,1]
	X = np.squeeze(X) # an mx1 vector
	M = np.squeeze((np.ones(xu.size),xu,yu,xu**2,xu*yu,yu**2,
		xu**3,xu**2*yu,xu*yu**2,yu**3)) # a mxn matrix
	M = M.transpose()
	print("solving for the polynomial fitting coefficients...")
	K,resid,rnk,svs = np.linalg.lstsq(M,X,rcond=-1) # k has size n
	print("residue:%0.8f	rank:%0.8f"%(np.sum(resid),rnk))
	return K

def beads(img,yps=None,thr=6,skip=5,nbead=0):
	ny,nx = np.shape(img)

	def search_peaks(yps):
		dI = ddx(img[yps,:])
		peaks = np.zeros(len(dI))
		i = 0
		while i < len(dI):
				if dI[i]>thr:
					peaks[i] = 1
					i+=skip
				i+=1
		return peaks, dI # a vector with 1 at positions where a peak was detected
	if yps == None: # scan the image vertically until a good result is found
		yps=0
		peaks = 0
		while np.sum(peaks)!=nbead:
			yps+=1
			peaks, dI = search_peaks(yps)
	else:
		peaks, dI = search_peaks(yps)
	#	plt.plot(peaks)
	#	plt.plot(dI)
	#	plt.show()
	return peaks, dI, yps

############# TIER 3 ###################################################

def tune_ghosal(fimg,K_s=5,k_m=None,N_k=5000,l_max=0.5,phi_min=1.45,outlier_sigma=2,blur=10):
	"""
	help make the selection of parameters for the function faster
	Machine learning on v2 of this code
	"""
	# open
	img = read_image(fimg)
	camera = fimg[-10]

	# ensure N_k is even
	if N_k%2!=0:
		N_k+=1
		print("N_k=%0.5f"%N_k)

	# blur image
	img = blurd_image(img,order=1,strength=blur,speed='fast')

	# calculate k,l and phi for the whole image
	_,_,k,l,phi = ghosal_edge_v2(img,Ks=K_s,debug=True)

	# find mean k
	if k_m == None:
		ij_m = select_points(img,n=1,prompt="choose a characteristic edge location")
		ij_m = np.array(ij_m[0],dtype=int)
		k_m = k[ij_m[1],ij_m[0]]
		print("k_m=",k_m)

	# find k limits
	k_sort = np.sort(k.flatten())
	k_sort[np.isnan(k_sort)]=0 # get rid of nans for argmin
	#print(np.shape(k_sort))
	#i_km = np.where(k_sort==k_m)[0][0]
	i_km = (np.abs(k_sort - k_m)).argmin()
	i_kmin = int(i_km-N_k/2)
	i_kmax = int(i_km+N_k/2)
	k_min = k_sort[i_kmin]
	k_max = k_sort[i_kmax]
	edg,org = ghosal_edge_v2(img,K_s,kmax=k_max,kmin=k_min,lmax=l_max,phimin=phi_min)

	# outlier removal
	pts,sig = line_fit(edg[:,1],edg[:,0])
	ptsy = np.mean(pts[:,1])
	accepted = ((edg[:,0]<(ptsy+outlier_sigma*sig)) & (edg[:,0]>(ptsy-outlier_sigma*sig)))
	edga = edg[accepted,:]

	# plotting
	vectors = edg-org # show the vectors also
	plt.imshow(img)
	plt.quiver(org[:,1],org[:,0],vectors[:,1],vectors[:,0],angles='xy',
		scale_units='xy', scale=1, color = 'orange')
	plt.scatter(edga[:,1],edga[:,0],c='blue',marker='.')
	plt.title("Validated Points")
	plt.show()

	# save
	print("Paramters:\n Camera %s:	K_s=%0.5f	k_min=%0.5f	k_max=%0.5f	l_max=%0.5f	phi_min=%0.5f outlier_sigma=%0.5f	blur=%0.5f"
		%(camera,K_s,k_min,k_max,l_max,phi_min,outlier_sigma, blur))
	save = input("save current parameters?(y/n)	")
	if save=="y":
		savename = fname_dir(fimg)
		savename = savename+os.sep+camera+"_ghosal_edge_parameters.txt"
		savematrix = np.squeeze([K_s,k_min, k_max, l_max, phi_min, outlier_sigma, blur])
		np.savetxt(savename,savematrix,delimiter="	")
		print("saved\n",savematrix)

############# EXTERNAL #################################################

def animate(directory,gifname,n_t,step=2,duration=0.2):
	"""
	n_t is the nuber of images you want
	step is how many images int the directory to skip each timestep
	duration is the duration of each frame in the final gif
	Based on the gif code of the SP classes of M Mendez 2019
	"""
	# create list of filenames
	fnames = dir_fname(directory,"*")
	# create list of plots
	images=[]
	for k in range(0,n_t):
		k = k*step
		print('Mounting Im '+ str(k))
		FIG_NAME=fnames[k]
		images.append(imageio.imread(FIG_NAME)) # read
	# Now we can assemble the video
	imageio.mimsave(gifname, images,duration=duration) # create gif
	print('Animation'+gifname+'Ready')
	return True


import cv2

import numpy as np
import matplotlib.pyplot as plt
img=cv2.imread("HALma.jpg")
gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
sobelx=cv2.Sobel(gray,cv2.CV_64F,1,0,ksize=3)
sobely=cv2.Sobel(gray,cv2.CV_64F,0,1,ksize=3)

canny=cv2.Canny(gray,100,200,apertureSize=3)
cv2.imshow("image",img)
cv2.imshow("sobel",sobelx)
cv2.imshow("sobel",sobely)
cv2.imshow("canny",canny)

cv2.waitKey()
cv2.destroyAllWindows()


# ###########################################
# Filename: diffsim_gpu_kernels.py
# Calculate diffraction pattern from first principles
# Feature to parameterise displaced atoms (strain).
#
# Author: Marcus Newton
# Version 1.0
# Licence: GNU GPL 3
#
# ###########################################

import numpy
from numba import cuda
import math
import numba

FLOAT = numba.float32

@cuda.jit(device=True)
def dotGPU(array1, array2):
	"""
	Calculate the dot product of two vectors.
	"""
	total = 0
	for i in range(len(array1)):
		total += array1[i]*array2[i]
	return total

@cuda.jit(device=True)
def crossGPU(array1, array2):
	"""
	Return cross product of two 3D vectors
	"""
	a1 = array1[0]
	a2 = array1[1]
	a3 = array1[2]
	b1 = array2[0]
	b2 = array2[1]
	b3 = array2[2]
	return a2*b3-a3*b2, a3*b1-a1*b3, a1*b2-a2*b1

@cuda.jit(device=True)
def RXGPU(vector, rvector, angle):
	"""
	Rotate a vector about X axis by angle (rad)
	"""
	rmatrix_x = cuda.local.array((3,3), FLOAT)

	rmatrix_x[0,0] = 1.0
	rmatrix_x[0,1] = 0.0
	rmatrix_x[0,2] = 0.0

	rmatrix_x[1,0] = 0.0
	rmatrix_x[1,1] = math.cos(angle)
	rmatrix_x[1,2] = -math.sin(angle)

	rmatrix_x[2,0] = 0.0
	rmatrix_x[2,1] = math.sin(angle)
	rmatrix_x[2,2] = math.cos(angle)

	rvector[0] = dotGPU(rmatrix_x[0,:],vector)
	rvector[1] = dotGPU(rmatrix_x[1,:],vector)
	rvector[2] = dotGPU(rmatrix_x[2,:],vector)

@cuda.jit(device=True)
def RYGPU(vector, rvector, angle):
	"""
	Rotate a vector about Y axis by angle (rad)
	"""
	rmatrix_y = cuda.local.array((3,3), FLOAT)

	rmatrix_y[0,0] = math.cos(angle)
	rmatrix_y[0,1] = 0.0
	rmatrix_y[0,2] = math.sin(angle)

	rmatrix_y[1,0] = 0.0
	rmatrix_y[1,1] = 1.0
	rmatrix_y[1,2] = 0.0

	rmatrix_y[2,0] = -math.sin(angle)
	rmatrix_y[2,1] = 0.0
	rmatrix_y[2,2] = math.cos(angle)

	rvector[0] = dotGPU(rmatrix_y[0,:],vector)
	rvector[1] = dotGPU(rmatrix_y[1,:],vector)
	rvector[2] = dotGPU(rmatrix_y[2,:],vector)

@cuda.jit(device=True)
def RZGPU(vector, rvector, angle):
	"""
	Rotate a vector about Z axis by angle (rad)
	"""
	rmatrix_z = cuda.local.array((3,3), FLOAT)

	rmatrix_z[0,0] = math.cos(angle)
	rmatrix_z[0,1] = -math.sin(angle)
	rmatrix_z[0,2] = 0.0

	rmatrix_z[1,0] = math.sin(angle)
	rmatrix_z[1,1] = math.cos(angle)
	rmatrix_z[1,2] = 0.0

	rmatrix_z[2,0] = 0.0
	rmatrix_z[2,1] = 0.0
	rmatrix_z[2,2] = 1.0

	rvector[0] = dotGPU(rmatrix_z[0,:],vector)
	rvector[1] = dotGPU(rmatrix_z[1,:],vector)
	rvector[2] = dotGPU(rmatrix_z[2,:],vector)

@cuda.jit(device=True)
def RotateToZGPU(vector):
	"""
	Rotate to z-axis and return rotation angles required to return back
	"""
	lxy = math.sqrt(vector[0]*vector[0]+vector[1]*vector[1])
	th_x = -math.atan2(lxy,vector[2])
	th_z = -math.atan2(vector[0],vector[1])
	mag = math.sqrt(dotGPU(vector,vector))
	return mag, th_x, th_z

@cuda.jit(device=True)
def CalcKfpGPU(k_i, k_f, alphakf, Q, k_fp_out):
	"""
	Create k_fp vector from k_f and alpha.
	1) find vector k_i * k_f
	2) Rotate to Z and use angle to rotate k_f about z
	3) rotate back
	4) repeat for the other alpha angle
	"""
	knorm = crossGPU(k_i, k_f)
	knormmag, thx, thz = RotateToZGPU(knorm)

	# Allocate memory for each step of the calculation, to avoid weird issues
	# with re-using the same memory
	k_fp_1 = cuda.local.array((3,), FLOAT)
	k_fp_2 = cuda.local.array((3,), FLOAT)
	k_fp_3 = cuda.local.array((3,), FLOAT)
	k_fp_4 = cuda.local.array((3,), FLOAT)
	k_fp_5 = cuda.local.array((3,), FLOAT)

	RZGPU(k_f, k_fp_1, -thz)
	RXGPU(k_fp_1, k_fp_2, -thx)
	RZGPU(k_fp_2, k_fp_3, alphakf[1])
	RXGPU(k_fp_3, k_fp_4, thx)
	RZGPU(k_fp_4, k_fp_5, thz)

	Qmag, thx, thz = RotateToZGPU(Q)
	RZGPU(k_fp_5, k_fp_1, -thz)
	RXGPU(k_fp_1, k_fp_2, -thx)
	RZGPU(k_fp_2, k_fp_3, alphakf[0])
	RXGPU(k_fp_3, k_fp_4, thx)
	RZGPU(k_fp_4, k_fp_5, thz)

	k_fp_out[0] = k_fp_5[0]
	k_fp_out[1] = k_fp_5[1]
	k_fp_out[2] = k_fp_5[2]

@cuda.jit(device=True)
def CalcDetAlphaGPU(idi, idj, pixelxy, detsize, R):
	"""Calculate K_ip rotation angles for the given (idi, idj) pixel."""
	alphakf_i = math.atan((10.0**(-6))*pixelxy[0] * (idi - detsize[0]/2.0)/R)
	alphakf_j = math.atan((10.0**(-6))*pixelxy[1] * (idj - detsize[1]/2.0)/R)
	return alphakf_i, alphakf_j

@cuda.jit()
def CalcPixelScatterGPU(atoms, k_i, k_f, atomR, Q, exposure, binbits, pixelxy,
												detsize, r0, flux, R, flatshapearray, gam_out):
	"""Calculate the scattering intensity for a pixel.

	Computes the intensity for the pixel given by the ID of the current GPU
	thread, determined by idi, idj and idk.	Units: photons.

	Parameters
	----------
	atoms: numpy float array (natoms, 4)
	k_i: numpy float array (nz, 3)
	k_f: numpy float array (nx, ny, nz, 3)
	atomR: numpy float array (natoms, nunitcells, 3)
	Q: numpy float array (nz, 3)
	exposure: float
	binbits: int
	pixelxy: numpy float array (2,)
	detsize: numpy int array (2,)
	r0: float
	flux: float
	R: float
	flatshapearray: numpy float array (nunitcells,)

	Return
	------
	gam_out: numpy float array (nx, ny, nz)
	"""
	idi = cuda.blockIdx.x*cuda.blockDim.x + cuda.threadIdx.x
	idj = cuda.blockIdx.y*cuda.blockDim.y + cuda.threadIdx.y
	idk = cuda.blockIdx.z*cuda.blockDim.z + cuda.threadIdx.z

	k_fp = cuda.local.array((3,), FLOAT)
	alphakf = CalcDetAlphaGPU(idi, idj, pixelxy, detsize, R)
	CalcKfpGPU(k_i[idk], k_f[idk], alphakf, Q[idk], k_fp)

	q0 = k_fp[0] - k_i[idk,0]
	q1 = k_fp[1] - k_i[idk,1]
	q2 = k_fp[2] - k_i[idk,2]

	expsumRe = 0.0
	expsumIm = 0.0
	n = len(atoms)
	m = len(atomR[0])
	for i in range(1,n,1):
		innersumRe = 0.0
		innersumIm = 0.0
		for j in range(m):
			atomDotProduct = atomR[i,j,0] * q0 + \
					atomR[i,j,1] * q1 + \
					atomR[i,j,2] * q2
			innersumRe += flatshapearray[j] * math.cos(atomDotProduct)
			innersumIm += flatshapearray[j] * math.sin(atomDotProduct)
		expsumRe += atoms[i, 0] * innersumRe
		expsumIm += atoms[i, 0] * innersumIm

	gam_out[idi,idj,idk] = exposure*pixelxy[0]*pixelxy[1]*r0*r0*binbits*binbits*\
		flux*(expsumRe*expsumRe+expsumIm*expsumIm)/(R*R)

@cuda.jit
def CalcPixelScatterNoAtomFuncGPU(k_f, k_i, Q, Rvec, exposure, binbits, pixelxy,
									detsize, r0, flux, R, flatshapearray, S, gam_out):
	"""Calculate the scattering intensity for a pixel, when there are no atom
	displacement functions.

	Computes the intensity for the pixel given by the ID of the current GPU
	thread, determined by idi, idj and idk.	Units: photons.

	Parameters
	----------
	atoms: numpy float array (natoms, 4)
	k_i: numpy float array (nz, 3)
	k_f: numpy float array (nx, ny, nz, 3)
	Q: numpy float array (nz, 3)
	Rvec: numpy float array (nunitcells, 3)
	exposure: float
	binbits: int
	pixelxy: numpy float array (2,)
	detsize: numpy int array (2,)
	r0: float
	flux: float
	R: float
	flatshapearray: numpy float array (nunitcells,)
	S: numpy complex float array (nz,)

	Return
	------
	gam_out: numpy float array (nx, ny, nz)
	"""
	idi = cuda.blockIdx.x*cuda.blockDim.x + cuda.threadIdx.x
	idj = cuda.blockIdx.y*cuda.blockDim.y + cuda.threadIdx.y
	idk = cuda.blockIdx.z*cuda.blockDim.z + cuda.threadIdx.z

	k_fp = cuda.local.array(3, FLOAT)
	alphakf = CalcDetAlphaGPU(idi, idj, pixelxy, detsize, R)
	CalcKfpGPU(k_i[idk], k_f[idk], alphakf, Q[idk], k_fp)

	q0 = k_fp[0] - k_i[idk,0]
	q1 = k_fp[1] - k_i[idk,1]
	q2 = k_fp[2] - k_i[idk,2]
	expsumRe = 0.0
	expsumIm = 0.0
	n = len(flatshapearray)
	for i in range(1,n,1):
		RVecDotProduct = Rvec[i,0]*q0+\
				Rvec[i,1]*q1+\
				Rvec[i,2]*q2
		expsumRe += flatshapearray[i]*math.cos(RVecDotProduct)
		expsumIm += flatshapearray[i]*math.sin(RVecDotProduct)

	gam_out[idi,idj,idk] = exposure*pixelxy[0]*pixelxy[1]*r0*r0*binbits*binbits*\
		flux*(S[idk].real*S[idk].real+S[idk].imag*S[idk].imag)*(expsumRe*expsumRe+expsumIm*expsumIm)/(R*R)