"""
Website: Leetcode
URL: https://leetcode.com/problems/add-two-numbers/

Problem statement:
You are given two non-empty linked lists representing two non-negative integers.
The digits are stored in reverse order and each of their nodes contain a single digit.
Add the two numbers and return it as a linked list.

You may assume the two numbers do not contain any leading zero, except the number 0 itself.

Example:

Input: (2 -> 4 -> 3) + (5 -> 6 -> 4)
Output: 7 -> 0 -> 8
Explanation: 342 + 465 = 807.
"""

# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        result = ListNode(0)
        rl = result
        carry = 0
        while True:
            if l1 == None and l2 == None:
                if carry == 1:
                    rl.next = ListNode(1)
                return result.next

            n1 = l1.val if l1 != None else 0
            n2 = l2.val if l2 != None else 0
            n = n1 + n2 + carry
            if n < 10:
                rl.next = ListNode(n)
                carry = 0
            else:
                rl.next = ListNode(n - 10)
                carry = 1

            l1 = l1.next if l1 != None else None
            l2 = l2.next if l2 != None else None
            rl = rl.next

# generate a ListNode
def genNodeList(*args):
    l = ListNode(args[0])
    a = l
    for i in args[1:]:
        a.next = ListNode(i)
        a = a.next
    return l

# prints a ListNode
def printNodeList(list: ListNode):
    while True:
        print(list.val, end=" ")
        if list.next != None:
            list = list.next
        else:
            break

if __name__ == "__main__":
    s = Solution()
    res = s.addTwoNumbers(
        genNodeList(9, 9),
        genNodeList(9),
    )
    printNodeList(res)


from gpytorch.models import ExactGP
from gpytorch.means import ConstantMean
from gpytorch.mlls import ExactMarginalLogLikelihood
from gpytorch.distributions import MultivariateNormal
from gpytorch.kernels import ScaleKernel
from hpo.casmo.kernels import CasmoKernel
from gpytorch.constraints.constraints import Interval
import numpy as np
from gpytorch.likelihoods import GaussianLikelihood
import ConfigSpace as CS
from ConfigSpace.util import deactivate_inactive_hyperparameters
import ConfigSpace.hyperparameters as CSH
import torch
from copy import deepcopy
from typing import Callable
import logging
import scipy.stats as ss
from typing import List

def is_large(config: CS.Configuration):
    """Roughly estimate the memory burden of a config to avoid OOM errors."""
    config_dict = config.get_dictionary()
    if 'unroll_length' in config_dict.keys() and config_dict['unroll_length'] >= 15:
        return True
    if 'NAS_policy_log2_width' in config_dict.keys() and config_dict['NAS_policy_log2_width'] >= 8 \
            and 'NAS_policy_num_layers' in config_dict.keys() and config_dict['NAS_policy_num_layers'] >= 4:
        # models larger than (256, 256, 256, 256)
        return True
    if 'NAS_policy_log2_width' in config_dict.keys() and config_dict['NAS_policy_log2_width'] >= 8 \
            and 'NAS_policy_num_layers' in config_dict.keys() and config_dict['NAS_policy_num_layers'] >= 4:
        # models larger than (256, 256, 256, 256)
        return True
    return False

def get_reward_from_trajectory(trajectory: np.array, use_last_fraction: float, risk_aversion: float = 0.):
    """
    Compute the final reward from a trajectory, based on drawdown (intead of standard deviation).
    use_last_fraction: 0 < .. < 1, the final reward will be computed as the simple average of the rewards.
    risk_aversion: (not currently used). Whether to penalize instabilities by considering the maximum drawdown
        from the max of the reward trajectory.
    """
    if not isinstance(trajectory, np.ndarray):
        trajectory = np.array(trajectory).astype(np.float)
    if use_last_fraction == 1:
        use_final_n = 1     # 1 is interpreted as using the last entry only
    else:
        use_final_n = max(
            1, int(np.round(use_last_fraction * len(trajectory))))
    rew = np.median(trajectory[-use_final_n:])
    if np.abs(risk_aversion) > 1e-3:
        rolling_max = np.array([0] + [np.max(trajectory[:i])
                               for i in range(1, len(trajectory + 1))]).astype(np.float)
        max_drawdown = max(rolling_max - trajectory)
    else:
        max_drawdown = 0.
    return rew - risk_aversion * max_drawdown

# GP Model

def order_stats(X):
    _, idx, cnt = np.unique(X, return_inverse=True, return_counts=True)
    obs = np.cumsum(cnt)  # Need to do it this way due to ties
    o_stats = obs[idx]
    return o_stats

def copula_standardize(X):
    X = np.nan_to_num(np.asarray(X))  # Replace inf by something large
    assert X.ndim == 1 and np.all(np.isfinite(X))
    o_stats = order_stats(X)
    quantile = np.true_divide(o_stats, len(X) + 1)
    X_ss = ss.norm.ppf(quantile)
    return X_ss

def latin_hypercube(n_pts, dim):
    """Basic Latin hypercube implementation with center perturbation."""
    X = np.zeros((n_pts, dim))
    centers = (1.0 + 2.0 * np.arange(0.0, n_pts)) / float(2 * n_pts)
    # random.seed(random.randint(0, 1e6))
    for i in range(dim):  # Shuffle the center locataions for each dimension.
        X[:, i] = centers[np.random.permutation(n_pts)]

    # Add some perturbations within each box
    pert = np.random.uniform(-1.0, 1.0, (n_pts, dim)) / float(2 * n_pts)
    X += pert
    return X

class GP(ExactGP):
    def __init__(self, train_x, train_y, kern, likelihood,
                 outputscale_constraint, ):
        super(GP, self).__init__(train_x, train_y, likelihood)
        self.dim = train_x.shape[1]
        self.mean_module = ConstantMean()
        self.covar_module = ScaleKernel(
            kern, outputscale_constraint=outputscale_constraint)

    def forward(self, x, x_mask=None, ):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x, x1_mask=x_mask, )
        return MultivariateNormal(mean_x, covar_x)

def train_gp(configspace: CS.ConfigurationSpace,
             train_x, train_y, use_ard, num_steps,
             time_varying: bool = False,
             train_t=None,
             lengthscale_scaling: float = 2.,
             hypers={},
             noise_variance=None,
             return_hypers=False,
             verbose: bool = False
             ):
    """Fit a GP model where train_x is in [0, 1]^d and train_y is standardized.
    (train_x, train_y): pairs of x and y (trained)
    noise_variance: if provided, this value will be used as the noise variance for the GP model. Otherwise, the noise
        variance will be inferred from the model.

    """
    from math import sqrt
    assert train_x.ndim == 2
    assert train_y.ndim == 1
    assert train_x.shape[0] == train_y.shape[0]
    if train_t is not None:
        if not isinstance(train_t, torch.Tensor):
            train_t = torch.tensor(train_t).to(dtype=train_x.dtype)

    # Create hyper parameter bounds
    if noise_variance is None:
        noise_variance = 0.001
        noise_constraint = Interval(1e-6, 0.1)
    else:
        if np.abs(noise_variance) < 1e-6:
            noise_variance = 0.02
            noise_constraint = Interval(1e-6, 0.05)
        else:
            noise_constraint = Interval(
                0.99 * noise_variance, 1.01 * noise_variance)
    if use_ard:
        lengthscale_constraint = Interval(0.01, 0.5)
    else:
        lengthscale_constraint = Interval(
            0.01, sqrt(train_x.shape[1]))  # [0.005, sqrt(dim)]
    # outputscale_constraint = Interval(0.05, 20.0)
    outputscale_constraint = Interval(0.5, 5.)

    # add in temporal dimension if t is not None
    if train_t is not None and time_varying:
        train_x = torch.hstack((train_t.reshape(-1, 1), train_x))

    # Create models
    likelihood = GaussianLikelihood(noise_constraint=noise_constraint).to(
        device=train_x.device, dtype=train_y.dtype)

    kern = CasmoKernel(cs=configspace, lamda=0.5, ard=use_ard,
                       time_varying=time_varying,
                       continuous_lengthscale_constraint=lengthscale_constraint,
                       categorical_lengthscale_constraint=lengthscale_constraint,
                       lengthscale_scaling=lengthscale_scaling)

    model = GP(
        train_x=train_x,
        train_y=train_y,
        likelihood=likelihood,
        kern=kern,
        outputscale_constraint=outputscale_constraint,
    ).to(device=train_x.device, dtype=train_x.dtype)

    # Find optimal model hyperparameters
    model.train()
    likelihood.train()

    # "Loss" for GPs - the marginal log likelihood
    mll = ExactMarginalLogLikelihood(likelihood, model)

    # Initialize model hypers
    loaded_hypers = False
    # if hyperparameters are already supplied, no need to optimize GP
    if hypers is not None and len(hypers):
        try:
            model.load_state_dict(hypers)
            loaded_hypers = True
        except Exception as e:
            logging.warning(
                f'Exception={e} occurred when loading the hyperparameters of the GP. Now training from scratch!')

    if not loaded_hypers:
        hypers = {}
        hypers["covar_module.outputscale"] = 1.0
        hypers["covar_module.base_kernel.lengthscale"] = np.sqrt(0.01 * 0.5)
        hypers["likelihood.noise"] = noise_variance if noise_variance is not None else 0.005
        model.initialize(**hypers)

        # Use the adam optimizer
        optimizer = torch.optim.Adam([{"params": model.parameters()}], lr=0.2)

        for _ in range(num_steps):
            optimizer.zero_grad()
            output = model(train_x, )
            try:
                loss = -mll(output, train_y).float()
                loss.backward()
                optimizer.step()
                if verbose and _ % 50 == 0:
                    logging.info(
                        f'Optimising GP log-likelihood: Iter={_}, Loss={loss.detach().numpy()}')

            except Exception as e:
                print(
                    f'RuntimeError={e} occurred due to non psd covariance matrix. returning the model at last successful iter')
                model.eval()
                likelihood.eval()
                return model
    # Switch to eval mode
    model.eval()
    likelihood.eval()
    if return_hypers:
        return model, model.state_dict()
    else:
        return model

def normalize(data, wrt):
    """ Normalize data to be in range (0,1), with respect to (wrt) boundaries,
        which can be specified.
    """
    return (data - np.min(wrt, axis=0)) / (
        np.max(wrt, axis=0) - np.min(wrt, axis=0))

def get_dim_info(cs: CS.ConfigurationSpace, x, return_indices=False):
    """Return the information on the categorical, integer and continuous spaces"""
    if len(cs) != len(x):
        # this is because x is longer than cs -- the final dimensions are the contextual info presented as fixed dimensions.
        x = deepcopy(x)[:len(cs)]
    cat_dims, cont_dims, int_dims = [], [], []
    cat_dims_idx, cont_dims_idx, int_dims_idx = [], [], []
    for i, variable in enumerate(range(len(x))):
        # do not sample an inactivated hyperparameter (such a hyperparameter has nan value imputed)
        if x[variable] != x[variable]:
            continue
        if type(cs[cs.get_hyperparameter_by_idx(variable)]) == CSH.CategoricalHyperparameter:
            cat_dims.append(cs.get_hyperparameter_by_idx(variable))
            cat_dims_idx.append(i)
        elif type(cs[cs.get_hyperparameter_by_idx(variable)]) in [CSH.UniformIntegerHyperparameter,
                                                                  CSH.NormalIntegerHyperparameter]:
            int_dims.append(cs.get_hyperparameter_by_idx(variable))
            int_dims_idx.append(i)
        elif type(cs[cs.get_hyperparameter_by_idx(variable)]) in [CSH.UniformFloatHyperparameter,
                                                                  CSH.NormalFloatHyperparameter]:
            cont_dims.append(cs.get_hyperparameter_by_idx(variable))
            cont_dims_idx.append(i)
    if return_indices:
        return cat_dims_idx, cont_dims_idx, int_dims_idx
    return cat_dims, cont_dims, int_dims

def sample_discrete_neighbour(cs: CS.ConfigurationSpace, x, frozen_dims: List[int] = None):
    """Sample a neighbour from x in one of the active hyperparameter.

    select type:

    frozen_dims: the frozen dimensions where neighbours that differ from them will be rejected.

    """
    # note that for acquisition function optimisation (which this def is likely used), integer-type variables are treated
    # as discrete.
    assert len(x) >= len(cs)
    if len(x) > len(cs):
        # this is because x is longer than cs -- the final dimensions are the contextual info presented as fixed dimensions.
        fixed_dims = x[len(cs):]
        x = deepcopy(x)[:len(cs)]
    else:
        fixed_dims = None
    cat_dims, _, int_dims = get_dim_info(cs, x)
    config = CS.Configuration(cs, vector=x.detach(
    ).numpy() if isinstance(x, torch.Tensor) else x)

    try:
        cs.check_configuration(config)
    except ValueError as e:
        # there seems to be a bug with ConfigSpace that raises error even when a config is valid
        # Issue #196: https://github.com/automl/ConfigSpace/issues/196
        # print(config)
        config = CS.Configuration(cs, config.get_dictionary())

    # print(config)
    config_pert = deepcopy(config)
    selected_dim = str(np.random.choice(cat_dims + int_dims, 1)[0])
    index_in_array = cs.get_idx_by_hyperparameter_name(selected_dim)
    while config_pert[selected_dim] is None or (frozen_dims is not None and index_in_array in frozen_dims):
        selected_dim = str(np.random.choice(cat_dims + int_dims, 1)[0])
        index_in_array = cs.get_idx_by_hyperparameter_name(selected_dim)

    # if the selected dimension is categorical, change the value to another variable
    if selected_dim in cat_dims:
        config_pert[selected_dim] = np.random.choice(
            cs[selected_dim].choices, 1)[0]
        while config_pert[selected_dim] == config[selected_dim]:
            config_pert[selected_dim] = np.random.choice(
                cs[selected_dim].choices, 1)[0]
    elif selected_dim in int_dims:
        lb, ub = cs[selected_dim].lower, cs[selected_dim].upper
        if selected_dim in ['NAS_policy_num_layers', 'NAS_q_num_layers']:
            candidates = list(
                {max(lb, config[selected_dim] - 1), min(ub, config[selected_dim] + 1)})
        else:
            candidates = list(range(max(lb, min(config[selected_dim] - 1, round(config[selected_dim] * 0.8))),
                                    min(ub, max(round(config[selected_dim] * 1.2), config[selected_dim] + 1)) + 1))
        config_pert[selected_dim] = np.random.choice(candidates, 1)[0]
        while config_pert[selected_dim] == config[selected_dim]:
            config_pert[selected_dim] = np.random.choice(candidates, 1)[0]
    config_pert = deactivate_inactive_hyperparameters(config_pert, cs)
    x_pert = config_pert.get_array()
    if fixed_dims is not None:
        x_pert = np.concatenate([x_pert, fixed_dims])
    return x_pert

def hamming_distance(x1, x2, normalize=False):
    diff = (x1 != x2).to(dtype=torch.float)
    dist = diff.sum()
    if normalize:
        dist /= x1.shape[0]
    return dist

def construct_bounding_box(cs: CS.ConfigurationSpace, x, tr_length, weights=None, ):
    """Construct a bounding box around x_cont with tr_length being the k-dimensional trust region size.
    The weights should be the learnt lengthscales of the GP surrogate model.
    """
    if weights is None:
        weights = 1. / len(x.shape[0]) * np.ones(x.shape[1])
    # non-ard lengthscales passed -- this must be a scalar input
    elif len(weights) != x.shape[0]:
        weights = weights * np.ones(x.shape[0])
    weights = weights / weights.mean()
    # We now have weights.prod() = 1
    weights = weights / np.prod(np.power(weights, 1.0 / len(weights)))
    lb, ub = np.zeros_like(x), np.ones_like(x)
    for i, dim in enumerate(x):
        if np.isnan(x[i]) or i >= len(cs):
            lb[i], ub[i] = 0., 1.
        else:
            hp = cs[cs.get_hyperparameter_by_idx(i)]
            if type(hp) == CSH.CategoricalHyperparameter:
                lb[i], ub[i] = 0, len(hp.choices)
            else:
                lb[i] = np.clip(x[i] - weights[i] * tr_length / 2.0, 0.0, 1.0)
                ub[i] = np.clip(x[i] + weights[i] * tr_length / 2.0, 0.0, 1.0)
                if type(hp) in [CSH.UniformIntegerHyperparameter, CSH.NormalIntegerHyperparameter,
                                CSH.NormalFloatHyperparameter, CSH.UniformFloatHyperparameter]:
                    lb[i] = max(hp._inverse_transform(hp.lower), lb[i])
                    ub[i] = min(hp._inverse_transform(hp.upper), ub[i])
    return lb, ub

def get_start_point(cs: CS.ConfigurationSpace, x_center, frozen_dims: List[int] = None, return_config=False):
    # get a perturbed starting point from x_center
    new_config_array = deepcopy(x_center)

    perturbation_factor = [0.8, 1.2]  # <- taken from PB2
    for i in range(new_config_array.shape[0]):
        # print(param_name)
        if np.isnan(new_config_array[i]) or (frozen_dims is not None and i in frozen_dims):
            continue
        param_name = cs.get_hyperparameter_by_idx(i)
        if type(cs[param_name]) == CSH.CategoricalHyperparameter:
            new_config_array[i] = np.random.choice(
                range(len(cs[param_name].choices)))
        elif type(cs[param_name]) in [CSH.UniformIntegerHyperparameter, CSH.NormalIntegerHyperparameter] and cs[param_name].lower == 0 and cs[param_name].upper == 1:   # binary
            new_config_array[i] = np.random.choice([0, 1])
        else:
            perturbation = np.random.uniform(*perturbation_factor)
            new_config_array[i] = np.clip(x_center[i] * perturbation, 0., 1.)
    # whether to change the network depth
    config = CS.Configuration(cs, vector=new_config_array)
    config = deactivate_inactive_hyperparameters(config, cs)

    try:
        cs.check_configuration(config)
    except ValueError:
        config = CS.Configuration(cs, config.get_dictionary())
    new_config_array = config.get_array()
    if return_config:
        return new_config_array, config
    return new_config_array

def grad_search(cs: CS.ConfigurationSpace,
                x_center,
                f: Callable,
                n_restart: int = 1,
                step: int = 40,
                batch_size: int = 1,
                dtype=torch.float,
                fixed_dims=None,
                verbose: bool = True,
                ):
    """Vanilla gradient-based search"""
    num_fixed_dims = x_center.shape[0] - \
        len(cs) if x_center.shape[0] > len(cs) else 0
    if num_fixed_dims > 0:
        fixed_dims = list(range(len(cs), x_center.shape[0]))
    else:
        fixed_dims = None

    x0s = []
    for _ in range(n_restart):
        p = cs.sample_configuration().get_array()
        if fixed_dims is not None:
            p = np.concatenate((p, x_center[fixed_dims]))
        x0s.append(p)
    x0 = np.array(x0s).astype(np.float)

    def _grad_search(x0):
        lb, ub = np.zeros(x0.shape[0]), np.ones(x0.shape[0])
        n_step = 0
        x = deepcopy(x0)
        acq_x = f(x).detach().numpy()
        x_tensor = torch.tensor(x, dtype=dtype).requires_grad_(True)
        optimizer = torch.optim.Adam([{"params": x_tensor}], lr=0.1)

        while n_step <= step:
            optimizer.zero_grad()
            acq = f(x_tensor)
            acq.backward()
            if num_fixed_dims:
                x_tensor.grad[fixed_dims] = 0.
            if verbose and n_step % 20 == 0:
                logging.info(
                    f'Acquisition optimisation: Step={n_step}: Value={x_tensor}. Acq={acq_x}.')
            optimizer.step()
            with torch.no_grad():
                x_tensor = torch.maximum(torch.minimum(x_tensor, torch.tensor(
                    ub).to(x_tensor.dtype)), torch.tensor(lb).to(x_tensor.dtype))
            n_step += 1
        x = x_tensor.detach().numpy().astype(x0.dtype)
        acq_x = f(x).detach().numpy()
        del x_tensor
        return x, acq_x

    X, fX = [], []
    for i in range(n_restart):
        res = _grad_search(x0[i, :])
        X.append(res[0])
        fX.append(res[1])
    top_idices = np.argpartition(
        np.array(fX).flatten(), batch_size)[:batch_size]
    return np.array([x for i, x in enumerate(X) if i in top_idices]).astype(np.float), np.array(fX).astype(np.float).flatten()[top_idices]

def interleaved_search(cs: CS.ConfigurationSpace,
                       n_dim,
                       x_center,
                       f: Callable,
                       max_dist_cont: float,
                       max_dist_cat: float = None,
                       cont_int_lengthscales: float = None,
                       n_restart: int = 1,
                       step: int = 40,
                       batch_size: int = 1,
                       interval: int = 1,
                       dtype=torch.float,
                       frozen_dims: List[int] = None,
                       frozen_vals: list = None,
                       num_fixed_dims: int = None,
                       verbose: bool = True
                       ):
    """
    x_center: the previous best x location that will be the centre of the bounding box
    f: the objective function of the interleaved_search. In this case, it is usually the acquisition function.
        This objective should be minimized.
    max_dist_cont: the bounding box length of the continuous trust region
    max_dist_cat: the bounding box length of the categorical trust region. This is in terms of normalized Hamming distance >0, <=1.
    cont_int_lengthscales: the lengthscales of the learnt GP model on the continuous and integer dimensions
    n_restart: number of restarts for the acquisition function optimization.
    """
    # when a x_center with a higher dimension than that specified by he configspace object, the additional dimensions
    #   are treated as "contextual" dimensions which are fixed during acquisition optimization.
    if max_dist_cat is None:
        max_dist_cat = 1.  # the normalized hamming distance is upper bounded by 1.
    num_fixed_dims = n_dim - len(cs) if n_dim > len(cs) else 0
    if num_fixed_dims > 0:
        fixed_dims = list(range(len(cs), n_dim))
    else:
        fixed_dims = None

    cat_dims, cont_dims, int_dims = get_dim_info(
        cs, cs.sample_configuration().get_array(), return_indices=True)

    if x_center is not None:
        assert x_center.shape[0] == n_dim
        x_center_fixed = deepcopy(
            x_center[-num_fixed_dims:]) if num_fixed_dims > 0 else None

        # generate the initially random points by perturbing slightly from the best location
        x_center_local = deepcopy(x_center)
        if frozen_dims is not None:
            x_center_local[frozen_dims] = frozen_vals   # freeze these values
        x0s = []
        lb, ub = construct_bounding_box(
            cs, x_center_local, max_dist_cont, cont_int_lengthscales)
        for _ in range(n_restart):
            if num_fixed_dims:
                p = get_start_point(
                    cs, x_center_local[:-num_fixed_dims], frozen_dims=frozen_dims)
                p = np.concatenate((p, x_center_fixed))
            else:
                p = get_start_point(cs, x_center_local,
                                    frozen_dims=frozen_dims)
            x0s.append(p)
    else:
        lb, ub = np.zeros(n_dim), np.ones(n_dim)
        x0s = [cs.sample_configuration().get_array() for _ in range(n_restart)]
        x_center_fixed = None

    x0 = np.array(x0s).astype(np.float)  # otherwise error on jade

    def _interleaved_search(x0):
        x = deepcopy(x0)
        acq_x = f(x).detach().numpy()
        n_step = 0
        while n_step <= step:
            # First optimise the continuous part, freezing the categorical part
            x_tensor = torch.tensor(x, dtype=dtype).requires_grad_(True)

            optimizer = torch.optim.Adam([{"params": x_tensor}], lr=0.1)
            for _ in range(interval):
                optimizer.zero_grad()
                acq = f(x_tensor)
                acq.backward()
                # freeze the grads of the non-continuous dimensions & the fixed dims
                for n, w in enumerate(x_tensor):
                    if n not in cont_dims or (fixed_dims is not None and n in fixed_dims) or (
                            frozen_dims is not None and n in frozen_dims):
                        x_tensor.grad[n] = 0.
                if verbose and n_step % 20 == 0:
                    logging.info(
                        f'Acquisition optimisation: Step={n_step}: Value={x_tensor}. Acq={acq_x}.')
                optimizer.step()
                with torch.no_grad():
                    x_nan_mask = torch.isnan(x_tensor)
                    # replace the data from the optimized tensor
                    x_tensor[cont_dims] = torch.maximum(torch.minimum(x_tensor[cont_dims], torch.tensor(ub[cont_dims])),
                                                        torch.tensor(lb[cont_dims])).to(x_tensor.dtype)
                    # enforces the nan entries remain nan
                    x_tensor[x_nan_mask] = torch.tensor(
                        np.nan, dtype=x_tensor.dtype)

                    # fixed dimensions should not be updated during the optimization here. Enforce the constraint below
                    if x_center_fixed is not None:
                        # the fixed dimensions are not updated according to the gradient information.
                        x_tensor[-num_fixed_dims:] = torch.tensor(
                            x_center_fixed, dtype=dtype)
                    # print(x_tensor)

            x = x_tensor.detach().numpy().astype(x0.dtype)
            del x_tensor

            # Then freeze the continuous part and optimise the categorical part
            for j in range(interval):
                neighbours = [sample_discrete_neighbour(
                    cs, x, frozen_dims=frozen_dims) for _ in range(10)]
                for i, neighbour in enumerate(neighbours):
                    neighbours[i][int_dims] = np.clip(
                        neighbour[int_dims], lb[int_dims], ub[int_dims])
                acq_x = f(x).detach().numpy()
                acq_neighbour = np.array(
                    [f(n).detach().numpy() for n in neighbours]).astype(np.float)
                acq_neighbour_argmin = np.argmin(acq_neighbour)
                acq_neighbour_min = acq_neighbour[acq_neighbour_argmin]
                if acq_neighbour_min < acq_x:
                    x = deepcopy(neighbours[acq_neighbour_argmin])
                    acq_x = acq_neighbour_min
            n_step += interval
        return x, acq_x

    def local_search(x):
        acq = np.inf
        x = deepcopy(x)
        logging.info(f'Bounds: {lb}, {ub}')

        if x_center_fixed is not None:
            x_config = CS.Configuration(cs, vector=x[:-num_fixed_dims])
        else:
            x_config = CS.Configuration(cs, vector=x)
        for _ in range(step):
            n_config = CS.util.get_random_neighbor(
                x_config, seed=int(np.random.randint(10000)))
            n = n_config.get_array()
            if x_center_fixed is not None:
                # the fixed dimensions are not updated according to the gradient information.
                n = np.concatenate((n, x_center_fixed))
            n = np.clip(n, lb, ub)
            acq_ = f(n).detach().numpy()
            if acq_ < acq:
                acq = acq_
                x = n
                x_config = n_config
        return x, acq

    X, fX = [], []
    for i in range(n_restart):
        res = _interleaved_search(x0[i, :])
        X.append(res[0])
        fX.append(res[1])
    top_idices = np.argpartition(
        np.array(fX).flatten(), batch_size)[:batch_size]
    return np.array([x for i, x in enumerate(X) if i in top_idices]).astype(np.float), np.array(fX).astype(np.float).flatten()[top_idices]


#! /usr/env/bin python
import os
import pickle
import numpy as np
from scipy.integrate import trapz
import pylab as plt

from pyboas import predictor, models

# Reconstruct location of directory containing data.
# TODO if possible make this work when script is called using execfile.
print(__file__)
basedir = os.path.dirname(__file__)
datadir = os.path.abspath(os.path.join(basedir, '..', 'data'))

def read_data():
    # Read data
    t, rv, erv = np.loadtxt(os.path.join(datadir,
                                         'HD222582_keck.rdb'),
                            skiprows=2, unpack=True)
    t -= 10000
    return t[:24], rv[:24], erv[:24]

def run():
    # np.random.seed(010605)

    # Read posterior samples from data directory.
    print('Reading data...')
    t, rv, erv = read_data()

    print('Reading posterior samples....')
    f = open(os.path.join(datadir, 'HD222582_emcee_chain.dat'))
    chain = pickle.load(f)
    f.close()

    # Burn-in and flatten chain
    s = chain.shape
    posterior = chain[:, -500:, :].reshape(s[0] * 500, s[2])
    # posterior = chain.reshape(s[0]*s[1], s[2])

    # Define array with times where to predict future measurement.
    timepred = np.linspace(1500, 2200, 50)
    epoch = t.mean()

    # The scale of the Gaussian likelihood is taken as the mean error + jitter.
    likescale = np.sqrt(erv.mean() ** 2 + posterior[:, -1] ** 2)

    # Instantianate GaussPredictor
    gpredictor = predictor.GaussPredictor(posterior, models.keplerian_fixed,
                                          extramodelargs=(epoch, 'k1'),
                                          likescale=likescale
                                          )

    print('Making predictions....')
    # Make predicions
    x, post_preds = gpredictor.make_prediction(timepred, verbose=True,
                                               scale=likescale)

    # Compute information gain # TODO replace with 'informer' methods.
    infos = np.zeros(len(x))
    for i in range(len(x)):
        y = post_preds[i][post_preds[i] > 0]
        infos[i] = trapz(y * np.log(y), x[i][post_preds[i] > 0])

    print('Drawing sample....')
    # Draw samples
    tsample = np.array([1600, 1700, 1800, 1870, 1920, 2020])

    postpred_sample = gpredictor.samplepredictive(tsample,
                                                  scale=likescale.mean())

    # Plot results
    plt.ion()
    fig = plt.figure()
    ax = fig.add_subplot(111)

    # Plot data.
    ax.errorbar(t, rv, erv, fmt='or', ms=8, zorder=4)

    # time array for plots
    tt = np.linspace(700, 2200, 1000)

    # Pick 20 random curves from posterior
    post = posterior.copy()
    np.random.shuffle(post)
    rvall = models.keplerian_fixed(post[:20], tt, epoch, 'k1')

    # Plot sample posterior curves
    ax.plot(tt, rvall, '-', color='0.45', alpha=0.3)

    # Plot samples from posterior predictive
    for i, t in enumerate(tsample):
        ax.plot(t + np.random.rand(100)*5-2.5, postpred_sample[i, :100], ',m')

    # Define axis for expected information gain
    ax2 = ax.twinx()
    ax2.plot(timepred, -infos, '-g', lw=2)

    # Labels
    ax.set_xlabel('BJD - 2450000', fontsize=16)
    ax.set_ylabel('RV [m/s]', fontsize=16)
    ax2.set_ylabel('E[$\Delta$I]', fontsize=16)
    plt.show()

    return gpredictor

"""
if __name__ == '__main__':
    run()
"""


import csv, sys, os, re #, urllib2
from bs4 import BeautifulSoup
import urllib
from urllib.request import urlopen

# colors
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def html2diff(html):
    prev_doc, next_doc = [],[]
    soup = BeautifulSoup(html, 'html')
    nodes = soup.find_all(class_=re.compile(r'(diff-deletedline)|(diff-addedline)|(diff-empty)'))
    div_p = re.compile(r'
(.*)
', re.DOTALL)

    for i in range(0, len(nodes), 2):
        # skip straddeling cases
        if i + 1 >= len(nodes):
            continue

        node_prev = nodes[i]
        node_next = nodes[i + 1]

        if not node_prev.div or not node_next.div:
            continue

        prev_match = re.match(div_p, node_prev.div.prettify(formatter=None))
        next_match = re.match(div_p, node_next.div.prettify(formatter=None))

        if prev_match and next_match:
            prev_doc.append(prev_match.group(1).strip())
            next_doc.append(next_match.group(1).strip())

    return prev_doc, next_doc

def url2diff(url):
    try:
        response = urlopen(url)
        html = response.read()
        return html2diff(html)
    except Exception as e:
        print(e, file=sys.stderr)
        return [], []

def print_withcolor(idx, l):
    l = l.replace('\n', '')
    ins_p = re.compile(r'(.*?)', re.DOTALL)
    del_p = re.compile(r'(.*?)', re.DOTALL)
    patterns = [ins_p, del_p]
    for i,p in enumerate(patterns):
        match = re.finditer(p, l)
        if match:
            new_l = ""
            last = 0
            for m in match:
                if i == 1:
                    color = bcolors.OKBLUE
                else:
                    color = bcolors.OKGREEN
                new_l = new_l + l[last:m.start(1)] + color + m.group(1) + bcolors.ENDC
                last = m.end(1)
            new_l = new_l + l[last:]
            l = new_l
    print (bcolors.HEADER+'line '+str(idx+1)+':'+bcolors.ENDC+l)


#!/usr/bin/env python
# coding: utf-8

# DEPENDENCIES: Import Splinter, BeautifulSoup, and Pandas
from splinter import Browser
from bs4 import BeautifulSoup as soup
import pandas as pd
from webdriver_manager.chrome import ChromeDriverManager

# Set the executable path and initialize Splinter (creating instance of a Splinter browser)
executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', **executable_path, headless=False)

# ### VISIT THE NASA MARS NEWS SITE
# Visit the mars nasa news site (tell Splinter which site I want to visit by assigning link to a URL)
url = 'https://redplanetscience.com/'
browser.visit(url)

# Optional delay for loading the page
browser.is_element_present_by_css('div.list_text', wait_time=1)

# Convert the browser html to a soup object and use BeautifulSoup to parse the HTML
html = browser.html
news_soup = soup(html, 'html.parser')

#assign slide_elem as the variable to pinpoint the div tag with the class of list_text
slide_elem = news_soup.select_one('div.list_text')

#look inside the slide_elem variable to find this specific data: a div with a class of content_title
slide_elem.find('div', class_='content_title')

# Use the parent element variable (slide_elem) & return only the title of the news article by chaining .get_text onto .find(), so only the text of the element is returned.
news_title = slide_elem.find('div', class_='content_title').get_text()
news_title

# Use the parent element to find the paragraph text
news_p = slide_elem.find('div', class_='article_teaser_body').get_text()
news_p

# ### JPL SPACE IMAGES FEATURED IMAGE
# Visit URL to begin scraping
url = 'https://spaceimages-mars.com'
browser.visit(url)

# Start getting code ready to automate all of the clicks it will take to get the full-size version of the Featured Image
#Button 1: Use indexing to stipulate that I want the browser to click the 2nd button ~ Find and click the full image button
full_image_elem = browser.find_by_tag('button')[1]
full_image_elem.click()

# Parse the new page that was loaded onto the automated browser, so you can continue scraping full-size image URL
html = browser.html
img_soup = soup(html, 'html.parser')
img_soup

# find the relative image url by using .get('src') to pull the link to the image. (telling BeautifulSoup this is where the image I want lives, so use the link that's inside these tags)
img_url_rel = img_soup.find('img', class_='fancybox-image').get('src')
img_url_rel

# Add the baes URL to the relative URL to get the complete link (Use the base url to create an absolute url)
img_url = f'https://spaceimages-mars.com/{img_url_rel}'
img_url

# ### MARS FACTS: Scrape a table from another website to add to the app
#Scrape entire table with Pandas' .read_html()
#Step 1: search for & return a list of tables. The index of 0 tells Pandas to ull only the 1st table it encounters. Then, turn it into a DataFrame
df = pd.read_html('https://galaxyfacts-mars.com')[0]
df.head()

#Step 2: Assign new columns to the df
df.columns=['Description', 'Mars', 'Earth']

#Step 3: turn the description column into the df's index. inPlace=True so that the updated index will remain in place.
df.set_index('Description', inplace=True)
df

# Add the table to the web app by Converting dataframe back into html-ready code
df.to_html()

#I have the image, the article, and the table. Now, automate the scraping by converting this code into a .py file

# # D1: SCRAPE HIGH-RES MARS' HEMISPHERE IMAGES & TITLES

# ### Hemispheres

# 1. Use browser to visit the URL
url = 'https://marshemispheres.com/'
browser.visit(url)

# # 2. Create a list to hold the images and titles.
# hemisphere_image_urls = []

# #3. Write code to retrieve the titles for each hemisphere (and then image urls)
# html = browser.html
# html_soup = soup(html, 'html.parser')
# html_soup

# a_tags = html_soup.find_all('a', class_ = 'itemLink')

# for i in a_tags:
#     hemisphere = {}
#     # Get Hemisphere title
#     hemisphere['title'] = browser.find_by_tag('h3')

#     #Get image urls
#     #links = browser.links.find_by_text('itemLink product-item')[i].click()
# #    links = browser.links.find_by_partial_text('.html')
#     #links = browser.find_by_css('a.product-item img')

# 2. Create a list to hold the images and titles.
hemisphere_image_urls = []

# 3. Write code to retrieve the image urls and titles for each hemisphere.
# First, get a list of all of the hemispheres
links = browser.find_by_css('a.product-item img')

# Next, loop through those links, click the link, find the sample anchor, return the href
for i in range(len(links)):
    hemisphere = {}

    # We have to find the elements on each loop to avoid a stale element exception
    browser.find_by_css('a.product-item img')[i].click()

    # Next, we find the Sample image anchor tag and extract the href
    sample_elem = browser.links.find_by_text('Sample').first
    hemisphere['img_url'] = sample_elem['href']

    # Get Hemisphere title
    hemisphere['title'] = browser.find_by_css('h2.title').text

    # Append hemisphere object to list
    hemisphere_image_urls.append(hemisphere)

    # Finally, we navigate backwards
    browser.back()

# 4. Print the list that holds the dictionary of each image url and title.
hemisphere_image_urls

# 5. Quit the browser
browser.quit()


from extensions.module_result.common.enums import NewSymptomAction, HealthIssueAction
from extensions.module_result.models.primitives import (
    BreastFeedingUpdate,
    BreastFeedingUpdateBabyDetails,
)
from extensions.module_result.models.primitives.primitive_questionnaire import (
    QuestionnaireMetadata,
    QuestionnaireAnswer,
)
from extensions.tests.module_result.UnitTests.primitives_tests import (
    COMMON_FIELDS,
    PrimitiveTestCase,
)

class BreastFeedingUpdateTestCase(PrimitiveTestCase):
    def set_primitive_values(self):
        breastfeeding_payload = {
            BreastFeedingUpdate.IS_BREASTFEEDING_NOW: True,
            BreastFeedingUpdate.STOP_DATE: "2020-06-30T00:00:00Z",
            BreastFeedingUpdate.NOW_OR_WORSE_SYMPTOM: [
                NewSymptomAction.DISABILITY_INCAPACITATION,
                NewSymptomAction.CONSULT_DOCTOR,
            ],
            BreastFeedingUpdate.BREASTFEEDING_BABY_DETAILS: [
                {
                    BreastFeedingUpdateBabyDetails.HEALTH_ISSUE: ["fever"],
                    BreastFeedingUpdateBabyDetails.HEALTH_ISSUE_ACTION: [
                        HealthIssueAction.CONSULT_DOCTOR,
                        HealthIssueAction.VISIT_ER,
                    ],
                    BreastFeedingUpdateBabyDetails.HAS_RECEIVED_DIAGNOSIS: True,
                    BreastFeedingUpdateBabyDetails.DIAGNOSIS_LIST: [],
                }
            ],
            BreastFeedingUpdate.METADATA: {
                QuestionnaireMetadata.ANSWERS: [
                    {
                        QuestionnaireAnswer.QUESTION_ID: "7c50c7d5-6f4e-4c0b-aeaf-c266eafdt587",
                        QuestionnaireAnswer.QUESTION: "Are you currently breastfeeding?",
                        QuestionnaireAnswer.ANSWER_TEXT: "Yes",
                    }
                ]
            },
        }

        COMMON_FIELDS.update(breastfeeding_payload)

    def test_success_creation(self):
        self.set_primitive_values()
        primitive = BreastFeedingUpdate.create_from_dict(
            COMMON_FIELDS, name="BreastFeedingUpdate"
        )
        self.assertIsNotNone(primitive)

    def test_failed_missing_required_fields_in_BreastFeedingUpdate(self):
        required_fields_keys = [
            BreastFeedingUpdate.IS_BREASTFEEDING_NOW,
            BreastFeedingUpdate.NOW_OR_WORSE_SYMPTOM,
            BreastFeedingUpdate.METADATA,
        ]

        for field in required_fields_keys:
            self.set_primitive_values()
            del COMMON_FIELDS[field]
            self._assert_convertible_validation_err(BreastFeedingUpdate)

    def test_failed_missing_required_fields_in_BreastFeedingUpdateBabyDetails(self):

        required_nested_field_keys = [
            BreastFeedingUpdateBabyDetails.HEALTH_ISSUE,
            BreastFeedingUpdateBabyDetails.HEALTH_ISSUE_ACTION,
            BreastFeedingUpdateBabyDetails.HAS_RECEIVED_DIAGNOSIS,
        ]

        for field in required_nested_field_keys:
            self.set_primitive_values()
            nested_field = COMMON_FIELDS[
                BreastFeedingUpdate.BREASTFEEDING_BABY_DETAILS
            ][0]
            del nested_field[field]
            self._assert_convertible_validation_err(BreastFeedingUpdate)

    def test_failed_invalid_breastfeeding_stop_date(self):
        self.set_primitive_values()
        COMMON_FIELDS[BreastFeedingUpdate.STOP_DATE] = "2019-12-3"
        self._assert_convertible_validation_err(BreastFeedingUpdate)


__author__    = "Individual contributors (see AUTHORS file)"
__date__      = "$DATE$"
__rev__       = "$REV$"
__license__   = "AGPL v.3"
__copyright__ = """
This file is part of the ESP Web Site
Copyright (c) 2007 by the individual contributors
  (see AUTHORS file)

The ESP Web Site is free software; you can redistribute it and/or
modify it under the terms of the GNU Affero General Public License
as published by the Free Software Foundation; either version 3
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

Contact information:
MIT Educational Studies Program
  84 Massachusetts Ave W20-467, Cambridge, MA 02139
  Phone: 617-253-4882
  Email: esp-webmasters@mit.edu
Learning Unlimited, Inc.
  527 Franklin St, Cambridge, MA 02139
  Phone: 617-379-0178
  Email: web-team@learningu.org
"""

import json
from datetime import datetime, timedelta

from django.db.models import Min
from django.db.models.query import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.utils.safestring import mark_safe

from esp.users.models    import ESPUser, Record, ContactInfo, StudentInfo, K12School
from esp.program.models import RegistrationProfile
from esp.program.class_status import ClassStatus

from esp.program.modules.base import ProgramModuleObj, needs_onsite, needs_student, main_call, aux_call
from esp.program.models import ClassSubject, ClassSection, StudentRegistration, ScheduleMap, Program
from esp.utils.web import render_to_response
from esp.cal.models import Event
from argcache import cache_function
from esp.users.models import ESPUser, Record
from esp.resources.models import ResourceAssignment
from esp.utils.models import Printer, PrintRequest
from esp.utils.query_utils import nest_Q
from esp.tagdict.models import Tag
from esp.accounting.controllers import IndividualAccountingController
from esp.program.controllers.studentclassregmodule import RegistrationTypeController as RTC

class OnSiteClassList(ProgramModuleObj):
    doc = """Display lists of classes for onsite registration purposes."""

    @classmethod
    def module_properties(cls):
        return {
            "admin_title": "Show Open Classes at Onsite Registration",
            "link_title": "List of Open Classes",
            "module_type": "onsite",
            "seq": 32,
            "choosable": 1,
            }

    @cache_function
    def section_data(sec):
        sect = {}
        sect['id'] = sec.id
        sect['emailcode'] = sec.emailcode()
        sect['title'] = sec.title()
        sect['teachers'] = ', '.join([t.name() for t in list(sec.teachers)])
        sect['rooms'] = (' ,'.join(sec.prettyrooms()))[:12]
        return sect
    section_data.depend_on_model('resources.ResourceAssignment')
    section_data.depend_on_cache(ClassSubject.get_teachers, lambda **kwargs: {})
    section_data=staticmethod(section_data)

    """ Warning: for performance reasons, these views are not abstracted away from
        the models.  If the schema is changed this code will need to be updated.
    """

    @aux_call
    @needs_onsite
    def catalog_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        #   Fetch a reduced version of the catalog to save time
        sections = list(ClassSection.objects.filter(parent_class__parent_program=prog, status__gt=0).extra({'event_ids':  """ARRAY(SELECT "cal_event"."id" FROM "cal_event", "program_classsection_meeting_times" WHERE ("program_classsection_meeting_times"."event_id" = "cal_event"."id" AND "program_classsection_meeting_times"."classsection_id" = "program_classsection"."id"))"""}).values('id', 'parent_class__id', 'enrolled_students', 'event_ids', 'registration_status'))
        for i in range(0, len(sections)):
            sections[i]['capacity'] = ClassSection.objects.get(id=sections[i]['id']).capacity
        data = {
            #   Todo: section current capacity ? (see ClassSection.get_capacity())
            'classes': list(ClassSubject.objects.filter(parent_program=prog, status__gt=0).extra({'teacher_names': """array_to_string(ARRAY(SELECT auth_user.first_name || ' ' || auth_user.last_name FROM auth_user,program_class_teachers WHERE program_class_teachers.classsubject_id=program_class.id AND auth_user.id=program_class_teachers.espuser_id), ', ')""", 'class_size_max_optimal': """SELECT program_classsizerange.range_max FROM program_classsizerange WHERE program_classsizerange.id = optimal_class_size_range_id"""}).values('id', 'class_size_max', 'class_size_max_optimal', 'class_info', 'prereqs', 'hardness_rating', 'grade_min', 'grade_max', 'title', 'teacher_names', 'category__symbol', 'category__id')),
            'sections': sections,
            'timeslots': list(prog.getTimeSlots().extra({'start_millis':"""EXTRACT(EPOCH FROM start) * 1000""",'label': """to_char("start", 'Dy HH:MI -- ') || to_char("end", 'HH:MI AM')"""}).values_list('id', 'label','start_millis').order_by("start")),
            'categories': list(prog.class_categories.all().order_by('-symbol').values('id', 'symbol', 'category')),
        }
        json.dump(data, resp)

        return resp

    @aux_call
    @needs_onsite
    def enrollment_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        data = StudentRegistration.valid_objects().filter(section__status__gt=0, section__parent_class__status__gt=0, section__parent_class__parent_program=prog, relationship__name='Enrolled').values_list('user__id', 'section__id')
        json.dump(list(data), resp)
        return resp

    @aux_call
    @needs_onsite
    def students_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        #   Try to ensure we don't miss anyone

        students_dict = self.program.students(QObjects=True)
        search_query = request.GET.get('q')

        student_types = ['student_profile']     #   You could add more list names here, but it would get very slow.
        students_Q = Q()

        for student_type in student_types:
            students_Q = students_Q | students_dict[student_type]

        students = ESPUser.objects.filter(students_Q)
        program_students_ids = []

        if search_query:
            #If user provided a search term then we want to expand search to the
            #entire student base

            search_tokens = search_query.split(' ')
            search_qset = Q()

            for token in search_tokens:
                search_qset = search_qset & (Q(last_name__icontains=token) | Q(first_name__icontains=token))

            program_students_ids = set(students.values_list('id', flat=True).distinct())
            students = ESPUser.objects.filter(search_qset)

        students = students.values_list('id', 'last_name', 'first_name') \
                           .distinct() \
                           .order_by('first_name', 'last_name')

        data = []
        for student in students:
            has_profile = not search_query or student[0] in program_students_ids
            data.append(list(student) + [has_profile])

        data.sort(key=lambda x: not x[3])
        if search_query:
            data = data[:20]

        json.dump(data, resp)
        return resp

    @aux_call
    @needs_onsite
    def register_student(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        program = self.program
        success = False
        student = get_object_or_404(ESPUser,pk=request.POST.get("student_id"))

        registration_profile = RegistrationProfile.getLastForProgram(student,
                                                                program)
        success = registration_profile.student_info is not None

        if success:
            registration_profile.save()

            for extension in ['attended','med','liab','onsite']:
                Record.createBit(extension, program, student)

            IndividualAccountingController.updatePaid(self.program, student, paid=True)

        json.dump({'status':success}, resp)
        return resp

    @aux_call
    @needs_onsite
    def checkin_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        students = prog.students(True)
        data = ESPUser.objects.filter(students['attended'] & ~students['checked_out']).distinct().values_list('id')
        json.dump(list(data), resp)
        return resp

    @aux_call
    @needs_onsite
    def counts_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        data = ClassSection.objects.filter(status__gt=0, parent_class__status__gt=0, parent_class__parent_program=prog).values_list('id', 'enrolled_students', 'attending_students')
        json.dump(list(data), resp)
        return resp

    @aux_call
    @needs_onsite
    def full_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        data = [[section.id, section.isFull(webapp=True)] for section in
                     ClassSection.objects.filter(status__gt=0, parent_class__status__gt=0,
                                                 parent_class__parent_program=prog)]
        json.dump(data, resp)
        return resp

    @aux_call
    @needs_onsite
    def rooms_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        data = ClassSection.objects.filter(status__gt=0, parent_class__status__gt=0, parent_class__parent_program=prog, resourceassignment__resource__res_type__name="Classroom").select_related('resourceassignment__resource__name').values_list('id', 'resourceassignment__resource__name', 'resourceassignment__resource__num_students')
        json.dump(list(data), resp)
        return resp

    @aux_call
    @needs_onsite
    def get_schedule_json(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        result = {'user': None, 'user_grade': 0, 'sections': [], 'messages': []}
        try:
            result['user'] = int(request.GET['user'])
        except:
            result['messages'].append('Error: no user specified.')
        if result['user']:
            result['user_grade'] = ESPUser.objects.get(id=result['user']).getGrade(program=prog)
            result['sections'] = list(ClassSection.objects.filter(nest_Q(StudentRegistration.is_valid_qobject(), 'studentregistration'), status__gt=0, parent_class__status__gt=0, parent_class__parent_program=prog, studentregistration__relationship__name='Enrolled', studentregistration__user__id=result['user']).values_list('id', flat=True).distinct())
        json.dump(result, resp)
        return resp

    @aux_call
    @needs_onsite
    def update_schedule_json(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')
        result = {'user': None, 'sections': [], 'messages': []}
        try:
            user = ESPUser.objects.get(id=int(request.GET['user']))
        except:
            user = None
            result['messages'].append('Error: could find user %s' % request.GET.get('user', None))
        try:
            desired_sections = json.loads(request.GET['sections'])
        except:
            result['messages'].append('Error: could not parse requested sections %s' % request.GET.get('sections', None))
            desired_sections = None

        #   Check in student if not currently checked in, since if they're using this view they must be onsite
        if not prog.isCheckedIn(user) and request.GET.get('check_in') == 'true':
            rec = Record(user=user, program=prog, event='attended')
            rec.save()

        if user and desired_sections is not None:
            override_full = (request.GET.get("override", "") == "true")

            current_sections = list(ClassSection.objects.filter(nest_Q(StudentRegistration.is_valid_qobject(), 'studentregistration'), status__gt=0, parent_class__status__gt=0, parent_class__parent_program=prog, studentregistration__relationship__name='Enrolled', studentregistration__user__id=user.id).values_list('id', flat=True).order_by('id').distinct())
            sections_to_remove = ClassSection.objects.filter(id__in=list(set(current_sections) - set(desired_sections)))
            sections_to_add = ClassSection.objects.filter(id__in=list(set(desired_sections) - set(current_sections)))

            failed_add_sections = []
            for sec in sections_to_add:
                if sec.isFull(webapp=True) and not override_full:
                    result['messages'].append('Failed to add %s (%s) to %s: %s (%s).  Error was: %s' % (user.name(), user.id, sec.emailcode(), sec.title(), sec.id, 'Class is currently full.'))
                    failed_add_sections.append(sec.id)

            if len(failed_add_sections) == 0:
                verbs = RTC.getVisibleRegistrationTypeNames(prog)
                #   Remove sections the student wants out of
                for sec in sections_to_remove:
                    sec.unpreregister_student(user, verbs)
                    result['messages'].append('Removed %s (%s) from %s: %s (%s)' % (user.name(), user.id, sec.emailcode(), sec.title(), sec.id))

                #   Remove sections that conflict with those the student wants into
                sec_times = sections_to_add.select_related('meeting_times__id').values_list('id', 'meeting_times__id').order_by('meeting_times__id').distinct()
                sm = ScheduleMap(user, prog)
                existing_sections = []
                for (sec, ts) in sec_times:
                    if ts and ts in sm.map and len(sm.map[ts]) > 0:
                        #   We found something we need to remove
                        for sm_sec in sm.map[ts]:
                            if sm_sec.id not in sections_to_add:
                                sm_sec.unpreregister_student(user, verbs)
                                result['messages'].append('Removed %s (%s) from %s: %s (%s)' % (user.name(), user.id, sm_sec.emailcode(), sm_sec.title(), sm_sec.id))
                            else:
                                existing_sections.append(sm_sec)

                #   Add the sections the student wants
                for sec in sections_to_add:
                    if sec not in existing_sections and sec.id not in failed_add_sections:
                        error = sec.cannotAdd(user, not override_full, webapp=True)
                        if not error:
                            reg_result = sec.preregister_student(user, overridefull=override_full, webapp=True)
                            if not reg_result:
                                error = 'Class is currently full.'
                        else:
                            reg_result = False
                        if reg_result:
                            result['messages'].append('Added %s (%s) to %s: %s (%s)' % (user.name(), user.id, sec.emailcode(), sec.title(), sec.id))
                        else:
                            result['messages'].append('Failed to add %s (%s) to %s: %s (%s).  Error was: %s' % (user.name(), user.id, sec.emailcode(), sec.title(), sec.id, error))

            result['user'] = user.id
            result['sections'] = list(ClassSection.objects.filter(nest_Q(StudentRegistration.is_valid_qobject(), 'studentregistration'), status__gt=0, parent_class__status__gt=0, parent_class__parent_program=prog, studentregistration__relationship__name='Enrolled', studentregistration__user__id=result['user']).values_list('id', flat=True).distinct())

        json.dump(result, resp)
        return resp

    """ End of highly model-dependent JSON views    """

    @aux_call
    @needs_onsite
    def printschedule_status(self, request, tl, one, two, module, extra, prog):
        resp = HttpResponse(content_type='application/json')

        result = {}

        try:
            user = int(request.GET.get('user', None))
            user_obj = ESPUser.objects.get(id=user)
        except:
            result['message'] = "Could not find user %s." % request.GET.get('user', None)

        printer = request.GET.get('printer',None)
        if printer is not None:
            # we could check that it exists and is unique first, but if not, that should be an error anyway, and it isn't the user's fault unless they're trying to mess with us, so a 500 is reasonable and gives us better debugging output.
            printer = Printer.objects.get(name=printer)
        req = PrintRequest.objects.create(user=user_obj, printer=printer)
        result['message'] = "Submitted %s's schedule for printing (print request #%s)." % (user_obj.name(), req.id)

        json.dump(result, resp)
        return resp

    @aux_call
    @needs_onsite
    def classchange_grid(self, request, tl, one, two, module, extra, prog):
        context = {}
        context['timeslots'] = prog.getTimeSlots()
        context['printers'] = Printer.objects.all().values_list('name', flat=True)
        context['initial_student'] = request.GET.get('student_id', '')
        context['check_in_default'] = datetime.today().date() in prog.dates()

        open_class_category = prog.open_class_category
        open_class_category = dict( [ (k, getattr( open_class_category, k )) for k in ['id','symbol','category'] ] )
        context['open_class_category'] = mark_safe(json.dumps(open_class_category))

        return render_to_response(self.baseDir()+'ajax_status.html', request, context)

    @aux_call
    @needs_onsite
    def classList(self, request, tl, one, two, module, extra, prog):
        return self.classList_base(request, tl, one, two, module, extra, prog, template_name='classlist.html')

    @aux_call
    @needs_student
    def classlist_public(self, request, tl, one, two, module, extra, prog):
        return self.classList_base(request, tl, one, two, module, extra, prog, options={}, template_name='allclass_fragment.html')

    def classList_base(self, request, tl, one, two, module, extra, prog, options=None, template_name='classlist.html'):
        """ Display a list of all classes that still have space in them """

        #   Allow options to be supplied as an argument to the view function, in lieu
        #   of request.GET (used by the public view just above)
        if options is None:
            options = request.GET.copy()

            #   Display options-selection page if this page is requested with no GET variables
            if len(options.keys()) == 0:
                return render_to_response(self.baseDir() + 'classlist_options.html', request, {'prog': prog})

        context = {}
        defaults = {'refresh': 120, 'scrollspeed': 1}
        for key_option in defaults.keys():
            if key_option in options:
                context[key_option] = options[key_option]
            else:
                context[key_option] = defaults[key_option]

        time_now = datetime.now()

        start_id = int(options.get('start', -1))
        if start_id != -1:
            curtime = Event.objects.filter(id=start_id)
        else:
            window_start = time_now + timedelta(-1, 85200)  # 20 minutes ago
            curtime = Event.objects.filter(start__gte=window_start, event_type__description='Class Time Block').order_by('start')
            # If there are no events after the current time, just pick the first event of the program
            if curtime.count() == 0:
                curtime = Event.objects.filter(program=self.program, event_type__description='Class Time Block').order_by('start')

        end_id = int(options.get('end', -1))
        if end_id != -1:
            endtime = Event.objects.filter(id=end_id)
        else:
            endtime = None

        sort_spec = options.get('sorting', None)
        if sort_spec is None:
            sort_spec = extra

        #   Enforce a maximum refresh speed to avoid server overload.
        min_refresh = int(Tag.getTag('onsite_classlist_min_refresh'))
        if int(context['refresh']) < min_refresh:
            context['refresh'] = min_refresh

        classes = []
        if curtime:
            curtime = curtime[0]
            if endtime:
                endtime = endtime[0]
                classes = self.program.sections().annotate(begin_time=Min("meeting_times__start")).filter(
                    status=ClassStatus.ACCEPTED, parent_class__status=ClassStatus.ACCEPTED,
                    begin_time__gte=curtime.start, begin_time__lte=endtime.start
                    )
            else:
                 classes = self.program.sections().annotate(begin_time=Min("meeting_times__start")).filter(
                     status=ClassStatus.ACCEPTED, parent_class__status=ClassStatus.ACCEPTED,
                     begin_time__gte=curtime.start
                     )
            if sort_spec == 'unsorted':
                classes = classes.order_by('begin_time', 'id').distinct()
            elif sort_spec == 'by_time':
                classes = classes.order_by('begin_time', 'parent_class__category', 'id').distinct()
            else:
                classes = classes.order_by('parent_class__category', 'begin_time', 'id').distinct()

        context.update({'prog': prog, 'current_time': curtime, 'classes': classes, 'one': one, 'two': two})

        if sort_spec == 'unsorted':
            context['use_categories'] = False
        else:
            context['use_categories'] = True

        return render_to_response(self.baseDir()+template_name, request, context)

    @main_call
    @needs_onsite
    def allClassList(self, request, tl, one, two, module, extra, prog):
        """ Display a list of all classes that still have space in them """

        #   This view still uses classes, not sections.  The templates show information
        #   for each section of each class.
        classes = [(i.num_students()/(i.class_size_max + 1), i) for i in self.program.classes() if i.class_size_max]
        classes.sort()
        classes = [i[1] for i in classes]

        categories = {}
        for cls in classes:
            categories[cls.category_id] = {'id': cls.category_id, 'category': cls.category.category}

        printers = [ x.name for x in Printer.objects.all() ]

        return render_to_response(self.baseDir()+'allclasslist.html', request,
            {'classes': classes, 'prog': self.program, 'one': one, 'two': two, 'categories': categories.values(), 'printers': printers})

    def makeLink(self):
        calls = [("classchange_grid","Grid-based Class Changes Interface"), ("classList","Scrolling Class List"), (self.get_main_view(),self.module.link_title)]
        strings = [u'<a href="%s" title="%s" class="vModuleLink" >%s</a>' % \
                ('/' + self.module.module_type + '/' + self.program.url + '/' + call[0], call[1], call[1]) for call in calls]
        return "</li><li>".join(strings)

    def makeButtonLink(self):
        calls = [("classchange_grid","Grid-based Class Changes Interface"), ("classList","Scrolling Class List"), (self.get_main_view(),self.module.link_title)]
        strings = [u'<a href="%s"><button type="button" class="module_link_large btn btn-default btn-lg"><div class="module_link_main">%s</div></button></a>' % \
                ('/' + self.module.module_type + '/' + self.program.url + '/' + call[0], call[1]) for call in calls]
        return "<br>".join(strings)

    class Meta:
        proxy = True
        app_label = 'modules'


from typing import Optional
from pydantic import BaseModel

import uvicorn
from fastapi import FastAPI

#버킷과 파일 이름은 여기서 결정된다. 다른 곳에서는 이 값을 받아와 사용
#
class ServerStatus(BaseModel):
    S3_bucket: str = 'ccl-fl-demo-model'
    S3_key: str = 'model.h5'  # 모델 가중치 파일 이름
    FLSeReady: bool = False
    Model_V: int = 0 #모델버전

app = FastAPI()

FLSe = ServerStatus()

@app.get("/")
def read_root():
    return {"Hello": "World"}

@app.get("/FLSe/info")
def read_status():
    return {"Server_Status": FLSe}

@app.put("/FLSe/FLSeUpdate")
def update_status(Se: ServerStatus):
    global FLSe
    FLSe = Se
    return {"Server_Status": FLSe}

@app.put("/FLSe/FLSeReady")
def update_ready(i: bool):
    global FLSe
    FLSe.FLSeReady = i
    if i==False:
        FLSe.Model_V += 1
    return {"Server_Status": FLSe}

if __name__ == "__main__":
    uvicorn.run("app:app", host="0.0.0.0", port=8000, reload=True)
    print('asdf')


import json
import logging

import odoo
from odoo import models, fields, api
from odoo.tools.translate import _
from odoo.exceptions import ValidationError,UserError

from .process import TimeConverterDate

_logger = logging.getLogger(__name__)

class Case(models.Model):
    _inherit = 'syd_bpm.case'

    pm_assigned_to = fields.Many2one('res.users',string="Assigned To")
    pm_case_id = fields.Char(string='Process Maker Task ID', required=False)
    activity_id = fields.Many2one('syd_bpm.activity',string="Activity")
    pm_element_id = fields.Char(string='Process Maker Element ID')
    pm_element_type = fields.Char(string='Process Maker Element Type')
    pm_element_name = fields.Char(string='Process Maker Element Name')
    related_model = fields.Char(string='Related Model')
    related_id = fields.Char(string='Related Record ID')
    is_assigned_to = fields.Boolean(string='Is Assigned To',compute='_compute_is_assigned_to')
    odoo_activity_id = fields.Many2one("mail.activity", string="Mail Activity")

    def _compute_is_assigned_to(self):
        for record in self:
            record.is_assigned_to = bool(record.pm_assigned_to.id == self.env.uid)

    def create(self, vals):
        case =  super(Case, self).create(vals)
        _vals = {
            "res_name": "",
            "date_deadline": case.date_deadline,
            "activity_type_id": self.env['ir.model.data']._xmlid_to_res_id('mail.mail_activity_data_todo', raise_if_not_found=False),
            "user_id": case.pm_assigned_to.id,
            "summary": "",
            "res_id": int(case.related_id),
            "res_model_id": self.env['ir.model'].search([('model', '=', case.related_model)]).id
        }
        if case.activity_id and "-" in case.activity_id.name:
            _vals.update({"res_name": case.activity_id.name.split('-')[1]})

        activity = self.env['mail.activity'].create(_vals)
        print(activity)
        case.write({'odoo_activity_id': activity.id})
        return case

    def confirm_case(self, result="pass"):
        """
        /tasks/{task_id} Update a task
        """
        # TODO 通过process_id找到对应Process的Dynamic Form, 在Dynamic Form找到同名的Case并获取所需的字段
        if result not in ['pass', 'refuse']:
            raise ValidationError("审批结论传值错误，必须是pass或者refuse！")

        _data = {
            "status": "COMPLETED",
            "data": {
                "result": result
            }
        }
        if self.process_id:
            res = self.process_id.process_group_id._call(f'tasks/{self.pm_case_id}', jsonobject=json.dumps(_data), method='PUT')
            if res:
                if res.get('status') and res.get('status')=='CLOSED':
                    self.state='completed'
                params = {
                    'process_request_id': int(self.activity_id.pm_activity_id)
                }
                pm_tasks = self.process_id.process_group_id._call('tasks', params, method='GET')
                if pm_tasks.get('data'):
                    for item in pm_tasks.get('data'):
                        print(item)
                        _domain = [
                            ('pm_case_id', '=', item.get('id')),
                        ]
                        task = self.env['syd_bpm.case'].search(_domain)
                        if not task:
                            _state='in_progress'
                            if item.get('status') and item.get('status')=='CLOSED':
                                _state = 'cancelled'
                            elif item.get('status') and item.get('status')=='COMPLETED':
                                _state = 'completed'
                            _val = {
                                'pm_case_id': item.get('id'),
                                'name': item.get('element_name'),
                                'activity_id': self.activity_id.id,
                                'process_id': self.process_id.id,
                                'related_model':self.related_model,
                                'related_id':int(self.related_id) if isinstance(self.related_id, int) else str(self.related_id),
                                'state':_state,
                                'date_deadline': TimeConverterDate(item.get("due_at")),
                            }
                            _user = self.env['res.users'].search([('pm_user_id', '=', item.get('user_id'))])

                            if _user:
                                _val.update({'pm_assigned_to': _user.id})
                            else:
                                _val.update({'pm_assigned_to': self.env['res.users'].search([],limit=1).id})
                            self.env['syd_bpm.case'].create(_val)
                else:
                    _logger.warning(pm_tasks)
            else:
                _logger.warning("更细task没有成功...")
            return True


import numpy as np
import pandas as pd
import cv2
import torch
import torch.nn as nn
import random
import os
from albumentations import (
    HorizontalFlip, VerticalFlip, CenterCrop, RandomRotate90, RandomCrop,
    PadIfNeeded, Normalize, Flip, OneOf, Compose, Resize, Transpose,
    IAAAdditiveGaussianNoise, GaussNoise, CLAHE, RandomBrightnessContrast, HueSaturationValue,
    HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, RandomBrightnessContrast, IAAPiecewiseAffine,
    IAASharpen, IAAEmboss, Flip, OneOf, Compose
)
import collections
from torch.utils.data import Dataset, DataLoader
from efficientnet_pytorch import EfficientNet

def crop_image_from_gray(img, tol=7):
    if img.ndim == 2:
        mask = img > tol
        return img[np.ix_(mask.any(1), mask.any(0))]
    elif img.ndim == 3:
        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
        mask = gray_img > tol

        check_shape = img[:, :, 0][np.ix_(mask.any(1), mask.any(0))].shape[0]
        if (check_shape == 0):  # image is too dark so that we crop out everything,
            return img  # return original image
        else:
            img1 = img[:, :, 0][np.ix_(mask.any(1), mask.any(0))]
            img2 = img[:, :, 1][np.ix_(mask.any(1), mask.any(0))]
            img3 = img[:, :, 2][np.ix_(mask.any(1), mask.any(0))]
            img = np.stack([img1, img2, img3], axis=-1)
        return img

def arguments(fold_idx, batch_size, model_name, image_size, head_n_epochs, head_lr, full_n_epochs, full_lr, exp_name):
	print('Will run with the following arguments: \n fold_idx : {}, model_name : {}, batch_size : {} \n image_size : {}, head_n_epochs : {}, head_lr : {}, full_n_epochs : {}, full_lr : {}, exp_name : {}'.format(fold_idx, model_name, batch_size, image_size, head_n_epochs, head_lr, full_n_epochs, full_lr, exp_name))
	return(fold_idx, batch_size, model_name, image_size, head_n_epochs, head_lr, full_n_epochs, full_lr, exp_name)

def prepare_model(model_name, n_classes):
	print('Will load {} with {} classes'.format(model_name, n_classes))
	model = EfficientNet.from_pretrained(model_name)
	in_features = model._fc.in_features
	model._fc = nn.Linear(in_features, n_classes)
	model.cuda()
	return(model)

def seed_everything(seed):
	random.seed(seed)
	os.environ['PYTHONHASHSEED'] = str(seed)
	np.random.seed(seed)
	torch.manual_seed(seed)
	torch.cuda.manual_seed(seed)
	torch.backends.cudnn.deterministic = True

class EyeDataset(Dataset):
    def __init__(self, dataset_path, labels, ids, albumentations_tr=False, shuffle=True):
        self.labels = labels
        self.ids = ids
        self.shuffle = shuffle
        self.dataset_path = dataset_path
        self.albumentations_tr = albumentations_tr

    def __len__(self):
        return len(self.ids)

    def __getitem__(self, index):
        image = cv2.imread(os.path.join(self.dataset_path, self.ids[index]))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image = crop_image_from_gray(image)
        if self.albumentations_tr:
            augmented = self.albumentations_tr(image=image)
            image = augmented['image']
        target = self.labels[index]
        return torch.from_numpy(image.transpose((2, 0, 1))).float(), torch.tensor(target).float()

def aug_train(resolution,p=1):
    return Compose([Resize(resolution,resolution),
                    OneOf([
                        HorizontalFlip(),
                        VerticalFlip(),
                        RandomRotate90(),
                        Transpose()],p=0.5),
                    Normalize()
                    ], p=p)
def aug_train_heavy(resolution, p=1.0):
    return Compose([
        Resize(resolution,resolution),
        OneOf([RandomRotate90(),Flip(),Transpose(),HorizontalFlip(),VerticalFlip()],p=1.0),
        OneOf([
            IAAAdditiveGaussianNoise(),
            GaussNoise(),
        ], p=0.5),
        OneOf([
            MotionBlur(p=.2),
            MedianBlur(blur_limit=3, p=0.1),
            Blur(blur_limit=3, p=0.1),
        ], p=0.2),
        ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2),
        OneOf([
            OpticalDistortion(p=0.3),
            GridDistortion(p=.1),
            IAAPiecewiseAffine(p=0.3),
        ], p=0.1),
        OneOf([
            CLAHE(clip_limit=2),
            IAASharpen(),
            IAAEmboss(),
            RandomBrightnessContrast(),
        ], p=0.3),
        HueSaturationValue(p=0.3),
        Normalize()
    ], p=p)

def aug_val(resolution,p=1):
    return Compose([Resize(resolution,resolution),Normalize()], p=p)


import pandas as pd
import numpy as np
import charlieTools.statistics as stats

def get_psth_pvals(R, spont, cellids=None):
    '''
    Determine if significant response for each cell, for each time bin,
         using jackknifed t-test.

    R: folded response matrix (reps X cells X time)
    R0: folded spont matrix (reps X cells X time)

    cellids: list of cellids corresponding to R.shape[1]
    '''

    if cellids is None:
        index = np.arange(0, R.shape[1])
    else:
        index = cellids

    bins = np.arange(0, R.shape[-1])
    sig_df = pd.DataFrame(index=index, columns=[bins])

    for i, idx in enumerate(index):
        x0 = spont[:, i, :].mean()
        for t in bins:
            x = R[:, i, t]
            res = stats._do_jackknife_ttest(x, x0)
            sig_df.loc[idx, t] = res.pvalue

    return sig_df