#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Disable pylint invalid module name complaint.
# pylint : disable = C0103

""" Pre-commit script for invoking linters and auto-formatters."""

# Make sure we use the new print function
from __future__ import print_function
from random import randint

import datetime
import imp
import os
import pylint.lint
import re
import subprocess

CLANG_FORMAT_DIFF_EXECUTABLE = "clang-format-diff-3.8"
AUTOPEP8_FORMAT_EXECUTABLE = "autopep8"

def run_command_in_folder(command, folder):
  """Run a bash command in a specific folder."""
  run_command = subprocess.Popen(command,
                                 shell=True,
                                 cwd=folder,
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE)
  stdout, _ = run_command.communicate()
  command_output = stdout.rstrip()
  return command_output

def get_number_of_commits(some_folder_in_root_repo='./'):
  command = "git shortlog -s -n --all --author=\"$(git config --get user.email)\" | grep -o '[0-9]\+' |  awk '{s+=$1} END {print s}'"
  return run_command_in_folder(command, some_folder_in_root_repo)

def get_staged_files(some_folder_in_root_repo='./'):
  """Get all staged files from git."""
  output = run_command_in_folder(
      "git diff --staged --name-only", some_folder_in_root_repo)
  if len(output) == 0:
    return []
  else:
    return output.split("\n")

def get_unstaged_files(some_folder_in_root_repo='./'):
  """Get all unstaged files from git."""
  output = run_command_in_folder(
      "git diff --name-only", some_folder_in_root_repo)
  return output.split("\n")

def check_cpp_lint(staged_files, cpplint_file, ascii_art, repo_root):
  """Runs Google's cpplint on all C++ files staged for commit,"""
  cpplint = imp.load_source('cpplint', cpplint_file)
  cpplint._cpplint_state.SetFilters(
      '-legal/copyright,-build/c++11')  # pylint: disable=W0212

  # Prevent cpplint from writing to stdout directly, instead
  # the errors will be stored in pplint.output as (line, message) tuples.
  cpplint.print_stdout = False

  total_error_count = 0
  for changed_file in staged_files:
    if not os.path.isfile(changed_file):
      continue
    if changed_file.lower().endswith(('.cc', '.h', '.cpp', '.cu', '.cuh')):
      # Search iteratively for the root of the catkin package.
      package_root = ''
      search_dir = os.path.dirname(changed_file)
      found_package_root = False
      MAX_DEPTH_OF_FILES = 100
      for _ in range(1, MAX_DEPTH_OF_FILES):
        if os.path.isfile(search_dir + '/package.xml'):
          package_root = search_dir
          found_package_root = True
          break
        search_dir = os.path.dirname(search_dir)
      assert found_package_root, ("Could not find the root of the "
                                  "catkin package that contains: "
                                  "{}".format(changed_file))

      cpplint._root = package_root + '/include'   # pylint: disable=W0212

      # Reset error count and messages:
      cpplint.output = []
      cpplint._cpplint_state.ResetErrorCounts()   # pylint: disable=W0212
      v_level = cpplint._cpplint_state.verbose_level  # pylint: disable=W0212

      cpplint.ProcessFile(changed_file, v_level)

      error_count = cpplint._cpplint_state.error_count  # pylint: disable=W0212
      if error_count > 0:
        total_error_count += error_count

        print("-" * 80)
        print("Found {} errors in : {}".format(
            error_count, changed_file))
        print("-" * 80)
        for line in cpplint.output:
          assert len(line) == 2
          print("line {: >4}:\t {}".format(line[0], line[1]))

  if total_error_count > 0:
    print("=" * 80)
    print("Found {} cpplint errors".format(total_error_count))
    print("=" * 80)

    if total_error_count > 50:
      print(ascii_art.AsciiArt.cthulhu)
    elif total_error_count > 20:
      print(ascii_art.AsciiArt.tiger)
    return False
  else:
    return True

def check_modified_after_staging(staged_files):
  """Checks if one of the staged files was modified after staging."""
  files_changed = get_unstaged_files()
  files_changed = filter(None, files_changed)  # pylint: disable=W0141

  staged_files_changed = 0

  staged_files_changed_list = []

  is_first = True
  for changed_file in files_changed:
    if changed_file in staged_files:

      if is_first:
        print("=" * 80)
        is_first = False

      print("\'{}\' was modified after staging".format(changed_file))
      staged_files_changed_list.append(changed_file)
      staged_files_changed += 1

  if staged_files_changed > 0:
    print("-" * 80)
    print("Found {} files that have been changed after staging".format(
        staged_files_changed))
    print("=" * 80)

  return staged_files_changed_list

def check_commit_against_master(repo_root):
  """Check if the current commit is intended to for the master branch."""
  current_branch = run_command_in_folder(
      "git branch | grep \"*\" | sed \"s/* //\"", repo_root)
  print("\nCurrent_branch: {}\n".format(current_branch))
  return current_branch == "master"

def check_if_merge_commit(repo_root):
  """Check if the current commit is a merge commit."""
  merge_msg_file_path = repo_root + "/.git/MERGE_MSG"
  return os.path.isfile(merge_msg_file_path)

def run_clang_format(repo_root, staged_files, list_of_changed_staged_files):
  """Runs clang format on all cpp files staged for commit."""

  clang_format_path = ("/tmp/" + os.path.basename(
      os.path.normpath(repo_root)) + "_" +
      datetime.datetime.now().isoformat() + ".clang.patch")

  run_command_in_folder("git diff -U0 --cached | " +
                        CLANG_FORMAT_DIFF_EXECUTABLE +
                        " -style=file -p1 > " +
                        clang_format_path, repo_root)

  if not os.stat(clang_format_path).st_size == 0:
    if list_of_changed_staged_files:
      print("=" * 80)
      print("Cannot format your code, because some files are \n"
            "only partially staged! Format your code or try \n"
            "stashing your unstaged changes...")
      print("=" * 80)
      exit(1)

    run_command_in_folder("git apply -p0 " + clang_format_path, repo_root)
    run_command_in_folder("git add " + ' '.join(staged_files), repo_root)

    print("=" * 80)
    print("Formatted staged C++ files with clang-format.\n" +
          "Patch: {}".format(clang_format_path))
    print("=" * 80)
  return True

def run_autopep8_format(repo_root, staged_files, list_of_changed_staged_files):
  """Runs autopep8 format on all python files staged for commit."""

  first_file_formatted = True
  for staged_file in staged_files:
    if not os.path.isfile(staged_file):
      continue
    if staged_file.endswith((".py")):

      # Check if the file needs formatting by applying the formatting and store
      # the results into a patch file.
      autopep8_format_path = ("/tmp/" +
                              os.path.basename(os.path.normpath(repo_root)) +
                              "_" + datetime.datetime.now().isoformat() +
                              ".autopep8.patch")
      task = (AUTOPEP8_FORMAT_EXECUTABLE +
              " -d --aggressive --aggressive --max-line-length=80 " +
              "--indent-size=2 --ignore=E24 " +
              staged_file + " > " + autopep8_format_path)
      run_command_in_folder(task, repo_root)

      if not os.stat(autopep8_format_path).st_size == 0:
        if first_file_formatted:
          print("=" * 80)
          print("Formatted staged python files with autopep8:")
          first_file_formatted = False

        print("-> " + staged_file)

        if staged_file in list_of_changed_staged_files:
          print("Cannot format your code, because this file is \n"
                "only partially staged! Format your code or try \n"
                "stashing your unstaged changes...")
          print("=" * 80)
          exit(1)
        else:
          run_command_in_folder(
              "git apply -p1 " + autopep8_format_path, repo_root)
          run_command_in_folder("git add " + staged_file, repo_root)

  if not first_file_formatted:
    print("=" * 80)
  return True

def check_python_lint(repo_root, staged_files, pylint_file):
  """Runs pylint on all python scripts staged for commit."""

  class TextReporterBuffer(object):
    """Stores the output produced by the pylint TextReporter."""

    def __init__(self):
      """init"""
      self.content = []

    def write(self, input_str):
      """write"""
      self.content.append(input_str)

    def read(self):
      """read"""
      return self.content

  # Parse each pylint output line individualy and searches
  # for errors in the code.
  pylint_errors = []
  for changed_file in staged_files:
    if not os.path.isfile(changed_file):
      continue
    if re.search(r'\.py$', changed_file):

      print("Running pylint on \'{}\'".format(repo_root + "/" + changed_file))
      pylint_output = TextReporterBuffer()
      pylint_args = ["--rcfile=" + pylint_file,
                     "-rn",
                     repo_root + "/" + changed_file]
      from pylint.reporters.text import TextReporter
      pylint.lint.Run(pylint_args,
                      reporter=TextReporter(pylint_output),
                      exit=False)

      for output_line in pylint_output.read():
        if re.search(r'^(E|C|W):', output_line):
          print(changed_file + ": " + output_line)
          pylint_errors.append(output_line)

  if len(pylint_errors) > 0:
    print("=" * 80)
    print("Found {} pylint errors".format(len(pylint_errors)))
    print("=" * 80)
    return False
  else:
    return True

def linter_check(repo_root, linter_subfolder):
  """ Main pre-commit function for calling code checking script. """

  cpplint_file = repo_root + "/" + linter_subfolder + "/cpplint.py"
  pylint_file = repo_root + "/" + linter_subfolder + "/pylint.rc"
  ascii_art_file = repo_root + "/" + linter_subfolder + "/ascii_art.py"

  print("Found linter subfolder: {}".format(linter_subfolder))
  print("Found ascii art file at: {}".format(ascii_art_file))
  print("Found cpplint file at: {}".format(cpplint_file))
  print("Found pylint file at: {}".format(pylint_file))

  # Run checks
  staged_files = get_staged_files()

  if len(staged_files) == 0:
    print("\n")
    print("=" * 80)
    print("No files staged...")
    print("=" * 80)
    exit(1)

  # Load ascii art.
  ascii_art = imp.load_source('ascii_art', ascii_art_file)

  if check_commit_against_master(repo_root):
    print(ascii_art.AsciiArt.grumpy_cat)
    exit(1)

  if not check_if_merge_commit(repo_root):
    # Do not allow commiting files that were modified after staging.This
    # avoids problems such as forgetting to stage fixes of cpplint complaints.
    list_of_changed_staged_files = check_modified_after_staging(staged_files)

    run_clang_format(repo_root, staged_files, list_of_changed_staged_files)
    run_autopep8_format(repo_root, staged_files, list_of_changed_staged_files)

    # Use Google's C++ linter to check for compliance with Google style guide.
    cpp_lint_success = check_cpp_lint(
        staged_files, cpplint_file, ascii_art, repo_root)

    # Use pylint to check for comimpliance with Tensofrflow python style guide.
    pylint_success = check_python_lint(repo_root, staged_files, pylint_file)

    if not(cpp_lint_success and pylint_success):
      print("=" * 80)
      print("Commit rejected! Please address the linter errors above.")
      print("=" * 80)
      exit(1)
    else:

      commit_number = get_number_of_commits(repo_root)
      lucky_commit = ((int(commit_number) + 1) % 42 == 0)

      if lucky_commit:
        print(ascii_art.AsciiArt.story)

        print("=" * 80)
        print("Commit accepted, well done! This is your {}th commit!".format(
            commit_number))
        print("This is a lucky commit! Please enjoy this free sheep story.")
        print("=" * 80)
      else:
        print(ascii_art.AsciiArt.commit_success)

        print("=" * 80)
        print("Commit accepted, well done! This is the {}th commit!".format(
            commit_number))
        print("=" * 80)
  else:
    print(ascii_art.AsciiArt.homer_woohoo)


import numpy as np
import cv2
import tensorflow as tf

# ce fichier va permettre d'extraire les caractéristiques des images comprise dans les boxs afin de pouvoir appliquer le tracking
def _run_in_batches(f, data_dict, out, batch_size):
    data_len = len(out) #taille de l'élément out
    num_batches = int(data_len / batch_size)

    s, e = 0, 0
    for i in range(num_batches):
        s, e = i * batch_size, (i + 1) * batch_size
        batch_data_dict = {k: v[s:e] for k, v in data_dict.items()}
        out[s:e] = f(batch_data_dict)
    if e < len(out):
        batch_data_dict = {k: v[e:] for k, v in data_dict.items()}
        out[e:] = f(batch_data_dict)

def extract_image_patch(image, bbox, patch_shape):

    #cette fonction permet de retrouner l'image de la box
    #image : l'image en entiére
    #bbox : la box dans le  format (x, y, width, height)
    #patch_shape : ce paramètre permet de donner des dimensions souhaitée , ce qui va donc modifier les dimensions de la box
    bbox = np.array(bbox) # création d'un vecteur avec les coordonées de la box, les coordonées sont en format (x,y, width, height)
    if patch_shape is not None: # si il y a un élément patch_shape

        # rapport hauteur/ largeur à la forme du patch
        target_aspect = float(patch_shape[1]) / patch_shape[0]
        new_width = target_aspect * bbox[3] #calcul de la nouvelle largeur
        bbox[0] -= (new_width - bbox[2]) / 2
        bbox[2] = new_width #enregistrement de la nouvelle largueur

    bbox[2:] += bbox[:2] #transforme les coodonées en coin  haut gauche (x,y) et bas droite (x,y)
    bbox = bbox.astype(np.int) # transormation en entier

    # clip aux limites de l'image
    bbox[:2] = np.maximum(0, bbox[:2]) #prend les valeurs de x,y de bbox si elles sont positives
    bbox[2:] = np.minimum(np.asarray(image.shape[:2][::-1]) - 1, bbox[2:])
    if np.any(bbox[:2] >= bbox[2:]): #si la box est vide
        return None
    sx, sy, ex, ey = bbox # enregistre les valeurs
    image = image[sy:ey, sx:ex] # enregistrement de la partie de l'image où se situe la box
    image = cv2.resize(image, tuple(patch_shape[::-1])) # resize de l'image au format voulu afin d'avoir que l'image de la box
    return image

class ImageEncoder(object):

    def __init__(self, checkpoint_filename, input_name="images",output_name="features"): #constructeur de la classe
        self.session = tf.Session() #création d'une session de tensorflow
        with tf.gfile.GFile(checkpoint_filename, "rb") as file_handle:
            #transformation du message en message binaire
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(file_handle.read())

        tf.import_graph_def(graph_def, name="net") #importation du nouveau message
        self.input_var = tf.get_default_graph().get_tensor_by_name("net/%s:0" % input_name) #obtention du nom du tensor d'input
        self.output_var = tf.get_default_graph().get_tensor_by_name("net/%s:0" % output_name) #obtention du nom du tensor d'output

        assert len(self.output_var.get_shape()) == 2  #vérification de la taille de output_var
        assert len(self.input_var.get_shape()) == 4   #vérification de la taille de input_var
        self.feature_dim = self.output_var.get_shape().as_list()[-1] #enregistre une liste d'intenger  du dernier élément
        self.image_shape = self.input_var.get_shape().as_list()[1:]  #enregistre une liste d'intenger de tout les éléments sauf le premier

    def __call__(self, data_x, batch_size=32): #fonction appeller durant l'utilisation de l'objet de la classe ImageEncoder
        out = np.zeros((len(data_x), self.feature_dim), np.float32) # création d'une matrice de zeros de taille (nombre d'image de box , feature_dime)

        _run_in_batches(lambda x: self.session.run(self.output_var, feed_dict=x),{self.input_var: data_x}, out, batch_size)
        return out

def create_box_encoder(model_filename, input_name="images",output_name="features", batch_size=32):
    image_encoder = ImageEncoder(model_filename, input_name, output_name) #création d'un objet de la classe Image encoder
    image_shape = image_encoder.image_shape #enregistrement de la taille de l'image

    def encoder(image, boxes):
        image_patches = []
        for box in boxes: # pour chaque élément du vecteurs boxes qui sont en format (x, y, width, height)
            patch = extract_image_patch(image, box, image_shape[:2]) #permet d'obtenir l'image de la box
            if patch is None: # si pas d'élément
                print("WARNING: Échec de l'extraction du patch d'image: %s." % str(box)) # affichage d'une erreur et ajout d'un élément random
                patch = np.random.uniform( 0., 255., image_shape).astype(np.uint8)
            image_patches.append(patch) #ajout de l'image à la liste des toutes les images de boxs
        image_patches = np.asarray(image_patches) #création d'un vecteur
        return image_encoder(image_patches, batch_size) #utilisation de la fct __call__ de la classe ImageEncoder

    return encoder #appele de la fonction encoder


from requests import get
from pandas import DataFrame as pdf, read_csv, concat, Series, set_option, options, option_context
from numpy import arange, array
import cvxpy

options.display.width = 0
set_option('display.max_columns', None)
#set_option('display.max_rows', None)

def get_player(x): return xpts.loc[x[0], 'Name']
def decay(row, rate): return row * rate**arange(0, len(row))
def x90(mins): return 0.000215171*(mins**2) - 0.04515889*mins + 2.342686553 + 1

base_path = '/Users/desmundhui/Downloads/'
xpts = read_csv(base_path + 'fplreview.csv')
xpts['Pos'] = xpts['Pos'].astype("category").cat.set_categories(['F', 'M', 'D', 'G'])

#approximate points assuming all xMin=90
xpts_90 = xpts.copy()
for i in range(5, xpts_90.shape[1], 2):
    xpts_90.iloc[:, i+1] *= xpts_90.iloc[:, i].map(x90)

#decay([1,1,1,1,1,1,1,1], 0.84)
#adjustment for form of player, using a factor of 0.8
#form = read_csv(base_path + 'fplreview_formFactor.csv', encoding= 'unicode_escape')
#form['FF_0.8'] = (((form['Form Factor'] - 1) * 0.8) + 1).fillna(1)

#check size of dataframe first
#if xpts.shape[0] != form.shape[0]:
#    print('New player(s) in fplreview.csv, please update form.csv first.')
#    quit()

#cvxpy
def optim(itb=100, incl=[], excl=[], preset=[], weeks_from=1, weeks_to=8, ft=1, use_form=False, use_90=False,
    bench_weight=[0, 0.2, 0.05, 0], BB=0, TC=0, WC=0, FH=0, decay_rate=0.84):
    global xpts
    #use form to estimate
    #if use_form: xpts.iloc[:, 5:] = xpts.iloc[:, 5:].mul(form['FF_0.8'], axis=0)
    #use 90xMin
    if use_90: xpts = xpts_90
    #Isolate the targeted GWs, and eliminate poor value options
    pts_df = xpts.filter(like='_Pts', axis=1).iloc[:,weeks_from-1:weeks_to]
    xpts['total'] = pts_df.sum(axis=1)
    #include v cheap players that may offer no value (in terms of ppm)
    cheap = xpts[xpts['SV']<=4.5].sort_values(['SV', 'total'], ascending=[True, False])
    cheap_idx = (
        cheap[cheap['SV']<=4].groupby(['Pos', 'SV']).head(5).index.union(  #take first 5 G & D of each price point <= 4.0
            cheap[(cheap['Pos']=='M') | (cheap['Pos']=='F')].groupby(['Pos', 'SV']).head(5).index #M & D, <= 4.5
        )
    )

    xpts['value'] = xpts['total'] / xpts['SV'] / pts_df.shape[1]
    xpts2 = concat([xpts[xpts['value'] > (0.35 if not use_90 else 0.5)],xpts.loc[cheap_idx, :],xpts.iloc[preset, :]]).drop_duplicates()
    pts_df2 = xpts2.filter(like='_Pts', axis=1).iloc[:,weeks_from-1:weeks_to]
    #time decay
    if decay_rate != 1:
        pts_df2 = pts_df2.apply(decay, axis=1, args=(decay_rate,))
        undecay = (1/decay_rate)**arange(0, 8)
        undecay_total = 0

    GW = int(pts_df2.columns[0][:-4]) #initial GW
    weeks_to = min(weeks_to, 38+weeks_from-GW) #in case weeks_to exceed GW38

    pts_dict = pts_df2.to_dict('list')
    sell_price = xpts2['SV'].tolist()
    buy_price = xpts2['BV'].tolist()

    var_dict = {}
    price_dict = {}
    total_points = 0
    constraints = []

    var_dict[f'{GW}_FT'] = ft
    var_dict[f'{GW}_itb'] = itb

    for gw in range(GW, GW - weeks_from + weeks_to + 1):#gw=34
        #variable construction
        var_dict[f'{gw}_selction'] = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_bench_0']  = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_bench_1']  = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_bench_2']  = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_bench_3']  = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_squad']    = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_captain']  = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_auxiliary']= cvxpy.Variable(boolean=True) #refer to Optimization for FPL (with Excel) - Part 3
        var_dict[f'{gw}_tra_used'] = cvxpy.Variable(integer=True)
        var_dict[f'{gw}_incoming'] = cvxpy.Variable(xpts2.shape[0], boolean=True)
        var_dict[f'{gw}_outgoing'] = cvxpy.Variable(xpts2.shape[0], boolean=True)

        #price dict
        price_dict[f'{gw}_sold_amt'] = cvxpy.sum(sell_price @ var_dict[f'{gw}_outgoing'])
        price_dict[f'{gw}_bought_amt']  = cvxpy.sum(buy_price @ var_dict[f'{gw}_incoming'])

        #construct total_points as target to maximise
        total_points += (
            pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_selction'] +
            pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_captain'] +
            (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_0']) * bench_weight[0] +
            (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_1']) * bench_weight[1] +
            (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_2']) * bench_weight[2] +
            (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_3']) * bench_weight[3]
        )
        #Add the remaining weight of bench if BB
        if gw==GW+BB-weeks_from:
            total_points += (
                (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_0']) * (1-bench_weight[0]) +
                (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_1']) * (1-bench_weight[1]) +
                (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_2']) * (1-bench_weight[2]) +
                (pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_bench_3']) * (1-bench_weight[3])
            )
        #TC
        elif gw==GW+TC-weeks_from: total_points += pts_dict[f'{gw}_Pts'] @ var_dict[f'{gw}_captain']

        #total_points deduction for too many transfers
        #WC or FH or no preset (i.e. a WC)
        if gw==GW+WC-weeks_from or gw==GW+FH-weeks_from or (not preset and gw==GW):
            #no transfer cost when WC/FH, so no deduction from total_points. Next GW always 1 FT
            var_dict[f'{gw+1}_FT'] = 1
            var_dict[f'{gw+1}_itb'] = var_dict[f'{gw}_itb'] + price_dict[f'{gw}_sold_amt'] - price_dict[f'{gw}_bought_amt']
        #in week FH+1, compare itb with week FH-1
        elif (FH!=0 and gw==GW+FH+1-weeks_from):
            total_points -= cvxpy.maximum((var_dict[f'{gw}_tra_used'] - var_dict[f'{gw}_FT']) * 4, 0)
            var_dict[f'{gw+1}_FT'] = var_dict[f'{gw}_auxiliary'] + 1
            var_dict[f'{gw+1}_itb'] = var_dict[f'{gw-1}_itb'] + price_dict[f'{gw}_sold_amt'] - price_dict[f'{gw}_bought_amt']
        else:
            total_points -= cvxpy.maximum((var_dict[f'{gw}_tra_used'] - var_dict[f'{gw}_FT']) * 4, 0)
            var_dict[f'{gw+1}_FT'] = var_dict[f'{gw}_auxiliary'] + 1
            var_dict[f'{gw+1}_itb'] = var_dict[f'{gw}_itb'] + price_dict[f'{gw}_sold_amt'] - price_dict[f'{gw}_bought_amt']

        #give more weight to rolling FT
        total_points += (var_dict[f'{gw}_FT']-1) * 0.5 #so 0.5 pts if you have 2FTs, else 0
        #doesnt matter if it deducts points when ft==0, since further weeks' results wont change

        #constraints
        #each team cannot have more than 3 players
        for team in set(xpts2['Team']):
            constraints += [array(xpts2['Team'] == team) @ var_dict[f'{gw}_squad'] <= 3]

        constraints += [
            var_dict[f'{gw}_squad'] == ( #so player cant be selected AND benched
                var_dict[f'{gw}_selction'] + var_dict[f'{gw}_bench_0'] + var_dict[f'{gw}_bench_1'] +
                var_dict[f'{gw}_bench_2'] + var_dict[f'{gw}_bench_3']
            ),
            sum(var_dict[f'{gw}_bench_0']) == 1,
            array(xpts2['Pos'] == 'G') @ var_dict[f'{gw}_bench_0'] == 1, #bench_0 = GK
            sum(var_dict[f'{gw}_bench_1']) == 1,
            sum(var_dict[f'{gw}_bench_2']) == 1,
            sum(var_dict[f'{gw}_bench_3']) == 1,
            sum(var_dict[f'{gw}_captain']) == 1,
            sum(cvxpy.pos(var_dict[f'{gw}_selction'] - var_dict[f'{gw}_captain'])) <= 10, #always 10 noncaptains from XI, fit DCP rule
            var_dict[f'{gw}_tra_used'] >= 0, #no negative transfers
            #squad limits
            array(xpts2['Pos'] == 'G') @ var_dict[f'{gw}_squad'] == 2,
            array(xpts2['Pos'] == 'D') @ var_dict[f'{gw}_squad'] == 5,
            array(xpts2['Pos'] == 'M') @ var_dict[f'{gw}_squad'] == 5,
            array(xpts2['Pos'] == 'F') @ var_dict[f'{gw}_squad'] == 3,
            #XI limits
            array(xpts2['Pos'] == 'G') @ var_dict[f'{gw}_selction'] == 1,
            array(xpts2['Pos'] == 'D') @ var_dict[f'{gw}_selction'] >= 3,
            array(xpts2['Pos'] == 'M') @ var_dict[f'{gw}_selction'] >= 2,
            array(xpts2['Pos'] == 'F') @ var_dict[f'{gw}_selction'] >= 1,
            #only 10 outfielders
            array(xpts2['Pos'] != 'G') @ var_dict[f'{gw}_selction'] == 10,

            #auxiliary for FT logic (cr. @Sertalpbial)
            var_dict[f'{gw}_FT'] - var_dict[f'{gw}_tra_used'] <= 2 * var_dict[f'{gw}_auxiliary'],
            var_dict[f'{gw}_FT'] - var_dict[f'{gw}_tra_used'] >= var_dict[f'{gw}_auxiliary'] + (-14) * (1-var_dict[f'{gw}_auxiliary'])
        ]

    #compare last week squad to this week squad, ensuring no. of transfers made and itb are correct
        constraints += [
            cvxpy.sum(var_dict[f'{gw}_incoming']) == var_dict[f'{gw}_tra_used'],
            var_dict[f'{gw+1}_itb'] >= 0
        ]
        if (FH!=0 and gw==GW+FH+1-weeks_from): #make sure in the week after FH, you get back original squad + transfer
            constraints += [
                var_dict[f'{gw}_squad'] == (
                    var_dict[f'{gw-2}_squad'] + var_dict[f'{gw}_incoming'] - var_dict[f'{gw}_outgoing']
            )]
        else:
            constraints += [
                var_dict[f'{gw}_squad'] == (
                    ([idx in preset for idx in xpts2.index.values] if gw==GW else var_dict[f'{gw-1}_squad']) +
                        var_dict[f'{gw}_incoming'] - var_dict[f'{gw}_outgoing']
            )]

    #force incl/excl player
    for id, gw_from_start in excl:
        constraints += [(xpts2.index==id) @ var_dict[f'{GW - weeks_from + gw_from_start}_squad'] == 0]
    for id, gw_from_start in incl:
        constraints += [(xpts2.index==id) @ var_dict[f'{GW - weeks_from + gw_from_start}_squad'] == 1]

    problem = cvxpy.Problem(cvxpy.Maximize(total_points), constraints)
    problem.solve(solver=cvxpy.CBC, allowablePercentageGap=0.1, verbose=False)

    if problem.status in ["infeasible", "unbounded"]:
        print(problem.status)
        quit()

    print(f'\nObjective value between GW{GW}-GW{GW - weeks_from + weeks_to} is {round(problem.value, 3)}. (Decay at {decay_rate})')
    if preset:  print(f'Preset used.')
    #if use_form: print(f'Form considered.')
    if incl: print(f'Force include: {", ".join(map(get_player, incl))}')
    if excl: print(f'Force exclude: {", ".join(map(get_player, excl))}')

    for gw in range(GW, GW - weeks_from + weeks_to + 1):
        res = concat([xpts2.iloc[:, :5], pts_df2[f'{gw}_Pts']], axis=1).reset_index(drop=True)
        #transfers in/out
        if gw==GW or (gw==GW + FH and FH==1): #no gw-1 for first week OR week 1 FH (so no gw-2)
            tra = concat([res, Series((var_dict[f'{gw}_squad'].value -
                [idx in preset for idx in xpts2.index.values]).round(), name='tra')],axis=1)
        elif gw==GW+FH-weeks_from+1: #for the week after FH is used, compare squad with 2 weeks ago
            tra = concat([res, Series((var_dict[f'{gw}_squad'].value - var_dict[f'{gw-2}_squad'].value).round(), name='tra')], axis=1)
        else:
            tra = concat([res, Series((var_dict[f'{gw}_squad'].value - var_dict[f'{gw-1}_squad'].value).round(), name='tra')], axis=1)
        tra = tra.sort_values(by='Pos', ascending=False)

        res = concat([res,
            Series(var_dict[f'{gw}_selction'].value, name=f'{gw}_Select'),
            Series(var_dict[f'{gw}_captain'].value, name=f'{gw}_Cap'),
            Series(var_dict[f'{gw}_squad'].value, name=f'{gw}_Squad'),
            Series(var_dict[f'{gw}_bench_0'].value, name=f'{gw}_Bench_0'),
            Series(var_dict[f'{gw}_bench_1'].value, name=f'{gw}_Bench_1'),
            Series(var_dict[f'{gw}_bench_2'].value, name=f'{gw}_Bench_2'),
            Series(var_dict[f'{gw}_bench_3'].value, name=f'{gw}_Bench_3'),
        ], axis=1)
        #select elements that are not equal to 0, then sort by selected, Pos
        res = res[res.iloc[:, 6:].ne(0).any(1)].sort_values(by=[res.columns[6], *res.columns[9:12], 'Pos'], ascending=False).iloc[:,:-4]
        prediction = res.iloc[:11, -4] @ (res.iloc[:11,-2]+1) #dot prod of pts and captain + 1 (so captain becomes 2)
        pred_bench = res.iloc[-4:,-4].sum()
        if decay_rate != 1:
            undecay_pred  = round(prediction * undecay[gw-GW], 3)
            undecay_bench = round(pred_bench * undecay[gw-GW], 3)

        if gw==GW and not preset: #for first week WC
            print(
                f'\nxPts: {round(prediction, 3)} ' + '(0) '+ f'Bench Points: {round(pred_bench, 3)} ' +
                f'TV used: {round(res["SV"].sum(), 1)} ' + 'WC'
            )
            if decay_rate != 1:
                print(f'un-decay xPts: {undecay_pred} Bench: {undecay_bench}')
                undecay_total += undecay_pred
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}') #no transfer out as no preset
            print(res.reset_index(drop=True))
        elif gw==GW+WC-weeks_from:
            print(
                f'\nxPts: {round(prediction, 3)} ' + '(0) '+ f'Bench Points: {round(pred_bench, 3)} ' +
                f'TV used: {round(res["SV"].sum(), 1)} ' +
                f'Transfer: {var_dict[f"{gw}_tra_used"].value.round()} ' + 'WC'
            )
            if decay_rate != 1:
                print(f'un-decay xPts: {undecay_pred} Bench: {undecay_bench}')
                undecay_total += undecay_pred
            print(f'Transfer(s) out: {", ".join(tra[tra["tra"].eq(-1)]["Name"])}')
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}')
            print(res.reset_index(drop=True))
        elif gw==GW+FH-weeks_from:
            print(
                f'\nxPts: {round(prediction, 3)} ' + '(0) '+
                f'Bench Points: {round(pred_bench, 3)} ' +
                f'TV used: {round(res["SV"].sum(), 1)} ' + 'FH'
            )
            if decay_rate != 1:
                print(f'un-decay xPts: {undecay_pred} Bench: {undecay_bench}')
                undecay_total += undecay_pred
            print(f'Transfer(s) out: {", ".join(tra[tra["tra"].eq(-1)]["Name"])}')
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}')
            print(res.reset_index(drop=True))
        elif gw==GW+BB-weeks_from:
            prediction = res.iloc[:, -4] @ (res.iloc[:,-2] + 1) #included bench
            try: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - var_dict[f'{gw}_FT'].value, 0) * -4) #pts hit
            except AttributeError: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - ft, 0) * -4)
            print(
                f'\nxPts: {round(prediction, 3)} ' + #included bench
                f'({pts_hit}) ' + f'Bench Boost TV used: {round(res["SV"].sum(), 1)} ' +
                f'Transfer: {var_dict[f"{gw}_tra_used"].value.round()}'
            )
            if decay_rate != 1:
                print(f'un-decay xPts: {round(prediction * undecay[gw-GW], 3)}')
                undecay_total += round(prediction * undecay[gw-GW], 3) + pts_hit
            print(f'Transfer(s) out: {", ".join(tra[tra["tra"].eq(-1)]["Name"])}')
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}')
            print(res.sort_values(by=[res.columns[5]], ascending=False).reset_index(drop=True))
        elif gw==GW+TC-weeks_from:
            prediction = res.iloc[:, -4] @ (2*(res.iloc[:,-2])+1) #2*captaincy + all 1
            try: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - var_dict[f'{gw}_FT'].value, 0) * -4) #pts hit
            except AttributeError: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - ft, 0) * -4)
            print(
                f'\nxPts: {round(prediction, 3)} ' + f'({pts_hit}) ' +
                f'Bench Points: {round(pred_bench, 3)} ' +
                f'TV used: {round(res["SV"].sum(), 1)} ' +
                f'Transfer: {var_dict[f"{gw}_tra_used"].value.round()} ' + 'Triple Cap'
            )
            if decay_rate != 1:
                print(
                    f'un-decay xPts: {round(prediction * undecay[gw-GW], 3)} ' +
                    f'Bench: {undecay_bench}'
                )
                undecay_total += round(prediction * undecay[gw-GW], 3) + pts_hit
            print(f'Transfer(s) out: {", ".join(tra[tra["tra"].eq(-1)]["Name"])}')
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}')
            print(res.reset_index(drop=True))
        else:
            try: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - var_dict[f'{gw}_FT'].value, 0) * -4) #pts hit
            except AttributeError: pts_hit = round(max(var_dict[f"{gw}_tra_used"].value - ft, 0) * -4)
            print(
                f'\nxPts: {round(prediction, 3)} ' + f'({pts_hit}) ' +
                f'Bench Points: {round(pred_bench, 3)} ' +
                f'TV used: {round(res["SV"].sum(), 1)} ' +
                f'Transfer: {var_dict[f"{gw}_tra_used"].value.round()}'
            )
            if decay_rate != 1:
                print(f'un-decay xPts: {undecay_pred} Bench: {undecay_bench}')
                undecay_total += round(prediction * undecay[gw-GW], 3) + pts_hit
            print(f'Transfer(s) out: {", ".join(tra[tra["tra"].eq(-1)]["Name"])}')
            print(f'Transfer(s) in: {", ".join(tra[tra["tra"].eq(1)]["Name"])}')
            print(res.reset_index(drop=True))

    print("Total xPts:", undecay_total)

#with option_context('display.max_rows', None):
#    print(xpts_90.iloc[:,:7].sort_values(by=xpts_90.columns[6], ascending=False))
#optim(101.1, preset=[11, 258, 103, 541, 389, 253, 300, 467, 387, 201, 232, 522, 23, 66, 60], FH=3)
#optim(itb=0.6, preset=[11, 258, 103, 541, 389, 253, 300, 467, 387, 201, 232, 522, 23, 66, 60])
#optim(102.8, FH=2, weeks_to=2, weeks_from=2, bench_weight=[0.01, 0.1, 0.01, 0], use_90=True)


"""
http://www.codewars.com/kata/54bf1c2cd5b56cc47f0007a1/train/python

Count the number of Duplicates

Write a function that will return the count of distinct case-insensitive alphabetic characters that occur more than once in the given string. The given string can be assumed to contain only uppercase and lowercase alphabets.

Example

"abcde" -> 0 # no characters repeats more than once
"aabbcde" -> 2 # 'a' and 'b'
"aabbcdeB" -> 2 # 'a' and 'b'
"indivisibility" -> 1 # 'i'
"Indivisibilities" -> 2 # 'i' and 's'
"""

def duplicate_count(text):

    dupl_list = []
    letter_list = []
    count = 0

    for letter in text.lower():

        if letter not in dupl_list:

            dupl_list.append(letter)

        else:

            if letter not in letter_list:

                count += 1
                letter_list.append(letter)

    return count

print(duplicate_count('aabbcdeB'))

"""
Lessons:
-use listA.count(x) to get the number of occurrences of x in listA

"""


'''
    commit:
            下载当前页面的所有图片。
    依赖：
        不使用BS模块。
'''

import urllib
import urllib.request
import re
import sys
import os

#获取当前的脚本路径

#打开网页，下载器
def open_html ( url):
 require=urllib.request.Request(url)
 reponse=urllib.request.urlopen(require)
 html=reponse.read()
 return html
#下载图片
def load_image(html):
 regx='http://[\S]*jpg'
 pattern=re.compile(regx)
 get_image=re.findall(pattern,repr(html))

 #文件命名从1开始
 num=1
 path = sys.path[0]+'/picture'
 if not os.path.exists(path):
  os.makedirs(path)
 for img in get_image:
  photo=open_html(img)
  with open(r'%s\%s.jpg'%(path,num),'wb') as f:
   print('开始下载图片')
   f.write(photo)
   print('正在下载第%s张图片'%num)
   f.close()
  num=num+1
 if num>1:
  print('下载成功！！！')
 else:
  print('下载失败！！！')

'''
    commit:
            创建保存资源的目录
'''
def _checkPath(foldername):
	#strip()方法用以消除字符串首尾的空格
	dirname = os.path.dirname(foldername.strip())

url='http://www.qiqipu.com/'
html=open_html(url)
load_image(html)


import xmltodict

with open('stations.xml') as file:
    stringStations = xmltodict.parse(file.read())

    stations = {}
    counter1 = 0
    counter2 = 0

    for element in stringStations['Stations']['Station']:
        stations.update({element['Code']: [element['Type'], [element['Namen']['Kort'], element['Namen']['Middel'], element['Namen']['Lang']], element['Land']]})

        try:
            list = []
            for synoniemen in stringStations['Stations']['Station'][counter1]['Synoniemen']['Synoniem']:
                list.append(synoniemen)
            stations[element['Code']].append(list)
        except TypeError:
            continue

        counter1 += 1

    print('Dit zijn de codes en types van de 4 stations:')

    for key, val in stations.items():
        print('{} - {}'.format(key, val[0]))

    print('\nDit zijn alle stations met één of meer synoniemen:')

    for key, val in stations.items():
        if len(val) > 3:
            print('{} - {}'.format(key, stringStations['Stations']['Station'][counter2]['Synoniemen']))
        else:
            continue

        counter2 += 1

    print('\nDit is de lange naam van elk station:')

    for key, val in stations.items():
        print('{} - {}'.format(key, val[1][2]))


cards = []

"""
cards.append({
    "name": "jack",
    "phone": "123456",
    "qq": "789",
    "email": "test@gmail.com",
})

"""

one = {
    "name": "jack",
    "phone": "123456",
    "qq": "789",
    "email": "test@gmail.com",
}

def collect_info(card):
    """
        收集用户输入的信息
    :param card:  如果不为None,那么就需要根据用户的输入来修改原有的内容。如果为None则新建一个card
    :return:  新建或修改后的card
    """
    new_card = {}
    hint = ""
    if card:
        hint = "【回车不修改】"

    name = input("请输入姓名" + hint + ":")
    if name.strip() == "":  # 回车不修改，将传入的card的对应数据拷贝一份
        new_card["name"] = card["name"]
    else:
        new_card["name"] = name

    phone = input("请输入电话" + hint + ":")
    if phone.strip() == "":
        new_card["phone"] = card["phone"]
    else:
        new_card["phone"] = phone

    qq = input("请输入QQ" + hint + ":")
    if qq.strip() == "":
        new_card["qq"] = card["qq"]
    else:
        new_card["qq"] = qq

    email = input("请输入邮箱" + hint + ":")
    if email.strip() == "":
        new_card["email"] = card["email"]
    else:
        new_card["email"] = email

    return new_card

def create_card():
    """
       新建名片
    :return:
    """
    print("-" * 50)
    print("功能：新建名片")
    card = collect_info(None)
    cards.append(card)

def print_cards(pcards):
    if pcards and len(pcards) > 0:
        print("名称\t\t\t电话\t\t\tQQ\t\t\t邮箱")
        for c in pcards:
            print("-" * 80)
            print("%s\t\t\t%s\t\t\t%s\t\t\t%s" % (c["name"], c["phone"], c["qq"], c["email"]))
            print("-" * 80)
    else:
        print("没有名片数据")

def list_card():
    """
       显示全部
    :return:
    """
    print("-" * 50)
    print("功能：显示全部")
    print_cards(cards)

def search_card():
    """
        查询名片
    :return:
    """
    print("-" * 50)
    print("功能：查询名片")

    search_name = input("请输入姓名:")

    find_card = None
    for c in cards:
        if c["name"] == search_name:
            find_card = c
            print_cards([find_card])
            break
    else:
        print("没有找到%s的名片" % search_name)

    if find_card:
        while True:
            card_choose = input("请选择对名片的操作:1修改，2删除，0返回")
            if card_choose == "1":
                modify_card(find_card)
                break
            elif card_choose == "2":
                cards.remove(find_card)
                print("删除%s名片成功" % find_card["name"])
                break
            elif card_choose == "0":
                break
            else:
                print("您的输入有误，请重新输入")

def modify_card(card):
    new_card = collect_info(card)
    cards.remove(card)
    cards.append(new_card)
    print("修改%s名片成功" % card["name"])


"""
#==================================Introduction==========================================
# Author:           Le Ngoc Huy - UIT K12
# Contact:
# 	Email:          huykingsofm@gmail.com
# 	Edu Email:      17520074@gm.uit.edu.vn
#   Github:         https://github.com/huykingsofm/ALU_Simulator.git
#
#
# File name:        server.py
# Language:         Python
# Modified Date:    Dec 10 2018
# PURPOSE:          provide a API to wait a connection from ALU laucher
#========================================================================================
"""
#import numpy as np
import socket
from utilities import dec2bin
from ALU import ALU

def call(data):
    """
    Purpose: Perform called function

    Parameter:
    data    string      a string which format "func_name|arg1,arg2,..."

    Return:
    If success, return a value responding called function
    If failure, return empty string
    """
    data = data.upper()
    data = data.split("|")

    func = data[0]
    arg = data[1].split(",")

    returnValue = ""

    if (func == "ALU" and len(arg) == 4):
        base = int(arg[2])
        nbit = int(arg[3])
        alu = ALU(arg[0], arg[1], base, nbit)
        alu.run()
        returnValue = alu.convertAll()

    elif (func == "DEC2BIN" and len(arg) == 2):
        dec = int(arg[0])
        nbit = int(arg[1])
        returnValue = dec2bin(dec, nbit = nbit, sep = int(nbit / 4))

    return returnValue

def openAPI():
    """
    Purpose: Open API to connect to another program
    No-Return
    """
    HOST = '127.0.0.1'  # (localhost)
    PORT = 65432        # Port to listen on (non-privileged ports are > 1023)

    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.bind((HOST, PORT))
    server.listen()
    print('Begin listening......')
    conn, addr = server.accept()

    print('Connected by', addr)
    while True:
        print("waiting.....")
        data = conn.recv(1024)
        if not data:
            conn.close()
            break

        data = data.decode()
        print("Receive : ", data)

        data = call(data)
        print("Solved")

        conn.sendall(data.encode())
        print("Sended")

    server.close()
    print("Close connection completely.....")

#=============MAIN================================
openAPI()


from collections import defaultdict
from datetime import time

from sqlalchemy import func

from fossir.core.db import db
from fossir.modules.rb.models.aspects import Aspect
from fossir.util.caching import memoize_request
from fossir.util.decorators import classproperty
from fossir.util.i18n import _
from fossir.util.locators import locator_property
from fossir.util.string import return_ascii

class Location(db.Model):
    __tablename__ = 'locations'
    __table_args__ = {'schema': 'roombooking'}

    # TODO: Turn this into a proper admin setting
    working_time_periods = ((time(8, 30), time(12, 30)), (time(13, 30), time(17, 30)))

    @classproperty
    @classmethod
    def working_time_start(cls):
        return cls.working_time_periods[0][0]

    @classproperty
    @classmethod
    def working_time_end(cls):
        return cls.working_time_periods[-1][1]

    id = db.Column(
        db.Integer,
        primary_key=True
    )
    name = db.Column(
        db.String,
        nullable=False,
        unique=True,
        index=True
    )
    is_default = db.Column(
        db.Boolean,
        nullable=False,
        default=False
    )
    default_aspect_id = db.Column(
        db.Integer,
        db.ForeignKey(
            'roombooking.aspects.id',
            use_alter=True,
            name='fk_locations_default_aspect_id',
            onupdate='CASCADE',
            ondelete='SET NULL'
        )
    )
    map_url_template = db.Column(
        db.String,
        nullable=False,
        default=''
    )

    aspects = db.relationship(
        'Aspect',
        backref='location',
        cascade='all, delete-orphan',
        primaryjoin=(id == Aspect.location_id),
        lazy='dynamic',
    )

    default_aspect = db.relationship(
        'Aspect',
        primaryjoin=default_aspect_id == Aspect.id,
        post_update=True,
    )

    rooms = db.relationship(
        'Room',
        backref='location',
        cascade='all, delete-orphan',
        lazy=True
    )

    attributes = db.relationship(
        'RoomAttribute',
        backref='location',
        cascade='all, delete-orphan',
        lazy='dynamic'
    )

    equipment_types = db.relationship(
        'EquipmentType',
        backref='location',
        lazy='dynamic',
        cascade='all, delete-orphan'
    )

    holidays = db.relationship(
        'Holiday',
        backref='location',
        cascade='all, delete-orphan',
        lazy='dynamic'
    )

    # relationship backrefs:
    # - breaks (Break.own_venue)
    # - contributions (Contribution.own_venue)
    # - events (Event.own_venue)
    # - session_blocks (SessionBlock.own_venue)
    # - sessions (Session.own_venue)

    @return_ascii
    def __repr__(self):
        return u'<Location({0}, {1}, {2})>'.format(
            self.id,
            self.default_aspect_id,
            self.name
        )

    @locator_property
    def locator(self):
        return {'locationId': self.name}

    @property
    @memoize_request
    def is_map_available(self):
        return self.aspects.count() > 0

    @classproperty
    @classmethod
    @memoize_request
    def default_location(cls):
        return cls.query.filter_by(is_default=True).first()

    def set_default(self):
        if self.is_default:
            return
        (Location.query
         .filter(Location.is_default | (Location.id == self.id))
         .update({'is_default': func.not_(Location.is_default)}, synchronize_session='fetch'))

    def get_attribute_by_name(self, name):
        return self.attributes.filter_by(name=name).first()

    def get_equipment_by_name(self, name):
        return self.equipment_types.filter_by(name=name).first()

    def get_buildings(self):
        building_rooms = defaultdict(list)
        for room in self.rooms:
            building_rooms[room.building].append(room)

        buildings = []
        for building_name, rooms in building_rooms.iteritems():
            room_with_lat_lon = next((r for r in rooms if r.longitude and r.latitude), None)
            if not room_with_lat_lon:
                continue
            buildings.append({'number': building_name,
                              'title': _(u'Building {}'.format(building_name)),
                              'longitude': room_with_lat_lon.longitude,
                              'latitude': room_with_lat_lon.latitude,
                              'rooms': [r.to_serializable('__public_exhaustive__') for r in rooms]})
        return buildings


#coding=utf-8
from __future__ import absolute_import

from flask import Blueprint, request, current_app
from .models import Group, Relation
from .routes import urlpatterns
from errors.base_errors import APIError
from utils.base_utils import make_json_response, route_inject
from utils.bp_users_utils import verify_token
from utils.bp_groups_utils import verify_member_permission, verify_owner_permission

bp_name = "group"

open_endpoints = [
    "{}.add_group".format(bp_name),
    "{}.get_my_groups".format(bp_name),
]

member_permission_endpoints = [
    "{}.get_group_role".format(bp_name),
    "{}.get_group_profile".format(bp_name),
    "{}.get_group_member".format(bp_name)]

owner_permission_endpoints = [
    "{}.add_group_member".format(bp_name),
    "{}.delete_group_member".format(bp_name),
    "{}.update_group_profile".format(bp_name),
    "{}.transfer_owner_permission".format(bp_name),
    "{}.delete_group".format(bp_name)]

blueprint = Blueprint(bp_name, __name__)

route_inject(blueprint, urlpatterns)

model_list = [Group, Relation]

@blueprint.before_app_first_request
def before_first_request():
    current_app.mongodb_database.register(model_list)
    return

@blueprint.before_request
def before_request():

    verify_token()

    if request.endpoint in member_permission_endpoints:
        verify_member_permission()
    if request.endpoint in owner_permission_endpoints:
        verify_owner_permission()
    return

#
# @blueprint.errorhandler(APIError)
# def blueprint_api_err(err):
#     return make_json_response(err)


"""Contains the definitions and utility functions for running a GP object.

NOTE on "CONVENTION":
	- pass around direct np.arrays, not lcf DataFrames;
		i.e X = lcf[['t', 'x', 'y']].values
			f = lcf.f.values
		The object here don't know how lcf objects work

NOTE on STRUCTURE:
	- Aim for a structure where we separate Kernel and GPDetrender objects.
	- GPDetrender objects may even be general for a kernel
	- Kernel must contain the GP kernels, as well as DE bounds and priors.
	- GPDetrender object must be able to calculate the lnposterior
	- GPDetrender object must also take care of the full detrending
	- GPDetrender contains the data only through GP

NOTE POSSIBILITIES:
	- LCGP object also automatically

NOTE on short cadence extension:
	- Will create an SCGPDetrender (or SCDetrended) which will overload
	  the necessary methods with the optimized short-cadence algorithms.

NOTE on paradigm:
	- TODO: go back and look at structure again. It is getting a little
	  bloated and "general." Keep this in terms of very specific objects
	  that to very specific jobs. I.e one is a prior/kernel (switch to
	  prior perhaps). Another is a GP wrapper, with detrend methods,
	  and log_likelihood/log_prior exposure; possibly two different
	  kinds of optimize methods (DE and powell), and finally, tied
	  to the detrending/predict methods; an outlier highlighter and
	  parameter conversion.
"""

import copy
from collections import OrderedDict

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from numpy.linalg.linalg import LinAlgError
from scipy.optimize import (minimize, differential_evolution,
							fmin_powell, OptimizeResult)

import george
from george import kernels

from . import model_objects
from .kernels import PriorInitialisationError
from .. import lcf_tools
from ..analysis import calc_cdpp, plot_model
from ..__init__ import DATA_DIR

# Ignore all the warning messages when optimising
np.seterr(under='ignore', invalid='ignore')

class K2Detrender(object):
	"""Contains the gp object, and optimization/detrending methods.

	Within the object; the lightcurve timeseries is stored as an
	lcf object, i.e a pandas.DataFrame. It ASSUMES that the data
	is stored in columns 'f', 'x', 'y', 't'.

	NOTE: very important to keep in mind; at the level of this
	object's internals, the parameters are known only in the
	george vector parametrisation. The prior/Kernel objects may
	give a meaning to this parametrisation, but internally this
	object doesn't care.

	Attributes:
		_kernel_type (str): 'SemiPeriodic' or 'Classic'
		_hp_names (list)
		_vector_dim (int)
		ocut (float, property): cached value of the last
			outlier clipping factor (gets updated when outliers are
			removed), and sets the next default clipping factor
		lcf (pd.DataFrame, property): contains the lcf internally
		lcf_basis (pd.DataFrame, property): cached lcf used as the
			basis from which to perform *detrending* by default.
		hp_vector (array, property): caches hyperparameter values.
			It is suggested not to rely solely on this; it will
			only be used as a default if no hp_vector is entered.
			Only values of the hp_vector which are produced by
			optimizations should be stored here (or manual writing).
		K2Kernel
		kernel (george.kernels.Kernel)
		gp (george.GP)

	TODO: currently a bit too much freedom with setting lcf_basis
	and lcf at various points. I believe it should be simpler:
	only lcf can be "set" or selected. lcf_basis is constrained to
	be a subset of lcf. Also makes it easier to highlight outliers
	in _lcf for example (when selecting basis). Investigate how
	this could work.

	Regression/detrending methods:
		get_spatial_component
		get_temporal_component
		get_full_prediction
		detrend_lightcurve
		mask_outliers
		select_basis
	Probabilitistic methods:
		log_likelihood
		grad_log_likelihood
		lnposterior
		neg_lnposterior
		grad_lnposterior
		neg_grad_lnposterior
	Optimization and inference methods:
		optimise_hp_powell
		optimise_hp_grad
		evolve_hp
		optimise_hp_global
	Utilities and internal methods:
		is_detrended
		plot_model
		calc_cdpp
	Properties:
		ocut (get_ocut, set_ocut)
		hp_vector (hp, set_hp)
		lcf (get_lcf, set_lcf)
		lcf_basis (get_basis, set_basis)
	"""

	_local_parameters = ('2ln_wn',)

	def __init__(self, lcf, k2_kernel_class, infer_wn=True, ocut=5,
				 model='y_offset', X_cols=None):
		"""Assign lightcurve and kernel to the GP object.

		Args:
			lcf (pd.DataFrame): columns used at 't', 'x', 'y', 'f'
			k2_kernel_class (K2Kernel): contains the
				prior and de_domain
			infer_wn (bool, True): to calculate a first estimate
				for the white noise hyperparamter. Without this,
				the first outlier mask will be wrong.
			ocut (float, optional): the outlier cutting factor
			model (str or ModelObject): options
				- 'y_offset' estimates y_offset and puts it in GP.
					This is not a free parameter by default
				- None same as y_offset
				- 'ramp' sets up a RampModel, with y_offset etc...
				- 1.0 or an numeric values forces that mean
				- otherwise give it a direct ModelObject
			X_cols (str or list of str): columns in the lcf to use
				as the independent timeseries parameters (X). If it's
				a single str, then it will be a one dimensional
				GP, otherwise multidimensional. If None, will attempt
				to estimate it from the kernel.

			**OBSOLETE**
			ramp (bool or RampModel): the ramp model to use; if
				True, will automatically initiate a ramp model.
				If given as a full ramp model, the y_offset contained
				will be used, whether it is correct or not.
				If false, the ramp will be taken as a constant y_offset

		"""

		# Set-up the kernel
		self.k2_kernel = k2_kernel_class
		self._kernel = k2_kernel_class.kernel
		self._kernel_type = str(type(k2_kernel_class))
		self.kernel_lnprior = k2_kernel_class.log_prior

		if X_cols is None:
			self.X_cols = list(self.k2_kernel.default_X_cols)
		else:
			self.X_cols = list(X_cols)

		# The column to use for the x axis of the model
		self.model_col = 't'

		# TODO x_ndim is obsolete

		# Set up the model
		if model is None or model == 'y_offset':
			self._model = None
		elif model == 'ramp':
			y_offset = np.nanmedian(lcf.f)
			self._model = model_objects.RampModel(lcf=lcf,
												  y_offset=y_offset,
												  x_ndim=len(self.X_cols))
		elif not isinstance(model, str):
			self._model = model
		else:
			raise NotImplementedError("model {} not recognised.".format(model))

		# Initialise the george.GP object
		if model is None or model == 'y_offset':
			self.gp = george.GP(kernel=self._kernel,
								white_noise=1.0,
								mean=np.nanmedian(lcf.f),
								fit_white_noise=True,
								fit_mean=True)
		elif isinstance(model, (int, float)):
			self.gp = george.GP(kernel=self._kernel,
								white_noise=1.0,
								mean=model,
								fit_white_noise=True,
								fit_mean=True)
		else:
			self.gp = george.GP(kernel=self._kernel,
								white_noise=1.0,
								mean=self._model,
								fit_white_noise=True,
								fit_mean=True)

		# Model parameters
		if (model == 'y_offset' or
			model is None or
			isinstance(model, (int, float))):
			self._model_parameters = ('y_offset',)
		else:
			self._model_parameters = tuple(self._model.get_parameter_names(include_frozen=True))

		self._hp_names_local = self._model_parameters + \
							   self._local_parameters + \
							   self.k2_kernel.parameter_names

		self._hp_names_gp = self.gp.parameter_names

		if np.shape(self._hp_names_local) != np.shape(self._hp_names_gp):
			# TODO BUG
			import pdb; pdb.set_trace()
			raise ValueError("Internal and GP parameter shape doesn't match.")

		# White noise parameter bounds are separate
		self.wn_bounds = [-20.0, 0.0]		# on 2ln_wn

		# Set initial value
		if infer_wn:
			self.set_parameter(name='2ln_wn', local_name=True,
							   value=lcf_tools.get_white_noise(lcf, chunk_size=10))

		# Freeze mask doesn't need to be set; it's already in gp
		if (model == 'ramp' or
			model is None or
			isinstance(model, (int, float))):
			self.freeze_parameter('y_offset')

		# Set-up the internal lightcurves (detached from input)
		self.set_lcf(lcf)
		self.set_basis(lcf[~self.mask_flares(lcf)])

		# Set internal values
		self.set_ocut(ocut)
		self.__detrending_hp = None		# Used to cache the latest hp which
										# were used to detrend the saved
										# lightcurve so can check if it's
										# detrended

		self.OversamplingPopulationError = OversamplingPopulationError
		self.NonPositiveDefiniteError = NonPositiveDefiniteError
		self.PriorInitialisationError = PriorInitialisationError

		X_0 = self.get_basis(full=True)[self.X_cols].values
		self.compute(X_0)

	# Basic method/decorators of GP
	# -----------------------------

	def compute(self, X_array, *args, **kwargs):
		"""Decorator around gp.compute to handle Exceptions."""

		try:
			self.gp.compute(X_array, *args, **kwargs)
		except LinAlgError:
			raise NonPositiveDefiniteError(
				X=X_array,
				hp=self.hp,
				message='hp_dict = {}'.format(self.get_parameter_dict()))

	def predict(self, f_basis, X_predict, *args, **kwargs):
		"""Decorator around gp.predict to handle Exceptions."""

		try:
			return self.gp.predict(f_basis, X_predict, *args, **kwargs)
		except Exception as e:
			import pdb; pdb.set_trace()
			raise LAPACKError(
				message="LAPACKError",
				exception=e,
				X_predict=X_predict,
				f_basis=f_basis,
				X_basis=self.gp._x,
				hp=self.get_parameter_dict(include_frozen=True))

	# Regression methods
	# ------------------

	def get_spatial_component(self, X_predict=None,
							  lcf_basis=None, hp=None):
		"""Spatially correlated component at points X_predict.

		Predicts the spatial component of the trend in the
		Gaussian Process, as:
			f_star_xy = K_star . K^-1 . f
		where
			K_star is the vector of coveriances between X_predict
			and lcf_basis, and the K's here to the spatial kernel.

		Args:
			X_predict (pd.DataFrame): the points on which to predict
				the trend. Must contain columns ['x', 'y', 't'].
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				['f', 'x', 'y', 't'].
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp

		Returns:
			f_predict (1D array): aka f_star_xy; the spatially
				correlated component in the lightcurve at points
				X_predict
		"""

		if hp is not None:
			self.set_hp(hp)
		if lcf_basis is None:
			lcf_basis = self.get_basis(True)
		if X_predict is None:
			X_predict = self.get_lcf()[self.X_cols].values
		elif isinstance(X_predict, pd.DataFrame):
			X_predict = X_predict[self.X_cols].values

		f_basis = lcf_basis['f'].values
		X_basis = lcf_basis[self.X_cols].values

		self.compute(X_basis)

		K_star = self.gp.kernel.k1.get_value(x1=X_predict, x2=X_basis)
		f_predict = np.matmul(K_star, self.gp.apply_inverse(f_basis))

		return f_predict

	def get_temporal_component(self, X_predict=None,
							   lcf_basis=None, hp=None):
		"""Temporally correlated component at points X_predict.

		Predicts the time component of the trend in the
		Gaussian Process, as:
			f_star_xy = K_star . K^-1 . f
		where
			K_star is the vector of covariances between X_predict
			and lcf_basis, and the K's here to the time kernel.

		Args:
			X_predict (pd.DataFrame): the points on which to predict
				the trend. Must contain columns ['x', 'y', 't'].
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				['f', 'x', 'y', 't'].
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp

		Returns:
			f_predict (1D array): aka f_star_xy; the spatially
				correlated component in the lightcurve at points
				X_predict
		"""

		if hp is not None:
			self.set_hp(hp)
		if lcf_basis is None:
			lcf_basis = self.get_basis(full=True)
		if X_predict is None:
			X_predict = self.get_lcf()[self.X_cols].values
			t_predict = self.get_lcf()[self.model_col].values
		elif isinstance(X_predict, pd.DataFrame):
			t_predict = X_predict[self.model_col].values
			X_predict = X_predict[self.X_cols].values
		else:
			# Assumes time is in the last column
			print("Check shape of X_predict!!!")
			print("t_predict = X_predict[:,-1]")
			import pdb; pdb.set_trace()
			t_predict = X_predict[:,-1]

		f_basis = lcf_basis['f'].values
		X_basis = lcf_basis[self.X_cols].values

		self.compute(X_basis)

		K_star = self.gp.kernel.k2.get_value(x1=X_predict, x2=X_basis)
		f_predict = np.matmul(K_star, self.gp.apply_inverse(f_basis))

		if self._model is not None:
			# Better use the proper time column
			f_model = self._model.get_value(t_predict)
		elif self._model is None:
			# The case of a constant offset
			f_model = self.get_parameter('y_offset')

		return f_predict + f_model

	def get_full_prediction(self, X_predict=None, lcf_basis=None, hp=None):
		"""Gets added temporal and spatial components.

		A bit useless on its own though.

		See get_temporal_component and get_spatial_component."""

		f_spatial = self.get_spatial_component(X_predict, lcf_basis, hp)
		f_temporal = self.get_temporal_component(X_predict, lcf_basis, hp)
		return f_spatial + f_temporal

	def detrend_lightcurve(self, lcf_predict=None, lcf_basis=None, hp=None, save=True):
		"""Produces a decorrelated lightcurve.

		Args:
			lcf_predict (pd.DataFrame): the points on which to predict
				the trend. Must contain columns ['f', 'x', 'y', 't'].
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				['f', 'x', 'y', 't'].
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp
			save (bool): if True, saves the lcf_predict to self._lcf.
				Default: False

		Returns:
			lcf_predict (pd.DataFrame): detrended values contain in
				'f_detrended', 'f_spatial', 'f_temporal'.
				Attention: f_spatial etc... ATTENTION: these have
				been changed to the more intuitive meaning or just
				the relevant component.
		"""

		if lcf_basis is None:
			lcf_basis = self.get_basis(full=True)
		if lcf_predict is None:
			lcf_predict = self._lcf.copy()
		if hp is not None:
			# Need to set the hp so that self._model works
			self.hp = hp

		f_spatial = self.get_spatial_component(lcf_predict, lcf_basis)
		f_temporal = self.get_temporal_component(lcf_predict, lcf_basis)

		lcf_predict['f_spatial'] = f_spatial
		lcf_predict['f_temporal'] = f_temporal

		# Special behaviour: ramp is supposed to be removed (detrender),
		# while other models aren't. So if the model is ramp, then
		# remove from detrended, otherwise keep in detrended
		f_detrended = lcf_predict['f'] - f_spatial - f_temporal

		if self._model is None:
			f_detrended = f_detrended + self.get_parameter('y_offset')
		elif isinstance(self._model, model_objects.RampModel):
			f_detrended = f_detrended + self.get_parameter('y_offset')
		else:
			f_detrended = f_detrended + self._model.get_value()

		lcf_predict['f_detrended'] = f_detrended

		if save:
			self.set_lcf(lcf_predict)
			# To be able to check if the saved lightcurve is
			# detrended with the most recent hyperparameters
			self.__detrending_hp = self.hp if hp is None else hp

		return lcf_predict

	def mask_outliers(self, ocut=None, lcf=None, lcf_basis=None, hp=None):
		"""Masks points further than ocut from the mean.

		Args:
			ocut (float): the number of sigmas that determine an outlier.
			lcf (pd.DataFrame): the points from which to mask outliers.
				Must contain columns 'f' and X_cols.
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				'f' and X_cols.
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp

		Returns:
			outlier_mask (1D array of bool): True if point is an outlier.
		"""

		if lcf_basis is None:
			lcf_basis = self.get_basis(full=True)
		if lcf is None:
			lcf = self.get_lcf()
		if hp is None:
			hp = self.hp
		if ocut is None:
			ocut = self.get_ocut()

		f_basis = lcf_basis['f'].values
		X_basis = lcf_basis[self.X_cols].values

		f = lcf['f'].values
		X = lcf[self.X_cols].values

		self.compute(X_basis)
		mu, _ = self.predict(f_basis, X)
		std = np.exp(self.get_parameter('2ln_wn')/2)

		outlier_mask = (f > (mu + ocut*std)) | (f < (mu - ocut*std))

		return outlier_mask

	def select_basis(self, N=None, lcf=None, cut_outliers=True,
					 cut_transits=True, save=True, save_o_flag=True,
					 quiet=True, cut_flares=False, **mask_kwargs):
		"""Removes outliers and samples an lcf_basis from a lightcurve.

		The lcf_basis has an column called 'sub_flag', determined by N.
		The full basis is that which has sub_flag = True; and
		determines the subset of the basis which is used for
		optimization. When retrieving the basis, to ask for
		the full basis or the subset, use .get_basis(full=...)

		Args:
			N (int, optional): number of points to use.
			lcf (pd.Dataframe, optional): uses self._lcf by default.
			cut_outliers (bool): if False, doesn't remove outliers.
			cut_transits (bool): if False, doesn't remove transit points,
				taken from the 't_flag' column in lcf.
			save (bool): if True, saves to self._basis
			quiet (bool): whether to throw an exception if N > len(lcf)
			cut_flates (bool): removes all flare like points that
				are massive outliers above when doing the first
				detrend.
			**mask_kwargs: ocut, hp, lcf_basis

		Returns:
			lcf_basis (pd.DataFrame): also saved to self._lcf_basis
		"""

		if lcf is None:
			lcf = self.get_lcf()
		if cut_flares:
			lcf = lcf[~self.mask_flares(lcf)]

		if cut_outliers:
			outlier_mask = self.mask_outliers(lcf=lcf, **mask_kwargs)
		else:
			outlier_mask = np.ones(len(lcf), dtype=bool)

		lcf_basis = lcf.copy()[~outlier_mask]

		if cut_transits and 't_flag' in lcf_basis.columns:
			lcf_basis = lcf_basis[~lcf_basis.t_flag]

		lcf_basis = lcf_basis.assign(sub_flag=True)

		# Flag the subselection (for a non-full basis)
		if N is not None:
			if N < len(lcf_basis):
				indices = lcf_basis.sample(N, replace=False).index
				lcf_basis.loc[~lcf_basis.index.isin(indices), 'sub_flag'] = False
			elif quiet:
				print("WARNING: basis set is undersampled.\n",
					"There were less non-outlier points than the expected\n",
					"number of samples.")
			else:
				raise OversamplingPopulationError

		if save:
			self.set_basis(lcf_basis)
		if save_o_flag and cut_outliers:
			self.set_lcf(lcf.assign(o_flag=outlier_mask))

		return lcf_basis[lcf_basis.sub_flag]

	def mask_flares(self, lcf=None, factor=6.0):
		"""Rough removal of flare-like points.
		Use only in the beginning."""

		n_largest = 20

		if lcf is None:
			lcf = self.get_lcf()

		rms = lcf_tools.estimate_sigma(lcf.f,
									   n_chunks=5,
									   chunk_size=min(2500, len(lcf)-3))
		mask = lcf.f > (np.nanmedian(lcf.f) + 6*rms)
		if np.sum(mask) > 20:
			idx = lcf.nlargest(n_largest, 'f').index
			mask = lcf.index.isin(idx)
		return mask

	# Probabilistic methods
	# ---------------------

	def log_likelihood(self, hp_vector=None, f=None):
		"""Returns the log_likelihood of the GP model.

		Updates the internal hp automatically.

		Args:
			hp_vector (1 array): values to use for the hyperparameters.
			f (1D array): the flux points where gp was last computed.
		"""

		if hp_vector is None:
			hp_vector = self.hp
		else:
			self.set_hp(hp_vector)

		if f is None:
			f = self.lcf_basis.f

		return self.gp.log_likelihood(f, quiet=True)

	def grad_log_likelihood(self, hp_vector, f):
		"""Returns the gradient of the log_likelihood of the GP model.

		Updates the internal hp automatically.

		Args:
			hp_vector (1 array): values to use for the hyperparameters.
			f (1D array): the flux points where gp was last computed.
		"""

		if hp_vector is None:
			hp_vector = self.hp
		else:
			self.set_hp(hp_vector)

		return self.gp.grad_log_likelihood(f, quiet=True)

	def log_prior(self, hp_vector=None):
		"""Calculates the log_prior of the constituent models.
		Updates the internal hp automatically."""

		if hp_vector is None:
			hp_vector = self.hp
		else:
			self.set_hp(hp_vector)

		return (self.k2_kernel.log_prior() \
			  + self.log_prior_wn() \
			 + (self._model.log_prior() if self._model is not None else 0))

	def log_prior_wn(self, value=None):
		"""Calculates the log prior in the wn hyperparameter."""

		value = self.get_parameter('2ln_wn') if value is None else value

		if value < self.wn_bounds[0] or value > self.wn_bounds[1]:
			return -np.inf
		else:
			return 0.0

	def lnposterior(self, hp_vector=None, f=None):
		"""Returns the posterior of the GP model at hp_vector.

		Updates the internal hp automatically.

		Args:
			hp_vector (1 array): values to use for the hyperparameters.
			f (1D array): the flux points where gp was last computed.
		"""

		if hp_vector is None:
			hp_vector = self.hp
		else:
			self.set_hp(hp_vector)

		if f is None:
			f = self.lcf_basis.f

		return self.log_likelihood(hp_vector, f) + self.log_prior(hp_vector)

	def neg_lnposterior(self, hp_vector=None, f=None):
		"""Negative of the lnposterior, for minimisation."""

		return -self.lnposterior(hp_vector, f)

	def grad_lnposterior(self, hp_vector=None, f=None):
		"""Returns the gradient of the lnposterior w.r.t hp_vector."""

		raise NotImplementedError

	def neg_grad_lnposterior(self, hp_vector=None, f=None):
		"""Negative gradient of lnposterior, for minimisation."""

		return -self.grad_lnposterior(hp_vector, f)

	# Optimization and inference methods
	# ----------------------------------

	def optimise_hp_powell(self, lcf=None, initial_hp=None, diagnose=False, save=True, **fmin_args):
		"""Minimises the posterior w.r.t hyperpameters with fmin_powell.

		Args:
			lcf (pd.DataFrame): the lightcurve to use as basis
				for optimization (default: self._lcf_basis)
			initial_hp (array, Optional): the initial points
				for optimization
			diagnose (bool): if True, prints and returns full output
			save (bool): whether to save result into hp
				Cannot be done if diagnose is True.
			**fmin_args (dict): other inputs into the minimise function
				choices: 'full_output', 'disp', 'retall', 'direc',
				'xtol', 'ftol', 'maxiter', 'maxfun'

		Returns:
			result (1D array): the output from fmin_powell, may be
				more complicated if diagnose==True.
		"""

		if lcf is None:
			lcf = self.get_basis(full=False)
		if initial_hp is None:
			initial_hp = self.hp
		if diagnose:
			for key in ('full_output', 'retall', 'disp'):
				fmin_args[key] = True
		else:
			for key in ('full_output', 'retall', 'disp'):
				fmin_args[key] = False

		f = lcf['f'].values
		X = lcf[self.X_cols].values

		self.compute(X)
		result = fmin_powell(self.neg_lnposterior, x0=initial_hp, args=(f,), **fmin_args)

		if not diagnose:
			if isinstance(result, OptimizeResult) and save:
				self.set_hp(result.x)
			elif save:
				self.set_hp(result)
		elif diagnose:
			if self.neg_lnposterior() != result[1][1]:
				print("Posterior mismatch.")
				print("Current value:", self.neg_lnposterior())
				print("Optimised value:", result[1][1])

		return result

	def optimise_hp_grad(self, method='BFGS', lcf=None, initial_hp=None,
						 save=True, **min_args):
		"""Minimise posterior w.r.t hyperpameters with a gradient method.

		Args:
			method (str): 'BFGS', etc...
				Currently, gradients are not possible
			lcf (pd.DataFrame): the lightcurve to use as basis
				for optimization (default: self._lcf_basis)
			initial_hp (array, Optional): the initial points
				for optimization
			save (bool): whether to save result into hp
				Cannot be done if diagnose is True.
			**min_args (dict): other inputs into the minimise function

		Returns:
			...
		"""

		if lcf is None:
			lcf = self.get_basis(full=False)
		if initial_hp is None:
			initial_hp = self.hp

		f = lcf['f'].values
		X = lcf[self.X_cols].values

		self.compute(X)

		result = minimize(self.neg_lnposterior,
						x0=initial_hp,
						args=(f,),
						method=method,
						jac=self.neg_grad_lnposterior,
						**min_args)

		# TODO: Check the form of result first, maybe take result.x
		print("First run of grad optimizer, check result.")
		import pdb; pdb.set_trace()

		if save:
			self.set_hp(result)

		return result

	def evolve_hp(self, ngen=50, npop=100, lcf=None, save=True, full_result=False):
		"""Estimates a posterior best-fit w.r.t hyperparameters with DE.*

		*Differential Evolution

		Args:
			ngen (int)
			npop (int)
			lcf (pd.DataFrame, Optional): default is **lcf_basis**;
				i.e the lightcurve on which we want to optimize the
				hyperparameters
			save (bool): to save hp into hp (default: True)
			full_result (bool): return the full scipy.OptimizationResult
				or just the best fit vector (default: False)

		Returns:
			The OptimizeResult object from scipy.optimize, where
			result.x contains the optimized values, and .success
			and .message contain the successful exit flag and any
			exit messages respectively.
		"""

		if lcf is None:
			lcf = self.get_basis(full=False)

		f = lcf['f'].values
		X = lcf[self.X_cols].values
		de_bounds = self.de_bounds

		self.compute(X)

		result = differential_evolution(self.neg_lnposterior,
										bounds=de_bounds,
										popsize=npop,
										maxiter=ngen,
										args=(f,))

		if save:
			self.set_hp(result.x)
		if full_result:
			return result
		else:
			return result.x

	def optimise_hp_global(self, lcf=None, ngen=None, npop=None, save=True, **kwargs):
		"""Performs a global minimisation by DE then fmin_powell.

		TODO: add gradient based final minimisation.

		Args:
			lcf
			ngen
			npop
			save
			**kwargs: currently get dumped into optimise_hp_powell()
		"""

		if lcf is None:
			lcf = self.get_basis(full=False)

		de_result = self.evolve_hp(lcf=lcf,
								   ngen=ngen,
								   npop=npop,
								   save=save)

		fmin_result = self.optimise_hp_powell(lcf=lcf,
											  initial_hp=de_result,
											  save=False,
											  **kwargs)

		if save:
			self.set_hp(fmin_result)

		return fmin_result

	# Utilities and internals
	# -----------------------

	@property
	def is_detrended(self):
		"""Returns True if saved lcf has been detrended
		with the most recent parameters."""
		if self.hp == self.__detrending_hp:
			if 'f_detrended' in self._lcf.columns:
				return True
			else:
				print("WARNING: The current lightcurve has no detrending\n"
				"column, but is flagged as 'is_detrended' with the\n"
				"current hyperparameters.")
				return False
		else:
			return False

	def plot_model(self, lcf=None, show=False, title=None):
		"""Plots the model, using `k2gp.analysis.plot_model`."""

		if lcf is None:
			lcf = self._lcf
		if 'f_detrended' in lcf:
			plot_model(lcf=lcf, show=show, title=title)
		else:
			raise AttributeError("f_detrended is not in the lightcurve.")

	def plot_lcf(self, lcf=None, show=False, title=None):
		"""Just plots the raw data in f."""
		if lcf is None:
			lcf = self._lcf
		fig, ax = plt.subplots(2)
		ax[0].plot(lcf.t, lcf.x, 'r.', lcf.t, lcf.y, 'b.')
		ax[1].plot(lcf.t, lcf.f, 'k.')
		fig.suptitle(title)
		if show:
			plt.show()
		else:
			fig.show()

	def calc_cdpp(self, columns='f_detrended', lcf=None, remove_outliers=False, remove_transits=True):
		"""Calculates the CDPP in columns.

		Args:
			columns (tuple/str): which columns to calculate the cdpp
				for, will return a dictionary unless a single
				column is specified.
			lcf (pd.DataFrame, optional)
			remove_outliers (bool, False): masks o_flag values
			remove_transits (bool, True): masks t_flag values

		Returns:
			dict() of cdpp's with keys = columns, or a single float
			if a single column is specified
		"""

		lcf = lcf if lcf is not None else self._lcf
		if remove_outliers:
			lcf = lcf[~lcf.o_flag]
		if remove_transits and 't_flag' in lcf.columns:
			lcf = lcf[~lcf.t_flag]

		if columns == 'all':
			columns = ('f', 'f_detrended')
		elif isinstance(columns, str):
			columns = (columns,)

		cdpps = dict()
		for col in columns:
			cdpps[col] = calc_cdpp(lcf, column=col)

		if len(columns) == 1:
			return cdpps[columns[0]]
		else:
			return cdpps

	def verify_prior(self, vector=None, quiet=False, include_frozen=True, info_str=None):
		"""Checks if the vector is inside the prior bounds.

		Args:
			vector (np.array): optional
			quiet (bool): if True, will only return a False if
				verification fails, otherwise raises a full error
			include_frozen (bool): if True, vector is only the
				unfrozen parameters
			info_str (str): str to append to error message if
				verification fails
		"""

		initial_vector = np.copy(self.get_hp(include_frozen=True))

		if vector is None:
			vector = np.copy(self.get_hp(include_frozen=include_frozen))
		self.set_hp(vector, include_frozen=include_frozen)

		if np.isfinite(self.log_prior()):
			self.set_hp(initial_vector, include_frozen=True)
			return True
		elif quiet:
			self.set_hp(initial_vector, include_frozen=True)
			return False
		else:
			raise PriorInitialisationError((
				"Initial hyperparameter values are out of "
				"prior bounds.\n"
				"hp: {}\n"
				"bounds: {}"
				"".format(self.get_hp(True), self.parameter_bounds)))

			# hp = self.get_parameter_dict(include_frozen=True)
			# for i, (val, n) in enumerate(zip(self.k2_kernel._default_hp,
			# 							self.k2_kernel._param_names)):
			# 	if self.unfrozen_mask[i]:
			# 		self.set_parameter(n, val)
			# self.set_parameter('A', 0.1)
			# self.set_parameter('alpha', 1.0)

			# parameter_dict = dict()
			# for i, n in enumerate(self.get_parameter_names(include_frozen)):
			# 	initial_value = self.get_parameter(n, False)
			# 	self.set_parameter(n, vector[i], False)
			# 	ln = self._pname_to_local(n)
			# 	if not np.isfinite(self.log_prior()):
			# 		parameter_dict[ln] = vector[i]
			# 	self.set_parameter(n, initial_value, False)
			# self.set_hp(initial_vector, include_frozen=True)

			# hpp = self.k2_kernel.hpp

			# raise PriorInitialisationError(
			# 	"\nHyperparameters are out of bounds. Info str: {}\n"
			# 	"kernel.log_prior(): {}\n"
			# 	"_ramp.log_prior(): {}\n"
			# 	"log_prior_wn(): {}\n".format(
			# 								info_str,
			# 								self.k2_kernel.log_prior(),
			# 								self._ramp.log_prior(),
			# 								self.log_prior_wn()),
			# 	hp=hp, hpp=hpp,
			# 	parameter_dict=parameter_dict
			# )

	# Internal parameter properties
	# -----------------------------

	def __len__(self):
		return len(self.gp)

	@property
	def _hp_active(self):
		return self.gp.unfrozen_mask
	unfrozen_mask = _hp_active

	def get_hp(self, *args, **kwargs):
		return self.gp.get_parameter_vector(*args, **kwargs)

	def set_hp(self, *args, **kwargs):
		self.gp.set_parameter_vector(*args, **kwargs)

	hp = property(get_hp, set_hp)

	def get_parameter_names(self, *args, **kwargs):
		return self.gp.get_parameter_names(*args, **kwargs)

	@property
	def parameter_names(self):
		return self.gp.parameter_names

	@parameter_names.setter
	def parameter_names(self, vector):
		self.gp.parameter_names = vector

	@property
	def parameter_vector(self):
		return self.gp.parameter_vector

	@parameter_vector.setter
	def parameter_vector(self, vector):
		self.gp.parameter_vector = vector

	def get_parameter_dict(self, include_frozen=False, local_names=True):
		"""Returns the parameter dict, local_names=True by default."""

		names = self.get_parameter_names(include_frozen)
		if local_names:
			names = self._pname_to_local(names)
		return OrderedDict(zip(names, self.get_hp(include_frozen)))

	def get_ocut(self):
		return self._ocut

	def set_ocut(self, value):
		if value <= 0:
			raise ValueError("Invalid ocut value, must be greater than zero.")
		self._ocut = value

	ocut = property(get_ocut, set_ocut)

	@property
	def de_bounds(self):
		"""Returns the de_bounds on the currently active parameters."""

		raise NotImplementedError

		# ramp_bounds = list(self._ramp.get_parameter_bounds())
		# kernel_bounds = []
		# for name in self.get_parameter_names():
		# 	key = self._pname_to_local(name)
		# 	if key in self.k2_kernel.parameter_names:
		# 		kernel_bounds.append(self.k2_kernel.de_domain[key])
		# return ramp_bounds + kernel_bounds

	# Internal parameter methods
	# --------------------------

	def _convert_name_to_gp(self, name):
		"""Converts the parameter name from local to gp."""

		# The indices corresponding (ordered)
		if not hasattr(name, '__len__') or isinstance(name, str):
			return self._hp_names_gp[self._hp_names_local.index(name)]
		else:
			return tuple(self._hp_names_gp[self._hp_names_local.index(n)]
						 for n in name)

	def _pname_to_local(self, name):
		"""Converts the parameter name from gp to local."""

		# The indices corresponding (ordered)
		if not hasattr(name, '__len__') or isinstance(name, str):
			return self._hp_names_local[self._hp_names_gp.index(name)]
		else:
			return tuple(self._hp_names_local[self._hp_names_gp.index(n)]
						 for n in name)

	def get_parameter(self, name, local_name=True):
		"""Gets a specific parameter; name is local by default."""
		name = self._convert_name_to_gp(name) if local_name else name
		return self.gp.get_parameter(name)

	def set_parameter(self, name, value, local_name=True):
		"""Sets a specific parameter; name is local by default."""
		name = self._convert_name_to_gp(name) if local_name else name
		self.gp.set_parameter(name, value)

	def freeze_parameter(self, name, local_name=True):
		"""By default the name is local."""
		name = self._convert_name_to_gp(name) if local_name else name
		if isinstance(name, str):
			self.gp.freeze_parameter(name)
		else:
			for n in name:
				self.gp.freeze_parameter(n)

	def thaw_parameter(self, name, local_name=True):
		"""By default the name is local."""
		name = self._convert_name_to_gp(name) if local_name else name
		if isinstance(name, str):
			self.gp.thaw_parameter(name)
		else:
			for n in name:
				self.gp.thaw_parameter(n)

	def freeze_model(self):
		"""Freezes the hyperparameters for the ramp."""
		self.freeze_parameter(self._model_parameters)

	def thaw_model(self):
		"""Activates the hyperparameters for the ramp."""
		self.thaw_parameter(self._model_parameters)

	@property
	def model_bounds(self):
		if self._model is not None:
			return self._model.parameter_bounds
		else:
			return [[-20, 20]]

	@property
	def parameter_bounds(self):
		"""Fused the various bounds into one array."""
		return np.concatenate([self.model_bounds,
							   [self.wn_bounds],
							   self.k2_kernel.bounds])

	# TODO: make a setter function so we can change the bounds

	def get_parameter_bounds(self, include_frozen=False):
		return self.parameter_bounds[self.unfrozen_mask]

	def get_bound_dict(self, local_names=True):
		if local_names:
			return dict(zip(self._hp_names_local, self.bounds))
		else:
			return dict(zip(self._hp_names_gp, self.bounds))

	# Lightcurves getters and setters
	# -------------------------------

	def get_lcf(self):
		return self._lcf.copy()

	def set_lcf(self, lcf):
		if not pd.Series(['f'] + self.X_cols).isin(lcf.columns).all():
			raise ValueError("The lcf must contain 'f' and X_cols.")
		self._lcf = lcf.copy()

	def get_basis(self, full=True):
		"""If full is False; then only gives a subset."""
		if full:
			return self._lcf_basis.copy()
		else:
			return self._lcf_basis[self._lcf_basis.sub_flag].copy()

	def set_basis(self, lcf_basis):
		if not pd.Series(['f'] + self.X_cols).isin(lcf_basis.columns).all():
			raise ValueError("The lcf must contain 'f' and X_cols.")
		if 'sub_flag' not in lcf_basis.columns:
			lcf_basis = lcf_basis.assign(sub_flag=True)
		self._lcf_basis = lcf_basis.copy()

	lcf = property(get_lcf, set_lcf)
	lcf_basis = property(get_basis, set_basis)

#
# One dimensional kernel (should be temporary)
# --------------------------------------------

class K2Detrender1D(K2Detrender):
	"""Redefines the detrending methods, removing spatial/temporal."""

	def get_spatial_component(self, X_predict=None,
							  lcf_basis=None, hp=None):
		raise NotImplementedError

	def get_temporal_component(self, X_predict=None,
							   lcf_basis=None, hp=None):
		"""Temporally correlated component at points X_predict.

		Predicts the time component of the trend in the
		Gaussian Process, as:
			f_star_xy = K_star . K^-1 . f
		where
			K_star is the vector of covariances between X_predict
			and lcf_basis, and the K's here to the time kernel.

		Args:
			X_predict (pd.DataFrame): the points on which to predict
				the trend. Must contain columns ['x', 'y', 't'].
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				['f', 'x', 'y', 't'].
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp

		Returns:
			f_predict (1D array): aka f_star_xy; the spatially
				correlated component in the lightcurve at points
				X_predict
		"""

		if hp is not None:
			self.set_hp(hp)
		if lcf_basis is None:
			lcf_basis = self.get_basis(full=True)
		if X_predict is None:
			t_predict = self.get_lcf()[self.model_col].values
			X_predict = self.get_lcf()[self.X_cols].values
		elif isinstance(X_predict, pd.DataFrame):
			t_predict = X_predict[self.model_col].values
			X_predict = X_predict[self.X_cols].values
		else:
			# Assumes time is in the last column
			print("Check shape of X_predict!!!")
			import pdb; pdb.set_trace()
			t_predict = X_predict[-1,:]

		f_basis = lcf_basis['f'].values
		X_basis = lcf_basis[self.X_cols].values

		self.compute(X_basis)

		f_predict, _ = self.predict(f_basis, X_predict)

		#K_star = self.gp.kernel.get_value(x1=X_predict, x2=X_basis)
		#f_predict = np.matmul(K_star, self.gp.apply_inverse(f_basis))

		#if self._model is not None:
		#	# Better use the proper time column
		#	f_model = self._model.get_value(t_predict)
		#elif self._model is None:
		#	# The case of a constant offset
		#	f_model = self.get_parameter('y_offset')

		return f_predict

	def get_full_prediction(self, X_predict=None, lcf_basis=None, hp=None):
		"""Gets added temporal and spatial components.

		A bit useless on its own though.

		See get_temporal_component and get_spatial_component."""

		f_temporal = self.get_temporal_component(X_predict, lcf_basis, hp)
		return f_temporal

	def detrend_lightcurve(self, lcf_predict=None, lcf_basis=None, hp=None, save=True):
		"""Produces a decorrelated lightcurve.

		Args:
			lcf_predict (pd.DataFrame): the points on which to predict
				the trend. Must contain columns ['f', 'x', 'y', 't'].
				Default: self._lcf.
			lcf_basis (pd.DataFrame): the points to use as the basis
				from which to regress the trend. Must contain columns
				['f', 'x', 'y', 't'].
				Default: self._lcf_basis.
			hp (1D array): vector for the GP hyperparameters.
				Default: self.hp
			save (bool): if True, saves the lcf_predict to self._lcf.
				Default: False

		Returns:
			lcf_predict (pd.DataFrame): detrended values contain in
				'f_detrended', 'f_spatial', 'f_temporal'.
				Attention: f_spatial etc... ATTENTION: these have
				been changed to the more intuitive meaning or just
				the relevant component.
		"""

		if lcf_basis is None:
			lcf_basis = self.get_basis(full=True)
		if lcf_predict is None:
			lcf_predict = self._lcf.copy()
		if hp is not None:
			# Need to set the hp so that self._model works
			self.hp = hp

		f_temporal = self.get_temporal_component(lcf_predict, lcf_basis)

		lcf_predict['f_temporal'] = f_temporal

		# Special behaviour: ramp is supposed to be removed (detrender),
		# while other models aren't. So if the model is ramp, then
		# remove from detrended, otherwise keep in detrended
		f_detrended = lcf_predict['f'] - f_temporal

		if self._model is None:
			f_detrended = f_detrended + self.get_parameter('y_offset')
		elif isinstance(self._model, model_objects.RampModel):
			f_detrended = f_detrended + self.get_parameter('y_offset')
		else:
			f_detrended = f_detrended + self._model.get_value()

		lcf_predict['f_detrended'] = f_detrended

		if save:
			self.set_lcf(lcf_predict)
			# To be able to check if the saved lightcurve is
			# detrended with the most recent hyperparameters
			self.__detrending_hp = self.hp if hp is None else hp

		return lcf_predict

# ----------
# Exceptions
# ----------

class OversamplingPopulationError(ValueError):
	"""Thrown when attempting to sample more values than there
	exist in a set, without replacement."""

	pass

class NonPositiveDefiniteError(LinAlgError):
	"""When the GP coordinate is non positive definite.

	i.e to catch the LinAlgError throw by gp.compute(X).

	Attributes:
		X: the """

	def __init__(self, X, message=None, hp=None):

		message = message if message is not None else ''
		super().__init__('NonPositiveDefiniteError - ' + message)
		self.X_basis = pd.DataFrame(X, columns=['x', 'y', 't'])
		self.hp = hp

	def attach_detrender(self, k2_detrender_class):
		"""Setter for packing the whole detrender class."""

		self.k2_detrender = k2_detrender_class

class LAPACKError(Exception):
	"""When the GP coordinate is non positive definite.

	i.e to catch the LinAlgError throw by gp.compute(X).

	Attributes:
		X: the """

	def __init__(self, X_predict, f_basis, exception, X_basis=None, message=None, hp=None):

		message = message if message is not None else ''
		message = ("{}\n"
				   "Received error message: {}".format(message,
				   									   str(exception)))
		super().__init__('LAPACKError - possible _flapack.error - ' + message)
		self.X_predict = pd.DataFrame(X_predict, columns=['x', 'y', 't'])
		self.f_predict = pd.Series(f_basis)
		if X_basis is not None:
			self.X_basis = pd.DataFrame(X_basis, columns=['x', 'y', 't'])
		if hp is not None:
			self.hp = hp
		self.exception = exception

# -------
# Testing
# -------

def test_init():
	"""Tests the different kernels, and the different models."""
	from . import kernels as k2_kernels
	from ..__init__ import LOCAL_K2

	lcf = pd.read_pickle("{}/ultracool/diagnostic_targets/201573260-C14/k2gp201573260-c14-detrended-track.pickle".format(LOCAL_K2))

	# Testing kernels
	# ---------------

	kernel_classic = k2_kernels.ClassicK2Kernel()
	kernel_k2qp = k2_kernels.QuasiPeriodicK2Kernel()
	kernel_1dqp = k2_kernels.QuasiPeriodic1DKernel()

	for k in [kernel_classic, kernel_k2qp, kernel_1dqp]:
		print("Initialising with kernel:", type(k))
		dt_test = K2Detrender(lcf, k, model=None)
		dt_test.get_parameter_names()
		dt_test.get_hp()
		dt_test.log_prior()
		dt_test.lnposterior()
		dt_test.parameter_bounds
		dt_test.select_basis(100)
		dt_test.get_full_prediction()

	# Testing models
	# --------------

	for model in [None, 'y_offset', 'ramp', 1.0]:
		print("Initialising with model:", model)
		dt_test = K2Detrender(lcf, kernel_classic, model)
		dt_test.get_parameter_names()
		dt_test.get_parameter_names(include_frozen=True)
		dt_test.get_hp()
		dt_test.get_hp(include_frozen=True)
		dt_test.log_prior()
		dt_test.lnposterior()
		dt_test.parameter_bounds
		dt_test.select_basis(100)
		dt_test.get_full_prediction()

# TODO: following is obsolete
# ---------------------------
# def test_main(qp=False, N=1000):
# 	"""Tests the initialisation and basic functioning of the detrender."""

# 	import time
# 	from ..__init__ import LOCAL_DATA
# 	from . import kernels as k2_kernels

# 	if not qp:
# 		k2_kernel = k2_kernels.ClassicK2Kernel()
# 	else:
# 		k2_kernel = k2_kernels.QuasiPeriodicK2Kernel(3)
# 	#ramp = model_objects.RampModel(A_0=1)

# 	lcf = pd.read_pickle('{}/trappist/k2gp200164267-c12-lcf-full.pickle'.format(LOCAL_DATA))
# 	lcf['x'] = lcf.x_pos
# 	lcf['y'] = lcf.y_pos
# 	lcf = lcf_tools.initialise_lcf(lcf, f_col='f_tpf_track')

# 	detrender = K2Detrender(lcf, k2_kernel, ramp=True)

# 	# Run basic tests
# 	print("Parameter vector: {}".format(detrender.get_hp(True)))
# 	# print("Ocut value: {}".format(detrender.ocut))
# 	# print("Local names: {}".format(detrender._pname_to_local(detrender.parameter_names)))
# 	# print("de_bounds: {}".format(detrender.de_bounds))
# 	# print("log_likelihood: {}".format(detrender.log_likelihood(
# 	# 	detrender.hp,
# 	# 	detrender.get_basis(full=True).f
# 	# )))
# 	# print("grad_log_likelihood: {}".format(detrender.grad_log_likelihood(
# 	# 	detrender.hp,
# 	# 	detrender.get_basis(full=True).f
# 	# )))
# 	# print("log_prior: {}".format(detrender.log_prior(detrender.hp)))

# 	print('y_offset: {}'.format(detrender.get_parameter('y_offset')))

# 	# Detrending test
# 	detrender.select_basis(N=N)
# 	detrender.detrend_lightcurve()
# 	detrender.plot_model(show=True)

# 	tstart = time.time()
# 	result1 = detrender.optimise_hp_powell(diagnose=True)
# 	print("Runtime fmin_powell: {}".format(time.time() - tstart))
# 	print(detrender.hp)
# 	detrender.detrend_lightcurve()
# 	detrender.plot_model(show=True)

# def test_prior(N=1000):
# 	"""Tests the initialisation and basic functioning of the detrender."""

# 	import time
# 	from ..__init__ import LOCAL_DATA
# 	from . import kernels as k2_kernels

# 	k2_kernel = k2_kernels.QuasiPeriodicK2Kernel(3)
# 	#ramp = model_objects.RampModel(A_0=1)

# 	lcf = pd.read_pickle('{}/trappist/k2gp200164267-c12-lcf-full.pickle'.format(LOCAL_DATA))
# 	lcf['x'] = lcf.x_pos
# 	lcf['y'] = lcf.y_pos
# 	lcf = lcf_tools.initialise_lcf(lcf, f_col='f_tpf_track')

# 	detrender = K2Detrender(lcf, k2_kernel, ramp=True)

# 	try:
# 		detrender.set_parameter('At', -100)
# 		detrender.verify_prior()
# 	except PriorInitialisationError as e:
# 		print("Verification flagged successfully.")
# 		print(e)
# 	else:
# 		print("Verification wasn't successful, At was not flagged.")
# 		import pdb; pdb.set_trace()
# 	detrender.set_parameter('At', -12)

# 	from IPython.terminal import embed
# 	embed.embed()

# 	# Detrending test
# 	detrender.select_basis(N=N)
# 	detrender.detrend_lightcurve()
# 	detrender.plot_model(show=True)

# 	tstart = time.time()
# 	result1 = detrender.optimise_hp_powell(diagnose=True)
# 	print("Runtime fmin_powell: {}".format(time.time() - tstart))
# 	print(detrender.hp)
# 	detrender.detrend_lightcurve()
# 	detrender.plot_model(show=True)

# def diagnose_objects():
# 	"""Tests the initialisation and basic functioning of the detrender."""

# 	from IPython.terminal import embed
# 	from ..__init__ import LOCAL_DATA
# 	from . import kernels as k2_kernels

# 	k2_kernel = k2_kernels.ClassicK2Kernel()
# 	#ramp = model_objects.RampModel(A_0=1)

# 	lcf = pd.read_pickle('{}/trappist/k2gp200164267-c12-lcf-full.pickle'.format(LOCAL_DATA))
# 	lcf['x'] = lcf.x_pos
# 	lcf['y'] = lcf.y_pos
# 	lcf = lcf_tools.initialise_lcf(lcf, f_col='f_tpf_track')

# 	k2 = K2Detrender(lcf, k2_kernel, ramp=True)

# 	# Run basic tests
# 	print("k2.parameter_names: {} - {}\n".format(len(k2.parameter_names), k2.parameter_names))
# 	print("gp.get_parameter_names: {} - {}\n".format(len(k2.gp.get_parameter_names()), k2.gp.get_parameter_names()))
# 	print("gp.parameter_names: {} - {}\n".format(len(k2.gp.parameter_names), k2.gp.parameter_names))
# 	print("gp.unfrozen_mask: {} - {}\n".format(len(k2.gp.unfrozen_mask), k2.gp.unfrozen_mask))
# 	print("\n" + "~"*50 + "\n")

# 	for key, model in k2.gp.models.items():
# 		print("{}, length: {} - {}".format(key, len(model.unfrozen_mask), model.unfrozen_mask))
# 		print("{}, length: {} - {}".format(key, len(model.parameter_names), model.parameter_names))
# 		print("{}, length: {} - {}".format(key, len(model.get_parameter_names()), model.get_parameter_names()))
# 		print("{}, length: {} - {}".format(key, len(model.get_parameter_vector()), model.get_parameter_vector()))
# 		print("\n\n")
# 	print("\n" + "~"*50 + "\n")

# 	print("Testing if sub-model freezing affects ModelSet.")
# 	print("{} - {}".format(k2._ramp.unfrozen_mask, k2.gp.unfrozen_mask))
# 	print("Freezing alpha in gp.")
# 	k2.freeze_parameter("alpha")
# 	print("{} - {}".format(k2._ramp.unfrozen_mask, k2.gp.unfrozen_mask))
# 	print("Freezing A in k2._ramp.")
# 	k2._ramp.freeze_parameter("A")
# 	print("{} - {}".format(k2._ramp.unfrozen_mask, k2.gp.unfrozen_mask))
# 	print("\n" + "~"*50 + "\n")
# 	print("Freezing A in gp.")
# 	k2.gp.freeze_parameter("mean:A")
# 	print("{} - {}".format(k2._ramp.unfrozen_mask, k2.gp.unfrozen_mask))
# 	print("\n" + "~"*50 + "\n")

# 	print("k2.get_hp(): {} - {}\n".format(len(k2.get_hp()), k2.get_hp()))
# 	print("k2.get_parameter_names: {} - {}\n".format(len(k2.get_parameter_names()), k2.get_parameter_names()))
# 	print("k2.parameter_names: {} - {}\n".format(len(k2.parameter_names), k2.parameter_names))
# 	print("\n" + "~"*50 + "\n")

# 	embed.embed()

# def diagnosis_plot(lcf, show=False, title=None):
# 	"""Plots a diagnosis style set of graphs to see result."""

# 	fig, ax = plt.subplots(2, sharex=True)
# 	ax[0].plot(lcf.t, lcf.f, '.', c='0.5', alpha=0.5, label='$f_{raw}$')
# 	ax[0].plot(lcf.t, lcf.f_detrended, 'r-', label='$f_d$')
# 	ax[0].plot(lcf.t, lcf.f - lcf.f_spatial - lcf.f_temporal,
# 			   'k.', label='$f - f_s - f_t$')
# 	ax[0].legend()
# 	ax[1].plot(lcf.t, lcf.f_spatial + lcf.f_temporal,
# 			   'k.', label='$f_s + f_t$')
# 	ax[1].plot(lcf.t, lcf.f_spatial, 'r-', label='$f_s$')
# 	ax[1].plot(lcf.t, lcf.f_temporal, 'b-', label='$f_t$')
# 	ax[1].plot(lcf.t, lcf.f - lcf.f_detrended, '-', c='0.5', label='$mu$', alpha=0.5, zorder=-1)
# 	ax[1].plot(lcf.t, lcf.f, '.', c='0.8', alpha=0.3, label='$f_{raw}$', zorder=-2)
# 	ax[1].legend()

# 	fig.suptitle(title)
# 	if show:
# 		plt.show()
# 	else:
# 		fig.show()

# def diagnose_hp(detrender_class, hp=None):
# 	"""Prints information for current hp value in context."""

# 	print("\nHyperparameter diagnosis\n~~~~~~~~~~~~~~~~~~~~~~~~")

# 	hp = detrender_class.hp if hp is None else hp
# 	php = detrender_class.k2_kernel.v2p(hp)
# 	hpp = detrender_class.k2_kernel.hpp
# 	hpn = detrender_class.k2_kernel.parameter_names
# 	for i, n in enumerate(hpn):
# 		print("`{}` value: {:.2f}".format(n, php[i]),
# 			  "\t- bounds: {:.2f}, {:.2f}\t".format(*hpp[n][:2]))
# 	print("Prior value: {}\n".format(detrender_class.log_prior(hp)))