import json
import os
import time
import traceback
from socket import gethostname
from threading import Thread

from framework.base.internal_msgQ import InternalMsgQ
from framework.base.module_thread import (ScheduledModuleThread, SensorThread,
    SensorThreadState)
from framework.base.sspl_constants import (OperatingSystem, cs_legacy_products,
    cs_products, enabled_products, DATA_PATH)
# Import modules to control
from framework.messaging.egress_processor import \
    EgressProcessor
from framework.messaging.ingress_processor import \
    IngressProcessor
from framework.utils.conf_utils import (
    GLOBAL_CONF, SSPL_CONF, Conf, SSPL_LL_SETTING, NODE_ID_KEY)
from framework.utils.service_logging import logger
from json_msgs.messages.actuators.thread_controller import ThreadControllerMsg
from json_msgs.messages.sensors.thread_monitor_msg import ThreadMonitorMsg
from message_handlers.disk_msg_handler import DiskMsgHandler
# Note that all threaded message handlers must have an import here to be controlled
from message_handlers.node_controller_msg_handler import \
    NodeControllerMsgHandler
from message_handlers.node_data_msg_handler import NodeDataMsgHandler
from message_handlers.real_stor_actuator_msg_handler import \
    RealStorActuatorMsgHandler
from message_handlers.real_stor_encl_msg_handler import RealStorEnclMsgHandler
from message_handlers.service_msg_handler import ServiceMsgHandler
from framework.utils.store_factory import store

module_persistent_data = {}
module_cache_dir = os.path.join(DATA_PATH, "modules")
node_id = Conf.get(GLOBAL_CONF, NODE_ID_KEY, "SN01")

def _get_recovery_config(module_name):
    """
    Read sspl config for corresponding module recovery configs.

    Common sensor recovery config will be overriden by individual
    module recovery config.
    """
    recovery_count = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_recovery_count", 0)
    recovery_interval = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_recovery_interval", 0)
    # Override common recovery config if individual module has it
    recovery_count = Conf.get(
        SSPL_CONF,
        f"{module_name.upper()}>sensor_recovery_count", recovery_count)
    recovery_interval = Conf.get(
        SSPL_CONF,
        f"{module_name.upper()}>sensor_recovery_interval", recovery_interval)
    return recovery_count, recovery_interval

# Global method used by Thread to capture and log errors.  This must be global.
def execute_thread(module, msgQlist, conf_reader, product, resume=True):
    """
    Run module as a thread. Recover the module if any error during
    initialization and run time of the module.

    If recovery count>0,
        module will be recovered from failure until the maximum recovery
        attempt. If not recoverable, corresponding module will be shutdown
        and failure alert will be raised due to its impact.
    If recovery count=0,
        no recovery attempt will be made.
    """
    module_name = module.name()
    # Suspend module threads
    if resume == False:
        module.suspend()

    # Initialize persistent cache for sensor status
    per_data_path = os.path.join(
        module_cache_dir, f"{module_name.upper()}_{node_id}")
    if not os.path.isfile(per_data_path):
        module_persistent_data[module_name] = {}
        store.put(module_persistent_data[module_name], per_data_path)

    is_sensor_thread = False
    recovery_count = recovery_interval = 0
    if isinstance(module, SensorThread):
        recovery_count, recovery_interval = _get_recovery_config(module_name)
        is_sensor_thread = True

    attempt = 0

    while attempt <= recovery_count:
        attempt += 1
        try:
            # Each module is passed a reference list to message queues so it
            # can transmit internal messages to other modules as desired
            module.start_thread(conf_reader, msgQlist, product)
        except Exception as err:
            curr_state = "fault"
            err_msg = f"{module_name}, {err}"
            logger.error(err_msg)
            if attempt > recovery_count:
                logger.debug(traceback.format_exc())
                description = f"{module_name} is stopped and unrecoverable. {err_msg}"
                impact = module.impact()
                recommendation = "Restart SSPL service"
                logger.critical(
                    f"{description}. Impact: {impact} Recommendation: {recommendation}")
                # Check previous state of the module and send fault alert
                if os.path.isfile(per_data_path):
                    module_persistent_data[module_name] = store.get(per_data_path)
                prev_state = module_persistent_data[module_name].get('prev_state')
                if is_sensor_thread and curr_state != prev_state:
                    module_persistent_data[module_name] = {"prev_state": curr_state}
                    store.put(module_persistent_data[module_name], per_data_path)
                    specific_info = Conf.get(SSPL_CONF, f"{module_name.upper()}")
                    info = {
                        "module_name": module_name,
                        "alert_type": curr_state,
                        "description": description,
                        "impact": impact,
                        "recommendation": recommendation,
                        "severity": "critical",
                        "specific_info": specific_info
                    }
                    jsonMsg = ThreadMonitorMsg(info).getJson()
                    module._write_internal_msgQ(EgressProcessor.name(), jsonMsg)
            else:
                logger.debug(f"Recovering {module_name} from failure, "
                             f"attempt: {attempt}")
                time.sleep(recovery_interval)

            # Shutdown if no recovery attempt
            logger.info(f"Terminating monitoring thread {module_name}")
            module.shutdown()
            retry = 5
            while module.is_running():
                module.shutdown()
                retry -= 1
                if not retry:
                    break
                time.sleep(2)

def _check_module_recovered(module):
    """
    Once SSPL is restarted, check current status of the module after
    certain recovery cycle time. If module is running and its previous
    state is fault, raise fault_resolved alert and update cache.
    """
    module_name = module.name()
    # Wait till sensor module completes few run cycle. Then
    # raise module recovery fault_resolved alert.
    polling_cycle_time = Conf.get(
        SSPL_CONF, f"{SSPL_LL_SETTING}>sensor_polling_cycle_time", 60)
    time.sleep(polling_cycle_time)
    if not module.is_running():
        return

    curr_state = "fault_resolved"
    per_data_path = os.path.join(
        module_cache_dir, f'{module_name.upper()}_{node_id}')
    if not os.path.isfile(per_data_path):
        module_persistent_data[module_name] = {}
        store.put(module_persistent_data[module_name], per_data_path)
    # Check previous state before sending fault resolved alert
    module_persistent_data[module_name] = store.get(per_data_path)
    prev_state = module_persistent_data[module_name].get('prev_state')
    if prev_state and curr_state != prev_state:
        module_persistent_data[module_name] = {"prev_state": curr_state}
        store.put(module_persistent_data[module_name], per_data_path)
        specific_info = Conf.get(SSPL_CONF, f"{module_name.upper()}")
        info = {
            "module_name": module_name,
            "alert_type": curr_state,
            "description": f"{module_name} is recovered",
            "impact": "",
            "recommendation": "",
            "severity": "info",
            "specific_info": specific_info
        }
        jsonMsg = ThreadMonitorMsg(info).getJson()
        module._write_internal_msgQ(EgressProcessor.name(), jsonMsg)

class ThreadController(ScheduledModuleThread, InternalMsgQ):

    MODULE_NAME = "ThreadController"
    PRIORITY = 1

    # Section and keys in configuration file
    THREADCONTROLLER = MODULE_NAME.upper()
    ALWAYS_ACTIVE_MODULES = [
        "EgressProcessor", "IngressProcessor",
        "ThreadController"
    ]

    # Constats for keys to read from conf file
    SSPL_SETTING = 'SSPL_LL_SETTING'
    DEGRADED_STATE_MODULES = 'degraded_state_modules'

    @staticmethod
    def name():
        """ @return: name of the monitoring module."""
        return ThreadController.MODULE_NAME

    def __init__(self):
        super(ThreadController, self).__init__(self.MODULE_NAME,
                                                  self.PRIORITY)
        self._threads_initialized = False
        self._thread_response = "N/A"
        self.debug_section = None

        # Location of hpi data directory populated by dcs-collector
        self._hpi_base_dir = "/tmp/dcs/hpi"
        self._start_delay = 10
        self._systemd_support = True
        self._hostname = gethostname()
        self._modules_to_resume = []

    def get_sspl_module(self, module):
        try:
            return self._sspl_modules[module]
        except KeyError:
            return None

    def initialize(self, conf_reader, msgQlist, product):
        """initialize configuration reader and internal msg queues"""
        # Initialize ScheduledMonitorThread
        super(ThreadController, self).initialize(conf_reader)

        # Initialize internal message queues for this module
        super(ThreadController, self).initialize_msgQ(msgQlist)
        self._modules_to_resume = self._get_degraded_state_modules_list()

    def initialize_thread_list(self, sspl_modules, operating_system, product,
                               systemd_support):
        """initialize list of references to all modules"""
        self._sspl_modules = sspl_modules
        self._product = product
        self._operating_system = operating_system
        self._systemd_support = systemd_support

        if operating_system == OperatingSystem.CENTOS7.value or operating_system == OperatingSystem.RHEL7.value or \
            operating_system.lower() in OperatingSystem.CENTOS7.value.lower() or operating_system.lower() in OperatingSystem.RHEL7.value.lower():
            # Note that all threaded sensors and actuators must have an
            # import here to be controlled
            from sensors.impl.centos_7.disk_monitor import DiskMonitor
            from sensors.impl.centos_7.service_monitor import ServiceMonitor
            from sensors.impl.centos_7.drive_manager import DriveManager
            from sensors.impl.centos_7.hpi_monitor import HPIMonitor

        # Handle configurations for specific products
        if product.lower() == "cs-a":
            from sensors.impl.generic.SMR_drive_data import SMRdriveData
        if product.lower() in [x.lower() for x in enabled_products]:
            from sensors.impl.platforms.realstor.realstor_disk_sensor \
                import RealStorDiskSensor
            from sensors.impl.platforms.realstor.realstor_psu_sensor \
                import RealStorPSUSensor
            from sensors.impl.platforms.realstor.realstor_fan_sensor \
                import RealStorFanSensor
            from sensors.impl.platforms.realstor.realstor_controller_sensor \
                import RealStorControllerSensor
            from sensors.impl.platforms.realstor.realstor_sideplane_expander_sensor \
                import RealStorSideplaneExpanderSensor
            from sensors.impl.platforms.realstor.realstor_dg_volume_sensor \
                import RealStorLogicalVolumeSensor
            from sensors.impl.platforms.realstor.realstor_enclosure_sensor \
                import RealStorEnclosureSensor
            from sensors.impl.generic.raid import RAIDsensor
            from sensors.impl.generic.raid_integrity_data import RAIDIntegritySensor

    def run(self):
        """Run the module periodically on its own thread."""
        if (self._product.lower() in [x.lower() for x in enabled_products]) and \
           not self._threads_initialized:
            if self._product.lower() in [x.lower() for x in cs_products]:
                # Wait for the dcs-collector to populate the /tmp/dcs/hpi directory
                while not os.path.isdir(self._hpi_base_dir):
                    logger.info(
                        "ThreadController, dir not found: %s " % self._hpi_base_dir)
                    logger.info(
                        "ThreadController, rechecking in %s secs" % self._start_delay)
                    time.sleep(int(self._start_delay))

            logger.debug("ThreadController._sspl_modules is {}".format(
                self._sspl_modules))
            # Allow other threads to initialize
            continue_waiting = False
            for (n,m) in self._sspl_modules.items():
                if not isinstance(m, SensorThread):
                    continue
                thread_init_status = m.get_thread_init_status()
                logger.debug("Thread status for {} is {}".format(
                    m.__class__, thread_init_status))
                if thread_init_status == SensorThreadState.WAITING:
                    continue_waiting = True

            if continue_waiting:
                logger.debug("ThreadController, waiting for all modules to initialize")
                self._scheduler.enter(10, self._priority, self.run, ())
                return

            # Notify external applications that've started up successfully
            startup_msg = "SSPL-LL service has started successfully"
            json_msg = ThreadControllerMsg(ThreadController.name(), startup_msg).getJson()
            self._write_internal_msgQ(EgressProcessor.name(), json_msg)
            self._threads_initialized = True

            # Check sensor module is recovered from previous failure
            for module in self._sspl_modules.values():
                if not isinstance(module, SensorThread):
                    continue
                module_checker = Thread(target=_check_module_recovered,
                                        args=(module,))
                module_checker.start()

            #self._set_debug(True)
            #self._set_debug_persist(True)
            self._log_debug("Start accepting requests")
        try:
            # Block on message queue until it contains an entry
            jsonMsg, _ = self._read_my_msgQ()
            if jsonMsg is not None:
                self._process_msg(jsonMsg)

            # Keep processing until the message queue is empty
            while not self._is_my_msgQ_empty():
                jsonMsg, _ = self._read_my_msgQ()
                if jsonMsg is not None:
                    self._process_msg(jsonMsg)
        except Exception as ex:
            # Log it and restart the whole process when a failure occurs
            logger.exception("ThreadController restarting: %r" % ex)

        self._scheduler.enter(1, self._priority, self.run, ())
        self._log_debug("Finished processing successfully")

    def _process_msg(self, jsonMsg):
        """Parses the incoming message and calls the appropriate method"""
        self._log_debug("_process_msg, jsonMsg: %s" % jsonMsg)

        # Check to see if debug mode is being globally turned off on all modules
        if self._check_reset_all_modules(jsonMsg) is True:
            return

        # Parse out the module name and request
        module_name    = jsonMsg.get("actuator_request_type").get("thread_controller").get("module_name")
        thread_request = jsonMsg.get("actuator_request_type").get("thread_controller").get("thread_request")

        # Parse out the uuid so that it can be sent back in Ack message
        uuid = None
        if jsonMsg.get("sspl_ll_msg_header") is not None and \
           jsonMsg.get("sspl_ll_msg_header").get("uuid") is not None:
            uuid = jsonMsg.get("sspl_ll_msg_header").get("uuid")
            self._log_debug("_processMsg, uuid: %s" % uuid)

        # Pass along the debug section to the module
        if jsonMsg.get("sspl_ll_debug") is not None:
            self.debug_section = { "sspl_ll_debug": {}}
            self.debug_section["sspl_ll_debug"] = jsonMsg.get("sspl_ll_debug")
        else:
            self.debug_section = None

        self._log_debug("_process_msg, self.debug_section: %s" % self.debug_section)

        # Parse out thread request and call the appropriate method
        if thread_request == "restart":
            self._restart_module(module_name)
        elif thread_request == "start":
            self._start_module(module_name)
        elif thread_request == "stop":
            # Don't let the outside world stop us or shut down this thread
            if module_name == "EgressProcessor" or \
                module_name == "IngressProcessor" or \
                module_name == "ThreadController":
                    logger.warn("Attempt to stop message processors or ThreadController Processors, \
                                    ignoring. Please try 'restart' instead.")
                    return
            self._stop_module(module_name)
        elif thread_request == "status":
            self._status_module(module_name)
        elif thread_request == "degrade":
            if module_name.lower() != "all":
                logger.warn(
                    "Invalid module_name {0}. Need 'all' in module_name"
                    .format(module_name))
                return
            self._switch_to_degraded_state(self._sspl_modules)
        elif thread_request == "active":
            if module_name.lower() != "all":
                logger.warn(
                    "Invalid module_name {0}. Need 'all' in module_name"
                    .format(module_name))
                return
            self._switch_to_active_state(self._sspl_modules)
        else:
            self._thread_response = "Error, unrecognized thread request"

        node_id = []
        if jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters") is not None and \
           jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id"):
            node_id = jsonMsg.get("actuator_request_type").get("thread_controller").get("parameters").get("node_id")

        ack_type = {}
        ack_type["hostname"] = self._hostname
        ack_type["node_id"] = node_id

        # Populate an actuator response message and transmit
        threadControllerMsg = ThreadControllerMsg(module_name, self._thread_response, \
                                                  json.dumps(ack_type))

        if uuid is not None:
            threadControllerMsg.set_uuid(uuid)
        msgString = threadControllerMsg.getJson()
        logger.info("ThreadController, response: %s" % str(msgString))
        if self._product.lower() in [x.lower() for x in enabled_products]:
            self._write_internal_msgQ(EgressProcessor.name(), msgString)
        elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
            self._write_internal_msgQ(EgressProcessor.name(), msgString)

    def _restart_module(self, module_name):
        """Restart a module"""
        self._log_debug("_restart_module, module_name: %s" % module_name)

        try:
            # Stop the module if it's running and let existing thread die gracefully
            if self._status_module(module_name) is True:
                self._stop_module(module_name)

            # Allow module a few seconds to shut down gracefully
            max_wait  = 10
            curr_wait = 1
            while self._status_module(module_name) is True:
                time.sleep(3)
                logger.info("Retrying: %s" % str(curr_wait))
                self._stop_module(module_name)
                curr_wait += 1
                if curr_wait > max_wait:
                    break

            # Start the module
            self._start_module(module_name)
        except Exception as ae:
            logger.warn("Restart thread failed: %s" % str(ae))
            self._thread_response = "Restart Failed"
        else:
            self._thread_response = "Restart Successful"

    def _switch_to_degraded_state(self, modules):
        """Shifts SSPL to degraded state. Essentially it calls a suspend of
           every module every running in an independent thread
        """
        self._log_debug("_switch_to_degraded_state, modules: %s" % modules)
        if not modules:
            raise TypeError("module parameter can't be None")
        try:
            for name, module_instance in modules.items():
                if not name in self._modules_to_resume:
                    module_instance.suspend()
            self._thread_response = "Degrade Successful"
        except Exception as e:
            logger.warn("Degrade operation failed: {0}".format(e))
            self._thread_response = "Degrade failed"

    def _switch_to_active_state(self, modules):
        """Shifts SSPL to active state. Essentially it calls a resume of
           every module every running in an independent thread
        """
        self._log_debug("_switch_to_active_state, modules: %s" % modules)
        if not modules:
            raise TypeError("module parameter can't be None")
        try:
            for name, module_instance in modules.items():
                module_instance.resume()
            self._thread_response = "Active Successful"
        except Exception as e:
            logger.error("Active operation failed: {0}".format(e))
            self._thread_response = "Active failed"

    def _stop_module(self, module_name):
        """Stop a module"""
        self._log_debug("_stop_module, module_name: %s" % module_name)

        try:
            if self._status_module(module_name) is False:
                self._log_debug("_stop_module, status: False")
                return

            self._thread_response = "Stop Successful"

            # Put a debug message on the module's queue before shutting it down
            if self.debug_section is not None:
                self._write_internal_msgQ(module_name, self.debug_section)

            # Call the module's shutdown method for a graceful halt
            self._sspl_modules[module_name].shutdown()
        except Exception as ae:
            logger.warn("Stop thread failed: %s" % str(ae))
            self._thread_response = "Stop Failed"

    def _start_module(self, module_name):
        """Start a module"""
        self._log_debug("_start_module, module_name: %s" % module_name)

        try:
            if self._status_module(module_name) is True:
                self._log_debug("_start_module, status: True")
                return

            self._thread_response = "Start Successful"

            # NOTE: This is internal code that is currently unused.
            # If this is brought into use again its interaction
            # with the init dependency code will need to be considered
            module_thread = Thread(target=execute_thread,
                                   args=(self._sspl_modules[module_name],
                                         self._msgQlist, self._conf_reader,
                                         self._product))

            # Put a configure debug message on the module's queue before starting it up
            if self.debug_section is not None:
                self._write_internal_msgQ(module_name, self.debug_section)

            module_thread.start()
        except Exception as ae:
            logger.warn("Start thread failed: %s" % str(ae))
            self._thread_response = "Start Failed"

    def _status_module(self, module_name):
        """Returns if the module is running or not"""
        if self._sspl_modules[module_name].is_running() is True:
            self._thread_response = "Status: Running"
        else:
            self._thread_response = "Status: Halted"

        self._log_debug("_status_module, module_name: %s, _thread_response: %s" %
                        (module_name, self._thread_response))
        return self._sspl_modules[module_name].is_running()

    def _check_reset_all_modules(self, jsonMsg):
        """Restarts all modules with debug mode off. Activated by internal_msgQ"""
        if jsonMsg.get("sspl_ll_debug") is not None and \
            jsonMsg.get("sspl_ll_debug").get("debug_component") is not None and \
            jsonMsg.get("sspl_ll_debug").get("debug_component") == "all":
                for module in self._sspl_modules:
                    self._log_debug("_check_reset_all_modules, module: %s" % module)
                    # Don't restart this thread or it won't complete the loop
                    if module != self.name():
                        self._restart_module(module)

                # Populate an actuator response message and transmit
                msgString = ThreadControllerMsg("All Modules", "Restarted with debug mode off").getJson()
                self._write_internal_msgQ(EgressProcessor.name(), msgString)
                return True

        return False

    def shutdown_all_modules(self):
        """Calls shutdown for all modules"""
        logger.info("Shutting down all modules")
        for name, other_module in list(self._sspl_modules.items()):
            other_module.shutdown()

    def shutdown(self):
        """Clean up scheduler queue and gracefully shutdown thread"""
        super(ThreadController, self).shutdown()

    def check_EgressProcessor_is_running(self):
        """Used by the shutdown_handler to allow queued egress msgs to complete"""
        if self._product.lower() in [x.lower() for x in enabled_products]:
            return self._sspl_modules[EgressProcessor.name()].is_running()
        elif self._product.lower() in [x.lower() for x in cs_legacy_products]:
            return self._sspl_modules[EgressProcessor.name()].is_running()

    def _get_degraded_state_modules_list(self):
        """Reads list of modules to run in degraded state and returns a list
           of those modules.
        """
        # List of modules to run in degraded mode
        modules_to_resume = []
        try:
            # Read list of modules from conf file to load in degraded mode
            modules_to_resume = Conf.get(SSPL_CONF,
                f"{self.SSPL_SETTING}>{self.DEGRADED_STATE_MODULES}", [])
        except Exception as e:
            logger.warn("ThreadController: Configuration not found,"
                "degraded_state_modules")
        return modules_to_resume


#!/usr/bin/env python
# coding: utf-8

# This is script enables visualization of the effects Conservation Practices (NRS) have on Nitrogen Load.
# Acknowledgement: The NRS document refers to Nitrate-N concentration loss. Our current model calculates Nitrogen Surplus instead of Nitrogen Concentration. Currently, we assume that the change in % Nitrate-N is the same change for % Nitrogen Surplus.
#
#
# ![image-2.png](attachment:image-2.png)

# ### 0. Libraries

# In[1]:

get_ipython().run_line_magic('matplotlib', 'widget')
import ipywidgets as widgets
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import datetime as dt
import matplotlib as mpl
import pandas as pd
import seaborn as sns
sns.set()
import re

import textwrap
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

import IPython.display
from IPython.display import display, clear_output
from __future__ import print_function
from ipywidgets import interact, interactive, fixed, interact_manual

# ### 1. Get data

# In[2]:

df_ifew = pd.read_csv('NRS_IFEWs.csv')
df_pract = pd.read_csv('NRS_py_conservation.csv')

# In[3]:

df_pract

# ### 2. Organize data format

# In[4]:

# No need of silage values
df_ifew = df_ifew.drop(['CornSilageAcresHarvested','CornSilageYield_tonspacre'], axis = 1)

# Deal with Nan values
df_ifew.isna().sum()

# In[5]:

# Calculate based on other around year values
df_ifew = df_ifew.fillna(method = "bfill")

# Deal with Nan values
df_ifew.isna().sum()

# In[6]:

df_pract = df_pract.replace(to_replace=r'^-$', value=0, regex=True)

for i in range(len(df_pract)):
    df_pract['Names List'][i] = " ".join(df_pract['Names List'][i].split())

df_pract

# In[7]:

# convert the 'Date' column to datetime format
df_ifew['Year']= pd.to_datetime(df_ifew['Year'])
df_ifew['Year'] = df_ifew['Year'].dt.year

# In[8]:

# Create dropdown lists
y_lst = (df_ifew['Year'].unique()).tolist() #year
r_lst = ((df_ifew['CountyName'].unique()).tolist()) # region
r_lst.sort() # alphabeticallly
r_lst.insert(0,'Iowa')
p_lst = (df_pract['Names List']).tolist() #practice
#len(r_lst)

# In[9]:

df_r = df_ifew.loc[df_ifew['Year'] == 2015]
nr_change = df_pract.loc[df_pract['Names List']=='Moving from Fall to Spring Pre-plant Application','% AVG Nitrate-N Reduction+'].item()
df_r
Ns = df_r['NitrogenSurplus_kg_ha'].sum()
Nnew = Ns-((int(nr_change)/100)*Ns)

# In[10]:

# Create a function takes the info on the dropdown lists (chosen region) and calculates the Nitrogen surplus based on the chosen practice.

def n_ifew(df_ifew, selectedRegion, practice, df_pract, perc):
    # Region Selection
    r = selectedRegion
    df = df_ifew
    if (r == 'Iowa'):
        df_r = df.groupby(['Year']).sum()
        df_r = df_r.reset_index()
    else:
        df_r = df[df['CountyName'] == r].copy()

    # WHAT IF ACCOUNT FOR MULTIPLE PRACTICES?
    nr_change = df_pract.loc[df_pract['Names List']==practice,'% AVG Nitrate-N Reduction+'].item()
    cy_change = df_pract.loc[df_pract['Names List']==practice,'% Corn Yield Change++']

    #calculate values produced by query
    df_r.loc[:,'Nnew'] = df_r['NitrogenSurplus_kg_ha']-((int(nr_change)/100)*df_r['NitrogenSurplus_kg_ha'])
    df_r.loc[:,'Cnew'] = df_r['CornGrainYield_bupacre']+((int(cy_change)/100)*df_r['CornGrainYield_bupacre'])

    # total area in hectares of application of nitrogen - do we consider Soybean crops?
    #df_r['total_area_ha'] = (df_r['CornAcresPlanted']+ df_r['SoybeansAcresPlanted'])/2.57
    # total area in hectares of application of nitrogen - do we not consider Soybean crops?
    df_r.loc[:,'total_area_ha'] = (df_r['CornAcresPlanted'])/2.57

    # Nitrogen Load Old in metric tons
    df_r.loc[:,'Old_N_tons'] = df_r['total_area_ha']*df_r['NitrogenSurplus_kg_ha']/1000
    # Nitrogen Load New in metric tons
    df_r.loc[:,'New_N_tons'] = df_r['total_area_ha']*df_r['Nnew']/1000
    # Corn yield Old in bushels
    df_r.loc[:,'Old_Corn_bushels'] = df_r['CornAcresPlanted']*df_r['CornGrainYield_bupacre']
    # Corn Yield New
    df_r.loc[:,'New_Corn_bushels'] = df_r['CornAcresPlanted']*df_r['Cnew']

    return df_r

# In[11]:

#%%capture --no-display #hide warning here
trial = (n_ifew(df_ifew, 'Monroe', 'Moving from Fall to Spring Pre-plant Application', df_pract, 50))
trial

# In[12]:

# Plot Nitrogen
# Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['Old_N_tons'],
                    mode='lines',
                    name='Historical Nitrogen Load (tons)'))
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['New_N_tons'],
                    mode='lines',
                    name='Possible Nitrogen Load (tons)'))

fig.show()

# In[13]:

# Plot Corn
# Create traces
fig = go.Figure()
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['Old_Corn_bushels'],
                    mode='lines',
                    name='Historical Harvested Corn (bushels)'))
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['New_Corn_bushels'],
                    mode='lines',
                    name='Possible Harvested Corn (bushels)'))

fig.show()

# In[14]:

# Create figure with secondary y-axis
fig = make_subplots(specs=[[{"secondary_y": True}]])

# Add traces
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['Old_N_tons'],
                    mode='lines',
                    name='Historical Nitrogen Load (tons)'),
             secondary_y=False,
             )
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['New_N_tons'],
                    mode='lines',
                    name='Possible Nitrogen Load (tons)'),
             secondary_y=False,
             )

fig.add_trace(go.Scatter(x=trial["Year"], y=trial['Old_Corn_bushels'],
                    mode='lines',
                    name='Historical Harvested Corn (bushels)'),
             secondary_y=True,
             )
fig.add_trace(go.Scatter(x=trial["Year"], y=trial['New_Corn_bushels'],
                    mode='lines',
                    name='Possible Harvested Corn (bushels)'),
             secondary_y=True,
             )

# Add figure title
fig.update_layout(
    title_text="Change from Conservation Practice"
)

# Set x-axis title
fig.update_xaxes(title_text="Year")

# Set y-axes titles
fig.update_yaxes(title_text="Tons of Nitrogen", secondary_y=False)
fig.update_yaxes(title_text="Corn Bushels", secondary_y=True)

fig.show()

# ## Define Sliders Widget

# ## Text Widget

# In[15]:

CP = widgets.Dropdown(
    options = p_lst,
    value='Moving from Fall to Spring Pre-plant Application',
    description='Chosen Conservation Practice',
    disabled=False,
)

CR = widgets.Dropdown(
    options = r_lst,
    value='Monroe',
    description='Observed Region',
    disabled=False,
)

# In[16]:

def f(CP, CR):
    # Region Selection
    r = CR
    df = df_ifew
    if (r == 'Iowa'):
        df_r = df.groupby(['Year']).sum()
        df_r = df_r.reset_index()
    else:
        df_r = df[df['CountyName'] == r].copy()

    # WHAT IF ACCOUNT FOR MULTIPLE PRACTICES?
    nr_change = df_pract.loc[df_pract['Names List']==CP,'% AVG Nitrate-N Reduction+'].item()
    cy_change = df_pract.loc[df_pract['Names List']==CP,'% Corn Yield Change++']

   #calculate values produced by query
    df_r.loc[:,'Nnew'] = df_r['NitrogenSurplus_kg_ha']-((int(nr_change)/100)*df_r['NitrogenSurplus_kg_ha'])
    df_r.loc[:,'Cnew'] = df_r['CornGrainYield_bupacre']+((int(cy_change)/100)*df_r['CornGrainYield_bupacre'])

    # total area in hectares of application of nitrogen - do we consider Soybean crops?
    #df_r['total_area_ha'] = (df_r['CornAcresPlanted']+ df_r['SoybeansAcresPlanted'])/2.57
    # total area in hectares of application of nitrogen - do we not consider Soybean crops?
    df_r.loc[:,'total_area_ha'] = (df_r['CornAcresPlanted'])/2.57

    # Nitrogen Load Old in metric tons
    df_r.loc[:,'Old_N_tons'] = df_r['total_area_ha']*df_r['NitrogenSurplus_kg_ha']/1000
    # Nitrogen Load New in metric tons
    df_r.loc[:,'New_N_tons'] = df_r['total_area_ha']*df_r['Nnew']/1000
    # Corn yield Old in bushels
    df_r.loc[:,'Old_Corn_bushels'] = df_r['CornAcresPlanted']*df_r['CornGrainYield_bupacre']
    # Corn Yield New
    df_r.loc[:,'New_Corn_bushels'] = df_r['CornAcresPlanted']*df_r['Cnew']

    # Create figure with secondary y-axis
    fig = make_subplots(specs=[[{"secondary_y": True}]])

    # Add traces
    fig.add_trace(go.Scatter(x=df_r["Year"], y=df_r['Old_N_tons'],
                        mode='lines',
                        name='Historical Nitrogen Load (tons)'),
                 secondary_y=False,
                 )
    fig.add_trace(go.Scatter(x=df_r["Year"], y=df_r['New_N_tons'],
                        mode='lines',
                        name='Possible Nitrogen Load (tons)'),
                 secondary_y=False,
                 )

    fig.add_trace(go.Scatter(x=df_r["Year"], y=df_r['Old_Corn_bushels'],
                        mode='lines',
                        name='Historical Harvested Corn (bushels)'),
                 secondary_y=True,
                 )
    fig.add_trace(go.Scatter(x=df_r["Year"], y=df_r['New_Corn_bushels'],
                        mode='lines',
                        name='Possible Harvested Corn (bushels)'),
                 secondary_y=True,
                 )

    # Add figure title
    fig.update_layout(
        title_text="Change from Conservation Practice"
    )

    # Set x-axis title
    fig.update_xaxes(title_text="Year")

    # Set y-axes titles
    fig.update_yaxes(title_text="Tons of Nitrogen", secondary_y=False)
    fig.update_yaxes(title_text="Corn Bushels", secondary_y=True)

    fig.show()

ui = widgets.HBox([CP, CR])

out = widgets.interactive_output(f, {'CP': CP, 'CR': CR})

display(ui, out)


import requests
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
from PyPDF2 import PdfFileWriter, PdfFileReader
from requests_html import HTMLSession

# initialize the set of links (unique links)
internal_urls = set()
external_urls = set()
urls = set()

# This will make sure that a proper scheme (protocol, e.g http or https) and domain name exists in the URL.
def is_valid(url):
    """Checks whether `url` is a valid URL"""
    parsed = urlparse(url)
    return bool(parsed.netloc) and bool(parsed.scheme)

def get_all_website_links(url):

    session = HTMLSession()
    try:
        response = session.get(url)
    except:
        print("Check your url")
        return {}

    try:
        response.html.render()
    except:
        pass
#     soup = BeautifulSoup(requests.get(url).content, "html.parser")
    domain_name = urlparse(url).netloc
    soup = BeautifulSoup(response.html.html, "html.parser")  # this is assumed for javascript loaded pages
    for a_tag in soup.findAll("a"):
        href = a_tag.attrs.get("href")
        if href == "" or href is None:
            # href empty tag
            continue

        href = urljoin(url, href)
#         print(href)
        parsed_href = urlparse(href)
        if parsed_href.query != "":
            href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path + "?" + parsed_href.query
        else:
            href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path

        if not is_valid(href):
            # not a valid URL
            continue
        if href in internal_urls:
            # already in the set
            continue
        if domain_name not in href:
            # its external link
            if href not in external_urls:
                external_urls.add(href)
            continue
        # else its internel links
        urls.add(href)
        internal_urls.add(href)
    return urls

def make_dir(dirName):
    if not os.path.exists(dirName):
        os.mkdir(dirName)
        print("Directory " , dirName ,  " Created ")
    else:
        print("Directory " , dirName ,  " already exists")

total_urls_visited = 0

# help to call give number of times inside the website
def crawl(url, max_urls=10):
    """
    Crawls a web page and extracts all links.
    You'll find all links in `external_urls` and `internal_urls` global set variables.
    params:
        max_urls (int): number of max urls to crawl, default is 10.
    """
    global total_urls_visited
    total_urls_visited += 1
    links = get_all_website_links(url)
    links = list(links)
    if len(links) > 0:
        for link in links:
            if total_urls_visited > max_urls:
                break
            crawl(link, max_urls=max_urls)

if __name__ == "__main__":

    total_urls_visited = 0

    root_url = input("Enter the url:")
    max_url = int(input("Max number internel url to crawl:"))
    select_number_of_pdf = int(input("Number of pdf to dowload:"))

    links = get_all_website_links(root_url)
    print(len(links))
    list_links = list(links)
    if len(list_links) < max_url:
    	max_url = len(list_links)
    for i in range(max_url):
        get_all_website_links(list_links[i])

    print("[+] Total External links:", len(external_urls))
    print("[+] Total Internal links:", len(internal_urls))
    print("[+] Total:", len(external_urls) + len(internal_urls))

    #Create pdfs and txts
    import os
    parsed = urlparse(root_url)
    netloc = parsed.netloc

    if not os.path.exists("pdfs"):
        os.mkdir("pdfs")

    if not os.path.exists("txts"):
        os.mkdir("txts")

    pdf_dirName = os.path.join("pdfs",netloc.split(".")[1])
    txt_dirName = os.path.join("txts",netloc.split(".")[1])

    make_dir(pdf_dirName)
    make_dir(txt_dirName)

    internal_urls = list(internal_urls)
    pdf_urls = [iu for iu in internal_urls if iu.endswith(".pdf")]

    total_pdfs = len(pdf_urls)
    print(f"Total pdfs {total_pdfs}!")

    def dowload_pdf_save_txt(url):
        pdf_name = url.split("/")[-1]

        print(pdf_name)
        try:
            response = requests.get(url)

            with open(f'{pdf_dirName}/{pdf_name}', 'wb') as f:
                f.write(response.content)
                f.close()
            output = PdfFileWriter()
            inputs = PdfFileReader(open(f'{pdf_dirName}/{pdf_name}', "rb"))
            print (f"{pdf_name} has {inputs.getNumPages()} pages.")
            total_pages = inputs.getNumPages()
            txt = url+"\n" + ""
            for i in range(total_pages):
                txt+=inputs.getPage(i).extractText()

            with open(f'{txt_dirName}/{pdf_name.split(".")[0]}.txt', 'w+') as f:
                f.write(txt)
                f.close()
        except:
            pass

    import time
    t1 = time.time()
    count = 0

    for pdf_url in pdf_urls[:select_number_of_pdf]:
        count+=1
        dowload_pdf_save_txt(pdf_url)
    print(f"Total {count} pdfs downloaded:")
    t2 = time.time()
    print(f'!{t2-t1} " seconds take!')
    print("Lets see pdfs and txts folder")


#from __future__ import absolute_import

from models_utility.param_gp import Param
from models_utility.function_gp import  cholesky, lt_log_determinant
from models_utility.likelihoods import Gaussian
from torch import triangular_solve

from models.gp import gpr
import numpy as np
import torch
import torch.nn as nn

import torch
from torch.autograd import Variable
import numpy as np

#torch.set_default_tensor_type(torch.DoubleTensor)
torch.set_default_tensor_type(torch.FloatTensor)

jitter = 1e-8
class gp_vfe(gpr):

    def __init__(self, kernel, likelihood , device, param_dict):
        super(gp_vfe,self).__init__(kernel, likelihood, device ,param_dict)
        self.device = torch.device("cuda") if device else torch.device('cpu')
        #self.jitter = torch.tensor(1e-8).to(self.device)
        self.x = None
        self.y = None

    def _set_data(self, batch_x, batch_y):
        self.x = batch_x
        self.y = batch_y
        return

    def _set_inducing_pt(self,num_inducing):
        assert(self.x is not None)
        self.num_inducing_pt = num_inducing
        idx = np.random.choice(len(self.x),num_inducing)
        self.inducing_pt = Param(torch.tensor(self.x[idx]).to(self.device), requires_grad=True, requires_transform=False, param_name = 'inducing_pt')
        return

    def compute_loss(self,batch_x,batch_y,kl_option):
        num_inducing_pt = self.num_inducing_pt
        dim_output = batch_y.shape[1]
        num_data = batch_x.shape[0]

        K_ff_diag = self._compute_Kxx_diag(self.x)
        K_uf = self._compute_Kxs(self.inducing_pt, self.x)
        K_uu = self._compute_Kxx(self.inducing_pt)

        L = cholesky(K_uu)
        A = triangular_solve(K_uf, L, upper=False)[0]
        A_At = A.matmul(A.t()).div(self.likelihood.variance.transform()**2)
        B = A_At + torch.eye(int(num_inducing_pt)).to(self.device)
        L_B = cholesky(B)
        C = triangular_solve(A.matmul(self.y), L_B, upper=False)[0].div(self.likelihood.variance.transform()**2)

        Elbo = 0.
        Elbo += torch.tensor([-0.5 * dim_output * num_data * np.log(2 * np.pi)]).to(self.device)
        Elbo -= dim_output * L_B.diag().log().sum()
        Elbo -= 0.5 * dim_output * num_data * 2*self.likelihood.variance
        Elbo -= (0.5 / (self.likelihood.variance.transform()**2)) * (self.y.pow(2).sum() + dim_output * K_ff_diag.sum())
        Elbo += 0.5 * C.pow(2).sum()
        Elbo += 0.5 * dim_output * A_At.diag().sum()

        return -Elbo.squeeze()

    def _predict(self, inputs_new, diag = True):
        if isinstance(inputs_new, np.ndarray):
            inputs_new = torch.Tensor(inputs_new).to(self.device)

        num_inducing_pt = self.num_inducing_pt
        dim_output = self.y.shape[1]

        K_ff_diag = self._compute_Kxx_diag(self.x)
        K_uf = self._compute_Kxs(self.inducing_pt, self.x)
        K_uu = self._compute_Kxx(self.inducing_pt)
        K_us = self._compute_Kxs(self.inducing_pt, inputs_new)

        L = cholesky(K_uu)
        A = triangular_solve(K_uf, L, upper=False)[0]
        A_At = A.matmul(A.t()).div(self.likelihood.variance.transform()**2)
        B = A_At + torch.eye(int(num_inducing_pt)).to(self.device)
        L_B = cholesky(B)
        C = triangular_solve(A.matmul(self.y), L_B, upper=False)[0].div(self.likelihood.variance.transform()**2)

        temp1 = triangular_solve(K_us, L, upper=False)[0]
        temp2 = triangular_solve(temp1, L_B, upper=False)[0]
        mean_f = temp2.t().matmul(C)

        if diag:
            var = self._compute_Kxx_diag(inputs_new) + temp2.t().pow(2).sum(1) - temp1.pow(2).sum(0).squeeze() +  (self.likelihood.variance.transform()**2).diag().diag()
        else:
            var = self._compute_Kxx(inputs_new) + temp2.t().matmul(temp2) - temp1.t().matmul(temp1) +  self.likelihood.variance.transform()**2
        return mean_f, var


import tensorflow as tf
import numpy as np

import utils

class StackedDenoisingAutoencoder:
    """
    Implementation of a stacked denoising autoencoder using tensorflow
    """

    def __init__(self, corruption_type, corruption_fraction, weight_stddev, bias_init, learning_rate, epochs, batch_size):
        # Input
        self.corruption_type = corruption_type
        self.corruption_fraction = corruption_fraction

        # Initialization
        self.weight_stddev = weight_stddev
        self.bias_init = bias_init

        # Training
        self.learning_rate = learning_rate
        self.epochs = epochs
        self.batch_size = batch_size

        # Model Variables
        self.input_data = None
        self.corrupted_input_data = None
        self.weights = None
        self.biases = None
        self.encode = None
        self.flat_pool = None
        self.decode = None
        self.cost = None
        self.train_step = None

        # Build the model
        self.build_model2()

    def build_model(self):
        self.weights = tf.Variable(tf.truncated_normal([5, 5, 3, 32], stddev=self.weight_stddev))
        self.biases = tf.Variable(tf.constant(self.bias_init, shape=[32]))

        n_features = 3072

        self.input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        samples = tf.shape(self.input_data)[0]
        self.corrupted_input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        corrupted_image = tf.reshape(self.corrupted_input_data, [samples, 32, 32, 3])

        # Encode
        self.encode = tf.nn.relu(
            tf.nn.conv2d(corrupted_image, self.weights, strides=[1, 1, 1, 1], padding='SAME') + self.biases
        )

        # Flat Pool Encoding
        self.flat_pool = tf.reshape(
            tf.nn.max_pool(self.encode, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME'), [samples, 8192]
        )

        # Decode
        self.decode = tf.nn.conv2d_transpose(self.encode, self.weights, output_shape=[samples, 32, 32, 3],
                                             strides=[1, 1, 1, 1])

        # Flatten
        flat_decode = tf.reshape(self.decode, [samples, n_features])

        # Cost
        self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.input_data - flat_decode)))

        # Optimizer
        self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.cost)

    def build_model2(self):
        self.weights = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=self.weight_stddev))
        self.biases = tf.Variable(tf.constant(self.bias_init, shape=[64]))

        n_features = 8192

        self.input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        samples = tf.shape(self.input_data)[0]
        self.corrupted_input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        corrupted_image = tf.reshape(self.corrupted_input_data, [samples, 16, 16, 32])

        # Encode
        self.encode = tf.nn.relu(
            tf.nn.conv2d(corrupted_image, self.weights, strides=[1, 1, 1, 1], padding='SAME') + self.biases
        )

        # Pool Encoding
        self.flat_pool = tf.reshape(
            tf.nn.max_pool(self.encode, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME'), [samples, 4096]
        )

        # Decode
        self.decode = tf.nn.conv2d_transpose(self.encode, self.weights, output_shape=[samples, 16, 16, 32],
                                             strides=[1, 1, 1, 1])

        # Flatten
        flat_decode = tf.reshape(self.decode, [samples, n_features])

        # Cost
        self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.input_data - flat_decode)))

        # Optimizer
        self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.cost)

    def train(self, training_x, validation_x):
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())

        corruption_ratio = np.round(self.corruption_fraction * training_x.shape[1]).astype(np.int)
        for epoch in range(self.epochs):
            print("Epoch: {}".format(epoch))

            x_corrupted = self.corrupt_input(training_x, corruption_ratio)

            shuff = list(zip(training_x, x_corrupted))
            np.random.shuffle(shuff)

            batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)]

            for batch in batches:
                x_batch, x_corrupted_batch = zip(*batch)
                sess.run(self.train_step, feed_dict={self.input_data: x_batch,
                                                     self.corrupted_input_data: x_corrupted_batch})

            if epoch % 5 == 0:
                if validation_x is not None:
                    error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
                                                           self.corrupted_input_data: validation_x})
                    print("Validation cost is {}".format(error))

        if validation_x is not None:
            error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
                                                   self.corrupted_input_data: validation_x})
            print("Validation cost is {}".format(error))

        # Save the model
        #saver = tf.train.Saver()
        #saver.save(sess, 'SDAE_model2')

    def corrupt_input(self, data, v):

        if self.corruption_type == 'masking':
            x_corrupted = utils.masking_noise(data, v)

        elif self.corruption_type == 'salt_and_pepper':
            x_corrupted = utils.salt_and_pepper_noise(data, v)

        elif self.corruption_type == 'gaussian':
            x_corrupted = utils.gaussian_noise(data, v)

        elif self.corruption_type == 'none':
            x_corrupted = data

        else:
            x_corrupted = None

        return x_corrupted

    def transform(self, x, model_name):
        # load saved model
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())
        saver = tf.train.import_meta_graph(model_name)
        saver.restore(sess, tf.train.latest_checkpoint('./'))

        # Run against some data until pooling step
        pool = sess.run(self.flat_pool, feed_dict={self.input_data: x, self.corrupted_input_data: x})

        return pool

    def save_weights(self, x, model_name):
        # load saved model
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())
        saver = tf.train.import_meta_graph(model_name)
        saver.restore(sess, tf.train.latest_checkpoint('./'))

        weights, biases = sess.run([self.weights, self.biases], feed_dict={self.input_data: x, self.corrupted_input_data: x})
        return weights, biases


import uuid

import requests
from django.http import Http404
from django.http.request import QueryDict
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.filters import OrderingFilter
from rest_framework.mixins import CreateModelMixin as BaseCreateModelMixin
from rest_framework.mixins import DestroyModelMixin
from rest_framework.mixins import UpdateModelMixin as BaseUpdateModelMixin
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import ModelSerializer
from rest_framework.serializers import Serializer
from rest_framework.serializers import UUIDField
from rest_framework.serializers import ValidationError
from rest_framework.status import HTTP_201_CREATED
from rest_framework.status import HTTP_503_SERVICE_UNAVAILABLE
from six import string_types

from .utils.portal import registerfacility
from kolibri.core.auth.models import Facility
from kolibri.core.utils.urls import join_url
from kolibri.utils import conf

class KolibriDataPortalViewSet(viewsets.ViewSet):
    @action(detail=False, methods=["post"])
    def register(self, request):
        facility = Facility.objects.get(id=request.data.get("facility_id"))
        try:
            response = registerfacility(request.data.get("token"), facility)
        except requests.exceptions.RequestException as e:  # bubble up any response error
            return Response(e.response.json(), status=e.response.status_code)
        return Response(status=response.status_code)

    @action(detail=False, methods=["get"])
    def validate_token(self, request):
        PORTAL_URL = conf.OPTIONS["Urls"]["DATA_PORTAL_SYNCING_BASE_URL"]
        try:
            # token is in query params
            response = requests.get(
                join_url(
                    PORTAL_URL, "portal/api/public/v1/registerfacility/validate_token"
                ),
                params=request.query_params,
            )
        except requests.exceptions.ConnectionError:
            return Response({"status": "offline"}, status=HTTP_503_SERVICE_UNAVAILABLE)
        # handle any invalid json type responses
        try:
            data = response.json()
        except ValueError:
            data = response.content
        return Response(data, status=response.status_code)

class ValuesViewsetOrderingFilter(OrderingFilter):
    def get_default_valid_fields(self, queryset, view, context=None):
        """
        The original implementation of this makes the assumption that the DRF serializer for the class
        encodes all the serialization behaviour for the viewset:
        https://github.com/encode/django-rest-framework/blob/version-3.12.2/rest_framework/filters.py#L208

        With the ValuesViewset, this is no longer the case so here we do an equivalent mapping from the values
        defined by the values viewset, with consideration for the mapped fields.

        Importantly, we filter out values that have not yet been annotated on the queryset, so if an annotated
        value is requried for ordering, it should be defined in the get_queryset method of the viewset, and not
        the annotate_queryset method, which is executed after filtering.
        """
        if context is None:
            context = {}
        default_fields = set()
        # All the fields that we have field maps defined for - this only allows for simple mapped fields
        # where the field is essentially a rename, as we have no good way of doing ordering on a field that
        # that is doing more complex function based mapping.
        mapped_fields = {
            v: k for k, v in view.field_map.items() if isinstance(v, string_types)
        }
        # All the fields of the model
        model_fields = {f.name for f in queryset.model._meta.get_fields()}
        # Loop through every value in the view's values tuple
        for field in view.values:
            # If the value is for a foreign key lookup, we split it here to make sure that the first relation key
            # exists on the model - it's unlikely this would ever not be the case, as otherwise the viewset would
            # be returning 500s.
            fk_ref = field.split("__")[0]
            # Check either if the field is a model field, a currently annotated annotation, or
            # is a foreign key lookup on an FK on this model.
            if (
                field in model_fields
                or field in queryset.query.annotations
                or fk_ref in model_fields
            ):
                # If the field is a mapped field, we store the field name as returned to the client
                # not the actual internal field - this will later be mapped when we come to do the ordering.
                if field in mapped_fields:
                    default_fields.add((mapped_fields[field], mapped_fields[field]))
                else:
                    default_fields.add((field, field))

        return default_fields

    def remove_invalid_fields(self, queryset, fields, view, request):
        """
        Modified from https://github.com/encode/django-rest-framework/blob/version-3.12.2/rest_framework/filters.py#L259
        to do filtering based on valuesviewset setup
        """
        # We filter the mapped fields to ones that do simple string mappings here, any functional maps are excluded.
        mapped_fields = {
            k: v for k, v in view.field_map.items() if isinstance(v, string_types)
        }
        valid_fields = [
            item[0]
            for item in self.get_valid_fields(queryset, view, {"request": request})
        ]
        ordering = []
        for term in fields:
            if term.lstrip("-") in valid_fields:
                if term.lstrip("-") in mapped_fields:
                    # In the case that the ordering field is a mapped field on the values viewset
                    # we substitute the serialized name of the field for the database name.
                    prefix = "-" if term[0] == "-" else ""
                    new_term = prefix + mapped_fields[term.lstrip("-")]
                    ordering.append(new_term)
                else:
                    ordering.append(term)
        if len(ordering) > 1:
            raise ValidationError("Can only define a single ordering field")
        return ordering

class BaseValuesViewset(viewsets.GenericViewSet):
    """
    A viewset that uses a values call to get all model/queryset data in
    a single database query, rather than delegating serialization to a
    DRF ModelSerializer.
    """

    # A tuple of values to get from the queryset
    # values = None
    # A map of target_key, source_key where target_key is the final target_key that will be set
    # and source_key is the key on the object retrieved from the values call.
    # Alternatively, the source_key can be a callable that will be passed the object and return
    # the value for the target_key. This callable can also pop unwanted values from the obj
    # to remove unneeded keys from the object as a side effect.
    field_map = {}

    def __init__(self, *args, **kwargs):
        super(BaseValuesViewset, self).__init__(*args, **kwargs)
        if not hasattr(self, "values") or not isinstance(self.values, tuple):
            raise TypeError("values must be defined as a tuple")
        self._values = tuple(self.values)
        if not isinstance(self.field_map, dict):
            raise TypeError("field_map must be defined as a dict")
        self._field_map = self.field_map.copy()

    def generate_serializer(self):
        queryset = getattr(self, "queryset", None)
        if queryset is None:
            try:
                queryset = self.get_queryset()
            except Exception:
                pass
        model = getattr(queryset, "model", None)
        if model is None:
            return Serializer
        mapped_fields = {
            v: k for k, v in self.field_map.items() if isinstance(v, string_types)
        }
        fields = []
        extra_kwargs = {}
        for value in self.values:
            try:
                model._meta.get_field(value)
                if value in mapped_fields:
                    extra_kwargs[mapped_fields[value]] = {"source": value}
                    value = mapped_fields[value]
                fields.append(value)
            except Exception:
                pass

        meta = type(
            "Meta",
            (object,),
            {
                "fields": fields,
                "read_only_fields": fields,
                "model": model,
                "extra_kwargs": extra_kwargs,
            },
        )
        CustomSerializer = type(
            "{}Serializer".format(self.__class__.__name__),
            (ModelSerializer,),
            {"Meta": meta},
        )

        return CustomSerializer

    def get_serializer_class(self):
        if self.serializer_class is not None:
            return self.serializer_class
        # Hack to prevent the renderer logic from breaking completely.
        self.__class__.serializer_class = self.generate_serializer()
        return self.__class__.serializer_class

    def _get_lookup_filter(self):
        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field

        if lookup_url_kwarg not in self.kwargs:
            raise AssertionError(
                "Expected view %s to be called with a URL keyword argument "
                'named "%s". Fix your URL conf, or set the `.lookup_field` '
                "attribute on the view correctly."
                % (self.__class__.__name__, lookup_url_kwarg)
            )

        return {self.lookup_field: self.kwargs[lookup_url_kwarg]}

    def annotate_queryset(self, queryset):
        return queryset

    def _map_fields(self, item):
        for key, value in self._field_map.items():
            if callable(value):
                item[key] = value(item)
            elif value in item:
                item[key] = item.pop(value)
            else:
                item[key] = value
        return item

    def consolidate(self, items, queryset):
        return items

    def serialize(self, queryset):
        queryset = self.annotate_queryset(queryset)
        values_queryset = queryset.values(*self._values)
        return self.consolidate(
            list(map(self._map_fields, values_queryset or [])), queryset
        )

    def serialize_object(self, **filter_kwargs):
        try:
            filter_kwargs = filter_kwargs or self._get_lookup_filter()
            queryset = self.get_queryset().filter(**filter_kwargs)
            return self.serialize(self.filter_queryset(queryset))[0]
        except (IndexError, ValueError, TypeError):
            raise Http404(
                "No %s matches the given query." % queryset.model._meta.object_name
            )

class QueryParamRequest(Request):
    def __init__(self, query_params, *args, **kwargs):
        super(QueryParamRequest, self).__init__(*args, **kwargs)
        self._query_params = QueryDict(mutable=True)
        for key, value in query_params.items():
            self._query_params[key] = (
                ",".join(value) if isinstance(value, (list, set)) else value
            )
        self._query_params._mutable = False

    @property
    def query_params(self):
        return self._query_params

def _generate_request(request, query_params, method="GET"):
    ret = QueryParamRequest(
        query_params,
        request=request._request,
        parsers=request.parsers,
        authenticators=request.authenticators,
        negotiator=request.negotiator,
        parser_context=request.parser_context,
    )
    ret._data = request._data
    ret._files = request._files
    ret._full_data = request._full_data
    ret._content_type = request._content_type
    ret._stream = request._stream
    ret.method = method
    if hasattr(request, "_user"):
        ret._user = request._user
    if hasattr(request, "_auth"):
        ret._auth = request._auth
    if hasattr(request, "_authenticator"):
        ret._authenticator = request._authenticator
    if hasattr(request, "accepted_renderer"):
        ret.accepted_renderer = request.accepted_renderer
    if hasattr(request, "accepted_media_type"):
        ret.accepted_media_type = request.accepted_media_type
    if hasattr(request, "version"):
        ret.version = request.version
    if hasattr(request, "versioning_scheme"):
        ret.versioning_scheme = request.versioning_scheme
    return ret

class ListModelMixin(object):
    def _get_list_queryset(self):
        queryset = self.filter_queryset(self.get_queryset())

        page_queryset = self.paginate_queryset(queryset)

        if page_queryset is not None:
            return page_queryset, True
        return queryset, False

    def list(self, request, *args, **kwargs):
        queryset, paginated = self._get_list_queryset()

        if paginated:
            return self.get_paginated_response(self.serialize(queryset))

        return Response(self.serialize(queryset))

    def serialize_list(self, request, query_params=None, *args, **kwargs):
        """
        A method to allow serialization of objects for use outside of a regular
        request/response cycle - useful for obtaining identical serialized responses
        in a composite view that returns data from multiple viewsets at once.
        """
        self.request = _generate_request(request, query_params or {})
        response = self.list(self.request)
        return response.data

class RetrieveModelMixin(object):
    def retrieve(self, request, *args, **kwargs):
        return Response(self.serialize_object())

class ReadOnlyValuesViewset(BaseValuesViewset, RetrieveModelMixin, ListModelMixin):
    pass

class CreateModelMixin(BaseCreateModelMixin):
    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)
        instance = serializer.instance
        return Response(self.serialize_object(pk=instance.pk), status=HTTP_201_CREATED)

class UpdateModelMixin(BaseUpdateModelMixin):
    def update(self, request, *args, **kwargs):
        partial = kwargs.pop("partial", False)
        instance = self.get_object()
        serializer = self.get_serializer(instance, data=request.data, partial=partial)
        serializer.is_valid(raise_exception=True)
        self.perform_update(serializer)

        return Response(self.serialize_object())

class ValuesViewset(
    ReadOnlyValuesViewset, CreateModelMixin, UpdateModelMixin, DestroyModelMixin
):
    pass

class HexUUIDField(UUIDField):
    def __init__(self, **kwargs):
        kwargs["format"] = "hex"
        super(HexUUIDField, self).__init__(**kwargs)

    def to_internal_value(self, data):
        return super(HexUUIDField, self).to_internal_value(data).hex

    def to_representation(self, value):
        if isinstance(value, uuid.UUID):
            return value.hex
        return value


'''
Created on 08.08.2017

@author: Laptop8460
'''

import threading
import time
import traceback
import socket
import json
from sys import platform

import logging
import stopAll

logger = logging.getLogger('subscription')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler("log/subscription.log")
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
fmtr = logging.Formatter("%(threadName)s %(asctime)s  %(levelname)s: %(message)s","%H:%M:%S")
ch.setFormatter(fmtr)
logger.addHandler(fh)
logger.addHandler(ch)

stopAllEvent=threading.Event()

startSubscriptionUpdateEvent=threading.Event()
publishEvent = threading.Event()
subscriberUpdaterDebug = threading.Event()
subscriberListenerDebug= threading.Event()
subscriberListenerStopEvent = threading.Event()
subscriberSenderStopEvent=threading.Event()

def startSubscriberListener(broadcastIp, broadcastPort):
    # start register interface
    subscriberListener = threading.Thread(name="subscriptionService",
                                          target=runSubscriberListener,
                                          args=[broadcastIp,broadcastPort])
    subscriberListener.start()
    stopAll.AddShutdownEvent(subscriberListenerStopEvent)   # triggers a stop
    stopAll.AddShutdownEvent(publishEvent)      # triggers a send

def runSubscriberListener(ip,port):
    subscriberListenerStopEvent.clear()
    sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
    if platform=="linux" or platform=="linux2":
        ipa=ip.split('.')[0:3]
        ipa.append('255')
        sock.bind(('.'.join(ipa), port))
        logger.info("START SUBSCRIBER LISTENER AT %s:%s (LINUX:%s)"%(str(ip),str(port),str(ipa)))
    else:
        sock.bind((ip, port))
        logger.info("START SUBSCRIBER LISTENER AT %s:%s (WINDOWS)"%(str(ip),str(port)))
    while True:
        # wait for message
        #logger.debug("Wait for receive")
        recv,info = sock.recvfrom(1024)
        if subscriberListenerStopEvent.isSet():
            subscriberListenerStopEvent.clear()
            logger.info("STOP SUBSCRIBER LISTENER")
            break

        #logger.debug("RECEIVED from %s:%s Data:%s"%(str(info[0]),str(info[1]), str(recv) ))
        if info[0]==ip:
            # own broadcast
            #logger.debug("OWN BROADCAST RECEIVED from %s:%s Data:%s"%(str(info[0]),str(info[1]), str(recv) ))
            if not broadcastSentEvent.isSet():
                logger.error("Error: own broadcast was not received")
            broadcastSentEvent.clear()
        else:
            jsonData = json.loads(recv.decode())
            action=jsonData["action"]
            paydeskObj=jsonData["paydesk"]
            paydesk = paydesk_ctrl.post(paydeskObj)


import csv
import math

from django.forms.models import BaseInlineFormSet
from django.contrib import admin
from django.conf import settings
from django.db.models import Sum, Count
from django.http import HttpResponse

from . import models
from meet.admin import MeetDependentAdmin
from registration.models import Level

from django.utils.translation import ugettext_lazy as _

LED_SIGN_CODES = """
    The following are special codes you can use to customize the LED sign display

    MODES (defaults to rotate)

    |MODE_ROTATE|
    |MODE_HOLD|
    |MODE_FLASH|
    |MODE_ROLL_UP|
    |MODE_ROLL_DOWN|
    |MODE_ROLL_LEFT|
    |MODE_ROLL_RIGHT|
    |MODE_WIPE_UP|
    |MODE_WIPE_DOWN|
    |MODE_WIPE_RIGHT|
    |MODE_SCROLL|
    |MODE_AUTO_MODE|
    |MODE_ROLL_IN|
    |MODE_ROLL_OUT|
    |MODE_WIPE_IN|
    |MODE_WIPE_OUT|
    |MODE_TWINKLE|
    |MODE_SPARKLE|
    |MODE_SNOW|
    |MODE_INTERLOCK|
    |MODE_SWITCH|

"""

def export_as_csv(self, request, queryset):
    """ Generic csv export admin action. """
    opts = self.model._meta
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename={}.csv'.format(str(opts).replace('.', '_'))
    writer = csv.writer(response)
    field_names = [field.name for field in opts.fields]
    # Write a first row with header information
    writer.writerow(field_names)
    # Write data rows
    for obj in queryset:
        writer.writerow([getattr(obj, field) for field in field_names])
    return response
export_as_csv.short_description = "Export selected objects as csv file"

# Filters

class SessionFilter(admin.SimpleListFilter):
    # Human-readable title which will be displayed in the
    # right admin sidebar just above the filter options.
    title = _('session')

    # Parameter for the filter that will be used in the URL query.
    parameter_name = 'session'

    def lookups(self, request, model_admin):
        return [(s.id, s.name) for s in models.Session.objects.all()]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(division__session__id=self.value())
        else:
            return queryset

class LevelFilter(admin.SimpleListFilter):
    title = _('level')
    parameter_name = 'level'

    def lookups(self, request, model_admin):
        return [(s['level'], s['group']) for s in Level.objects.all().values('level', 'group').distinct().order_by('level')]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(level__level=self.value())
        else:
            return queryset

class LevelDivisionFilter(admin.SimpleListFilter):
    title = _('level division')
    parameter_name = 'level_division'

    def lookups(self, request, model_admin):
        return [(s.id, s.name) for s in Level.objects.all()]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(level__id=self.value())
        else:
            return queryset

class AgeDivisionFilter(admin.SimpleListFilter):
    title = _('age division')
    parameter_name = 'age_division'

    def lookups(self, request, model_admin):
        return [(s.id, s) for s in models.Division.objects.all()]

    def queryset(self, request, queryset):
        value = self.value()
        if queryset and value is not None:
            return queryset.filter(division__id=value)
        else:
            return queryset

class TeamFilter(admin.SimpleListFilter):
    title = _('team')
    parameter_name = 'team'

    def lookups(self, request, model_admin):
        return [(s.id, s.team) for s in models.Team.objects.all()]

    def queryset(self, request, queryset):
        value = self.value()
        if value is not None:
            return queryset.filter(team__id=value)
        else:
            return queryset

class StartingEventFilter(admin.SimpleListFilter):
    title = _('starting event')
    parameter_name = 'starting_event'

    def lookups(self, request, model_admin):
        lookups = [(s.id, s.name) for s in models.Event.objects.filter(active=True)]  # competition.Event
        lookups.append(('', '(None)'))
        return lookups

    def queryset(self, request, queryset):
        if self.value() is '':
            return queryset.filter(starting_event__isnull=True)
        elif self.value() is not None:
            return queryset.filter(starting_event__id=self.value())
        else:
            return queryset

class GymnastEventFilter(admin.SimpleListFilter):
    title = _('gymnast event')
    parameter_name = 'gymnast_event'

    def lookups(self, request, model_admin):
        lookups = [(s.id, s.name) for s in models.Event.objects.filter(active=True)]  # competition.Event
        return lookups

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(event__id=self.value())
        else:
            return queryset

# Admins

class GymnastEventInlineFormset(BaseInlineFormSet):
    def __init__(self, *args, **kwargs):
        super(GymnastEventInlineFormset, self).__init__(*args, **kwargs)
        self.can_delete = False

class GymnastEventInlineAdmin(admin.TabularInline):
    model = models.GymnastEvent
    formset = GymnastEventInlineFormset
    extra = 0
    max_num = 0
    readonly_fields = ('event', )
    fields = ('event', 'score',)
    verbose_name_plural = 'Gymnast Events'

@admin.register(models.Gymnast)
class GymnastAdmin(MeetDependentAdmin):
    ''' Base admin for the Mens Artistic and Women's Artistic admins. '''
    search_fields = ['athlete_id', 'last_name', 'first_name']
    inlines = (GymnastEventInlineAdmin, )
    readonly_fields = ('overall_score', 'rank', 'place', 'tie_break', 'age', 'team')
    list_filter = (TeamFilter, SessionFilter, StartingEventFilter, AgeDivisionFilter, LevelFilter, LevelDivisionFilter)
    list_per_page = 50
    list_display = ['athlete_id', 'last_name', 'first_name', 'show_team', 'division', 'session', 'starting_event']

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(GymnastAdmin, self).get_fieldsets(request, obj)
        fieldsets += (
            (None, {'fields': ('discipline', 'athlete_id', 'usag', 'last_name', 'first_name', 'team', ), }),
            ('Meet',
                {'fields':
                    ('is_scratched',
                     'age',
                     'division',
                     'level',
                     'starting_event',
                     'overall_score',
                     'rank',
                     'place',
                     'tie_break', ), }), )
        return fieldsets

    def get_readonly_fields(self, request, obj=None):
        # copying this from MeetDependentAdmin because I need the empty value display to be different here (empty string)
        # models.Meet.objects.filter(is_current_meet=True)[0]
        # if self.fieldsets and self.fieldsets[0][1]['fields'][0] == 'meet' and request.session.get('meet', ''):
        if self.fieldsets and self.fieldsets[0][1]['fields'][0] == 'meet' and models.Meet.objects.filter(is_current_meet=True).count() == 1:
            self.readonly_fields += ('meet',)
            self.empty_value_display = ''
            return self.readonly_fields
        else:
            return []

    def session(self, gymnast):
        if gymnast.division:
            return models.Session.objects.get(divisions=gymnast.division).name
        return '-'
    session.admin_order_field = 'division__session__name'

    def get_queryset(self, request):
        qs = super(GymnastAdmin, self).get_queryset(request)
        qs = qs.annotate(aa=Sum('events__score'))
        return qs

    def all_around(self, obj):
        return '{:.1f}'.format(obj.aa)
    all_around.admin_order_field = 'aa'
    all_around.short_description = 'AA'

    def show_team(self, obj):
        return obj.team.team
    show_team.short_description = "Team"
    show_team.admin_order_field = 'team__team'

    def has_add_permission(self, request, obj=None):
            return False

@admin.register(models.MensArtisticGymnast)
class MensArtisticGymnastAdmin(GymnastAdmin):
    pass

    def get_list_display(self, request):
        result = ['athlete_id',
                  'last_name',
                  'first_name',
                  'show_team',
                  'division',
                  'session',
                  'starting_event',
                  'fx',
                  'ph',
                  'sr',
                  'vt',
                  'pb',
                  'hb']
        result += ['all_around', ]
        return result

@admin.register(models.WomensArtisticGymnast)
class WomensArtisticGymnastAdmin(GymnastAdmin):
    pass

    def get_list_display(self, request):
        result = ['athlete_id',
                  'last_name',
                  'first_name',
                  'show_team',
                  'division',
                  'session',
                  'starting_event',
                  'vt',
                  'ub',
                  'bb',
                  'fx']
        result += ['all_around', ]
        return result

@admin.register(models.TeamAward)
class TeamAwardAdmin(MeetDependentAdmin):
    list_display = ('name', 'award_count', 'number_qualified_teams', 'order',)
    filter_horizontal = ('levels',)
    list_editable = ('order',)
    actions = ['meet_awards_percentage']

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(TeamAwardAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('name', 'levels', 'award_count', 'order',),
                'description': ''}), )
        return fieldsets

    def number_qualified_teams(self, obj):
        qualified_teams = obj.qualified_teams()
        return qualified_teams.count()
    number_qualified_teams.short_description = '# Qualified Teams'

    def meet_awards_percentage(self, request, queryset):
        ''' Count how many teams, with at least 3 gymnasts, registered to be considered for team awards.
            Calculate percentage of that number, using value stored in Meet Admin.
            Save calculated percentage to the award.

            gymnasts__level__group__in=award.levels.all().values('group') needed to handle the
            edge case of an award that spans two levels, which the BWI does for levels 9/10
        '''
        for award in queryset:
            awards = models.Team.objects.filter(
                team_awards=award, gymnasts__is_scratched=False,
                gymnasts__level__group__in=award.levels.all().values('group'))\
                .annotate(num_gymnasts=Count('id')).exclude(num_gymnasts__lt=3)

            if awards.count() == 1:
                award.award_count = 1
            else:
                award.award_count = math.ceil(awards.count() * award.meet.team_award_percentage)
            award.save()
    meet_awards_percentage.short_description = "Calculate team awards from %% in Meet admin"

@admin.register(models.TeamAwardRank)
class TeamAwardRankAdmin(MeetDependentAdmin):
    list_display = ('team', 'team_award', 'rank', 'score')
    list_filter = ['team', 'team_award']

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.TeamAwardRankEvent)
class TeamAwardRankEventAdmin(MeetDependentAdmin):
    list_display = ('team_award_rank', 'event', 'gymnast_event', 'rank')
    list_filter = ['team_award_rank__team', GymnastEventFilter]
    search_fields = ['gymnast_event__gymnast__first_name', 'gymnast_event__gymnast__last_name']

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.GymnastEvent)
class GymnastEventAdmin(MeetDependentAdmin):
    list_display = ('gymnast', 'event', 'score', 'rank', 'place')
    list_filter = [GymnastEventFilter]
    search_fields = ['gymnast__first_name', 'gymnast__last_name', 'id', ]

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(GymnastEventAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('gymnast', 'event', 'score', 'rank', 'place'),
                'description': ''}), )
        return fieldsets

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.Division)
# TODO: Come back to this one
class DivisionAdmin(MeetDependentAdmin):
    list_display = ('name', 'level', 'num_gymnasts', 'min_age', 'max_age', 'event_award_count', 'all_around_award_count')
    list_editable = ('min_age', 'max_age', 'event_award_count', 'all_around_award_count')
    ordering = ('level', 'min_age')
    actions = ['meet_awards_percentage']
    list_filter = ['level', 'level__group']

    def meet_awards_percentage(self, request, queryset):
        # Make sure to exclude scratched gymnasts when running the award count
        for division in queryset:
            division.event_award_count = math.ceil(division.gymnasts.filter(is_scratched=False).count() * division.meet.event_award_percentage)
            division.all_around_award_count = math.ceil(division.gymnasts.filter(is_scratched=False).count() * division.meet.all_around_award_percentage)
            division.save()
    meet_awards_percentage.short_description = "Calculate AA & event awards from %%s in Meet admin"

    def get_queryset(self, request):
        # TODO: annotate query with num_gymnasts, so the column can be sortable in the admin
        qs = super(DivisionAdmin, self).get_queryset(request)
        return qs

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(DivisionAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('level', 'name', 'short_name', 'min_age', 'max_age', 'event_award_count', 'all_around_award_count'),
                'description': ''}), )
        return fieldsets

@admin.register(models.Event)
class EventAdmin(admin.ModelAdmin):
    list_display = ('name', 'initials', 'order', 'is_mag', 'is_wag', 'active')
    actions = ['toggle_mag_wag']

    def toggle_mag_wag(self, request, queryset):
        for event in queryset:
            # Don't let this work on trampoline events
            if not (event.is_wag is False and event.is_mag is False):
                event.is_wag = False if event.is_wag else True
                event.is_mag = False if event.is_mag else True
                event.active = False if event.active else True
                event.save()
    toggle_mag_wag.short_description = "Switch active events, mens and womens"

@admin.register(models.Session)
class SessionAdmin(MeetDependentAdmin):
    list_display = ('name', 'num_gymnasts', 'warmup', 'session_start', 'timed_warmup_start', 'presentation_start', 'competition_start', )
    filter_horizontal = ('divisions',)

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(SessionAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('name',
                           'divisions',
                           'warmup',
                           'session_start',
                           'timed_warmup_start',
                           'presentation_start',
                           'competition_start'),
                'description': ''}), )
        return fieldsets

@admin.register(models.ScoreRankEvent)
class ScoreRankEventAdmin(MeetDependentAdmin):
    list_display = ('gymnast', 'fx', 'ph', 'sr', 'vt', 'pb', 'hb', 'ub', 'bb')
    search_fields = ['gymnast__last_name', 'gymnast__first_name', 'gymnast__usag']

@admin.register(models.LEDSign)
class LEDSignAdmin(admin.ModelAdmin):
    list_display = ('name', 'device', 'connect')

    def connect(self, obj):
        return 'Connect Sign'.format(obj.device)
    connect.allow_tags = True

    class Media:
        js = ("{}js/ledsign.js".format(settings.STATIC_URL),)

class LEDShowMessageInline(admin.TabularInline):
    model = models.LEDShowMessage
    fields = ('led_sign', 'message')
    extra = 0
    min_num = 6

@admin.register(models.LEDShow)
class LEDShowAdmin(admin.ModelAdmin):
    model = models.LEDShow
    list_display = ('name',)
    inlines = (LEDShowMessageInline, )
    fieldsets = ((None, {
        'fields': ('name',),
        'description': LED_SIGN_CODES}), )


import datetime
from typing import List

from . import setup, data_classes
from ..resources import values
from ..settings import hyper_parameters

def usd_to_quote(amount_usd, product_index):
    return amount_usd/setup.quote_currency_prices[product_index]

def usd_to_base(amount_usd, price_per_unit_usd):
    return amount_usd/price_per_unit_usd

def quote_to_usd(amount_quote, product_index):
    return amount_quote*setup.quote_currency_prices[product_index]

def base_to_usd(amount_base, price_per_unit_usd):
    return amount_base/price_per_unit_usd

def get_forecast(prediction_graph) -> int:
    dates = prediction_graph['dates']
    forecast = round(dates[-1] - dates[-2]) / 24 / 3600
    return int(forecast)

def investment_condition_satisfied(prediction_graph):
    investment_condition = hyper_parameters.investment_condition_factor*get_forecast(prediction_graph)
    currency_values = prediction_graph['predictions']
    prediction = currency_values[-1] / currency_values[-2] - 1
    if prediction > investment_condition:
        return True
    else:
        return False

def get_prediction_from_prediction_graph(prediction_graph):
    currency_values = prediction_graph['predictions']
    prediction = currency_values[-1] / currency_values[-2] - 1
    return prediction

def get_current_timestamp():
    current_datetime = datetime.datetime.combine(datetime.date.today(), values.min_time)
    current_timestamp = datetime.datetime.timestamp(current_datetime)
    return current_timestamp

def setup_investment_sizes(pending_investments: List[data_classes.PendingInvestment]):
    if len(pending_investments) == 0:
        return []
    investments = []
    forecast = pending_investments[0].get_forecast()
    for pending_investment in pending_investments:
        investment_condition = hyper_parameters.investment_condition_factor*forecast
        split_for_investment = hyper_parameters.investment / investment_condition * pending_investment.prediction / forecast
        investments.append(split_for_investment)
    capital = hyper_parameters.capital / forecast
    if sum(investments) > capital:
        factor = capital / sum(investments)
        investments = [n * factor for n in investments]
    for pending_investment, split_usd in zip(pending_investments, investments):
        pending_investment.setup_size(split_usd)
    return pending_investments

def get_product_id(base_currency):
    try:
        index = setup.base_currencies.index(base_currency)
        return setup.product_ids[index]
    except ValueError:
        return None

def update_trade_data_entry(trade_data, order_id, size, sell_date):
    for trade_data_entry in trade_data:
        if trade_data_entry['order_id'] == order_id:
            trade_data_entry['size'] = size
            trade_data_entry['sell_date'] = sell_date
    return trade_data

def remove_trade_data_entry(trade_data, order_id):
    trade_data = list(filter(lambda it: it['order_id'] != order_id, trade_data))
    return trade_data

def create_new_trade_data_entry(order_id, pending_investment: data_classes.PendingInvestment):
    trade_data_entry = {
        'order_id': order_id,
        'ticker': pending_investment.ticker,
        'size': None,
        'product_id': pending_investment.product_id,
        'buy_date': pending_investment.buy_date,
        'sell_date': pending_investment.sell_date
    }
    return trade_data_entry

def get_account_balance(currency):
    balance = None
    for account in setup.portfolio:
        if account['currency'] == currency:
            balance = account['balance']
    return float(balance)

def trade_history_entry_from_pending_order(pending_order, prediction_graphs, order_id):
    if isinstance(pending_order, data_classes.PendingInvestment):
        side = 'buy'
    elif isinstance(pending_order, data_classes.PendingDump):
        side = 'sell'
    else:
        raise ValueError('Input type {} not supported.'.format(type(pending_order)))
    current_market_price = None
    if get_current_market_price_usd(pending_order.ticker[0], prediction_graphs) is not None:
        current_market_price = \
            get_current_market_price_usd(pending_order.ticker[0], prediction_graphs)*pending_order.size
    trade_history_entry = {
        'order_id': order_id,
        'ticker': pending_order.ticker,
        'product_id': pending_order.product_id,
        'side': side,
        'size': pending_order.size,
        'value_usd': current_market_price,
        'date': datetime.datetime.now().timestamp()
    }
    return trade_history_entry

def get_current_market_price_usd(ticker, prediction_graphs):
    for prediction_graph in prediction_graphs:
        if prediction_graph['ticker'][0] == ticker:
            return prediction_graph['predictions'][-2]
    else:
        return None

def get_current_portfolio(portfolio_history, prediction_graphs):
    value_history = []
    for portfolio_history_entry in portfolio_history:
        value = 0
        for account in portfolio_history_entry['portfolio']:
            market_price_usd = get_current_market_price_usd(account['currency'], prediction_graphs)
            if market_price_usd is not None:
                value += market_price_usd*float(account['balance'])
            elif account['currency'] == 'USD':
                value += float(account['balance'])
            elif account['currency'] == 'EUR':
                value += float(account['balance'])*setup.euro_price_usd
        value_history.append({'value': value, 'date': datetime.datetime.now().timestamp()})

    return {'portfolio': portfolio_history[-1], 'value_history': value_history}

def get_filled_size(client, order_id):
    fills = client.get_fills(order_id=order_id)
    size = 0
    for fill in fills:
        size += float(fill['size'])
    return size

def float_to_string(value, decimal_places):
    return '{:.{}f}'.format(value, decimal_places)


from django.urls import path
from .views import LoginView, RegisterView, UpdateView, user_logout, gallery

urlpatterns = [
    path('signin/', LoginView.as_view(), name='login'),
    path('signup/', RegisterView.as_view(), name='register'),
    path('userupdate/', UpdateView.as_view(), name='update'),
    path('logout/', user_logout, name='logout'),
    path('gallery/', gallery)

]


try:
    from methods.regular.regular_api import *
    from methods.source_control.file.remove import remove_core as file_remove_core
    from methods.source_control.file.file_browser import File_Browser

except:
    from default.methods.regular.regular_api import *
    from default.methods.source_control.file.remove import remove_core as file_remove_core
    from default.methods.source_control.file.file_browser import File_Browser

from sqlalchemy.orm.session import Session
from flasgger import swag_from

@routes.route('/api/v1/project/<string:project_string_id>/file/<int:file_id>/update-metadata', methods = ['PUT'])
@Project_permissions.user_has_project(
    Roles = ["admin", "Editor"],
    apis_user_list = ['api_enabled_builder', 'security_email_verified'])
@swag_from('../../../docs/files/file_update_metadata.yml')
def api_file_update_metadata(project_string_id: str, file_id: int):
    """
        Updates a file ID metadata.
    :param project_string_id: the project where the file belongs.
    :param file_id: the file ID to update
    :return:
    """

    spec_list = [
        {'rotation_degrees': {'type': int, 'required': False}},
        {'ordinal': {'type': int, 'required': False}}
    ]

    log, input, untrusted_input = regular_input.master(request = request,
                                                       spec_list = spec_list)
    if len(log["error"].keys()) >= 1:
        return jsonify(log = log), 400

    with sessionMaker.session_scope() as session:

        member = get_member(session)
        project = Project.get(session, project_string_id)

        updated_file, log = file_update_metadata_core(
            session = session,
            project = project,
            file_id = file_id,
            rotation_degrees = input['rotation_degrees'],
            ordinal = input['ordinal'],
            log = log
        )

        if len(log["error"].keys()) >= 1:
            return jsonify(log = log), 400
        log['success'] = True
        return jsonify(log = log, file = updated_file), 200

def file_update_metadata_core(session: Session,
                              project: Project,
                              file_id: int,
                              rotation_degrees: int,
                              ordinal: int,
                              log: dict = regular_log.default()):
    file = File.get_by_id(session = session, file_id = file_id)
    if file.project_id != project.id:
        log['error']['project'] = f'File does not belong to project {project.project_string_id}'
        return None, log

    if file.type == 'image':
        if rotation_degrees is not None:
            valid_degrees = [0, 90, 180, 270]
            if rotation_degrees not in valid_degrees:
                log['error']['rotation_degrees'] = f'Invalid rotation degrees can only be {valid_degrees}'
                return None, log
            file.image.rotation_degrees = rotation_degrees
            session.add(file.image)
    if ordinal is not None:
        file.ordinal = ordinal
    session.add(file)
    result = file.serialize_with_type(session = session, regen_url = False)
    return result, log