#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
June 24, 2019

Figure 8. A) Alongshore beach width change for Bogue Banks for 1997-2016 (green), 1997-2010 (red),
and 2010-2016 (blue). B) Density distributions of beach width change for Bogue Banks for 1997-2016,
1997-2010, and 2010-2016. Positive values indicate progradation, negative values indicate erosion.
C) Mean beach width change for the entire island, Fort Macon, non-fenced areas, and fenced areas
colored by time period (with 95% confidence intervals).

- Creates the figure and a .txt file with the statistics used

@author: michaelitzkin
"""

import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import scipy.signal as sig
import scipy.stats as stats
import seaborn as sns
import pandas as pd
import numpy as np
import os

pd.set_option('mode.chained_assignment', None)

"""
Utility functions
"""

def confidence_intervals(data):
    """
    Calculate the 95% confidence interval
    """

    x_bar = np.nanmean(data)    # Mean value
    s = np.nanstd(data)         # Standard deviation
    n = len(data)               # Sample size

    lo_conf = x_bar - (1.96 * (s / np.sqrt(n)))  # Lower bound of confidence interval
    hi_conf = x_bar + (1.96 * (s / np.sqrt(n)))  # Upper bound of confidence interval

    conf_range = hi_conf - lo_conf  # Size of the 95% confidence interval

    return lo_conf, hi_conf, conf_range

def chunks(l, n):
    """
    Split list l into n sized chunks

    https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def estimated_autocorrelation(x):
    """
    http://stackoverflow.com/q/14297012/190597
    http://en.wikipedia.org/wiki/Autocorrelation#Estimation

    Called by:
    - rate_uncertainty()
    """
    n = len(x)
    variance = np.nanvar(x)
    x -= np.nanmean(x)
    r = np.correlate(x, x, mode='full')[-n:]
    # assert np.allclose(r, np.array([(x[:n - k]*x[-(n - k):]).sum() for k in range(n)]))
    result = r / (variance * np.arange(n, 0, -1))
    return result

def fft_smooth(y, points):
    """
    Smooth noisy data with an FFT filter

    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way
    """
    box = np.ones(points)/points
    y_smooth = np.convolve(y, box, mode='same')
    return y_smooth

def lidar_vertical_error(yr):
    """
    Return the vertical error of the lidar data for the
    given year. Returns in meters
    """

    if yr in ['1997', '1998', '1999', '2000', '2004', '2010', '2011']:
        return 0.15
    elif yr == '2005':
        return 0.20
    elif yr == '2014':
        return 0.062
    elif yr == '2016':
        return 0.20

def mean_residual_error(df, delta):
    """
    Calculate the mean residual error for the initial and final
    time periods. Than use that to calculate an average rate error.
    - df: Dataframe being worked on
    - delta: Dataframe of change rates
    - window: Window to average over
    """

    # Perform an autocorrelation to determine the window size
    autocorr = estimated_autocorrelation(delta)
    sig_vals = np.where(autocorr <= 0.05)
    n_star = np.nanmin(sig_vals)

    row = 0
    for beta in chunks(delta, n_star):

        # Calculate the average change rate of the
        # current chunk of profiles
        mu_beta = np.nanmean(beta)

        # Calculate the residuals for the current chunk
        residuals = beta - mu_beta

        # Calculate the MRE for the current chunk
        mre = np.sqrt((1 / len(beta)) * np.nansum(residuals ** 2))

        # Add the error to the dataframe
        df[row:row+len(beta)] = mre

        # Increment the row
        row += len(beta)

    return df

def rate_uncertainty(t0, tf, pos_uncertainty):
    """
    Calculate uncertainty in the change rate, based on
    methods in Hapke et al. (2011)

    - t0: beginning time
    - tf: end time
    - pos_uncertainties: positional uncertainty

    Called by:
    - shoreline_change_df()
    """

    # Convert the times to ints
    t0, tf = int(t0), int(tf)

    # Find the rate uncertainty based on the positional uncertainties
    ur = pos_uncertainty / (tf - t0)

    return ur

def regional_uncertainty(r, ur, alpha=0.05):
    """
    Calculate reigonal uncertainty. From Hapke et al (2011),
    equation 9

    - r: Calculated rates of change
    - alpha: significance value (default = 0.05)
    """

    # Perform an autocorrelation on r to determine
    # the number of independent samples
    r_acr = estimated_autocorrelation(ur[~np.isnan(ur)])
    r_acr_vals = np.where(r_acr <= alpha)
    n_star = np.nanmin(r_acr_vals)

    # Calculate the mean uncertainty
    Ur_bar = np.nanmean(ur)

    # Calculate the uncertainty
    Urq_bar = (1 / np.sqrt(n_star)) * Ur_bar
    Urq_bar = np.around(Urq_bar, decimals=2)
    return Urq_bar

"""
Functions to load an format data
"""

def beach_width_change_df(widths, rate=False):
    """
    Calculate the average rate of change for the beach width for each interval and its associated
    uncertainty. The uncertainty method is based on that used in Hapke et al (2012)
    but modified for the data outputted from Automorph.
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    new_df = widths[['1997', '2010', '2016', '1997-2010', '1997-2016', '2010-2016']]

    # At this point, there is a column for the x-position of the shoreline for every year.
    # Add columns for the rates
    for interval in intervals:

        # Create new columns
        orig_df, begin_df, end_df, begin_time, end_time, dt = setup_change_df(interval, new_df)

        # Calculate the change rate between the years being analyzed
        change = end_df - begin_df
        new_df[interval + ' Change'] = change
        new_df[interval + ' Mean'] = np.nanmean(change)

        # Calculate uncertainty in the MHW position
        up1, up2 = mhw_uncertainty(begin_time, end_time)

        # Calculate uncertainty in the dune toe position
        up3, up4 = dlow_uncertainty(begin_df, end_df, begin_time, end_time)

        # Calculate the rate uncertainty and regional uncertainty
        # based on the positional uncertainty
        position_uncertainty = np.sqrt((up1 ** 2) + (up2 ** 2) + (up3 ** 2) + (up4 ** 2))
        ur = rate_uncertainty(begin_time, end_time, position_uncertainty)
        region_uncertainty = regional_uncertainty(change, ur)
        new_df[interval + ' Mean Residual Error'] = mean_residual_error(new_df[interval + ' Mean Residual Error'], change)
        new_df[interval + ' Uncertainty'] = abs(ur)
        new_df[interval + ' Regional Uncertainty'] = abs(region_uncertainty)

    # Save the dataframe as a .csv
    save_name = os.path.join('..', 'Data', 'Beach Widths.csv')
    new_df.to_csv(save_name)

    # Return the new dataframe
    return new_df

def change_df(d1997, d2010, d2016):
    """
    Make a dataframe of change rates between lidar
    surveys and their smoothed trends

    - d1997: Dataframe with values from 1997
    - d2010: Dataframe with values from 2010
    - d2016: Dataframe with values from 2016
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    # Concatenate the data into one dataframe
    new_df = pd.concat([d1997, d2010, d2016], axis=1, keys=['1997', '2010', '2016'])

    # Add columns for the rates
    for interval in intervals:

        # Add a column with the change rate
        new_df[interval] = (new_df[interval[-4:]] - new_df[interval[0:4]])

        # Add a column with the smoothed trend
        new_col_name = interval + ' Trend'
        new_df[new_col_name] = sig.savgol_filter(x=new_df[interval], window_length=1001, polyorder=3)
        new_df[interval + ' FFT'] = fft_smooth(new_df[interval], 2002)

    return new_df

def dlow_uncertainty(d0, df, t0, tf):
    """
    Calculate uncertainty in the position of the dune
    toe, based on methods in Hapke et al (2011)

    - d0: Data at time 0
    - df: Data at time f
    - t0: time 0
    - tf: time f
    """

    # Load the y_mhw values and beach slopes for the current years to calculate uncertainty
    data_begin = load_data(year=int(t0), variable=['x_toe', 'Foreshore Slope'])
    data_end = load_data(year=int(tf), variable=['x_toe', 'Foreshore Slope'])

    # Find the uncertainty in the position of the dune toe
    toe_error_yr1 = lidar_vertical_error(str(t0)) / data_begin['Foreshore Slope']
    toe_error_yr2 = lidar_vertical_error(str(tf)) / data_begin['Foreshore Slope']

    # Find uncertainty based on deviation from the average
    toe_mean_yr1 = np.nanmean(data_begin['x_toe'].dropna())
    toe_mean_yr2 = np.nanmean(data_end['x_toe'].dropna())
    toe_error_2_yr1 = abs(data_begin['x_toe'] - toe_mean_yr1)
    toe_error_2_yr2 = abs(data_end['x_toe'] - toe_mean_yr2)

    # Find uncertainty based on the confidence interval
    _, _, ci0 = confidence_intervals(d0)
    _, _, cif = confidence_intervals(df)

    # Find the positional uncertainty for each year
    up1 = np.sqrt((toe_error_yr1 ** 2) + (toe_error_2_yr1 ** 2) + (ci0 ** 2))
    up2 = np.sqrt((toe_error_yr2 ** 2) + (toe_error_2_yr2 ** 2) + (cif ** 2))

    return up1, up2

def load_data(year, variable):
    """
    Load the data into Pandas dataframes. Supply the years needed
    as the input argument

    - year: Int for the year to look at
    - variable: String with the name of the column to use
    """

    # Set a path to the data
    fname = os.path.join('..', 'Data', 'Morphometrics for Bogue ' + str(year) + '.csv')

    # Load the data and pull the needed column
    if variable is not None:
        data = pd.read_csv(fname, header=0, delimiter=',')[variable]
    else:
        data = pd.read_csv(fname, header=0, delimiter=',')

    return data

def load_fence_locations(y):
    """
    Create a dataframe with profile numbers, category,
    and an associated y-value. The y-value is passed to the
    alongshore plotting function and is the y-axis position

    0 - Fort Macon
    1 - Non-fenced
    2 - Fenced
    """

    # Load the data. Can be from any year 2010-2016, results
    # won't change for this application
    data = load_data(2010, None)

    # Create a fence category column and initialize everything to
    # a value of 0
    data['Category'] = 'darkred'
    data['Number'] = 0

    # Set the rest of the categories
    fort_macon = 2229  # Profiles in from the end where Fort Macon starts
    data['Category'].iloc[:-fort_macon].loc[np.isnan(data['x_fence'])] = 'dimgray'
    data['Category'].iloc[:-fort_macon].loc[~np.isnan(data['x_fence'])] = 'cornflowerblue'

    data['Number'].iloc[:-fort_macon].loc[np.isnan(data['x_fence'])] = 1
    data['Number'].iloc[:-fort_macon].loc[~np.isnan(data['x_fence'])] = 2

    # Add a y-value column
    data['Y'] = y

    # Sort out fences using chunks
    for n in chunks(data[:fort_macon], 5):
        if np.any(data['x_fence'].iloc[n.index] > 0):
            data['Number'].iloc[n.index] = 2
        else:
            data['Number'].iloc[n.index] = 1

    # Just take the needed data
    data = data[['Profile No.', 'Category', 'Number', 'Y']]

    # Return the dataframe
    return data

def mhw_uncertainty(t0, tf):
    """
    Calculate uncertainty in the position of MHW on the profile
    based on methods in Hapke et al. (2011)

    - begin: dataframe of shoreline positions for the beginning time
    - end: dataframe of shoreline positions for the end time
    - t0: beginning time
    - tf: end time

    Called by:
    - shoreline_change_df()
    """

    # Load the y_mhw values and beach slopes for the current years to calculate uncertainty
    data_begin = load_data(year=int(t0), variable=['y_mhw', 'Beach Slope'])
    data_end = load_data(year=int(tf), variable=['y_mhw', 'Beach Slope'])

    # Use the confidence intervals as the first component of the error
    ci_df = pd.read_csv(os.path.join('..', 'Data', 'Shoreline Confidence Intervals.csv'))

    # Find the uncertainty in Y as the difference between the measured y-value
    # and the accepted value of MHW (0.34m). Do this for both years
    mhw_real = 0.34
    y_error_yr1 = abs(data_begin['y_mhw'] - mhw_real)
    y_error_yr2 = abs(data_end['y_mhw'] - mhw_real)

    # Find the uncertainty in X as the vertical error in the LiDAR
    # data divided by the beach slope
    x_error_yr1 = lidar_vertical_error(str(t0)) / data_begin['Beach Slope']
    x_error_yr2 = lidar_vertical_error(str(tf)) / data_end['Beach Slope']

    # Incorporate 95% confidence interval from the regression into the error analysis. There are more
    # confidence intervals than profiles, remove the unnecessary ones
    ci_error_1 = ci_df[str(t0) + ' CI'].iloc[:len(x_error_yr1)]
    ci_error_2 = ci_df[str(tf) + ' CI'].iloc[:len(x_error_yr1)]

    # Convert to arrays
    x_error_yr1, x_error_yr2 = np.asarray(x_error_yr1, dtype=np.float64), np.asarray(x_error_yr2, dtype=np.float64)
    y_error_yr1, y_error_yr2 = np.asarray(y_error_yr1, dtype=np.float64), np.asarray(y_error_yr2, dtype=np.float64)
    ci_error_1, ci_error_2 = np.asarray(ci_error_1, dtype=np.float64), np.asarray(ci_error_2, dtype=np.float64)

    # Find the positional uncertainty based on the uncertainty in X and Y
    up1 = np.sqrt((x_error_yr1 ** 2) + (y_error_yr1 ** 2) + (ci_error_1 ** 2))
    up2 = np.sqrt((x_error_yr2 ** 2) + (y_error_yr2 ** 2) + (ci_error_2 ** 2))

    return up1, up2

def setup_change_df(interval, orig_df):
    """
    Prepare the dataframe for the change dataframes
    with uncertainty values

    Called by:
    - shoreline_change_df()
    """

    # Get the two times associated with the interval
    begin_time = str(interval[0:4])  # Initial time as a string
    end_time = str(interval[-4:])  # Final time as a string
    dt = (int(end_time) - int(begin_time))  # Time span as an int
    begin_df = orig_df[begin_time]  # Create a dataframe of shoreline positions from the initial time
    end_df = orig_df[end_time]  # Create a dataframe of shoreline positions from the final time

    # Create new column names
    new_col_name_1 = interval + ' Change'
    new_col_name_2 = interval + ' Mean'
    new_col_name_3 = interval + ' Uncertainty'
    new_col_name_4 = interval + ' Regional Uncertainty'
    new_col_name_5 = interval + ' Mean Residual Error'

    # Need to copy other columns before working
    orig_df[new_col_name_1] = orig_df['1997']
    orig_df[new_col_name_2] = orig_df['1997']
    orig_df[new_col_name_3] = orig_df['1997']
    orig_df[new_col_name_4] = orig_df['1997']
    orig_df[new_col_name_5] = orig_df['1997']

    return orig_df, begin_df, end_df, begin_time, end_time, dt

"""
Make the figure
"""

def bar_plots(data, fences, ax3, ax4, ax5, ax6, rate, text_y, bar_y, color, param):
    """
    This function makes a series of bar plots showing the change rates broken down by
    Fort Macon, non-fenced, and fenced locations as well as the island overall. Each location will
    have three bars for 1997-2016, 1997-2010, 2010-2016

    Called by;
    - make_overall_change_plot_w_kde()
    """

    conf_type = 95

    # Set colors and locations
    colors = ['red', 'blue', 'green']
    locations = ['Fort Macon', 'Non-fenced', 'Fenced']

    # Make a dataframe for all locations
    all_locs = pd.DataFrame({'1997-2010': data['1997-2010'].dropna(),
                             '2010-2016': data['2010-2016'].dropna(),
                             '1997-2016': data['1997-2016'].dropna()})
    all_locs['ID'] = 'All'
    all_locs = pd.melt(all_locs, id_vars=['ID'], value_vars=['1997-2010', '2010-2016', '1997-2016'])

    # Make a barplot of the data for all locations
    sns.barplot(x='ID', y='value', ci=conf_type, hue='variable', data=all_locs,
                palette=colors, estimator=np.nanmedian, errcolor='black', errwidth=2, ax=ax3)

    # Loop through and plot Fort Macon, non-fenced areas, and fenced areas
    for ii, ax in enumerate([ax4, ax5, ax6]):

        # Set temporary data
        use_data = data.loc[fences['Number'] == ii]

        # Make a dataframe for the location
        temp_locs = pd.DataFrame({'1997-2010': use_data['1997-2010'].dropna(),
                                  '2010-2016': use_data['2010-2016'].dropna(),
                                  '1997-2016': use_data['1997-2016'].dropna()})
        temp_locs['ID'] = locations[ii]
        temp_locs = pd.melt(temp_locs, id_vars=['ID'], value_vars=['1997-2010', '2010-2016', '1997-2016'])

        # Make a barplot of the data for the location
        sns.barplot(x='ID', y='value', ci=conf_type, hue='variable', data=temp_locs,
                    palette=colors, estimator=np.nanmedian, errcolor='black', errwidth=2, ax=ax)

    # Set common axis properties
    for ax in [ax3, ax4, ax5, ax6]:

        # Set grids
        add_grid([ax], xax=False)

        # Add a zero line
        ax.axhline(y=0, color='black', linewidth=2, linestyle='--', zorder=10)

        # Set the y-limits
        ax.set_ylim(bottom=-bar_y, top=bar_y)
        ax.set_yticks([-bar_y, (-bar_y)/2, 0, bar_y/2, bar_y])

        # ax.set_ylim(bottom=-25, top=bar_y)
        # ax.set_yticks([(-bar_y) / 2, 0, bar_y / 2, bar_y])

        # Set the y-axis
        if ax == ax3:
            if rate:
                ax.set_ylabel('Median ' + param + '\nChange Rate (m/yr)',
                              color=color,
                              fontname='Arial',
                              fontsize=12,
                              fontweight='bold')
            else:
                ax.set_ylabel('Median ' + param + '\nChange (m)',
                              color=color,
                              fontname='Arial',
                              fontsize=12,
                              fontweight='bold')
        else:
            ax.set_ylabel('')
            plt.setp(ax.get_yticklabels(), visible=False)
            ax.tick_params(left=False)

        # Remove the x-axis label
        ax.set_xlabel('')

        # Remove the legend
        ax.legend_.remove()

        # Set the spines
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['bottom'].set_linewidth(2)
        ax.spines['bottom'].set_color(color)
        ax.tick_params(width=2)
        if ax == ax3:
            ax.spines['left'].set_linewidth(2)
            ax.spines['left'].set_color(color)
        else:
            ax.spines['left'].set_visible(False)

    # Add a "C" to ax3
    ax3.text(x=-0.35,
             y=bar_y - (bar_y * 0.20),
             s='C.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')

def tight_and_transparent(fig, axs, color, **kwargs):
    """
    Give the figure a tight layout and a transparent backgorund
    """
    for ax in axs:
        ax.tick_params(colors=color)
        for tick in ax.get_xticklabels():
            tick.set_fontname('Arial')
        for tick in ax.get_yticklabels():
            tick.set_fontname('Arial')

    plt.tight_layout(**kwargs)
    fig.patch.set_color('w')
    fig.patch.set_alpha(0.0)

def set_spines(axs, color):
    """
    Set spines on the lower and left borders. Remove the others
    """

    for ax in axs:

        # Set the spines
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_linewidth(2)
        ax.spines['bottom'].set_linewidth(2)
        ax.spines['left'].set_color(color)
        ax.spines['bottom'].set_color(color)

        # Make the tick widths match
        ax.tick_params(width=2)

def add_grid(axs, xax=True):
    """
    Add a grid to the plot. Set xax=False
    if you only want horizontal lines
    """

    for ax in axs:
        ax.grid(color='grey', linewidth=0.25, zorder=0)
        ax.xaxis.grid(xax)

def make_overall_change_plot_w_kde(data, fences, param, y_min, y_max, y_ticks, y_tick_labels,
                                   y_text, text_y, bar_y, spacing, rate=False, color='black'):
    """
    Make a plot of change rates for the pre-fencing (1997-2010),
    post-fencing (2010-2016), and overall (1997-2016) time periods

    Add a KDE plot in the second subplot

    Called by:
    - Shoreline_Change.py
    """

    # Setup the plot
    fig = plt.figure(dpi=300)
    categories = ['Fort Macon', 'Non-Fenced', 'Fenced']
    gs = gridspec.GridSpec(nrows=2, ncols=4, wspace=0.0, hspace=0.3)
    ax1 = fig.add_subplot(gs[0, 0:3])
    ax2 = fig.add_subplot(gs[0, 3], sharey=ax1)
    ax3 = fig.add_subplot(gs[1, 0])
    ax4 = fig.add_subplot(gs[1, 1], sharey=ax3)
    ax5 = fig.add_subplot(gs[1, 2], sharey=ax3)
    ax6 = fig.add_subplot(gs[1, 3], sharey=ax3)
    lwidth = 2
    max_x = 0
    marker_shape = 'X'
    marker_alpha = 0.075
    x_max = 40000
    exes = [5000, 19300, 32000]
    yrs = ['1997-2016', '1997-2010', '2010-2016']
    labs = ['\n(Overall)', '\n(Pre-Fencing)', '\n(Post-Fencing)']
    cols = ['darkred', 'darkgreen', 'darkblue']
    markers = ['lightsalmon', 'lightgreen', 'lightblue']

    text_years = ['1997-2010', '2010-2016', '1997-2016']
    text_labs = ['\n(Pre-Fencing)', '\n(Post-Fencing)', '\n(Overall)']
    text_cols = ['darkred', 'darkblue', 'green']

    # Add a grid for the top plots
    add_grid([ax1, ax2], xax=True)

    # Plot the data
    for ii, years in enumerate(['1997-2010', '1997-2016', '2010-2016']):

        # Scatter plot of the raw data
        sns.scatterplot(x=data.index,
                        y=years,
                        data=data,
                        linewidth=0,
                        color=markers[ii],
                        marker=marker_shape,
                        alpha=marker_alpha,
                        ax=ax1,
                        zorder=2)

        # Trend line over the raw data
        sns.lineplot(x=data.index,
                     y=years + ' Trend',
                     data=data,
                     linewidth=2,
                     color=cols[ii],
                     ax=ax1,
                     zorder=5)

        # KDE Plot in the second axis
        kde1 = sns.kdeplot(data[years].dropna(),
                           shade=True,
                           color=cols[ii],
                           linewidth=lwidth,
                           legend=False,
                           vertical=True,
                           ax=ax2,
                           zorder=2)

        # Add text labels
        ax1.text(x=exes[ii],
                 y=y_text - (2 * spacing),
                 s=text_years[ii] + text_labs[ii],
                 color=text_cols[ii],
                 fontname='Arial',
                 fontsize=12,
                 fontweight='bold',
                 ha='center',
                 va='center',
                 zorder=8)

        # Get the largest value in the KDE plot to set the axis limit later
        x1, y1 = kde1.get_lines()[0].get_data()
        if np.nanmax(x1) > max_x:
            max_x = np.around(a=np.nanmax(x1), decimals=2)

    # Add a zero-line
    ax1.axhline(y=0,
                xmin=0, xmax=x_max,
                color='black',
                linewidth=2,
                linestyle='--',
                zorder=7)
    ax2.axhline(y=0,
                xmin=0,
                xmax=max_x + 1,
                color='black',
                linewidth=2,
                linestyle='--',
                zorder=7)

    # Add an "A" and "B" label
    ax1.text(x=-0.5, y=y_max - (y_max * 0.20),
             s='A.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')
    ax2.text(x=max_x - (max_x * 0.91), y=y_max - (y_max * 0.20),
             s='B.',
             color='black',
             fontname='Arial',
             fontsize=14,
             fontweight='bold')

    # Set the x-axis
    ax1.set_xlim(right=x_max)
    ax1.set_xticks([0, 5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000])
    ax1.set_xticklabels(['0', '5', '10', '15', '20', '25', '30', '35', ''])
    ax1.set_xlabel('Alongshore Distance (km)',
                   color=color,
                   fontname='Arial',
                   fontsize=12,
                   fontweight='bold')

    # Set the y-axis
    ax1.set_ylim(bottom=y_min, top=y_max)
    ax1.set_yticks(y_ticks)
    ax1.set_yticklabels(y_tick_labels)
    if rate:
        ax1.set_ylabel(param + ' Change\nRate (m/yr)',
                       color=color,
                       fontname='Arial',
                       fontsize=12,
                       fontweight='bold')
    else:
        ax1.set_ylabel(param + ' Change (m)',
                       color=color,
                       fontname='Arial',
                       fontsize=12,
                       fontweight='bold')
    plt.setp(ax2.get_yticklabels(), visible=False)

    # Add a barplot of change rates on the bottom of the figure
    bar_plots(data, fences, ax3, ax4, ax5, ax6, rate, text_y, bar_y, color, param)

    # Set the spines for the top plots
    set_spines([ax1, ax2], color)

    # Set a tight layout and a transparent background
    tight_and_transparent(fig, [ax1, ax2, ax3, ax4, ax5, ax6], color)

    # Save the figure
    title = 'Figure 8'
    save_name = os.path.join('..', 'Figures', title + '.png')
    plt.savefig(save_name, bbox_inches='tight', dpi='figure')
    plt.close()
    print(f'Figure saved: {save_name}\n')

"""
Functions to make a .txt file
"""

def make_stats(data, avg, fences, param):
    """
    Calculate change statistics about Bogue Banks

    - Mean, median rates for 1997-2010, 2010-2016, and 1997-2016
    - T-Test for 1997-2010 v. 2010-2016 v. 1997-2016
    """

    # Set the intervals to work with
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    # Create a .txt file
    partial_fname = f'(Figure 8) {param} Change Statistics.txt'
    fname = os.path.join('..', 'Text Files', partial_fname)
    f = open(fname, 'w')

    # Write a header
    f.write('%s Change Statistics for Bogue Banks between 1997-2010, 2010-2016, 1997-2016\r\n' % param)
    f.write('\t- Summary statistics between Surveys\r\n')
    f.write('\t- KS-tests to compare pre- and post- fencing rates\r\n')
    f.write('\t- Mann-Whitney U-Tests between pre-fence, post-fence, and overall periods\r\n\r\n\r\n')

    # Write statistics for all of the profiles
    make_stats_section(f, data, avg, 'All Profiles', intervals)

    # Write stats for the different sections
    for ii, sect in enumerate(['Fort Macon', 'Non-Fenced', 'Fenced']):
        make_stats_section(f, data.loc[fences['Number'] == ii], avg.loc[fences['Number'] == ii], sect, intervals)

    # Compare Fort Macon to non-fenced to fenced
    mann_whitney_test_by_location(f, data, fences, bonferroni=3)

    # Close the file
    f.close()

def make_stats_section(f, data, avg, section, intervals):
    """
    Write a section in the stats file. This function is called by
    make_stats()

    Called by:
    - make_stats()
    """

    # Write a section header
    header_len = 35
    section_part = section + ' (n = ' + str(len(data)) + ')'
    lead_in = '#' * 2
    end_out = '#' * (header_len - len(section_part) - 4)
    f.write('%s\n' % ('#' * header_len))
    f.write('%s %s %s\n' % (lead_in, section_part, end_out))
    f.write('%s\n\n' % ('#' * header_len))

    # Write summary statistics for the data
    f.write('Summary Statistics (m/yr):\r\n')
    for yrs in intervals:
        f.write('%s' % (summary_statistics_alongshore(data, avg, yrs)))
    f.write('\r\n\r\n')

    # Perform an unequal variances T-test to compare the pre- and post-fencing rates to
    # each other. It is not important if the 1997-2016 is similar or dissimilar to either
    ks_test(f, data)

    f.write('\r\n\r\n')

def mann_whitney_test_by_location(f, data, fences, bonferroni=None):
    """
    Perform an unequal variances T-test and print the
    results to the stats file. This version compares
    Fort Macon v Non-Fenced v Fenced

    f: open text file with stats
    data: data to work on
    fences: dataframe with fence locations

    Called by:
    - make_stats()
    """

    bonferroni_alpha = 0.05 / 3

    # Set a list of intervals
    intervals = ['1997-2010', '1997-2016', '2010-2016']

    f.write('\r\n%s\r\n' % ('#' * 50))
    f.write('%s %s %s\r\n' % ('#' * 13, 'Compare locations', '#' * 18))
    f.write('%s\r\n\r\n' % ('#' * 50))

    # Write H-Test header
    f.write('Mann-Whitney U-Test with Bonferroni Correction (alpha = 0.05, bonf alpha = %f)\r\n' % bonferroni_alpha)
    f.write('H0: The sample populations have the same distribution\r\n')
    f.write('H1: The samples populations do not have the same distribution\r\n\r\n')

    for yrs in intervals:

        # Write a header for the years
        header_len = 25
        section_part = yrs
        lead_in = '#' * 2
        end_out = '#' * (header_len - len(section_part) - 4)
        f.write('%s\n' % ('#' * header_len))
        f.write('%s %s %s\n' % (lead_in, section_part, end_out))
        f.write('%s\n\n' % ('#' * header_len))

        # Perform tests
        u1, p1 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 0], y=data[yrs].loc[fences['Number'] == 1],
                                    alternative='two-sided')
        u2, p2 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 0], y=data[yrs].loc[fences['Number'] == 2],
                                    alternative='two-sided')
        u3, p3 = stats.mannwhitneyu(x=data[yrs].loc[fences['Number'] == 1], y=data[yrs].loc[fences['Number'] == 2],
                                    alternative='two-sided')

        # Print results
        if p1 < bonferroni_alpha:
            f.write('Fort Macon v. Non-Fenced:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u1, p1))
        else:
            f.write('Fort Macon v. Non-Fenced:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u1, p1))
        if p2 < bonferroni_alpha:
            f.write('Fort Macon v. Fenced\t:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u2, p2))
        else:
            f.write('Fort Macon v. Fenced\t:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n' % (u2, p2))
        if p3 < bonferroni_alpha:
            f.write('Non-Fenced v. Fenced\t:\tReject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n\r\n' % (u3, p3))
        else:
            f.write('Non-Fenced v. Fenced\t:\tFail to reject the null hypothesis\t(U = %0.4f\tp = %0.4f)\r\n\r\n' % (u3, p3))

def ks_test(f, data):
    """
    Perform a Kolmogorov-Smirnov test and print
    the results to the stats file. This version only compares
    years within one area (Fort Macon, non-fenced, fenced)

    f: open text file with stats
    data: data to compare

    Called by:
    - make_stats_section()
    """

    # Write KS-Test header
    f.write('Kolmogorov-Smirnov Test (alpha = 0.05)\r\n')
    f.write('H0: The sample populations come from the same distribution\r\n')
    f.write('H1: The samples populations do not come from the same distribution\r\n\r\n')

    d_stat, p_stat = stats.mannwhitneyu(data['1997-2010'], data['2010-2016'])
    d_round, p_round = str(np.around(d_stat, decimals=4)), str(np.around(p_stat, decimals=4))
    if p_stat <= 0.05:
        f.write('Reject the null hypothesis (D = %s\tp = %s)\r\n' % (d_round, p_round))
    else:
        f.write('Fail to reject the null hypothesis (D = %s\tp = %s)\r\n' % (d_round, p_round))

def summary_statistics_alongshore(data, avg, yrs):
    """
    Calculate summary statistics and format in a string

    - data: dataframe being worked on
    - avg: dataframe with uncertainties
    - yrs: years being looked at

    Called by:
    - make_stats_section()
    """

    # Set the number of decimals to round to
    dec = 1

    # Calculate the mean
    average = np.around(np.nanmean(avg[yrs + ' Change']), decimals=dec)

    # Calculate 2*sd
    sd_2 = np.around(np.std(avg[yrs + ' Change']), decimals=dec)

    # Calculate the median
    med = np.around(np.median(avg[yrs + ' Change']), decimals=dec)

    # Calculate the mode
    most = np.around(stats.mode(avg[yrs + ' Change'])[0][0], decimals=dec)

    # Calculate the mean uncertainty
    unc = np.around(np.nanmean(avg[yrs + ' Mean Residual Error']), decimals=2)
    min_unc = np.around(np.nanmin(avg[yrs + ' Mean Residual Error']), decimals=2)
    max_unc = np.around(np.nanmax(avg[yrs + ' Mean Residual Error']), decimals=2)
    avg_unc = np.around(np.nanmean(avg[yrs + ' Uncertainty']), decimals=2)

    # Calculate the percent increasing and decreasing
    pos = np.around((len(data[yrs][data[yrs] > 0]) / len(data[yrs])) * 100, decimals=dec)
    neg = np.around((len(data[yrs][data[yrs] < 0]) / len(data[yrs])) * 100, decimals=dec)

    # Calculate the 95% confidence interval
    x_bar = np.nanmean(data[yrs])  # Mean value
    s = np.nanstd(data[yrs])  # Standard deviation
    n = len(data[yrs])  # Sample size
    lo_conf = x_bar - (1.96 * (s / np.sqrt(n)))  # Lower bound of confidence interval
    hi_conf = x_bar + (1.96 * (s / np.sqrt(n)))  # Upper bound of confidence interval
    conf_int = np.around(hi_conf - lo_conf, decimals=dec)

    # Calculate the standard error on the mean
    # autocorr = estimated_autocorrelation(data[yrs])
    # sig_vals = np.where(autocorr <= 0.05)
    # n_star = np.nanmin(sig_vals)
    # sem = np.around(sd_2 / np.sqrt(n_star), decimals=dec)
    sem = np.around(stats.sem(data[yrs], nan_policy='omit'), decimals=dec)

    # Create a nicely formatted string
    res = f'{yrs}:\tMean: {average}\tSD: {sd_2}\tMedian: {med}\tMode: {most}\tMean Uncertainty: {avg_unc}\tMean MRE: {unc} ({min_unc} - {max_unc})\t95 Pct: {conf_int}\tPrograding Pct: {pos}\tEroding Pct: {neg}\tSEM: {sem}\r\n'

    return res

"""
Main function
"""

def main():
    """
    Main program function
    """

    # Load the data
    data_1997 = load_data(year=1997, variable='Beach Width')
    data_2010 = load_data(year=2010, variable='Beach Width')
    data_2016 = load_data(year=2016, variable='Beach Width')

    # Make a dataframe of Dhighs
    widths = change_df(d1997=data_1997, d2010=data_2010, d2016=data_2016)
    widths_errors = beach_width_change_df(widths=widths, rate=False)

    # Load a dataframe with fence locations identified
    fences = load_fence_locations(y=-7.5)

    # Make the figure
    y_min = -75
    y_text = y_min - (y_min * 0.35) + 3
    yticks = [y_min, -50, -25, 0, 25, 50, -y_min]
    make_overall_change_plot_w_kde(data=widths,
                                   fences=fences,
                                   param='Beach Width',
                                   y_min=y_min,
                                   y_max=y_min * -1,
                                   y_ticks=yticks,
                                   y_tick_labels=[str(ii) for ii in yticks],
                                   text_y=40 * 0.80,
                                   bar_y=60,
                                   y_text=y_text,
                                   spacing=1.5,
                                   rate=False,
                                   color='black')

    # Calculate statistics
    make_stats(widths, widths_errors, fences, 'Beach Width')

if __name__ == '__main__':
    main()


from __future__ import absolute_import
import sys
from extras.Error_Code import *
from extras.Required_Fields import required_api_dict

sys.path.append('../')
import os
import unittest
import Main
from API_NAME import *
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from extras.Utils import *
from extras.Check_Invalid import *
from extras.Random_Models import *

class TestUtils(unittest.TestCase):

    def test_keys_missing_1(self):
        #Case 1: missing keys
        expected_keys = ['authToken', 'userId']
        post = {}
        errors, values = keys_missing(expected_keys,post)
        #Errors should contain both keys and values should conatin none.
        self.assertEquals(errors, {'missingToken': 'authToken is Missing',
                                   'missingUserId': 'userId is Missing'})
        self.assertEquals(values, {})

    def test_keys_missing_2(self):
        #Case 2: correct keys
        expected_keys = ['authToken', 'userId']
        post ={'authToken': 'TestAuthToken',
                'userId': 'TestUserId'}
        errors, values = keys_missing(expected_keys,post)
        # Errors should contain no values and values should contain both.
        self.assertEquals(errors, {})
        self.assertEquals(values, {'authToken': 'TestAuthToken',
                                   'userId': 'TestUserId'})

    def test_keys_missing_3(self):
        # Case 2: Incorrect keys + one correct
        expected_keys = ['authToken', 'userId']
        post = {'authToken': 'TestAuthToken',
                'incorrectKey:': 'Incorrect Value'}
        errors, values = keys_missing(expected_keys, post)
        # Errors should contain the missing value,
        # values should contain all input keys.
        self.assertEquals(errors, {'missingUserId': 'userId is Missing'})
        self.assertEquals(values, {'authToken': 'TestAuthToken',
                                   'incorrectKey:': 'Incorrect Value'})

    def test_convert_to_bool(self):
        check_bool = 'True'
        self.assertTrue(convert_to_bool(check_bool))
        check_bool = 'true'
        self.assertTrue(convert_to_bool(check_bool))
        check_bool = 'False'
        self.assertFalse(convert_to_bool(check_bool))
        check_bool = 'anything else'
        self.assertFalse(convert_to_bool(check_bool))

    def test_is_existing_and_non_empty(self):
        test_dict = {'key1': 'value1',
                     'second_key:': 'value2'}
        exists = is_existing_and_non_empty('key1', test_dict)
        self.assertTrue(exists)
        #TODO why does this test not work? I am very confused by this one.
        #exists = is_existing_and_non_empty('second_key', test_dict)
        #self.assertTrue(exists)

        exists = is_existing_and_non_empty('not_key', test_dict)
        self.assertFalse(exists)

    def test_is_valid_read_del(self):
        self.assertTrue(is_valid_read_del('r'))
        self.assertTrue(is_valid_read_del('R'))
        self.assertTrue(is_valid_read_del('d'))
        self.assertFalse(is_valid_read_del('q'))

    def test_scale_province(self):
        self.assertEquals(scale_province('Saskatchewan'),'SK')
        self.assertEquals(scale_province('saskatchewan'), 'SK')
        self.assertNotEquals(scale_province('Alberta'), 'SK')

    def test_are_two_lists_same(self):
        list1 = ['element1', 'element2']
        list2 = ['element1', 'element2']
        self.assertTrue(are_two_lists_same(list1,list2))

        list2 = ['element2', 'element1']
        self.assertTrue(are_two_lists_same(list1, list2))

        list2 = ['element1', 'element2','element3']
        self.assertFalse(are_two_lists_same(list1, list2))

        list2 = ['element1']
        self.assertFalse(are_two_lists_same(list1, list2))

    def test_is_empty(self):
        self.assertTrue(is_empty({}))
        self.assertTrue(is_empty(''))
        self.assertTrue(is_empty(None))
        self.assertTrue(is_empty(' '))

        self.assertFalse(is_empty('not nothing'))
        self.assertFalse(is_empty(1))


import os, time

from stellaris.files import get_settings, list_jsonzip_paths, get_output_folder_path
from stellaris.model import Model
from stellaris.render import Render

def get_pairs():
	dest_folder_path = os.path.join(get_output_folder_path(), "images")

	if not os.path.exists(dest_folder_path):
		os.makedirs(dest_folder_path)

	for src_path in list_jsonzip_paths():
		name = os.path.splitext(os.path.basename(src_path))[0]
		dest_path = os.path.join(dest_folder_path, name + ".png")
		if not os.path.exists(dest_path):
			yield (name, src_path, dest_path)

def main():
	settings = get_settings()

	pairs = list(get_pairs())

	print("Current game is '" + settings["current"] + "'.")
	print(f"Rendering {len(pairs)} images.")
	print()

	for (name, src_path, dest_path) in pairs:

		t_start = time.time()
		model = Model.from_file(src_path)
		t_model = time.time()
		render = Render(model, settings["render"])
		render.steps()
		render.export(dest_path)
		t_end = time.time()

		t_parse_ms = int(1000*(t_model - t_start))
		t_render_ms = int(1000*(t_end - t_model))
		print(name + " (parse: " + str(t_parse_ms) + ", render: " + str(t_render_ms) + ")")

if __name__=="__main__":
	main()


"""AudioToolkit for processing audio data."""

from pathlib import Path
from typing import Union

import numpy as np
import librosa
from sox import Transformer

class AudioToolkit:
    """Audio toolkit for processing audio data."""

    sample_rate = 16000
    preemph = 0.97
    fft_window_ms = 25
    fft_hop_ms = 10
    n_mels = 40

    @classmethod
    def preprocess_wav(cls, fpath: Union[str, Path]) -> np.ndarray:
        """Load, resample, normalize and trim a waveform."""
        transformer = Transformer()
        transformer.norm()
        transformer.silence(silence_threshold=1, min_silence_duration=0.1)
        transformer.set_output_format(rate=cls.sample_rate, bits=16, channels=1)
        wav = transformer.build_array(input_filepath=str(fpath))
        wav = wav / (2 ** 15)
        return wav.astype(np.float32)

    @classmethod
    def wav_to_logmel(cls, wav: np.ndarray) -> np.ndarray:
        """Derives a log mel spectrogram from a preprocessed waveform."""
        wav = librosa.effects.preemphasis(wav)
        mel = librosa.feature.melspectrogram(
            wav,
            cls.sample_rate,
            n_fft=int(cls.sample_rate * cls.fft_window_ms / 1000),
            hop_length=int(cls.sample_rate * cls.fft_hop_ms / 1000),
            n_mels=cls.n_mels,
        )
        mel = np.log(mel + 1e-9)
        return mel.astype(np.float32).T


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    Copyright 2010, 2011 C Sommer, C Straehle, U Koethe, FA Hamprecht. All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without modification, are
#    permitted provided that the following conditions are met:
#
#       1. Redistributions of source code must retain the above copyright notice, this list of
#          conditions and the following disclaimer.
#
#       2. Redistributions in binary form must reproduce the above copyright notice, this list
#          of conditions and the following disclaimer in the documentation and/or other materials
#          provided with the distribution.
#
#    THIS SOFTWARE IS PROVIDED BY THE ABOVE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED
#    WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
#    FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ABOVE COPYRIGHT HOLDERS OR
#    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
#    ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#    ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#    The views and conclusions contained in the software and documentation are those of the
#    authors and should not be interpreted as representing official policies, either expressed
#    or implied, of their employers.

from PyQt4.QtCore import QObject

import numpy

#*******************************************************************************
# H i s t o r y M a n a g e r                                                  *
#*******************************************************************************

class HistoryManager(QObject):
    """
    DEPRECATED
    Stores the history of the various brush strokes (including erasing)
    that the user performed, such that undo/redo is possible.
    """

    def __init__(self, parent, maxSize = 3000):
        QObject.__init__(self)
        self.volumeEditor = parent
        self.maxSize = maxSize
        self._history = []
        self.current = -1

    def append(self, state):
        if self.current + 1 < len(self._history):
            self._history = self._history[0:self.current+1]
        self._history.append(state)

        if len(self._history) > self.maxSize:
            self._history = self._history[len(self._history)-self.maxSize:len(self._history)]

        self.current = len(self._history) - 1

    def undo(self):
        if self.current >= 0:
            self._history[self.current].restore(self.volumeEditor)
            self.current -= 1

    def redo(self):
        if self.current < len(self._history) - 1:
            self._history[self.current + 1].restore(self.volumeEditor)
            self.current += 1

    def serialize(self, grp, name='_history'):
        histGrp = grp.create_group(name)
        for i, hist in enumerate(self._history):
            histItemGrp = histGrp.create_group('%04d'%i)
            histItemGrp.create_dataset('labels',data=hist.labels)
            histItemGrp.create_dataset('axis',data=hist.axis)
            histItemGrp.create_dataset('slice',data=hist.num)
            histItemGrp.create_dataset('labelNumber',data=hist.labelNumber)
            histItemGrp.create_dataset('offsets',data=hist.offsets)
            histItemGrp.create_dataset('time',data=hist.time)
            histItemGrp.create_dataset('erasing',data=hist.erasing)
            histItemGrp.create_dataset('clock',data=hist.clock)

    def removeLabel(self, number):
        tobedeleted = []
        for index, item in enumerate(self._history):
            if item.labelNumber != number:
                item.dataBefore = numpy.where(item.dataBefore == number, 0, item.dataBefore)
                item.dataBefore = numpy.where(item.dataBefore > number, item.dataBefore - 1, item.dataBefore)
                item.labels = numpy.where(item.labels == number, 0, item.labels)
                item.labels = numpy.where(item.labels > number, item.labels - 1, item.labels)
            else:
                tobedeleted.append(index - len(tobedeleted))
                if index <= self.current:
                    self.current -= 1

        for val in tobedeleted:
            it = self._history[val]
            self._history.__delitem__(val)
            del it

    def clear(self):
        self._history = []


import datetime

import pytz

from django.db import models
from django.db.models import F
from django.utils.translation import ugettext_lazy as _

from smart_selects.db_fields import ChainedForeignKey

from tracpro.polls.models import Answer, Question, Poll

class BaselineTerm(models.Model):
    # e.g., 2015 Term 3 Attendance for P3 Girls
    # A term of time to gather statistics for a baseline chart
    # baseline_question: the answer to this will determine our baseline
    #                    information for all dates
    #                    ie. How many students are enrolled?
    # follow_up_question: the answers to this question will determine the
    #                     follow-up information, over the date range
    #                     (start_date -> end_date)
    #                     ie. How many students attended today?

    org = models.ForeignKey("orgs.Org", verbose_name=_(
        "Organization"), related_name="baseline_terms")
    name = models.CharField(max_length=255, help_text=_(
        "For example: 2015 Term 3 Attendance for P3 Girls"))
    start_date = models.DateField()
    end_date = models.DateField()

    baseline_poll = models.ForeignKey(Poll, related_name="baseline_terms")
    baseline_question = ChainedForeignKey(
        Question,
        chained_field='baseline_poll',
        chained_model_field='poll',
        auto_choose=True,
        related_name="baseline_terms",
        help_text=_("All baseline poll results over time will display in chart.")
    )

    follow_up_poll = models.ForeignKey(Poll)
    follow_up_question = ChainedForeignKey(
        Question,
        chained_field='follow_up_poll',
        chained_model_field='poll',
        auto_choose=True,
        help_text=_("Follow up poll responses over time to compare to the baseline values.")
    )
    y_axis_title = models.CharField(max_length=255, null=True, blank=True,
                                    help_text=_("The title for the y axis of the chart."))

    @classmethod
    def get_all(cls, org):
        baseline_terms = cls.objects.filter(org=org)
        return baseline_terms

    def _get_answers(self, question, regions, region_selected):
        """
        Retrieve answers to the question that are relevant for this
        BaselineTerm.
        """
        midnight = datetime.time(0, 0, 0, tzinfo=pytz.utc)
        start = datetime.datetime.combine(self.start_date, midnight)
        end = datetime.datetime.combine(self.end_date + datetime.timedelta(days=1), midnight)

        answers = Answer.objects.filter(
            question=question, submitted_on__gte=start, submitted_on__lt=end)
        answers = answers.annotate(region_name=F('response__contact__region__name'))
        answers = answers.select_related('response', 'response__contact')

        all_regions = answers.values('response__contact__region',
                                     'region_name').distinct('response__contact__region')

        if regions:
            answers = answers.filter(response__contact__region__in=regions)
            all_regions = answers.values('response__contact__region',
                                         'region_name').distinct('response__contact__region')

        if region_selected:
            answers = answers.filter(response__contact__region__in=[region_selected])

        return answers, all_regions

    def get_baseline(self, regions, region_selected):
        """ Get all baseline responses """
        answers, all_regions = self._get_answers(self.baseline_question, regions, region_selected)

        # Separate out baseline values per region
        region_answers = {}
        dates = []
        for region_name in set(a.region_name.encode('ascii') for a in answers):
            answers_by_region = answers.filter(region_name=region_name)
            answer_sums, dates = answers_by_region.numeric_sum_group_by_date()
            region_answers[region_name] = {'values': answer_sums}
        return region_answers, dates

    def get_follow_up(self, regions, region_selected):
        """ Get all follow up responses summed by region """
        answers, all_regions = self._get_answers(self.follow_up_question, regions, region_selected)

        # Loop through all regions in answers and create
        # a dict of values and dates per Region
        # ex.
        # { 'Kumpala': {'values': [35,...], 'dates': [datetime.date(2015, 8, 12),...]} }
        region_answers = {}
        dates = []
        for region_name in set(a.region_name.encode('ascii') for a in answers):
            answers_by_region = answers.filter(region_name=region_name)
            answer_sums, dates = answers_by_region.numeric_sum_group_by_date()
            region_answers[region_name] = {'values': answer_sums}
        return region_answers, dates, all_regions

    def check_for_data(self, regions):
        answers, all_regions = self._get_answers(self.baseline_question, regions, 0)
        if answers:
            return True
        else:
            return False


# -*- coding=utf-8 -*-
from __future__ import absolute_import, print_function

import functools
import os
import sys

import pytest
import six
from packaging.version import Version

import pythonfinder

from .testutils import (
    is_in_ospath,
    normalize_path,
    normalized_match,
    print_python_versions,
)

@pytest.mark.skipif(sys.version_info < (3,), reason="Must run on Python 3")
def test_python_versions(monkeypatch, special_character_python):
    def mock_version(*args, **kwargs):
        version_output = "2.7.15+ (default, Jun 28 2018, 13:15:42)\n[GCC 7.2.0]"

        class FakeObj(object):
            def __init__(self, out):
                self.out = out

            def communicate(self):
                return self.out, ""

            def kill(self):
                pass

        c = FakeObj(version_output.split()[0])
        return c

    os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = str("1")
    with monkeypatch.context() as m:
        m.setattr("subprocess.Popen", mock_version)
        parsed = pythonfinder.models.python.PythonVersion.from_path(
            special_character_python.as_posix()
        )
        assert isinstance(parsed.version, Version)

@pytest.mark.parametrize(
    "path, version_output, version",
    [
        (
            "/fake/path/3.6.2/bin/python",
            "3.6.2 (default, Jun 28 2018, 13:15:42)",
            "3.6.2",
        ),
        (
            "/fake/path/3.7.0/bin/python",
            "3.7.0 (default, Jun 28 2018, 13:15:42)",
            "3.7.0",
        ),
        ("/fake/path/3.6.20/bin/python", "3.6.20 :: Continuum Analytics, Inc.", "3.6.20"),
        (
            "/fake/path/3.5.3/bin/python",
            "3.5.3 (fdd60ed87e941677e8ea11acf9f1819466521bf2, Apr 27 2018, 15:39:57)\n[PyPy 5.10.1 with GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)]",
            "3.5.3",
        ),
        (
            "/fake/path/2.7.15+/bin/python",
            "2.7.15+ (default, Jun 28 2018, 13:15:42)",
            "2.7.15",
        ),
    ],
)
def test_python_version_output_variants(monkeypatch, path, version_output, version):
    def mock_version(*args, **kwargs):
        class FakeObj(object):
            def __init__(self, out):
                self.out = out

            def communicate(self):
                return self.out, ""

            def kill(self):
                pass

        c = FakeObj(".".join([str(i) for i in version_output.split()[0].split(".")]))
        return c

    def get_python_version(path, orig_fn=None):
        if not os.path.exists(path):
            return mock_version(version_output)
        return orig_fn(path)

    with monkeypatch.context() as m:
        os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = str("1")
        m.setattr("subprocess.Popen", mock_version)
        orig_run_fn = pythonfinder.utils.get_python_version
        get_pyversion = functools.partial(get_python_version, orig_fn=orig_run_fn)
        m.setattr("pythonfinder.utils.get_python_version", get_pyversion)
        # m.setattr("pythonfinder.utils.get_python_version", mock_version)
        parsed = pythonfinder.models.python.PythonVersion.from_path(path)
        assert isinstance(parsed.version, Version)

@pytest.mark.skip_nt
def test_shims_are_kept(monkeypatch, no_pyenv_root_envvar, setup_pythons, no_virtual_env):
    with monkeypatch.context() as m:
        os.environ["PATH"] = "{0}:{1}".format(
            normalize_path("~/.pyenv/shims"), os.environ["PATH"]
        )
        f = pythonfinder.pythonfinder.Finder(
            global_search=True, system=False, ignore_unsupported=True
        )
        f.rehash()
        # assert pythonfinder.environment.get_shim_paths() == []
        assert is_in_ospath("~/.pyenv/shims")
        shim_paths = pythonfinder.environment.get_shim_paths()
        # Shims directories are no longer added to the system path order
        # but instead are used as indicators of the presence of the plugin
        # and used to trigger plugin setup -- this is true only if ``PYENV_ROOT`` is set`
        if shim_paths:
            assert (
                os.path.join(normalize_path("~/.pyenv/shims"))
                not in f.system_path.path_order
            ), (
                pythonfinder.environment.get_shim_paths()
            )  # "\n".join(f.system_path.path_order)
        else:
            assert (
                os.path.join(normalize_path("~/.pyenv/shims")) in f.system_path.path_order
            ), "\n".join(f.system_path.path_order)
        python_versions = f.find_all_python_versions()
        anaconda = f.find_python_version("anaconda3-5.3.0")
        assert anaconda is not None, python_versions
        assert "shims" in anaconda.path.as_posix(), [
            f.system_path.path_order,
            f.system_path.pyenv_finder.roots,
        ]
        # for docker, use just 'anaconda'
        for path in f.system_path.path_order:
            print(
                "path: {0}    Entry: {1}".format(path, f.system_path.get_path(path)),
                file=sys.stderr,
            )
        which_anaconda = f.which("anaconda3-5.3.0")
        if shim_paths:
            assert "shims" not in which_anaconda.path.as_posix()
        else:
            assert "shims" in which_anaconda.path.as_posix()

@pytest.mark.skip_nt
def test_shims_are_removed(monkeypatch, no_virtual_env, setup_pythons):
    with monkeypatch.context() as m:
        pyenv_dir = pythonfinder.utils.normalize_path("~/.pyenv")
        asdf_dir = pythonfinder.utils.normalize_path("~/.asdf")
        six.moves.reload_module(pythonfinder.environment)
        six.moves.reload_module(pythonfinder.models.path)
        m.setattr(
            pythonfinder.environment,
            "SHIM_PATHS",
            pythonfinder.environment.get_shim_paths(),
        )
        f = pythonfinder.pythonfinder.Finder(
            global_search=True, system=False, ignore_unsupported=True
        )
        f.rehash()
        python_versions = f.find_all_python_versions()
        assert os.environ["PYENV_ROOT"] == os.path.abspath(
            os.path.join(os.path.expanduser("~"), ".pyenv")
        )
        assert os.environ["PYENV_ROOT"] == pythonfinder.environment.PYENV_ROOT
        assert pythonfinder.environment.PYENV_INSTALLED
        assert f.system_path.pyenv_finder is not None
        python_version_paths = list(
            v.path
            for v in python_versions
            if normalized_match(str(v.path), os.environ["PYENV_ROOT"])
        )
        # Make sure we have an entry for every python version installed
        python_names = set(list(v.parent.parent.name for v in python_version_paths))
        # this is how we test in docker / on local machines with python installs
        # setup_pythons = {version.name for version in all_python_versions if any(plugin in version.path for plugin in ("pyenv", "asdf"))}
        # this is the test implementation when we are simulating
        setup_key_list = [
            set(list(finder.keys())) for finder in list(setup_pythons.values())
        ]
        setup_pythons = {python for finder in setup_key_list for python in finder}
        # this calculates the pythons not present when we ran `find_all_python_versions`
        missing_from_finder = python_names ^ setup_pythons
        if missing_from_finder:
            print_python_versions(python_versions)
            for p in python_version_paths:
                print("path: {0}".format(p), file=sys.stderr)
            for p in sorted(python_names):
                print("python_name: {0}".format(p), file=sys.stderr)
            for p in sorted(list(setup_pythons)):
                print("setup python key: {0}".format(p), file=sys.stderr)
        assert not missing_from_finder, missing_from_finder
        anaconda = f.find_python_version("anaconda3-5.3.0")
        assert anaconda is not None, os.listdir(
            os.path.join(pyenv_dir, "versions", "anaconda3-5.3.0", "bin")
        )
        assert "shims" not in anaconda.path.as_posix()
        which_anaconda = f.which("anaconda3-5.3.0")
        # for docker, use just 'anaconda'
        # which_anaconda = f.which("anaconda")
        assert "shims" not in which_anaconda.path.as_posix()

@pytest.mark.skip_nt
def test_windows_pythonfinder(expected_python_versions, all_python_versions):
    assert sorted(expected_python_versions) == sorted(all_python_versions)


from django.urls import path
from .views import primera,segunda,login, formulario,admin,agregarcanciones,agregarartistas,linkeimagen,lista
urlpatterns = [
path ('Inicio/', primera , name='homes'),
path ('Secundario/', segunda, name='dosh'),
path ('Login/', login, name='login'),
path ('Formulario/', formulario, name='Form'),
path ('Admin/', admin, name='Admin'),
path ('AgregarCanciones/', agregarcanciones, name='Agregacanciones'),
path ('AgregarArtistas/', agregarartistas, name='Agregaartistas'),
path ('LinkeImagen/', linkeimagen, name='Admin'),
path ('Link/', lista, name='lista'),

]


# Python program to illustrate HoughLine
# method for line detection
import cv2
import numpy as np

# Reading the required image in
# which operations are to be done.
# Make sure that the image is in the same
# directory in which this python program is
#img = cv2.imread('sudoku-original.jpg')
img = cv2.imread('index.jpeg')

# filter colour range we want
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
low_yellow = np.array([18, 94, 140])
up_yellow = np.array([48, 255, 255])
mask = cv2.inRange(hsv, low_yellow, up_yellow)

cv2.imshow('mask',mask)
cv2.waitKey(0)
cv2.destroyAllWindows()

# Apply edge detection method on the image
edges = cv2.Canny(mask,75,150)

# This returns an array of r and theta values
lines = cv2.HoughLinesP(edges, 1, np.pi/180, 30, maxLineGap=250)

# The below for loop runs till r and theta values
# are in the range of the 2d array
for line in lines:
    x1, y1, x2, y2 = line[0]
    cv2.line(img, (x1, y1), (x2, y2), (0, 255, 0), 3)

cv2.imshow('bexiga',edges)
cv2.imshow('imagem',img)
cv2.waitKey(0)
cv2.destroyAllWindows()


import argparse, os
import yaml as Y
import pandas as pd
import utils as U
import numpy as np
from attrdict import AttrDict
from keras.models import load_model
from data_generator import DataGenerator, joint_flip
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

import matplotlib
matplotlib.use('agg')

# To prevent crash on Mac OS
os.environ['KMP_DUPLICATE_LIB_OK']='True'

from utils import plot_inference

def inferPlot(cModel, testGenerator, outF, testList, inpShape, plotLabel):
  # Create model folder -- if it does not exist already
  U.checkFolder(os.path.dirname(outF))
  nLabels = testGenerator.nLabels()
  idx = 0

  # Contains scaling -- always pass through joint flip
  testFlow = joint_flip(testGenerator, hflip=False, vflip=False, loop=False)

  for testX, testy in testFlow:
    predy  = cModel.predict(testX)
    if nLabels > 2:
      cpred = predy.argmax(axis=-1)
    else:
      cpred = predy > 0.5

    predy  = testGenerator.label_inverse_transform(cpred).tolist()

    # Create file name
    # Valid because shuffle is off for test Generator
    cF = outF + '_%s'%(os.path.basename(testList[idx]))

    # Now plot and move on
    plot_inference(testX, predy, inpShape, plotLabel, cF, 10)
    idx = idx + 1

  return

def inferEval(cModel, testGenerator):
  ## Perform inference
  true = []
  pred = []
  nLabels = testGenerator.nLabels()

  # Contains scaling -- always pass through joint flip
  testFlow = joint_flip(testGenerator, hflip=False, vflip=False, loop=False)

  for testX, testy in testFlow:
    predy  = cModel.predict(testX)
    if nLabels > 2:
      cpred = predy.argmax(axis=-1)
    else:
      cpred = predy > 0.5

    truey  = testGenerator.label_inverse_transform(testy).tolist()
    true.extend(truey)

    predy  = testGenerator.label_inverse_transform(cpred).tolist()
    pred.extend(predy)

  # Accuracy
  curr_acc   = accuracy_score(true, pred)

  # Confusion matrix
  confMat = confusion_matrix(true, pred)
  confMat = np.ravel(confMat).tolist()
  return (curr_acc, confMat, pred)

def createYamlData(data, fname):
  '''
  Yaml creation; useed in doc test; irony ? only if it fails
  '''
  with open(fname, 'w') as outfile:
    Y.dump(data, outfile, default_flow_style=False)
  return

def main_inference(configF):
  '''
  >>> params = U.readInput(configF, display=False)

  >>> main_inference('random/file')
  Traceback (most recent call last):
      ...
  ValueError: Config file does not exist

  # Missing parameter
  >>> tmpF = '/tmp/data.yaml'
  >>> inpFolder = params['inpFolder']
  >>> del params['inpFolder']
  >>> createYamlData(params, tmpF)
  >>> main_inference(tmpF)
  Traceback (most recent call last):
      ...
  AttributeError: 'AttrDict' instance has no attribute 'inpFolder'

  # Random model prefix
  >>> params['inpFolder'] = inpFolder
  >>> params['modelFprefix'] = 'random_model'
  >>> createYamlData(params, tmpF)
  >>> main_inference(tmpF)
  Traceback (most recent call last):
      ...
  IOError: Unable to open model file
  '''
  if not os.path.isfile(configF):
    raise ValueError('Config file does not exist')

  # Read input configuration file
  params = U.readInput(configF)
  params = AttrDict(params)

  inpFolder       = params.inpFolder
  inputShape      = params.inputShape
  wholeImg        = params.wholeImg
  patchShape      = params.patchShape
  infer           = params.inference
  modelFname      = params.modelFname
  confMatFile     = params.confMatFile
  accOutFile      = params.accOutFile

  if infer:
    outFolder     = params.outFolder
    plotLabel     = params.plotLabel

  # Read input list for testing
  testList = U.getList(inpFolder)

  # Data handler for test data
  metadataF = None
  try:
    cModel = load_model(modelFname)
  except:
    raise IOError('Unable to open model file')

  if infer:
    metadataF = modelFname.strip('.h5') + '.meta.p'
    if not os.path.isfile(metadataF):
      raise ValueError('meta data does not exist')

  # This component has been extensivelt tested
  testGenerator = DataGenerator(testList, inputShape, batchSize=1, shuffle=False, \
                                patchShape=patchShape, balance=False, inference=infer,
                                metadataF=metadataF, wholeImg=wholeImg, sequentialPatch=True)

  # Performance inference on test dataset
  if infer:
    outPrefix = outFolder + '/' + os.path.basename(modelFname).strip('.h5')
    inferPlot(cModel, testGenerator, outPrefix, testList, inputShape, plotLabel)
  else:
    curr_acc, confMat, pred = inferEval(cModel, testGenerator)

    # Write Performance
    with open(accOutFile, 'w') as F:
      if not wholeImg:
        F.write('Patch Accuracy\n')
      else:
        F.write('Whole Image Accuracy\n')
      F.write('%.5f\n'%(curr_acc))

    # Confusion matrix write out
    np.savetxt(confMatFile, confMat, delimiter=',', fmt='%d')

    print('Performance logged: %s'%accOutFile)
    print('Confusion Matrix logged: %s'%confMatFile)

  return

def firstPassCommandLine():

  # Creating the parser for the input arguments
  parser = argparse.ArgumentParser(description='Neural network based classification')

  # Positional argument - Input XML file
  parser.add_argument('-configF', '--i', type=str, default='config.yaml',
                       help='Input configuration file', dest='configF')
  parser.add_argument('-test', '--t', action='store_true', dest='test')

  # Parse input
  args = parser.parse_args()
  return args

if __name__ == '__main__':
  import doctest
  args = firstPassCommandLine()
  if args.test:
    doctest.testmod(extraglobs={'configF': args.configF})

  main_inference(args.configF)


class Scoop:
    def __init__(self, flavor):
        self.flavor = flavor

class Bowl:
    max_scoops = 3

    def __init__(self):
        self.scoops = []
    def add_scoops(self, *new_scoops):
        for one_scoop in new_scoops:
            if len(self.scoops) >= self.max_scoops:
                break
            self.scoops.append(one_scoop)
    def __repr__(self):
        return '\n'.join(one_scoop.flavor
                          for one_scoop in self.scoops)

s1 = Scoop('chocolate')
s2 = Scoop('vanilla')
s3 = Scoop('persimmon')

s4 = Scoop('flavor 4')
s5 = Scoop('flavor 5')
s6 = Scoop('flavor 6')

print("**** Bowl")
b = Bowl()
b.add_scoops(s1, s2)
b.add_scoops(s3, s4)
b.add_scoops(s5)
print(b)

class BigBowl(Bowl):
    max_scoops = 5

print("**** BigBowl")
bb = BigBowl()
bb.add_scoops(s1, s2)
bb.add_scoops(s3, s4)
bb.add_scoops(s5)
bb.add_scoops(s6)
print(bb)


from __future__ import unicode_literals

from django.db import models
from django.contrib.auth.models import User

class Article(models.Model):
    title = models.CharField(max_length=255)
    text = models.TextField()
    author = models.ForeignKey(User)
    created_at = models.DateTimeField()
    image = models.ImageField(upload_to='article_images/', blank=True)

    def __unicode__(self):
        return self.title

class Comment(models.Model):
    article = models.ForeignKey(Article, related_name='comments')
    text = models.TextField()
    created_at = models.DateTimeField(auto_now_add=True)
    author = models.ForeignKey(User)
    image = models.ImageField(upload_to='comments_images/', blank=True)

    def __unicode__(self):
        return self.article.title

class Period(models.Model):
    start_date = models.DateField()
    end_date = models.DateField()


from datetime import date

from pandas.io import json
from attractions_recc import *
import pandas as pd
import re
import datetime as dt
import json

low = 50
high = 899
province = 'british_columbia'
start = dt.date.today()
end = start + dt.timedelta(6)
att_df = pd.read_json('etl/attractions.json',orient='records')

category_df = att_df.groupby('category').size().reset_index().sort_values([0],ascending=False)[:20]
categories = list(category_df.category.values)
cat_rating = {'luxury_&_special_occasions': 5.0, 'outdoor_activities': 4.0, 'recommended_experiences': 5.0, 'food,_wine_&_nightlife': 4.0, 'cruises,_sailing_&_water_tours': 3.0, 'cultural_&_theme_tours': 5.0}

def feed_input(price_low,price_high,province,days,att_df,categories,cat_rating):
    filename, user, rbm_att = get_recc(att_df, cat_rating)
    with_url = filter_df(filename, user, low, high, province, att_df)

    final = dict()
    final['timeofday'] = []
    final['image'] = []
    final['name'] = []
    final['location'] = []
    final['price'] = []
    final['rating'] = []
    final['category'] = []
    total_days = end - start
    for i in range(1,total_days.days+2):
        for j in range(2):
            final['timeofday'].append('Morning')
        for j in range(2):
            final['timeofday'].append('Evening')

    for i in range(len(final['timeofday'])):
        if i%4 == 0:
            final = top_recc(with_url, final)
        else:
            final = find_closest(with_url, final['location'][-1],final['timeofday'][i], final)

    #print(final)
    t = final_output(days,final)
    return json.dumps(t)