"""
This script is listing all of the best starting hands for a given number of players
 - This will be useful for constructing ranges as percentage of hands
"""

import eval7
import numpy as np
import pandas as pd

# Cards for range appending
all_combos = ['AA', 'AKo', 'AQo', 'AJo', 'ATo', 'A9o', 'A8o', 'A7o', 'A6o', 'A5o', 'A4o', 'A3o', 'A2o',
              'AKs', 'AQs', 'AJs', 'ATs', 'A9s', 'A8s', 'A7s', 'A6s', 'A5s', 'A4s', 'A3s', 'A2s',
              'KK', 'KQo', 'KJo', 'KTo', 'K9o', 'K8o', 'K7o', 'K6o', 'K5o', 'K4o', 'K3o', 'K2o',
              'KQs', 'KJs', 'KTs', 'K9s', 'K8s', 'K7s', 'K6s', 'K5s', 'K4s', 'K3s', 'K2s',
              'QQ', 'QJo', 'QTo', 'Q9o', 'Q8o', 'Q7o', 'Q6o', 'Q5o', 'Q4o', 'Q3o', 'Q2o',
              'QJs', 'QTs', 'Q9s', 'Q8s', 'Q7s', 'Q6s', 'Q5s', 'Q4s', 'Q3s', 'Q2s',
              'JJ', 'JTo', 'J9o', 'J8o', 'J7o', 'J6o', 'J5o', 'J4o', 'J3o', 'J2o',
              'JTs', 'J9s', 'J8s', 'J7s', 'J6s', 'J5s', 'J4s', 'J3s', 'J2s',
              'TT', 'T9o', 'T8o', 'T7o', 'T6o', 'T5o', 'T4o', 'T3o', 'T2o',
              'T9s', 'T8s', 'T7s', 'T6s', 'T5s', 'T4s', 'T3s', 'T2s',
              '99', '98o', '97o', '96o', '95o', '94o', '93o', '92o',
              '98s', '97s', '96s', '95s', '94s', '93s', '92s',
              '88', '87o', '86o', '85o', '84o', '83o', '82o',
              '87s', '86s', '85s', '84s', '83s', '82s',
              '77', '76o', '75o', '74o', '73o', '72o',
              '76s', '75s', '74s', '73s', '72s',
              '66', '65o', '64o', '63o', '62o',
              '65s', '64s', '63s', '62s',
              '55', '54o', '53o', '52o',
              '54s', '53s', '52s',
              '44', '43o', '42o',
              '43s', '42s',
              '33', '32o',
              '32s',
              '22']

# Making a Hand-range Class for each hand combination
all_class_combos = []
for current_combo in all_combos:
     all_class_combos.append(eval7.HandRange(current_combo))

# Declaring Dataframe to store hand-strength
holecard_equity = pd.DataFrame(columns=['hole_cards', 'average_equity'])

# Testing all the combinations
board = []
hero_index = 0
for test_hand in all_class_combos:
    # Declaring Data-Frame to store equity for current hand
    test_hand_df = pd.DataFrame(columns=['hero', 'villain', 'equity'])
    hero_cards = all_combos[hero_index]
    villain_index = 0
    for villain_hand in all_class_combos:
        villain_cards = all_combos[villain_index]
        equity_map = eval7.py_all_hands_vs_range(test_hand, villain_hand, board, 800000)
        average_equity = np.average(list(equity_map.values()))

        # Appending necessary info to Dataframe
        print(hero_cards+" vs "+villain_cards+" : "+str(average_equity))
        test_hand_df.loc[villain_index, 'hero'] = hero_cards
        test_hand_df.loc[villain_index, 'villain'] = villain_cards
        test_hand_df.loc[villain_index, 'equity'] = average_equity

        # Adding counters
        villain_index = villain_index +1

    # Exporting current Hands
    holecard_equity.loc[hero_index, 'hole_cards'] = hero_cards
    holecard_equity.loc[hero_index, 'average_equity'] = test_hand_df['equity'].mean()
    test_hand_df.to_csv(hero_cards+'_vs_unknown1.csv')
    hero_index = hero_index+1

#Exporting the final csv
holecard_equity.to_csv('holecard_equity_unknown1.csv')


#!/usr/bin/env python3

import argparse
import os
import shutil

def add_version(image, path):
    if not os.path.isfile('.env'):
        if os.path.isfile('.env.example'):
            shutil.copyfile('.env.example', '.env')
        else:
            with open('.env', 'w') as f:
                f.write('')

    with open('.env', 'r') as f:
        lines = f.read().split('\n')

    out = []

    found = False

    for line in lines:
        if (not found) and line.startswith(image+'='):
            line = image + '=' + path
            found = True

        if line.strip():
            out.append(line)

    if not found:
        line = image + '=' + path
        out.append(line)

    shutil.copyfile('.env', '.env.old')
    with open('.env', 'w') as f:
        f.write('\n'.join(out))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Add image path to .env')
    parser.add_argument("--file", required=True, type=str, help="Path to txt file")
    parser.add_argument("--name", required=True, type=str, help="Environment variable name")
    args = parser.parse_args()

    add_version(args.name, args.file)


'''
Programme that creates a financial profiles for a group of projects i.e. can produce the portfolio profile or a chosen
set of projects profile.

Output document is an excel wb with four tabs containing the aggregate financial profile for 'latest quarter',
'last quarter', 'baseline 1' (i.e. nearest baseline of approvals) 'baseline 2' (second nearest baselines).

Just data handling for now. No graphics produced. Graph needs to be done manually.

See instructions below.

'''

from openpyxl import Workbook
from analysis.data import cost_list, income_list, year_list, latest_cost_profiles, last_cost_profiles, \
    baseline_1_cost_profiles, baseline_2_cost_profiles, latest_income_profiles, list_of_masters_all, \
    last_income_profiles, baseline_1_income_profiles, baseline_2_income_profiles, dont_double_count, root_path
from analysis.engine_functions import filter_project_group, calculate_group_project_total

def place_in_excel(project_name_list):
    wb = Workbook()

    financial_profile_list = ['Latest Profile', 'Last quarter profile', 'Baseline 1 Profile', 'Baseline 2 Profile']
    cost_profile_data_list = [latest_cost_profiles, last_cost_profiles, baseline_1_cost_profiles, \
                              baseline_2_cost_profiles]
    income_profile_data_list = [latest_income_profiles, last_income_profiles, baseline_1_income_profiles, \
                                baseline_2_income_profiles]

    for p, profile in enumerate(financial_profile_list):
        '''worksheet is created for each project'''
        ws = wb.create_sheet(profile, p)  # creating worksheets
        ws.title = profile  # title of worksheet

        '''place information in each sheet'''
        ws.cell(row=1, column=1).value = 'Project'
        for i, project_name in enumerate(project_name_list):
            '''lists project names in row one'''
            ws.cell(row=1, column=i + 2).value = project_name

            '''iterates through financial dictionary - placing financial data in ws'''
            row_number = 1
            for cost in cost_list:
                for year in year_list:
                    try:
                        ws.cell(row=row_number + 1, column=i+2).value = \
                            cost_profile_data_list[p][project_name][year + cost]
                    except KeyError:
                        pass
                    row_number += 1

        '''places totals in final column'''

        cost_totals = calculate_group_project_total(project_name_list, cost_profile_data_list[p], dont_double_count,
                                                    cost_list, year_list)

        ws.cell(row=1, column=len(project_name_list) + 2).value = 'Total'
        for i, value in enumerate(cost_totals.values()):
            ws.cell(row=i + 2, column=len(project_name_list)+2).value = value

        '''places keys into the chart in the first column'''
        for i, key in enumerate(cost_totals.keys()):
            ws.cell(row=i+2, column=1).value = key

        '''information on which projects are not included in totals'''
        ws.cell(row=1, column=len(project_name_list) + 4).value = 'Projects that have been removed to avoid double counting'
        for i, project in enumerate(dont_double_count):
            ws.cell(row=i + 2, column=len(project_name_list) + 4).value = project

        '''total cost data for output chart'''
        for z, year in enumerate(year_list):
            for x, type in enumerate(cost_list):
                ws.cell(row=z + 41, column=x + 2, value=cost_totals[year + type])

        '''labeling for total figure table '''
        ws.cell(row=40, column=1, value='Year')
        labeling_list_type = ['RDEL', 'CDEL', 'Non-Gov', 'Total']
        for i, label in enumerate(labeling_list_type):
            ws.cell(row=40, column=2 + i, value=label)

        '''labeling of years down the side'''
        for i, label in enumerate(year_list):
            ws.cell(row=41 + i, column=1, value=label)

        '''income total data'''
        income_totals = calculate_group_project_total(project_name_list, income_profile_data_list[p], dont_double_count,
                                                    income_list, year_list)

        for y, year in enumerate(year_list):
            for t, type in enumerate(income_list):
                ws.cell(row=y + 55, column=t + 2, value=income_totals[year + type])

        '''labeling for total figure table '''
        ws.cell(row=54, column=1, value='Year')
        ws.cell(row=54, column=2, value='Income Totals')

        '''labeling of years down the side'''
        for i, label in enumerate(year_list):
            ws.cell(row=55 + i, column=1, value=label)

    return wb

''' RUNNING PROGRAMME'''

'''To run the programme place list of project names into function.
NOTE: Default option is list of all current projects in portfolio. In majority of cases user should just run the
default programme'''
output = place_in_excel(list_of_masters_all[0].projects)
output.save(root_path/'output/portfolio_financial_profile_q4_1920.xlsx')


from flask import Flask, render_template
from datasource.mysql import MySQLDataSource
import seaborn as sns
import matplotlib.pyplot as plt

app = Flask(__name__)

@app.route('/cohort-analysis')
def show_analysis():
    query1 = """
        SELECT DATE_FORMAT(sign_period,'%Y-%m') AS Period,
            FORMAT((MONTH0/qtd_organizations),2) AS MONTH0,
            FORMAT((MONTH1/qtd_organizations),2) AS MONTH1,
            FORMAT((MONTH2/qtd_organizations),2) AS MONTH2,
            FORMAT((MONTH3/qtd_organizations),2) AS MONTH3,
            FORMAT((MONTH4/qtd_organizations),2) AS MONTH4,
            FORMAT((MONTH5/qtd_organizations),2) AS MONTH5,
            FORMAT((MONTH6/qtd_organizations),2) AS MONTH6,
            FORMAT((MONTH7/qtd_organizations),2) AS MONTH7,
            FORMAT((MONTH8/qtd_organizations),2) AS MONTH8,
            FORMAT((MONTH9/qtd_organizations),2) AS MONTH9
            FROM WzOn7Gbgmv.cohort;
            """

    cohort_table = MySQLDataSource(query1)
    df1 = cohort_table.extract_data_from_database()
    df2 = df1.set_index('Period')
    df2 = df2.astype(float)

    # plot heatmap
    sns.set(style='white')
    plt.figure(figsize=(12, 8))
    plt.title('Cohorts: Organizations Creation')
    ax1 = sns.heatmap(df2, mask=df2.isnull(), annot=True, cmap='Blues',\
                     linewidths=1, fmt='.0%')

    figure1 = ax1.get_figure()
    figure1.savefig('static/images/heatmap.png', dpi = 400)

#-----------------------------------------------
    query2 = """
            SELECT DATE_FORMAT(signDate,'%Y-%m') AS Period,
            		SUM(signedOrganizationsCount) AS TotalSignOrganizations
            FROM accountants AC
            GROUP BY 1
            ORDER BY 1;
            """
    qtd_organization_by_accountants1 = MySQLDataSource(query2)
    qtd_organization_by_accountants2 = qtd_organization_by_accountants1.extract_data_from_database()
    qtd_organization_by_accountants3 = qtd_organization_by_accountants2.set_index('Period')

    # plot heatmap
    sns.set(style='white')
    plt.figure(figsize=(10, 8))
    plt.title('Quantity Signed Organizations/Period')
    ax2 = sns.heatmap(qtd_organization_by_accountants3, \
                      mask=qtd_organization_by_accountants3.isnull(), annot=True, cmap='Blues',\
                      linewidths=1, fmt='.0f')
    figure2 = ax2.get_figure()
    figure2.savefig('static/images/table.png', dpi = 400)

    return render_template('simple.html', message="Hello Flask!")

if __name__ == "__main__":
    app.run(debug = True, host = '0.0.0.0', port = 8080)


import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

from sklearn.manifold import TSNE

from tfvae.data_readers import read_encoded_hdf5

nfloats = 49
prefix = 'stargalaxy_sim_20190214'

datafile = prefix + 'encoded_test.hdf5'
images, labels = read_encoded_hdf5(datafile, nfloats)

feat_cols = ['element' + str(i) for i in range(images.shape[1])]
df = pd.DataFrame(images, columns=feat_cols)
df['labels'] = labels
# df['labels'] = df['labels'].apply(lambda i: str(i))

np.random.seed(42)
rndperm = np.random.permutation(df.shape[0])

plt.gray()
fig = plt.figure(figsize=(16, 7))
for i in range(0, 15):
    ax = fig.add_subplot(3, 5, i + 1,
                         title='{}'.format(str(df.loc[rndperm[i], 'labels'])))
    ax.matshow(
        df.loc[rndperm[i], feat_cols].values.reshape((7, 7)).astype(float)
    )
    ax.axis('off')
# plt.show()
plt.savefig('vae_encodeds.pdf', bbox_inches='tight')

N = 10000
df_subset = df.loc[rndperm[:N], :].copy()
data_subset = df_subset[feat_cols].values

tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(data_subset)

df_subset['tsne-2d-one'] = tsne_results[:, 0]
df_subset['tsne-2d-two'] = tsne_results[:, 1]
plt.figure(figsize=(16, 10))
sns.scatterplot(
    x="tsne-2d-one", y="tsne-2d-two",
    hue="labels",
    palette=sns.color_palette("hls", 2),
    data=df_subset,
    legend="full",
    alpha=0.3
)
# plt.show()
plt.axis('off')
plt.savefig('vae_tsne.pdf', bbox_inches='tight')


import logging
import numpy as np
import os
import pickle
from scipy import sparse as sp
import six
import tensorflow as tf

from .loss_graphs import AbstractLossGraph, RMSELossGraph
from .prediction_graphs import AbstractPredictionGraph, DotProductPredictionGraph
from .recommendation_graphs import (
    project_biases, split_sparse_tensor_indices, bias_prediction_dense, bias_prediction_serial, rank_predictions,
    densify_sampled_item_predictions, collapse_mixture_of_tastes, predict_similar_items
)
from .representation_graphs import AbstractRepresentationGraph, LinearRepresentationGraph
from .session_management import get_session
from .util import sample_items, calculate_batched_alpha

class TensorRec(object):

    def __init__(self,
                 n_components=100,
                 n_tastes=1,
                 user_repr_graph=LinearRepresentationGraph(),
                 item_repr_graph=LinearRepresentationGraph(),
                 attention_graph=None,
                 prediction_graph=DotProductPredictionGraph(),
                 loss_graph=RMSELossGraph(),
                 biased=True,
                 normalize_users=False,
                 normalize_items=False):
        """
        A TensorRec recommendation model.
        :param n_components: Integer
        The dimension of a single output of the representation function. Must be >= 1.
        :param n_tastes: Integer
        The number of tastes/reprs to be calculated for each user. Must be >= 1.
        :param user_repr_graph: AbstractRepresentationGraph
        An object which inherits AbstractRepresentationGraph that contains a method to calculate user representations.
        See tensorrec.representation_graphs for examples.
        :param item_repr_graph: AbstractRepresentationGraph
        An object which inherits AbstractRepresentationGraph that contains a method to calculate item representations.
        See tensorrec.representation_graphs for examples.
        :param attention_graph: AbstractRepresentationGraph or None
        An object which inherits AbstractRepresentationGraph that contains a method to calculate user attention. Any
        valid repr_graph is also a valid attention graph. If None, no attention process will be applied.
        :param prediction_graph: AbstractPredictionGraph
        An object which inherits AbstractPredictionGraph that contains a method to calculate predictions from a pair of
        user/item reprs.
        See tensorrec.prediction_graphs for examples.
        :param loss_graph: AbstractLossGraph
        An object which inherits AbstractLossGraph that contains a method to calculate the loss function.
        See tensorrec.loss_graphs for examples.
        :param biased: bool
        If True, a bias value will be calculated for every user feature and item feature.
        :param normalize_users: bool
        If True, every row of the user features will be l2 normalized.
        :param normalize_items: bool
        If True, every row of the item features will be l2 normalized.
        """

        # Arg-check
        if (n_components is None) or (n_tastes is None) or (user_repr_graph is None) or (item_repr_graph is None) \
                or (prediction_graph is None) or (loss_graph is None):
            raise ValueError("All arguments to TensorRec() must be non-None")
        if n_components < 1:
            raise ValueError("n_components must be >= 1")
        if n_tastes < 1:
            raise ValueError("n_tastes must be >= 1")
        if not isinstance(user_repr_graph, AbstractRepresentationGraph):
            raise ValueError("user_repr_graph must inherit AbstractRepresentationGraph")
        if not isinstance(item_repr_graph, AbstractRepresentationGraph):
            raise ValueError("item_repr_graph must inherit AbstractRepresentationGraph")
        if not isinstance(prediction_graph, AbstractPredictionGraph):
            raise ValueError("prediction_graph must inherit AbstractPredictionGraph")
        if not isinstance(loss_graph, AbstractLossGraph):
            raise ValueError("loss_graph must inherit AbstractLossGraph")
        if attention_graph is not None:
            if not isinstance(attention_graph, AbstractRepresentationGraph):
                raise ValueError("attention_graph must be None or inherit AbstractRepresentationGraph")
            if n_tastes == 1:
                raise ValueError("attention_graph must be None if n_tastes == 1")

        self.n_components = n_components
        self.n_tastes = n_tastes
        self.user_repr_graph_factory = user_repr_graph
        self.item_repr_graph_factory = item_repr_graph
        self.attention_graph_factory = attention_graph
        self.prediction_graph_factory = prediction_graph
        self.loss_graph_factory = loss_graph
        self.biased = biased
        self.normalize_users = normalize_users
        self.normalize_items = normalize_items

        # A list of the attr names of every graph hook attr
        self.graph_tensor_hook_attr_names = [

            # Top-level API nodes
            'tf_user_representation', 'tf_item_representation', 'tf_prediction_serial', 'tf_prediction', 'tf_rankings',
            'tf_predict_similar_items', 'tf_rank_similar_items',

            # Training nodes
            'tf_basic_loss', 'tf_weight_reg_loss', 'tf_loss',

            # Feed placeholders
            'tf_n_users', 'tf_n_items', 'tf_user_feature_indices', 'tf_user_feature_values', 'tf_item_feature_indices',
            'tf_item_feature_values', 'tf_interaction_indices', 'tf_interaction_values', 'tf_learning_rate', 'tf_alpha',
            'tf_sample_indices', 'tf_n_sampled_items', 'tf_similar_items_ids'
        ]
        if self.biased:
            self.graph_tensor_hook_attr_names += ['tf_projected_user_biases', 'tf_projected_item_biases']
        if self.attention_graph_factory is not None:
            self.graph_tensor_hook_attr_names += ['tf_user_attention_representation']

        self.graph_operation_hook_attr_names = [

            # AdamOptimizer
            'tf_optimizer',

        ]
        self._clear_graph_hook_attrs()

        # A map of every graph hook attr name to the node name after construction
        # Tensors and operations are stored separated because they are handled differently by TensorFlow
        self.graph_tensor_hook_node_names = {}
        self.graph_operation_hook_node_names = {}

    def _clear_graph_hook_attrs(self):
        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            self.__setattr__(graph_tensor_hook_attr_name, None)
        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            self.__setattr__(graph_operation_hook_attr_name, None)

    def _attach_graph_hook_attrs(self):
        session = get_session()

        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            graph_tensor_hook_node_name = self.graph_tensor_hook_node_names[graph_tensor_hook_attr_name]
            node = session.graph.get_tensor_by_name(name=graph_tensor_hook_node_name)
            self.__setattr__(graph_tensor_hook_attr_name, node)

        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            graph_operation_hook_node_name = self.graph_operation_hook_node_names[graph_operation_hook_attr_name]
            node = session.graph.get_operation_by_name(name=graph_operation_hook_node_name)
            self.__setattr__(graph_operation_hook_attr_name, node)

    def _create_feed_dict(self, interactions_matrix, user_features_matrix, item_features_matrix,
                          extra_feed_kwargs=None):

        # Check that input data is of a sparse type
        if (interactions_matrix is not None) and (not sp.issparse(interactions_matrix)):
            raise Exception('Interactions must be a scipy sparse matrix')
        if not sp.issparse(user_features_matrix):
            raise Exception('User features must be a scipy sparse matrix')
        if not sp.issparse(item_features_matrix):
            raise Exception('Item features must be a scipy sparse matrix')

        n_users, user_feature_indices, user_feature_values = self._process_matrix(user_features_matrix,
                                                                                normalize_rows=self.normalize_users)
        n_items, item_feature_indices, item_feature_values = self._process_matrix(item_features_matrix,
                                                                                normalize_rows=self.normalize_items)

        feed_dict = {self.tf_n_users: n_users,
                     self.tf_n_items: n_items,
                     self.tf_user_feature_indices: user_feature_indices,
                     self.tf_user_feature_values: user_feature_values,
                     self.tf_item_feature_indices: item_feature_indices,
                     self.tf_item_feature_values: item_feature_values, }

        if interactions_matrix is not None:
            _, interaction_indices, interaction_values = self._process_matrix(interactions_matrix)
            feed_dict[self.tf_interaction_indices] = interaction_indices
            feed_dict[self.tf_interaction_values] = interaction_values

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _create_user_feed_dict(self, user_features_matrix, extra_feed_kwargs=None):

        if not sp.issparse(user_features_matrix):
            raise Exception('User features must be a scipy sparse matrix')

        n_users, user_feature_indices, user_feature_values = self._process_matrix(user_features_matrix,
                                                                                normalize_rows=self.normalize_users)

        feed_dict = {self.tf_n_users: n_users,
                     self.tf_user_feature_indices: user_feature_indices,
                     self.tf_user_feature_values: user_feature_values}

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _create_item_feed_dict(self, item_features_matrix, extra_feed_kwargs=None):

        if not sp.issparse(item_features_matrix):
            raise Exception('Item features must be a scipy sparse matrix')

        n_items, item_feature_indices, item_feature_values = self._process_matrix(item_features_matrix,
                                                                                normalize_rows=self.normalize_items)

        feed_dict = {self.tf_n_items: n_items,
                     self.tf_item_feature_indices: item_feature_indices,
                     self.tf_item_feature_values: item_feature_values}

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _process_matrix(self, features_matrix, normalize_rows=False):

        if normalize_rows:
            if not isinstance(features_matrix, sp.csr_matrix):
                features_matrix = sp.csr_matrix(features_matrix)
            mag = np.sqrt(features_matrix.power(2).sum(axis=1))
            features_matrix = features_matrix.multiply(1.0 / mag)

        if not isinstance(features_matrix, sp.coo_matrix):
            features_matrix = sp.coo_matrix(features_matrix)

        # "Actors" is used to signify "users or items" -- unused for interactions
        n_actors = features_matrix.shape[0]
        feature_indices = [pair for pair in six.moves.zip(features_matrix.row, features_matrix.col)]
        feature_values = features_matrix.data

        return n_actors, feature_indices, feature_values

    def _create_batch_feed_dicts(self, interactions, user_features, item_features, extra_feed_kwargs,
                                 user_batch_size=None):

        if not isinstance(interactions, sp.csr_matrix):
            interactions = sp.csr_matrix(interactions)
        if not isinstance(user_features, sp.csr_matrix):
            user_features = sp.csr_matrix(user_features)

        n_users = user_features.shape[0]

        # Infer the batch size, if necessary
        if user_batch_size is None:
            user_batch_size = n_users

        feed_dicts = []

        start_batch = 0
        while start_batch < n_users:

            # min() ensures that the batch bounds doesn't go past the end of the index
            end_batch = min(start_batch + user_batch_size, n_users)

            batch_interactions = interactions[start_batch:end_batch]
            batch_user_features = user_features[start_batch:end_batch]

            # TODO its inefficient to make so many copies of the item features
            feed_dict = self._create_feed_dict(interactions_matrix=batch_interactions,
                                               user_features_matrix=batch_user_features,
                                               item_features_matrix=item_features,
                                               extra_feed_kwargs=extra_feed_kwargs)
            feed_dicts.append(feed_dict)

            start_batch = end_batch

        return feed_dicts

    def _build_tf_graph(self, n_user_features, n_item_features):

        # Initialize placeholder values for inputs
        self.tf_n_users = tf.placeholder('int64')
        self.tf_n_items = tf.placeholder('int64')
        self.tf_n_sampled_items = tf.placeholder('int64')

        # SparseTensor placeholders
        self.tf_user_feature_indices = tf.placeholder('int64', [None, 2])
        self.tf_user_feature_values = tf.placeholder('float', [None])
        self.tf_item_feature_indices = tf.placeholder('int64', [None, 2])
        self.tf_item_feature_values = tf.placeholder('float', [None])
        self.tf_interaction_indices = tf.placeholder('int64', [None, 2])
        self.tf_interaction_values = tf.placeholder('float', [None])
        self.tf_similar_items_ids = tf.placeholder('int64', [None])

        self.tf_sample_indices = tf.placeholder('int64', [None, None])
        self.tf_learning_rate = tf.placeholder('float', None)
        self.tf_alpha = tf.placeholder('float', None)

        # Construct the features and interactions as sparse matrices
        tf_user_features = tf.SparseTensor(self.tf_user_feature_indices, self.tf_user_feature_values,
                                           [self.tf_n_users, n_user_features])
        tf_item_features = tf.SparseTensor(self.tf_item_feature_indices, self.tf_item_feature_values,
                                           [self.tf_n_items, n_item_features])
        tf_interactions = tf.SparseTensor(self.tf_interaction_indices, self.tf_interaction_values,
                                          [self.tf_n_users, self.tf_n_items])

        # Collect the weights for normalization
        tf_weights = []

        # Build the item representations
        self.tf_item_representation, item_weights = \
            self.item_repr_graph_factory.connect_representation_graph(tf_features=tf_item_features,
                                                                      n_components=self.n_components,
                                                                      n_features=n_item_features,
                                                                      node_name_ending='item')
        tf_weights.extend(item_weights)

        tf_x_user, tf_x_item = split_sparse_tensor_indices(tf_sparse_tensor=tf_interactions, n_dimensions=2)
        tf_transposed_sample_indices = tf.transpose(self.tf_sample_indices)
        tf_x_user_sample = tf_transposed_sample_indices[0]
        tf_x_item_sample = tf_transposed_sample_indices[1]

        # These lists will hold the reprs and predictions for each taste
        tastes_tf_user_representations = []
        tastes_tf_predictions = []
        tastes_tf_prediction_serials = []
        tastes_tf_sample_prediction_serials = []

        # If this model does not use attention, Nones are used as sentinels in place of the attentions
        if self.attention_graph_factory is not None:
            tastes_tf_attentions = []
            tastes_tf_attention_serials = []
            tastes_tf_sample_attention_serials = []
            tastes_tf_attention_representations = []
        else:
            tastes_tf_attentions = None
            tastes_tf_attention_serials = None
            tastes_tf_sample_attention_serials = None
            tastes_tf_attention_representations = None

        # Build n_tastes user representations and predictions
        for taste in range(self.n_tastes):
            tf_user_representation, user_weights = \
                self.user_repr_graph_factory.connect_representation_graph(tf_features=tf_user_features,
                                                                          n_components=self.n_components,
                                                                          n_features=n_user_features,
                                                                          node_name_ending='user_{}'.format(taste))
            tastes_tf_user_representations.append(tf_user_representation)
            tf_weights.extend(user_weights)

            # Connect attention, if applicable
            if self.attention_graph_factory is not None:
                tf_attention_representation, attention_weights = \
                    self.attention_graph_factory.connect_representation_graph(tf_features=tf_user_features,
                                                                              n_components=self.n_components,
                                                                              n_features=n_user_features,
                                                                              node_name_ending='attn_{}'.format(taste))
                tf_weights.extend(attention_weights)

                tf_attention = self.prediction_graph_factory.connect_dense_prediction_graph(
                    tf_user_representation=tf_attention_representation,
                    tf_item_representation=self.tf_item_representation
                )
                tf_attention_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                    tf_user_representation=tf_attention_representation,
                    tf_item_representation=self.tf_item_representation,
                    tf_x_user=tf_x_user,
                    tf_x_item=tf_x_item,
                )
                tf_sample_attention_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                    tf_user_representation=tf_user_representation,
                    tf_item_representation=self.tf_item_representation,
                    tf_x_user=tf_x_user_sample,
                    tf_x_item=tf_x_item_sample,
                )

                tastes_tf_attentions.append(tf_attention)
                tastes_tf_attention_serials.append(tf_attention_serial)
                tastes_tf_sample_attention_serials.append(tf_sample_attention_serial)
                tastes_tf_attention_representations.append(tf_attention_representation)

            # Connect the configurable prediction graphs for each taste
            tf_prediction = self.prediction_graph_factory.connect_dense_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation
            )
            tf_prediction_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation,
                tf_x_user=tf_x_user,
                tf_x_item=tf_x_item,
            )
            tf_sample_predictions_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation,
                tf_x_user=tf_x_user_sample,
                tf_x_item=tf_x_item_sample,
            )

            # Append to tastes
            tastes_tf_predictions.append(tf_prediction)
            tastes_tf_prediction_serials.append(tf_prediction_serial)
            tastes_tf_sample_prediction_serials.append(tf_sample_predictions_serial)

        # If attention is in the graph, build the API node
        if self.attention_graph_factory is not None:
            self.tf_user_attention_representation = tf.stack(tastes_tf_attention_representations)

        self.tf_user_representation = tf.stack(tastes_tf_user_representations)
        self.tf_prediction = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_predictions,
            tastes_attentions=tastes_tf_attentions
        )
        self.tf_prediction_serial = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_prediction_serials,
            tastes_attentions=tastes_tf_attention_serials
        )
        tf_sample_predictions_serial = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_sample_prediction_serials,
            tastes_attentions=tastes_tf_sample_attention_serials
        )

        # Add biases, if this is a biased estimator
        if self.biased:
            tf_user_feature_biases, self.tf_projected_user_biases = project_biases(
                tf_features=tf_user_features, n_features=n_user_features
            )
            tf_item_feature_biases, self.tf_projected_item_biases = project_biases(
                tf_features=tf_item_features, n_features=n_item_features
            )

            tf_weights.append(tf_user_feature_biases)
            tf_weights.append(tf_item_feature_biases)

            self.tf_prediction = bias_prediction_dense(
                tf_prediction=self.tf_prediction,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases)

            self.tf_prediction_serial = bias_prediction_serial(
                tf_prediction_serial=self.tf_prediction_serial,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases,
                tf_x_user=tf_x_user,
                tf_x_item=tf_x_item)

            tf_sample_predictions_serial = bias_prediction_serial(
                tf_prediction_serial=tf_sample_predictions_serial,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases,
                tf_x_user=tf_x_user_sample,
                tf_x_item=tf_x_item_sample)

        tf_interactions_serial = tf_interactions.values

        # Construct API nodes
        self.tf_rankings = rank_predictions(tf_prediction=self.tf_prediction)
        self.tf_predict_similar_items = predict_similar_items(prediction_graph_factory=self.prediction_graph_factory,
                                                              tf_item_representation=self.tf_item_representation,
                                                              tf_similar_items_ids=self.tf_similar_items_ids)
        self.tf_rank_similar_items = rank_predictions(tf_prediction=self.tf_predict_similar_items)

        # Compose loss function args
        # This composition is for execution safety: it prevents loss functions that are incorrectly configured from
        # having visibility of certain nodes.
        loss_graph_kwargs = {
            'tf_prediction_serial': self.tf_prediction_serial,
            'tf_interactions_serial': tf_interactions_serial,
            'tf_interactions': tf_interactions,
            'tf_n_users': self.tf_n_users,
            'tf_n_items': self.tf_n_items,
        }
        if self.loss_graph_factory.is_dense:
            loss_graph_kwargs.update({
                'tf_prediction': self.tf_prediction,
                'tf_rankings': self.tf_rankings,
            })
        if self.loss_graph_factory.is_sample_based:
            tf_sample_predictions = densify_sampled_item_predictions(
                tf_sample_predictions_serial=tf_sample_predictions_serial,
                tf_n_sampled_items=self.tf_n_sampled_items,
                tf_n_users=self.tf_n_users,
            )
            loss_graph_kwargs.update({'tf_sample_predictions': tf_sample_predictions,
                                      'tf_n_sampled_items': self.tf_n_sampled_items})

        # Build loss graph
        self.tf_basic_loss = self.loss_graph_factory.connect_loss_graph(**loss_graph_kwargs)

        self.tf_weight_reg_loss = sum(tf.nn.l2_loss(weights) for weights in tf_weights)
        self.tf_loss = self.tf_basic_loss + (self.tf_alpha * self.tf_weight_reg_loss)
        self.tf_optimizer = tf.train.AdamOptimizer(learning_rate=self.tf_learning_rate).minimize(self.tf_loss)

        # Get node names for each graph hook
        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            hook = self.__getattribute__(graph_tensor_hook_attr_name)
            self.graph_tensor_hook_node_names[graph_tensor_hook_attr_name] = hook.name
        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            hook = self.__getattribute__(graph_operation_hook_attr_name)
            self.graph_operation_hook_node_names[graph_operation_hook_attr_name] = hook.name

    def fit(self, interactions, user_features, item_features, epochs=100, learning_rate=0.1, alpha=0.00001,
            verbose=False, out_sample_interactions=None, user_batch_size=None, n_sampled_items=None):
        """
        Constructs the TensorRec graph and fits the model.
        :param interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :param epochs: Integer
        The number of epochs to fit the model.
        :param learning_rate: Float
        The learning rate of the model.
        :param alpha:
        The weight regularization loss coefficient.
        :param verbose: boolean
        If true, the model will print a number of status statements during fitting.
        :param out_sample_interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        If not None, and verbose == True, the model will be evaluated on these interactions on every epoch.
        :param user_batch_size: int or None
        The maximum number of users per batch, or None for all users.
        :param n_sampled_items: int or None
        The number of items to sample per user for use in loss functions. Must be non-None if
        self.loss_graph_factory.is_sample_based is True.
        """

        # Pass-through to fit_partial
        self.fit_partial(interactions=interactions,
                         user_features=user_features,
                         item_features=item_features,
                         epochs=epochs,
                         learning_rate=learning_rate,
                         alpha=alpha,
                         verbose=verbose,
                         out_sample_interactions=out_sample_interactions,
                         user_batch_size=user_batch_size,
                         n_sampled_items=n_sampled_items)

    def fit_partial(self, interactions, user_features, item_features, epochs=1, learning_rate=0.1,
                    alpha=0.00001, verbose=False, out_sample_interactions=None, user_batch_size=None,
                    n_sampled_items=None):
        """
        Constructs the TensorRec graph and fits the model.
        :param interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :param epochs: Integer
        The number of epochs to fit the model.
        :param learning_rate: Float
        The learning rate of the model.
        :param alpha:
        The weight regularization loss coefficient.
        :param verbose: boolean
        If true, the model will print a number of status statements during fitting.
        :param out_sample_interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        If not None, and verbose == True, the model will be evaluated on these interactions on every epoch.
        :param user_batch_size: int or None
        The maximum number of users per batch, or None for all users.
        :param n_sampled_items: int or None
        The number of items to sample per user for use in loss functions. Must be non-None if
        self.loss_graph_factory.is_sample_based is True.
        """

        session = get_session()

        # Arg checking
        if self.loss_graph_factory.is_sample_based:
            if (n_sampled_items is None) or (n_sampled_items <= 0):
                raise ValueError("n_sampled_items must be an integer >0")
        if (n_sampled_items is not None) and (not self.loss_graph_factory.is_sample_based):
            logging.warning('n_sampled_items was specified, but the loss graph is not sample-based')

        # Check if the graph has been constructed by checking the dense prediction node
        # If it hasn't been constructed, initialize it
        if self.tf_prediction is None:
            # Numbers of features are learned at fit time from the shape of these two matrices and cannot be changed
            # without refitting
            self._build_tf_graph(n_user_features=user_features.shape[1], n_item_features=item_features.shape[1])
            session.run(tf.global_variables_initializer())

        if verbose:
            logging.info('Processing interaction and feature data')

        batched_feed_dicts = self._create_batch_feed_dicts(interactions=interactions,
                                                           user_features=user_features,
                                                           item_features=item_features,
                                                           user_batch_size=user_batch_size,
                                                           extra_feed_kwargs={self.tf_learning_rate: learning_rate})

        # This scales down the alpha based on the number of batches and inserts the new alpha in all feed dicts
        batched_alpha = calculate_batched_alpha(num_batches=len(batched_feed_dicts), alpha=alpha)
        for feed_dict in batched_feed_dicts:
            feed_dict[self.tf_alpha] = batched_alpha

        if verbose:
            logging.info('Beginning fitting')

        for epoch in range(epochs):
            for batch, feed_dict in enumerate(batched_feed_dicts):

                # Handle random item sampling, if applicable
                if self.loss_graph_factory.is_sample_based:
                    sample_indices = sample_items(n_users=feed_dict[self.tf_n_users],
                                                  n_items=feed_dict[self.tf_n_items],
                                                  n_sampled_items=n_sampled_items,
                                                  replace=self.loss_graph_factory.is_sampled_with_replacement)
                    feed_dict[self.tf_sample_indices] = sample_indices
                    feed_dict[self.tf_n_sampled_items] = n_sampled_items

                # TODO find something more elegant than these cascaded ifs
                if not verbose:
                    session.run(self.tf_optimizer, feed_dict=feed_dict)

                else:
                    _, loss, serial_predictions, wr_loss = session.run(
                        [self.tf_optimizer, self.tf_basic_loss, self.tf_prediction_serial, self.tf_weight_reg_loss],
                        feed_dict=feed_dict
                    )
                    mean_loss = np.mean(loss)
                    mean_pred = np.mean(serial_predictions)
                    weight_reg_l2_loss = alpha * wr_loss
                    logging.info('EPOCH {} BATCH {} loss = {}, weight_reg_l2_loss = {}, mean_pred = {}'.format(
                        epoch, batch, mean_loss, weight_reg_l2_loss, mean_pred
                    ))
                    if out_sample_interactions:
                        os_feed_dict = self._create_feed_dict(out_sample_interactions, user_features, item_features)
                        os_loss = self.tf_basic_loss.eval(session=session, feed_dict=os_feed_dict)
                        logging.info('Out-Sample loss = {}'.format(os_loss))

    def predict(self, user_features, item_features):
        """
        Predict recommendation scores for the given users and items.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The predictions in an ndarray of shape [n_users, n_items]
        """
        feed_dict = self._create_feed_dict(interactions_matrix=None,
                                           user_features_matrix=user_features,
                                           item_features_matrix=item_features)

        predictions = self.tf_prediction.eval(session=get_session(), feed_dict=feed_dict)

        return predictions

    def predict_similar_items(self, item_features, item_ids, n_similar):
        """
        Predicts the most similar items to the given item_ids.
        :param item_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_ids: list or np.array
        The ids of the items of interest.
        E.g. [4, 8, 12] to get sims for items 4, 8, and 12.
        :param n_similar: int
        The number of similar items to get per item of interest.
        :return: list of lists of tuples
        The first level list corresponds to input arg item_ids.
        The second level list is of length n_similar and contains tuples of (item_id, score) for each similar item.
        """
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        feed_dict[self.tf_similar_items_ids] = np.array(item_ids)

        sims = self.tf_predict_similar_items.eval(session=get_session(), feed_dict=feed_dict)

        results = []
        for i in range(len(item_ids)):
            item_sims = sims[i]
            best = np.argpartition(item_sims, -n_similar)[-n_similar:]
            item_results = sorted(zip(best, item_sims[best]), key=lambda x: -x[1])
            results.append(item_results)

        return results

    def predict_rank(self, user_features, item_features):
        """
        Predict recommendation ranks for the given users and items.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The ranks in an ndarray of shape [n_users, n_items]
        """
        feed_dict = self._create_feed_dict(interactions_matrix=None,
                                           user_features_matrix=user_features,
                                           item_features_matrix=item_features)

        rankings = self.tf_rankings.eval(session=get_session(), feed_dict=feed_dict)

        return rankings

    def predict_user_representation(self, user_features):
        """
        Predict latent representation vectors for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The latent user representations in an ndarray of shape [n_users, n_components]
        """
        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        user_repr = self.tf_user_representation.eval(session=get_session(), feed_dict=feed_dict)

        # If there is only one user repr per user, collapse from rank 3 to rank 2
        if self.n_tastes == 1:
            user_repr = np.sum(user_repr, axis=0)

        return user_repr

    def predict_user_attention_representation(self, user_features):
        """
        Predict latent attention representation vectors for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The latent user attention representations in an ndarray of shape [n_users, n_components]
        """

        if self.attention_graph_factory is None:
            raise ValueError("This TensorRec model does not use attention. Try re-building TensorRec with a valid "
                             "'attention_graph' arg.")

        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        user_attn_repr = self.tf_user_attention_representation.eval(session=get_session(), feed_dict=feed_dict)

        # If there is only one user attn repr per user, collapse from rank 3 to rank 2
        if self.n_tastes == 1:
            user_attn_repr = np.sum(user_attn_repr, axis=0)

        return user_attn_repr

    def predict_item_representation(self, item_features):
        """
        Predict representation vectors for the given items.
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The latent item representations in an ndarray of shape [n_items, n_components]
        """
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        item_repr = self.tf_item_representation.eval(session=get_session(), feed_dict=feed_dict)
        return item_repr

    def predict_user_bias(self, user_features):
        """
        Predict bias values for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The user biases in an ndarray of shape [n_users]
        """
        if not self.biased:
            raise NotImplementedError('Cannot predict user bias for unbiased model')
        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        predictions = self.tf_projected_user_biases.eval(session=get_session(), feed_dict=feed_dict)
        return predictions

    def predict_item_bias(self, item_features):
        """
        Predict bias values for the given items.
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The item biases in an ndarray of shape [n_items]
        """
        if not self.biased:
            raise NotImplementedError('Cannot predict item bias for unbiased model')
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        predictions = self.tf_projected_item_biases.eval(session=get_session(), feed_dict=feed_dict)
        return predictions

    def save_model(self, directory_path):
        """
        Saves the model to files in the given directory.
        :param directory_path: str
        The path to the directory in which to save the model.
        :return:
        """

        if not os.path.exists(directory_path):
            os.makedirs(directory_path)

        saver = tf.train.Saver()
        session_path = os.path.join(directory_path, 'tensorrec_session.cpkt')
        saver.save(sess=get_session(), save_path=session_path)

        # Break connections to the graph before saving the python object
        self._clear_graph_hook_attrs()
        tensorrec_path = os.path.join(directory_path, 'tensorrec.pkl')
        with open(tensorrec_path, 'wb') as file:
            pickle.dump(file=file, obj=self)

        # Reconnect to the graph after saving
        self._attach_graph_hook_attrs()

    @classmethod
    def load_model(cls, directory_path):
        """
        Loads the TensorRec model and TensorFlow session saved in the given directory.
        :param directory_path: str
        The path to the directory containing the saved model.
        :return:
        """

        graph_path = os.path.join(directory_path, 'tensorrec_session.cpkt.meta')
        saver = tf.train.import_meta_graph(graph_path)

        session_path = os.path.join(directory_path, 'tensorrec_session.cpkt')
        saver.restore(sess=get_session(), save_path=session_path)

        tensorrec_path = os.path.join(directory_path, 'tensorrec.pkl')
        with open(tensorrec_path, 'rb') as file:
            model = pickle.load(file=file)
        model._attach_graph_hook_attrs()
        return model


###########################################################
### Configuration file to run cfA in CMSSW 7+ on miniAOD
###########################################################

### General assignments
import FWCore.ParameterSet.Config as cms
process = cms.Process("MinicfA")
process.load("FWCore.MessageService.MessageLogger_cfi")
process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(100) )

process.source = cms.Source("PoolSource",
    fileNames = cms.untracked.vstring(
        #'/store/cmst3/user/gpetrucc/miniAOD/v1/TT_Tune4C_13TeV-pythia8-tauola_PU_S14_PAT.root'
        #'/store/mc/Phys14DR/TT_Tune4C_13TeV-pythia8-tauola/MINIAODSIM/PU20bx25_tsg_PHYS14_25_V1-v1/00000/06843FC5-8370-E411-9B8C-0025905A60AA.root'
        '/store/mc/Phys14DR/TT_Tune4C_13TeV-pythia8-tauola/MINIAODSIM/PU40bx25_tsg_PHYS14_25_V1-v1/00000/06E41ADB-7870-E411-8850-0025905A605E.root'
        #'/store/mc/Spring14miniaod/TTJets_MSDecaysCKM_central_Tune4C_13TeV-madgraph-tauola/MINIAODSIM/PU20bx25_POSTLS170_V5-v2/00000/004C6DA7-FB03-E411-96BD-0025905A497A.root'
        #'file:/home/users/manuelf/cmssw/cfa/CMSSW_7_2_2_patch1/src/CfANtupler/minicfa/python/TT_Tune4C_13TeV-pythia8-tauola_MINIAODSIM.root'
    )
)

process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
process.GlobalTag.globaltag = 'PLS170_V7AN1::All'   # GT for 25ns (asymptotic alignment and calibration scenario)

### Loading branches
process.load("CfANtupler/minicfa.branchesminicfA_cfi")

process.InputTagDistributorService = cms.Service("InputTagDistributorService")
process.VariableHelperService = cms.Service("VariableHelperService")
process.TFileService = cms.Service("TFileService",
    fileName = cms.string('cfA.root')
)

process.trackIsolationMaker = cms.EDProducer("TrackIsolationMaker",
                                             pfCandidatesTag = cms.InputTag("packedPFCandidates"),
                                             vertexInputTag = cms.InputTag("offlineSlimmedPrimaryVertices"),
                                             dR_ConeSize = cms.double(0.3),
                                             dz_CutValue = cms.double(0.05),
                                             minPt_PFCandidate = cms.double(5.0), #looser than the likely analysis selection
                                             maxIso_PFCandidate = cms.double(0.25) #very loose
)

process.p = cms.Path(process.trackIsolationMaker)
process.outpath = cms.EndPath(cms.ignore(process.cfA))


import dateutil.parser
import pytz

def normalise_dict(data, defaults):
    """
    Normalise the given data dictionary to have the same structure as defaults dict.
    Logic is;
        a. only include keys in defaults
        b. if key is not set in data, set it to the value in defaults
    :param data:
    :param defaults:
    """

    normalised_data = {}

    if isinstance(data, dict) and isinstance(defaults, dict):

        for key in defaults:

            if key in data:
                normalised_data[key] = data[key]
            else:
                normalised_data[key] = defaults[key]

    return normalised_data

def parse_datetime(datetime_string):
    """
    Parse a date time string, in the format used by ASX.com.au
    :return: Parsed datetime object if valid, None if invalid date
    """

    datetime_parsed = None

    try:

        if datetime_string is not None and len(datetime_string):

            datetime_parsed = dateutil.parser.parse(
                str(datetime_string)
            )

    except ValueError:
        # date parse failed

        datetime_parsed = None

    return datetime_parsed

def format_mysql_datetime(datetime_obj):
    """
    Format datetime to MySQL date & time format YYYY-MM-DD HH:MM;SS
    """

    datetime_string = ''

    if datetime_obj is not None:
        datetime_string = datetime_obj.strftime('%Y-%m-%d %H:%M:%S')

    return datetime_string

def format_mysql_date(datetime_obj):
    """
    Format datetime to MySQL format YYYY-MM-DD
    """

    datetime_string = ''

    if datetime_obj is not None:
        datetime_string = datetime_obj.strftime('%Y-%m-%d')

    return datetime_string


'''
This code is based on the one available at https://pypi.org/project/pytorch-categorical/.
We make few change to make it works with torch>1.3.
'''
import torch
import pdb

class Categorical:
    def __init__(
        self, probs, device=None, normalized=False
    ):

        self.validate_probs_type_and_shape(probs)

        # Copy probs tensor; Enforce float32 dtype.  Preserve device.
        self.device = device or probs.device
        self.probs = probs.clone().detach().to(self.device, dtype=torch.float32)

        if not normalized:
            self.probs /= self.probs.sum()

        # Setup the alias outcomes and probabilities.
        self.alias_probs = None
        self.alias_outcomes = None
        self.setup()

    def validate_probs_type_and_shape(self, probs):
        if not isinstance(probs, torch.Tensor):
            raise ValueError("``probs`` should be a tensor.  Got {}.".format(
                type(probs).__name__
            ))
        if len(probs.shape) != 1:
            raise ValueError("``probs`` should be a 1D-tensor. Got {}.".format(
                probs.shape
            ))

    def __len__(self):
        return self.probs.shape[0]

    def setup(self):
        self.alias_probs = self.probs * len(self)
        self.alias_outcomes = torch.zeros(
            (len(self), 2), dtype=torch.long, device=self.device)
        self.alias_outcomes[:,0] = torch.arange(len(self), device=self.device)
        small_idxs, large_idxs = self.split_idxs(
            self.alias_probs, torch.arange(len(self), device=self.device))
        while small_idxs.shape[0] > 0 and large_idxs.shape[0] > 0:

            overlap_size = min(small_idxs.shape[0], large_idxs.shape[0])
            # Available large_idxs get allocated to the secondary
            # outcomes for aliases holding small outcomes
            covered_small = small_idxs[:overlap_size]
            used_large = large_idxs[:overlap_size]
            self.alias_outcomes[covered_small, 1] = used_large

            remaining_small = small_idxs[overlap_size:]
            self.alias_probs[used_large] -= (1-self.alias_probs[covered_small])

            new_small, large_idxs = self.split_idxs(
                self.alias_probs[large_idxs], large_idxs)

            small_idxs = torch.cat((remaining_small, new_small))

    def split_idxs(self, vector, idxs=None):
        is_large = vector > 1
        num_large = is_large.sum()
        sorted_positions = torch.sort(is_large.byte())[1]
        sorted_idxs = idxs[sorted_positions]
        large_idxs = sorted_idxs[-num_large:]
        small_idxs = sorted_idxs[:-num_large]

        return small_idxs, large_idxs

    def slow_setup(self):
        """
        For reference, this is an approach to assembling the alias probabilities
        and alias outcome map without taking advantage of easy parallelization
        using tensor-based operations.
        """
        self.alias_probs = torch.zeros(
            len(self), dtype=self.dtype, device=self.device)
        self.alias_outcomes = torch.zeros(
            (len(self), 2), dtype=torch.long, device=self.device)

        # Sort the data into the outcomes with probabilities
        # that are larger and smaller than 1/K.
        smaller = []
        larger  = []
        for outcome in range(len(self)):
            self.alias_probs[outcome] = len(self) * self.probs[outcome]
            self.alias_outcomes[outcome,0] = outcome
            if self.alias_probs[outcome] < 1.0:
                smaller.append(outcome)
            else:
                larger.append(outcome)
        while len(smaller) > 0 and len(larger) > 0:
            small_outcome = smaller.pop()
            large_outcome = larger.pop()
            self.alias_outcomes[small_outcome, 1] = large_outcome
            self.alias_probs[large_outcome] = (
                self.alias_probs[large_outcome] -
                (1.0 - self.alias_probs[small_outcome])
            )
            if self.alias_probs[large_outcome] < 1.0:
                smaller.append(large_outcome)
            else:
                larger.append(large_outcome)

    def sample(self, sample_shape=(1,), dtype='int64'):
        primary_outcomes = torch.randint(
            0, len(self), sample_shape, dtype=torch.long)
        secondary_outcomes = (
            torch.rand(sample_shape, device=self.device) >
            self.alias_probs[primary_outcomes]
        ).long()
        return self.alias_outcomes[primary_outcomes, secondary_outcomes]


#use class salary of empolyee in use of def function ...???

n = int(input("user choice depend how many entry got it in company "))
class Emp_Salary():
	def name(self):
		self.name = input("enter the name of empolyee : ")
	def father_name(self):
		self.father_name = input("enter the father_name of empolyee : ")
	def company(self):
		self.company_name = input("enter the company_name : ")
		self.company_address = input("enter the company_address : ")
	def company_salary(self):
		self.basicsalary = int(input("enter the basic salary : "))
		hra = self.basicsalary * (0.06)
		da = self.basicsalary * (0.05)
		ta = self.basicsalary * (0.06)
		fund  = self.basicsalary * (0.07)
		self.salary = self.basicsalary + hra + da + ta - fund
	def show(self):
		print("the empolyee name : ",self.name)
		print("the empolyee father_name : ",self.father_name)
		print("the company name : ",self.company_name)
		print("the company_address : ",self.company_address)
		print("total salary of empolyee hand to hand Givin by company : ",round(self.salary,2))

for i in range(n):
	a = Emp_Salary()
	a.name()
	a.father_name()
	a.company()
	a.company_salary()
	a.show()

'''
output ==

user choice depend how many entry got it in company 2
enter the name of empolyee : ram
enter the father_name of empolyee : shyam
enter the company_name : hcl
enter the company_address : noida62
enter the basic salary : 12000
the empolyee name :  ram
the empolyee father_name :  shyam
the company name :  hcl
the company_address :  noida62
total salary of empolyee hand to hand Givin by company :  13200.0
enter the name of empolyee : shyam
enter the father_name of empolyee : hardev pasad
enter the company_name : tcs
enter the company_address : newdelhi
enter the basic salary : 12500
the empolyee name :  shyam
the empolyee father_name :  hardev pasad
the company name :  tcs
the company_address :  newdelhi
total salary of empolyee hand to hand Givin by company :  13750.0

'''


from cloudmesh_data.data.Provider import Provider
from cloudmesh_data.database import mongo
from deprecated import google_cloud_upload, google_cloud_delete, s3_delete, s3_download, s3_upload, \
    google_cloud_download
from cloudmesh_data.data.Config import Config

def get_files(service):
    config = Config()
    kind = config['cloud']['data'][service]
    provider = Provider()
    provider = provider.get_provider(kind)
    files_list = provider.list(provider["location"])

    return files_list

def get_file_by_name(provider, bucketname, filename, user_uuid):
    if provider == 'aws':
        file_path = s3_download.download_file(bucketname, filename)
        mongo.save_file_to_db('AWS', file_path, filename, user_uuid)
    elif provider == 'google':
        file_path = google_cloud_download.download_blob(bucketname, filename)
        mongo.save_file_to_db('GOOGLE', file_path, filename, user_uuid)

def upload_file_by_name(provider, bucketname, filename):
    if provider == 'aws':
        s3_upload.upload_file(bucketname, filename)
    elif provider == 'google':
        google_cloud_upload.upload_blob(bucketname, filename)

def copy_file(filename, provider, provider_bucket, dest, dest_bucket):
    if provider == dest:
        print("Target cloud needs to different than the source cloud")
        exit
    else:
        get_file_by_name(provider, provider_bucket, filename)
        upload_file_by_name(dest, dest_bucket, filename)

def rsync_file(filename, source, dest):
    print('')

def delete_file(provider, bucketname, filename):
    if provider == 'aws':
        s3_delete.delete_file(bucketname, filename)
    elif provider == 'google':
        google_cloud_delete.delete_blob(bucketname, filename)

def update_user_for_file(user_uuid, filename):
    mongo.update_user_for_file(user_uuid, filename)


import os
import json
import datetime
import time
import subprocess

ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

p = subprocess.Popen(['netstat','-i'],stdout=subprocess.PIPE)
out,err = p.communicate()
p = subprocess.Popen(['grep','wlan0'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
out,err = p.communicate(out)
p = subprocess.Popen(['sed','-r','s/wlan0[\\t ]+1500[\\t ]+([0-9]+).+/\\1/'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
rx,err = p.communicate(out)
p = subprocess.Popen(['sed','-r','s/wlan0[\\t ]+1500[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+[0-9]+[\\t ]+([0-9]+).+/\\1/'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
tx,err = p.communicate(out)

rx = int(rx.decode('utf-8'))
tx = int(tx.decode('utf-8'))

rx_json_string = '{"id": "comms.recd","timestamp": "'+ts+'","value":'+str(rx)+'}'
tx_json_string = '{"id": "comms.sent","timestamp": "'+ts+'","value":'+str(tx)+'}'

print(rx_json_string)
print(tx_json_string)


from .mongo import db

def get_all_column_names():
    collections = db.list_collection_names()
    fields = {}

    for collection in collections:
        fields[collection] = get_column_names(collection)

    return fields

def get_column_names(col):
    return get_fields(db[col].find_one())

def get_fields(d):
    """
    Return a list of keys in a given dict.
    Nested keys store the whole path as a dot-seperated string
    """
    x = []
    for key, val in d.items():
        if isinstance(val, dict):
            for key_nested in get_fields(val):
                x.append("{}.{}".format(key, key_nested))
        elif isinstance(val, list) and len(val) > 0:
            for key_nested in get_fields(val[0]):
                x.append("{}.{}".format(key, key_nested))
        else:
            x.append(key)
    return x


from django.test import TestCase

import httpretty
import json

usage_statistics = [
    {
        "target": "core_count",
        "datapoints": [
            [20.0, 1324216800],
        ]
    },
    {
        "target": "instance_count",
        "datapoints": [
            [10.0, 1324303200],
        ]
    }
]

class AllocationViewTest(TestCase):

    fixtures = ['allocation_requests']
    multi_db = True

    # Web pages

    def test_page_index(self):
        response = self.client.get("/allocations/")
        self.assertEqual(200, response.status_code)

    def test_page_visualisation(self):
        response = self.client.get(
            "/allocations/applications/approved/visualisation")
        self.assertEqual(200, response.status_code)

    def test_page_project(self):
        response = self.client.get(
            "/allocations/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    def test_page_project_allocations(self):
        response = self.client.get(
            "/allocations/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    # Web services with JSON pay loads.

    def test_rest_allocation_tree(self):
        response = self.client.get(
            "/allocations/rest/applications/approved/tree")
        self.assertEqual(200, response.status_code)

    def test_rest_for_code(self):
        response = self.client.get("/allocations/rest/for_codes")
        self.assertEqual(200, response.status_code)

    @httpretty.activate
    def test_rest_for_project_from_allocation_request_id(self):
        httpretty.register_uri(
            httpretty.GET,
            'http://graphite.dev.rc.nectar.org.au/render/',
            body=json.dumps(usage_statistics),
            content_type="application/json")

        response = self.client.get(
            "/allocations/rest/applications/1654/approved")
        self.assertEqual(200, response.status_code)

    @httpretty.activate
    def test_rest_project_summary_response(self):
        httpretty.register_uri(
            httpretty.GET,
            'http://graphite.dev.rc.nectar.org.au/render/',
            body=json.dumps(usage_statistics),
            content_type="application/json")

        response = self.client.get(
            "/allocations/rest/applications/1654/approved")
        result = json.loads(response.content)

        assert result['used_cores'] == 20
        assert result['used_instances'] == 10

    def test_rest_for_project_allocations_from_allocation_request_id(self):
        response = self.client.get(
            "/allocations/rest/applications/1654/history")
        self.assertEqual(200, response.status_code)

    # Site map.

    def test_sitemap(self):
        response = self.client.get(
            "/sitemap.xml")
        self.assertContains(
            response,
            'http://example.com/allocations/applications', 2, 200)


N=int(input())
inl=list(map(lambda x: int(x)*2,input().split()))
vs=list(map(int,input().split()))

sumtime = sum(inl)

limits = [0]
for i, t in enumerate(inl):
    for _ in range(t):
        limits.append(vs[i])

resv = [0]
for i in range(1, sumtime+1):
    resv.append(min(resv[-1]+0.5, limits[i]))

resv[sumtime] = 0
for i in reversed(range(1, sumtime+1)):
    resv[i-1] = min(resv[i]+0.5, resv[i-1], limits[i])

print(limits)
print(resv)
res = 0.0
for i in range(sumtime):
    res += (resv[i] + resv[i+1])*0.5

print(res*0.5)


# -*- coding: utf-8 -*-
#
# authors: Vipin
#
# Created on Fri June 7 2019

# -- External Libraries
from django.test import TestCase
from django.contrib.auth.models import User

# -- Local Libraries
from comments.models import Comment

class CommentCreateViewTest(TestCase):

	@classmethod
	def setUpTestData(cls):
		# Set up non-modified objects used by all test methods
		user = User.objects.create(username='testuser')
		user.set_password('testpass')
		user.save()

	def test_response_status_on_post_data(self):
		post_data = {
			"comment_text": "This is comment 1",
			"user": 1,
			"parent": None
		}
		response = self.client.post('/comments/create', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 201)

class CommentDetailViewTest(TestCase):

	@classmethod
	def setUpTestData(cls):
		# Set up non-modified objects used by all test methods
		user_two = User.objects.create(username='testuser')
		user_two.set_password('testpass')
		user_two.save()

		# Crating another user
		user_three = User.objects.create(username='anotheruser')
		user_three.set_password('testpass')
		user_three.save()

		# Creating a comment to test GET operation
		comment = Comment.objects.create(comment_text="This is a test comment", user=user_two)
		comment.save()

	def test_response_status_on_get_request(self):
		response = self.client.get('/comments/2')
		self.assertEqual(response.status_code, 200)

	def test_response_data_received_on_get_request(self):
		response = self.client.get('/comments/2')
		data = {
			"id": 2,
			"comment_text": "This is a test comment",
			"user": 2,
			"parent": None,
			"child": []
		}
		self.assertEqual(response.data, data)

	def test_response_status_on_patch_request(self):
		post_data = {
			"comment_text": "This comment is patched",
			"user": 2,
			"parent": None,
		}
		response = self.client.patch('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 200)

	def test_response_status_on_patch_request_if_user_does_not_have_permission(self):
		post_data = {
			"comment_text": "This comment is patched",
			"user": 3,
			"parent": None
		}
		response = self.client.patch('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 403)

	def test_response_status_on_delete_request_if_user_does_not_have_permission(self):
		post_data = {
			"user": 3,
		}
		response = self.client.delete('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 403)

	def test_response_status_on_delete_request(self):
		post_data = {
			"user": 2,
		}
		response = self.client.delete('/comments/2', data=post_data, content_type='application/json')
		self.assertEqual(response.status_code, 204)


__all__ = [
    "__name__", "__version__", "__author__",
    "__email__", "__license__", "__copyright__",
    "__summary__", "__url__",
]

__name__ = "pyinsta"
__version__ = "1.0"
__author__ = "Tian Zhang"
__email__ = "zhangtian@utexas.edu"
__license__ = "BSD"
__summary__ = __doc__
__url__ = "https://github.com/HugoTian/pyinsta"


#!/usr/bin/env python3
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), 'trim-psbt'))
from bitcoin import psbt
from bitcoin.networks import NETWORKS
# base64 encoding
from binascii import a2b_base64, b2a_base64

def main():
    # parse psbt transaction
    # From sign-psbt.p2wsh.psbt:
    orig_psbt = "cHNidP8BAFICAAAAATrA+nAjorbvWhg+VE1ql6DnmRMibJRuJjHVq1I1b72fAQAAAAD/////AQxKTAAAAAAAFgAUpBDHZKyNoStXigYCfxHgLwe34dMAAAAAAAEAfQIAAAABEw04mQpL5Ny5AZwHaJeEoqjk51OwcCOHc7VnnAD9294AAAAAAP////8C2goOAAAAAAAWABRPf0Rx/SsT/Uez2y+ve3H6+M4a/UBLTAAAAAAAIgAgZchjx1AzHMA++02sohfnklz/h/XpypknVkI0vg1ieOgAAAAAAQErQEtMAAAAAAAiACBlyGPHUDMcwD77TayiF+eSXP+H9enKmSdWQjS+DWJ46AEFi1IhA9FN3PtoF/VXlpW7s+s+GFiHvylCsDHm9xY0O4/n6ejiIQKPzUb4YUssvzGAlpaCQqHiK8+22PK23JOcjCfDR7KTeyEDFay1UBIPTNy0YNXEkICrUIykzNje3srCL+6sjNAX1MEhAisGPuLyL54ZgsFA/neGcsaDERp5bczNvUe7ZePWCKmDVK4AAA=="

    # first convert it to binary
    raw = a2b_base64(orig_psbt)
    # then parse
    tx = psbt.PSBT.parse(raw)

    # Since PSBT has both witness and non-witness input descriptions:
    if not all(x.non_witness_utxo for x in tx.inputs):
        raise ValueError("Expected every input to have non_witness_utxo")
    if not all(x.witness_utxo for x in tx.inputs):
        raise ValueError("Expected every input to have witness_utxo")

    def amount_for(idx):
        """Return satoshis on input {idx}. Uses non-witness inputs."""
        vout = tx.tx.vin[idx].vout
        outp = tx.inputs[idx].non_witness_utxo.vout[vout]
        return outp.value

    # Calculate and display transaction fee.
    input_total = sum(amount_for(idx) for idx in range(len(tx.tx.vin)))
    output_total = sum(out.value for out in tx.tx.vout)

    print("Inputs total", input_total, "sats from", len(tx.tx.vin), "inputs")
    print("Outputs total", output_total, "sats")
    print("Fee equals", input_total - output_total, "sats")

    # print how much we are spending and where
    for out in tx.tx.vout:
        print(out.value,"to",out.script_pubkey.address(NETWORKS["test"]))

    # Corrupt the PSBT by modifying the amount on the first input
    tx.inputs[0].witness_utxo.value = 1

    save_to_file(tx, 'sign-psbt.corrupted-witness-utxo.psbt')

def save_to_file(psbt, filename):
    """Given a PSBT object, write it in base64 to filename."""
    raw = psbt.serialize()
    # convert to base64
    b64_psbt = b2a_base64(raw)
    # somehow b2a ends with \n...
    if b64_psbt[-1:] == b"\n":
        b64_psbt = b64_psbt[:-1]
    # print
    new_psbt = b64_psbt.decode('utf-8')
    print("Creating", filename)
    with open(filename, 'wt') as outfile:
        print(new_psbt, file=outfile)

if __name__ == '__main__':
    main()


import ael, amb

# Default Instrument types that are supported
# In order to exclude some Instrument types you are allowed to
# reduce a number of items in this list (do not add anything)
valid_instrument_types = ['Future/Forward', 'FRA', 'Swap', 'Cap', 'IndexLinkedSwap', 'Floor', 'CurrSwap', 'Option', 'Curr', 'Deposit']

def trade_message_can_be_skippped(amb_message):
    skip = False
    for trade_obj in objects_by_name(amb_message, ['', '+', '!'], 'TRADE'):
        #Instrument types that can be skipped
        insaddr_obj = trade_obj.mbf_find_object('INSADDR')
        if insaddr_obj:
           insaddr = insaddr_obj.mbf_get_value()
           if insaddr:
               instrument = ael.Instrument[int(insaddr)]
               if instrument:
                   instrument_type = instrument.instype
                   if instrument_type not in valid_instrument_types:
                       skip = True
        #If updated Premium is the same as the original Premium then skip message
        updat_premium_obj = trade_obj.mbf_find_object('!PREMIUM')
        if updat_premium_obj:
            new_premium_obj = trade_obj.mbf_find_object('PREMIUM')
            if new_premium_obj:
                updat_premium = round(float(updat_premium_obj.mbf_get_value()), 2)
                new_premium = round(float(new_premium_obj.mbf_get_value()), 2)
                if updat_premium == new_premium:
                    skip = True
        #If updated Price is the same as the original Price then skip message
        updat_price_obj = trade_obj.mbf_find_object('!PRICE')
        if updat_price_obj:
            new_price_obj = trade_obj.mbf_find_object('PRICE')
            if new_price_obj:
                updat_price = round(float(updat_price_obj.mbf_get_value()), 2)
                new_price = round(float(new_price_obj.mbf_get_value()), 2)
                if updat_price == new_price:
                    skip = True

        #If the execution time of the trade is prior to 08/11/2010 then skip
        exec_time_obj = trade_obj.mbf_find_object('EXECUTION_TIME')
        if exec_time_obj:
            exec_time = exec_time_obj.mbf_get_value()
            if exec_time != '0':
                time_time = ael.date_from_string(exec_time[:10], '%Y-%m-%d')
                if time_time < ael.date_from_string('2010-11-08', '%Y-%m-%d'):
                    skip = True
            else:
                skip = True

    return skip

def objects_by_name(parent_obj, name_prefixes, name):
    obj = parent_obj.mbf_first_object()
    names = list()
    for name_prefix in name_prefixes:
        names.append(name_prefix + name)
    while obj:
        if obj.mbf_get_name() in names:
            yield obj
        obj = parent_obj.mbf_next_object()

def object_by_name(parent_obj, name_prefixes, name):
    for obj in objects_by_name(parent_obj, name_prefixes, name):
        return obj
    return None

def modify_instrument_message(m, s):
    instrument_obj = object_by_name(m, ['', '+', '!'], 'INSTRUMENT')
    for leg_obj in objects_by_name(instrument_obj, ['', '+', '!'], 'LEG'):
        for cashflow_obj in objects_by_name(leg_obj, ['', '+', '!'], 'CASHFLOW'):
            for reset_obj in objects_by_name(cashflow_obj, ['', '+', '!'], 'RESET'):
                cashflow_obj.mbf_remove_object()
    return (m, s)

def modify_sender(m, s):

    result = (m, s)
    type_obj = m.mbf_find_object('TYPE', 'MBFE_BEGINNING')
    type_value = type_obj.mbf_get_value()

    if type_value in ['INSERT_INSTRUMENT', 'UPDATE_INSTRUMENT']:
        result = modify_instrument_message(m, s)
    elif type_value in ['INSERT_TRADE', 'UPDATE_TRADE']:
        if trade_message_can_be_skippped(m):
            return None
    return result


import random
import time

total = 100
num = 10
# 初始化数据列表
listA = [x for x in range(total)]
random.shuffle(listA)

# 1
def fun1():
    res = listA[: num]
    res.sort(reverse=True)
    for i in listA[num:]:
        if i > res[-1]:
            res.pop()
            res.append(i)
            res.sort(reverse=True)
    return res

# 2
def fun2():
    res = listA[: num]
    res.sort(reverse=True)
    for i in listA[num:]:
        if i > res[-1]:
            res.pop()
            j = 0
            while j < len(res):
                if i > res[j]:
                    res.insert(j, i)
                    break
                j += 1
    return res

# 3
class MinHeap(object):
    """最小堆"""
    def __init__(self):
        self.data = []  # 创建堆
        self.count = len(self.data)  # 元素数量

    # def __init__(self, arr):
    #   self.data = copy.copy(arr)
    #   self.count = len(self.data)
    #   i = self.count / 2
    #   while i >= 1:
    #       self.shiftDown(i)
    #       i -= 1

    def size(self):
        return self.count

    def isEmpty(self):
        return self.count == 0

    def insert(self, item):
        # 插入元素入堆
        self.data.append(item)
        self.count += 1
        self.shiftup(self.count)

    def shiftup(self, count):
        # 将插入的元素放到合适位置，保持最小堆
        while count > 1 and self.data[(count // 2)] > self.data[count - 1]:
            self.data[(count // 2)], self.data[count - 1] = self.data[count - 1], self.data[(count // 2)]
            count = count // 2

    def extractMin(self):
        # 出堆
        if self.count > 0:
            ret = self.data[0]
            self.data[0], self.data[self.count-1] = self.data[self.count-1], self.data[0]
            self.data.pop()
            self.count -= 1
            self.shiftDown(1)
            return ret

    def shiftDown(self, count):
        # 将堆的索引位置元素向下移动到合适位置，保持最小堆
        while 2 * count <= self.count :
            # 证明有孩子
            j = 2 * count
            if j + 1 <= self.count:
                # 证明有右孩子
                if self.data[j] < self.data[j-1]:
                    j += 1
            if self.data[count-1] <= self.data[j-1]:
                # 堆的索引位置已经小于两个孩子节点，不需要交换了
                break
            self.data[count-1], self.data[j-1] = self.data[j-1], self.data[count-1]
            count = j

def fun3():
    minheap = MinHeap()
    for i in listA[: num]:
        minheap.insert(i)
    print(minheap.data)
    for j in listA[num: ]:
        if j > minheap.data[0]:
            minheap.extractMin()
            minheap.insert(j)
    return minheap.data

if __name__ == '__main__':
    startTime = time.time()
    # res = fun1()
    # res = fun2()
    res = fun3()
    print('Time:', time.time() - startTime)
    print(res)


import os
import glob

# https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives
def find_vcs_root(test, dirs=(".git",)):
  prev, test = None, os.path.abspath(test)
  while prev != test:
    if any(os.path.isdir(os.path.join(test, d)) for d in dirs):
      return test
    prev, test = test, os.path.abspath(os.path.join(test, os.pardir))

  sys.exit('File not in git repository')
  return

contents = """{
  "className": "Camera"
}"""

root = find_vcs_root(__file__)
for server_folder in glob.glob(os.path.join(root, "src") + "/*/src/Server"):
  init_meta_json = os.path.join(server_folder, "init.meta.json")
  if os.path.exists(init_meta_json):
    print(init_meta_json)
  else:
    with open(init_meta_json, "w") as f:
      f.write(contents)


'''
Problem: Similar Char
Problem Description:
Tahir and Mamta are woking in a project in TCS.
Tahir being a problem solver came up with an interesting problem for his friend Mamta.
Problem consists of a string of length N and contains only small case alphabets.
It will be followed by Q queries, in which each query will contain an integer P (1<=P<=N) denoting a position within the string.
Mamta's task is to find the alphabet present at that location and determine the number of occurrence of same alphabet
preceding the given location P.
Mamta is busy with her office work. Therefore, she asked you to help her.
Constraints
1 <= N <= 500000
S consisting of small case alphabets
1 <= Q <= 10000
1 <= P <= N
Input Format
First line contains an integer N, denoting the length of string.
Second line contains string S itself consists of small case alphabets only ('a' - 'z').
Third line contains an integer Q denoting number of queries that will be asked.
Next Q lines contains an integer P (1 <= P <= N) for which you need to find the number occurrence of character
present at the Pth location preceeding P.

Output
For each query, print an integer denoting the answer on single line.
Test Case

Explanation
Example 1
Input
9
abacsddaa
2
9
3
Output
3
1
Explanation
Here Q = 2
For P=9, character at 9th location is 'a'. Number of occurrences of 'a' before P i.e., 9 is three.
Similarly for P=3, 3rd character is 'a'. Number of occurrences of 'a' before P. i.e., 3 is one.
'''

import sys

print("Enter inputs:")
print("Enter value of N")
try:
	N=int(input())
except ValueError:
	print("Please enter an integer")
	sys.exit()
if N>50000:
	print("Please enter a number smaller than 50000")
	sys.exit()

print("Enter value of S")
S=input()
if len(S)!=N:
	print("The length of string was",len(S),"please enter string with length=N")
	sys.exit()
for x in S:
	if ord(x)>122 or ord(x)<97:
		print("please enter all lower-case letters as the value of S")
		sys.exit()

print("Enter value of Q")
try:
	Q=int(input())
except ValueError:
	print("Please enter a number")
	sys.exit()
if Q>1000:
	print("Please enter a value less than 1000")
	sys.exit()

print("You entered following inputs")
print("N=",N)
print("S=",S)
print("Q=",Q)

print("Enter queries, i.e. the values of P")
counter=0
collect=[]
while(counter<Q):
	try:
		query=int(input())
		if query>N:
			print("P has exceeded the value of N, please try again")
			continue
		counter+=1
		collect.append(query)
	except ValueError:
		print("Please enter a number")

print("You entered following queries")
print(collect)

def char_counter(x):
	char=S[x-1]
	substr=S[:x-1]
	return len(substr.split(char))-1

output=[]
for x in collect:
	output.append(char_counter(x))

print("The answer is")
for x in output:
	print(x)


import vtk
import numpy as np
import sys

global lastNormal
global lastAxis1
global axes

from vtkUtils import AxesToTransform

def MyFunc2(obj, ev):
  global lastNormal
  global lastAxis1
  global axes

  normal0 = lastNormal
  first0  = lastAxis1
  origin0 = axes.GetOrigin()

  normal1 = np.array(obj.GetNormal())
  first1 = np.array(obj.GetPoint1()) - np.array(obj.GetOrigin())
  origin1 = obj.GetCenter()

  trans = AxesToTransform(normal0, first0, origin0,
                          normal1, first1, origin1)

  if axes.GetUserTransform() is not None:
    axes.GetUserTransform().Concatenate(trans)
  else:
    transform = vtk.vtkTransform()
    transform.SetMatrix(trans)
    transform.PostMultiply()
    axes.SetUserTransform(transform)

  # Only for book keeping
  axes.SetOrigin(obj.GetCenter()) # Not modified by SetUserTransform
  axes.Modified()

  # Update last axes
  lastAxis1[0] = first1[0]
  lastAxis1[1] = first1[1]
  lastAxis1[2] = first1[2]

  lastNormal = (normal1[0], normal1[1], normal1[2])

def MyFunc(obj, ev):
  global lastNormal
  global lastAxis1
  global axes

  saxis = np.array(obj.GetPoint1()) - np.array(obj.GetOrigin())
  axis = np.array(obj.GetNormal())

  vec = vtk.vtkVector3d() # Axis of rotation
  vtk.vtkMath.Cross(lastNormal, axis, vec)
  costheta = vtk.vtkMath.Dot(axis, lastNormal)
  sintheta = vtk.vtkMath.Norm(vec)
  theta = np.arctan2(sintheta, costheta)

  if sintheta != 0.0:
    vec[0] = vec[0]/sintheta
    vec[1] = vec[1]/sintheta
    vec[2] = vec[2]/sintheta

  # Convert to Quaternion
  costheta = np.cos(0.5*theta)
  sintheta = np.sin(0.5*theta)
  quat0 = vtk.vtkQuaterniond(costheta, vec[0]*sintheta, vec[1]*sintheta, vec[2]*sintheta)
  rot0 = np.ones((3,3),dtype=np.float)
  vtk.vtkMath.QuaternionToMatrix3x3(quat0, rot0)

  if 1:
    newAxis1 = vtk.vtkVector3d()
    vtk.vtkMath.Multiply3x3(rot0, lastAxis1, newAxis1)

    # Rotate newAxis1 into saxis
    vec = vtk.vtkVector3d() # Axis of rotation
    vtk.vtkMath.Cross(newAxis1, saxis, vec)
    costheta = vtk.vtkMath.Dot(saxis, newAxis1)
    sintheta = vtk.vtkMath.Norm(vec)
    theta = np.arctan2(sintheta, costheta)
    if sintheta != 0.0:
      vec[0] = vec[0]/sintheta
      vec[1] = vec[1]/sintheta
      vec[2] = vec[2]/sintheta

    # Convert to Quaternion
    costheta = np.cos(0.5*theta)
    sintheta = np.sin(0.5*theta)
    quat1 = vtk.vtkQuaterniond(costheta, vec[0]*sintheta, vec[1]*sintheta, vec[2]*sintheta)
    rot1 = np.ones((3,3),dtype=np.float)
    vtk.vtkMath.QuaternionToMatrix3x3(quat1, rot1)

    # Concatenate rotations
    rot = np.dot(rot1, rot0)

    # Rotate rot1 * rot0 (0, ix, jy, kz) rot0^-1 rot1^-1 = rot1*rot0 (0,x,y,z) (rot1*rot0)^-1
    #rot2 = np.ones((3,3),dtype=np.float)
    #quat2 = vtk.vtkQuaterniond()
    #vtk.vtkMath.MultiplyQuaternion(quat1, quat0, quat2)
    #vtk.vtkMath.QuaternionToMatrix3x3(quat2, rot2)

  mat = np.zeros((4,4), dtype=np.float)
  mat[:3,:3] = rot
  mat[3,3] = 1.0

  tmp = vtk.vtkVector3d()
  vtk.vtkMath.Multiply3x3(rot, axes.GetOrigin(), tmp)
  mat[:3,3] = np.array(obj.GetCenter()) - np.array(tmp)

  # Construct 4x4 matrix
  trans = vtk.vtkMatrix4x4()
  trans.DeepCopy(mat.flatten().tolist())

  if axes.GetUserTransform() is not None:
    axes.GetUserTransform().Concatenate(trans)
  else:
    transform = vtk.vtkTransform()
    transform.SetMatrix(trans)
    transform.PostMultiply()
    axes.SetUserTransform(transform)

  # Only for book keeping
  axes.SetOrigin(obj.GetCenter()) # Not modified by SetUserTransform
  axes.Modified()

  # Update last axes
  lastAxis1[0] = saxis[0]
  lastAxis1[1] = saxis[1]
  lastAxis1[2] = saxis[2]

  lastNormal = (axis[0], axis[1], axis[2])

renderer = vtk.vtkRenderer()
renderWindow = vtk.vtkRenderWindow()
renderWindow.SetWindowName("Test")
renderWindow.SetSize(600, 600)

renderWindow.AddRenderer(renderer);
renderWindowInteractor = vtk.vtkRenderWindowInteractor()
renderWindowInteractor.SetRenderWindow(renderWindow)

planeWidget = vtk.vtkPlaneWidget()
planeWidget.SetInteractor(renderWindowInteractor)
#planeWidget.AddObserver(vtk.vtkCommand.EndInteractionEvent, MyFunc, 1.0)
planeWidget.AddObserver(vtk.vtkCommand.EndInteractionEvent, MyFunc2, 1.0)

initialMovement = True

if initialMovement:
  planeWidget.SetOrigin(1.5, 0.5, 2)
  planeWidget.SetPoint1(2.5, 0.5, 2)
  planeWidget.SetPoint2(1.5, 1.5, 2)

planeWidget.Modified()
planeWidget.On()

lastNormal = planeWidget.GetNormal()
lastAxis1 = vtk.vtkVector3d()

vtk.vtkMath.Subtract(planeWidget.GetPoint1(),
                     planeWidget.GetOrigin(),
                     lastAxis1)

renderWindowInteractor.Initialize()

renderer.ResetCamera()

# Crazy behavior for origin
axes = vtk.vtkAxesActor()
if initialMovement:
  tf = vtk.vtkTransform()

  rot = np.diag(np.ones(3, dtype=np.float))
  mat = np.zeros((4,4), dtype=np.float)
  mat[:3,:3] = rot
  mat[3,3] = 1.0
  tmp = vtk.vtkVector3d()
  vtk.vtkMath.Multiply3x3(rot, axes.GetOrigin(), tmp)
  mat[:3,3] = np.array(planeWidget.GetCenter()) - np.array(tmp)

  # Construct 4x4 matrix
  tfm = vtk.vtkMatrix4x4()
  tfm.DeepCopy(mat.flatten().tolist())
  tfm.Modified()
  tf.SetMatrix(tfm)
  tf.PostMultiply()
  tf.Update()
  axes.SetUserTransform(tf)
  axes.SetOrigin(planeWidget.GetCenter())
  axes.Modified()

renderer.AddActor(axes)

renderWindow.Render()
renderWindowInteractor.Start()


"""Train the constructed cnn model with the training dataset
"""

import sys
sys.path.append("../modelConstruct")
sys.path.append("../dataPreprocess")
sys.path.append("../modelEvaluate")
#from model_construction import *
import dataRead
import dataVectorization
import dataPartition
import model_construction
import tensorflow as tf
import numpy as np
import model_evaluate
import math

# hyperparameters
LR = 0.001       #learning rate
TRAINING_ITER = 10000   #iteration times
BATCH_SIZE = 18        #batch size of input

SEQUENCE_LENGTH = 164   #sequence length of input
EMBEDDING_SIZE = 4      #char embedding size(sequence width of input)
# CONV_SIZE = 3    #first filter size
 # CONV_DEEP = 128   #number of first filter(convolution deepth)

STRIDES = [1,1,1,1]  #the strid in each of four dimensions during convolution
KSIZE = [1,164,1,1]    #pooling window size

FC_SIZE = 128     #nodes of full-connection layer
NUM_CLASSES = 2   # classification number

DROPOUT_KEEP_PROB = 0.5   #keep probability of dropout

FILE_PATH = "../../data/miRBase_set.csv"
FILE_PATH_PUTATIVE = "../../data/putative_mirtrons_set.csv"
all_data_array = dataRead.read_data(FILE_PATH,FILE_PATH_PUTATIVE)
vectorized_dataset = dataVectorization.vectorize_data(all_data_array)
X_train, y_train, X_test, y_test = dataPartition.data_partition(vectorized_dataset)
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
print(y_train.shape)
print(X_train.shape)
print(y_test.shape)
print(X_test.shape)
print("dataset vectorization finished!")
print("iteration",TRAINING_ITER)
dataset_size = len(X_train)  #number of training dataset

input_X = tf.placeholder(tf.float32,[None,SEQUENCE_LENGTH,EMBEDDING_SIZE,1])
input_y = tf.placeholder(tf.float32,[None, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)

def train_model_conv5(log_path,model_path):
    conv5_output = model_construction.model_conv5_output\
                                   (input_X,EMBEDDING_SIZE,keep_prob)

    #loss_and_optimization(conv6_output,y_train)
    cross_entropy = tf.nn.softmax_cross_entropy_with_logits\
                                (logits = conv5_output,labels = input_y)
    cross_entropy_mean = tf.reduce_mean(cross_entropy)
    # optimization
    train_op = tf.train.AdamOptimizer(LR).minimize(cross_entropy_mean)

    # calculate the accuracy of the model
    correct_prediction = tf.equal(tf.argmax(conv5_output,1), tf.argmax(input_y,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction,"float"))

    tf.summary.scalar("loss",cross_entropy_mean)
    tf.summary.scalar("accuracy",accuracy)

    tp_op,tn_op,fp_op,fn_op = model_evaluate.evaluation_op(conv5_output,input_y)

    merged = tf.summary.merge_all()
    saver = tf.train.Saver()

    # train the model with the training dataset
    with tf.Session() as sess:  # run the session
        writer = tf.summary.FileWriter(log_path, sess.graph)
        sess.run(tf.global_variables_initializer())
        for i in range(TRAINING_ITER):
            start = (i * BATCH_SIZE)% dataset_size
            end = min(start + BATCH_SIZE,dataset_size)
            batch_xs = X_train[start:end]
            batch_ys = y_train[start:end]
            #sess.run(train_op,feed_dict={input_X:batch_xs,input_y:batch_ys,\
            #                              keep_prob:DROPOUT_KEEP_PROB})

            _,rs = sess.run([train_op,merged],\
                            feed_dict={input_X:batch_xs,input_y:batch_ys,\
                                      keep_prob:DROPOUT_KEEP_PROB})
            writer.add_summary(rs,i)

          #  print loss and accuracy during the training process
            if(i%1000==0):
                print("The {} iteration:".format(i))
                print("The cross_entropy_mean为：",end='')
                print(sess.run(cross_entropy_mean,\
                               feed_dict={input_X:batch_xs,input_y:batch_ys,\
                                          keep_prob:DROPOUT_KEEP_PROB}))
                #print(sess.run(cross_entropy_mean,\
                #               feed_dict={input_X:batch_xs,input_y:batch_ys}))
              #  print("The accuracy on batch data:",end='')
              #  print(sess.run(accuracy,feed_dict={input_xs:batch_xs,\
              #                  input_ys:batch_ys,keep_prob:1}))
                print("The accuracy on training data:",end='')
                print(sess.run(accuracy,feed_dict={input_X:X_train,\
                               input_y:y_train,keep_prob:1}))
               # print(sess.run(accuracy,feed_dict={input_X:X_train,\
                #               input_y:y_train}))
                print("The accuracy on test data:",end='')
                print(sess.run(accuracy,feed_dict={input_X:X_test,\
                                           input_y:y_test,keep_prob:1}))
                print("==================")

            saver.save(sess,model_path)

        print("*********training finished********")
        print("performance on the test dataset:")
        tp,tn,fp,fn = sess.run([tp_op, tn_op,fp_op, fn_op],\
                                feed_dict={input_X:X_test,\
                                input_y:y_test,keep_prob:1})
        print("tp:{},tn:{},fp:{},fn:{}".format(tp,tn,fp,fn))

        model_evaluate.print_test_evaluation(tp,tn,fp,fn)

if __name__ == "__main__":
    train_model_conv5("../../logs/filter5","../../logs/filter5/filter5.ckpt")