import numpy as np
from sklearn.cluster import KMeans

# Global variables.
transition_metals = ['Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Zn',
                     'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd',
                     'Hf', 'Ta', 'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg']
metals = ['Li', 'Be', 'Na', 'Mg', 'K', 'Ca', 'Rb', 'Sr', 'Cs', 'Ba', 'Fr',
          'Ra', 'La', 'Ce', 'Pr', 'Nd', 'Pm', 'Sm',
          'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu', 'Ac', 'Th',
          'Pa', 'U', 'Np', 'Pu', 'Am', 'Cm', 'Bk',
          'Cf', 'Es', 'Fm', 'Md', 'No', 'Lw', 'B', 'Al', 'Si', 'Ga', 'Ge',
          'In', 'Sn', 'Tl', 'Pb']
metals.extend(transition_metals)

def stoichiometry(atoms):
    """Return a number of layers given a slab.

    Parameters
    ----------
    atoms : object
        ASE atoms object.

    Returns
    -------
    num_dict : dictionary
        First entry is total number of atoms.
        Then key =  element and entry = number
    """
    symbols = atoms.get_chemical_symbols()
    num_dict = {}
    elements = list(set(symbols))
    for element in elements:
        count = symbols.count(element)
        num_dict[element] = count
    return(num_dict)

def is_metal(chemical_symbol):
    """Checks whether string is a metal elementary symbol.

    Parameters
    ----------
    chemical_symbol : string
        The element name.

    Returns
    -------
    metal : Boolean
        Whether it's a metal.
    """
    metal = False
    if chemical_symbol in metals:
        metal = True
    return(metal)

def is_oxide(atoms):
    """Checks whether atms object is an oxide.

    Parameters
    ----------
    atoms : object
        ASE atoms object.

    Returns
    -------
    oxide : Boolean
        Whether it is likely an oxide.
    """
    oxide = False
    num_dict = stoichiometry(atoms)
    elements = list(num_dict.keys())[1:]
    all_count = list(num_dict.values())[0]
    for element in elements:
        if element in transition_metals:
            if num_dict[element]/all_count*1.0 > 0.25:
                try:
                    if num_dict['O']/num_dict[element] >= 1:
                        # print("Materials class likely a metal oxide.")
                        oxide = True
                except KeyError as e:
                    pass
        elif element in metals:
            if num_dict[element]/all_count*1.0 > 0.25:
                try:
                    if num_dict['O']/num_dict[element] >= 1:
                        # print("Materials class likely a metal oxide.")
                        oxide = True
                except KeyError as e:
                    pass
    return(oxide)

def slab_layers(atoms, max_layers=20, tolerance=0.5):
    """Return a number of layers given a slab.

    Parameters
    ----------
    atoms : object
        ASE atoms object.
    max_layers : maximum number of layers expected.
    tolerance : convergence criterion for clustering
    based on the pooled standard deviation of z-coordinates.
    Suggested : 0.5 for oxides, 0.2 for metals.

    Returns
    -------
    layer_avg_z : list
        List of average z-values of all layers.
    layer_atoms : list of list
        Each sublist contains the atom indices of the atoms in that layer.
    """
    oxide = is_oxide(atoms)
    if oxide:
        stoi = stoichiometry(atoms)
        elements = list(stoi.keys())[1:]
        for element in elements:
            if element in metals:
                indices = [atom.index for atom in atoms if
                           atom.symbol == element]
                zpos = [atoms.positions[atom.index][2] for atom in atoms if
                        atom.index in indices]
    else:
        indices = [atom.index for atom in atoms]
        zpos = [atoms.positions[atom.index][2] for atom in atoms]
    X = [[i] for i in zpos]

    for n in range(1, max_layers + 1):
        kmeans = KMeans(n_clusters=n, random_state=0).fit(X)
        clusters = kmeans.labels_
        results = list(zip(indices, zpos, clusters))
        layer_numbers = list(set(clusters))

        # check convergence
        var_i_n_1 = []
        n_i = []
        layer_atoms = []
        layer_avg_z = []
        for i in layer_numbers:
            layer = [entry for entry in results if entry[2] == i]
            layer_atoms.append([entry[0] for entry in layer])
            layer_avg_z.append(np.mean([entry[1] for entry in layer]))
            var_i_n_1.append(np.var([entry[1] for entry in layer]) *
                             (len(layer) - 1))
            n_i.append(len(layer))
        k = len(n_i)
        pooled_standart_deviation = np.sqrt(sum(var_i_n_1) / (sum(n_i) - k))

        if pooled_standart_deviation <= tolerance:
            final_results = list(zip(layer_numbers, layer_avg_z, layer_atoms))
            final_results = sorted(final_results, key=lambda tup: tup[1])
            layer_numbers = list(range(len(final_results)))
            layer_avg_z = [tup[1] for tup in final_results]
            layer_atoms = [tup[2] for tup in final_results]
            # print('Found ' + str(n) + ' layers.')
            return(layer_avg_z, layer_atoms)


#A complete set of volatility estimators based on Euan Sinclair's Volatility Trading
#
# For each of the estimators, plot:
# * Probability cones
# * Rolling quantiles
# * Rolling extremes
# * Rolling descriptive statistics
# * Histogram
# * Comparison against arbirary comparable
# * Correlation against arbirary comparable
# * Regression against arbirary comparable
#
# Also creates a term sheet with all the metrics printed to a PDF

# #variables for the instances
# window=30
# windows=[30, 60, 90, 120]
# quantiles=[0.25, 0.75]
# bins=100
# normed=True
# bench='^GSPC'
# # call plt.show() on any of the below
# fig, plt = vol.cones(windows=windows, quantiles=quantiles)
# fig, plt = vol.rolling_quantiles(window=window, quantiles=quantiles)
# fig, plt = vol.rolling_extremes(window=window)
# fig, plt = vol.rolling_descriptives(window=window)
# fig, plt = vol.histogram(window=window, bins=bins, normed=normed)
# fig, plt = vol.benchmark_compare(window=window, bench=bench)
# fig, plt = vol.benchmark_correlation(window=window, bench=bench)
# # prints regression statistics
# print vol.benchmark_regression(window=window, bench=bench)
# # creates a pdf term sheet with all metrics
# vol.term_sheet(window, windows, quantiles, bins, normed, bench)

import datetime
import math
import os
import matplotlib
import numpy as np
import pandas
import statsmodels.api as sm

matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
# from matplotlib.pyplot import *
import matplotlib.mlab as mlab
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt, mpld3
import matplotlib.pyplot as pyplt
from persist.sqlite_methods import read_market_data_from_sqllite, save_graph_to_db, save_lineplot_data_to_db
from bokeh.models.annotations import Legend
from bokeh.embed import components
from bokeh.layouts import layout
import persist.sqlite_methods as md
# references
# http://stackoverflow.com/questions/4700614/how-to-put-the-legend-out-of-the-plot
# http://www.blog.pythonlibrary.org/2010/09/04/python-101-how-to-open-a-file-or-program/

from core import config
globalconf = config.GlobalConfig()
log = globalconf.log

class VolatilityEstimator(object):
    def __init__(self, db_type, symbol, expiry, last_date, num_days_back, resample, estimator, clean):
        if symbol is None or symbol == '':
            raise ValueError('symbol symbol required')
        if num_days_back is None or num_days_back == '':
            raise ValueError('num_days_back required')
        if last_date is None or last_date == '':
            raise ValueError('last_date required')
        if estimator not in ["GarmanKlass", "HodgesTompkins", "Kurtosis", "Parkinson", "Raw", "RogersSatchell", "Skew",
                             "YangZhang"]:
            raise ValueError('Acceptable volatility model is required')

        self._symbol = symbol
        self._num_days_back = num_days_back
        self._last_date = last_date
        self._estimator = estimator
        self._db_type = db_type
        self._expiry = expiry
        self._resample = resample
        self._clean = clean
        self._prices = read_market_data_from_sqllite(db_type=self._db_type, symbol=self._symbol, expiry=self._expiry,
                                                     last_date=self._last_date, num_days_back=self._num_days_back,
                                                     resample=self._resample)

        last_record_stored = np.max(self._prices.index).replace(hour=0, minute=59, second=59)
        df1 = md.get_last_bars_from_rt(symbol=symbol, last_date=last_date, last_record_stored=last_record_stored)
        self._prices = self._prices.append(df1)

        matplotlib.rc('image', origin='upper')
        matplotlib.rcParams['font.family'] = 'Times New Roman'
        matplotlib.rcParams['font.size'] = '11'
        matplotlib.rcParams['grid.color'] = 'lightgrey'
        matplotlib.rcParams['grid.linestyle'] = '-'
        matplotlib.rcParams['figure.subplot.left'] = 0.1
        matplotlib.rcParams['figure.subplot.bottom'] = 0.13
        matplotlib.rcParams['figure.subplot.right'] = 0.9
        matplotlib.rcParams['figure.subplot.top'] = 0.9

    def _get_estimator(self, window):
        """Selector for volatility estimator
        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        clean : boolean
            Set to True to remove the NaNs at the beginning of the series
        Returns
        -------
        y : pandas.DataFrame
            Estimator series values
        """
        if self._estimator is "GarmanKlass":
            return self.get_GarmanKlass_estimator(window=window)
        elif self._estimator is "HodgesTompkins":
            return self.get_HodgesTompkins_estimator(window=window)
        elif self._estimator is "Kurtosis":
            return self.get_Kurtosis_estimator(window=window)
        elif self._estimator is "Parkinson":
            return self.get_Parkinson_estimator(window=window)
        elif self._estimator is "Raw":
            return self.get_Raw_estimator(window=window)
        elif self._estimator is "RogersSatchell":
            return self.get_RogersSatchell_estimator(window=window)
        elif self._estimator is "Skew":
            return self.get_Skew_estimator(window=window)
        elif self._estimator is "YangZhang":
            return self.get_YangZhang_estimator(window=window)

    def get_GarmanKlass_estimator(self, window=30):
        prices = self._prices
        log_hl = (prices['high'] / prices['low']).apply(np.log)
        log_co = (prices['close'] / prices['open']).apply(np.log)
        rs = 0.5 * log_hl ** 2 - (2 * math.log(2) - 1) * log_co ** 2
        def f(v):
            return math.sqrt(252 * v.mean())
        result = rs.rolling(window=window, center=False).apply(func=f)
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_HodgesTompkins_estimator(self, window=30):
        prices = self._prices
        log_return = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
        vol = log_return.rolling(window=window, center=False).std() * math.sqrt(252)
        adj_factor = math.sqrt((1.0 / (1.0 - (window / (log_return.count() - (window - 1.0))) + (window ** 2 - 1.0) / (
        3.0 * (log_return.count() - (window - 1.0)) ** 2))))
        result = vol * adj_factor
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_Kurtosis_estimator(self, window=30):
        prices = self._prices
        log_return = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
        result = log_return.rolling(window=window, center=False).kurt()
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_Parkinson_estimator(self, window=30):
        prices = self._prices
        rs = (1 / (4 * math.log(2))) * ((prices['High'] / prices['Low']).apply(np.log)) ** 2
        def f(v):
            return math.sqrt(252 * v.mean())
        result = rs.rolling(window=window, center=False).apply(func=f)
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_Raw_estimator(self, window=30):
        prices = self._prices
        log_return = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
        result = log_return.rolling(window=window, center=False).std() * math.sqrt(252)
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_RogersSatchell_estimator(self, window=30):
        prices = self._prices
        log_ho = (prices['High'] / prices['Open']).apply(np.log)
        log_lo = (prices['Low'] / prices['Open']).apply(np.log)
        log_co = (prices['Close'] / prices['Open']).apply(np.log)
        rs = log_ho * (log_ho - log_co) + log_lo * (log_lo - log_co)
        def f(v):
            return math.sqrt(252 * v.mean())
        result = rs.rolling(window=window, center=False).apply(func=f)
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_Skew_estimator(self, window=30):
        prices = self._prices
        log_return = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
        result = log_return.rolling(window=window, center=False).skew()
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def get_YangZhang_estimator(self, window=30):
        prices = self._prices
        log_ho = (prices['High'] / prices['Open']).apply(np.log)
        log_lo = (prices['Low'] / prices['Open']).apply(np.log)
        log_co = (prices['Close'] / prices['Open']).apply(np.log)
        log_oc = (prices['Open'] / prices['Close'].shift(1)).apply(np.log)
        log_oc_sq = log_oc ** 2
        log_cc = (prices['Close'] / prices['Close'].shift(1)).apply(np.log)
        log_cc_sq = log_cc ** 2
        rs = log_ho * (log_ho - log_co) + log_lo * (log_lo - log_co)
        close_vol = log_cc_sq.rolling(window=window, center=False).sum() * (1.0 / (window - 1.0))
        open_vol = log_oc_sq.rolling(window=window, center=False).sum() * (1.0 / (window - 1.0))
        window_rs = rs.rolling(window=window, center=False).sum() * (1.0 / (window - 1.0))
        result = (open_vol + 0.164333 * close_vol + 0.835667 * window_rs).apply(np.sqrt) * math.sqrt(252)
        result[:window - 1] = np.nan
        if self._clean:
            return result.dropna()
        else:
            return result

    def cones_prepare_data(self, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75]):
        if len(windows) < 2:
            raise ValueError('Two or more window periods required')
        if len(quantiles) != 2:
            raise ValueError('A two element list of quantiles is required, lower and upper')
        if quantiles[0] + quantiles[1] != 1.0:
            raise ValueError('The sum of the quantiles must equal 1.0')
        if quantiles[0] > quantiles[1]:
            raise ValueError(
                'The lower quantiles (first element) must be less than the upper quantile (second element)')
        max = []
        min = []
        top_q = []
        median = []
        bottom_q = []
        realized = []
        data = []

        for w in windows:
            estimator = self._get_estimator(w)

            max.append(estimator.max())
            top_q.append(estimator.quantile(quantiles[1]))
            median.append(estimator.median())
            bottom_q.append(estimator.quantile(quantiles[0]))
            min.append(estimator.min())
            realized.append(estimator[-1])
            data.append(estimator)

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        return top_q, median, bottom_q, realized, min, max, f, data

    def cones_data(self, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75]):
        top_q, median, bottom_q, realized, min, max, f, data = self.cones_prepare_data(windows, quantiles)
        xs = [windows, windows, windows, windows, windows, windows]
        ys = [top_q, median, bottom_q, realized, min, max]
        ys_labels = ["top_q", "median", "bottom_q", "realized", "min", "max"]
        title = self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(
            self._num_days_back) + ')'
        save_lineplot_data_to_db(xs=xs, ys=ys, ys_labels=ys_labels, title=title, symbol=self._symbol,
                                 expiry=self._expiry, last_date=self._last_date, num_days_back=self._num_days_back,
                                 resample=self._resample, estimator=self._estimator, name="VOLEST")

    def cones_bokeh(self, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75]):
        """Plots volatility cones
        Parameters
        ----------
        windows : [int, int, ...]
            List of rolling windows for which to calculate the estimator cones
        quantiles : [lower, upper]
            List of lower and upper quantiles for which to plot the cones
        """
        top_q, median, bottom_q, realized, min, max, f, data = self.cones_prepare_data(windows, quantiles)
        colors_list = ['orange','blue','pink','black','red','green']
        methods_list = ['x','diamond','x','square','inverted_triangle','inverted_triangle']
        line_dash_list = ['dotted', 'dotdash', 'dotted', 'solid', 'dashed', 'dashed']
        xs = [windows, windows, windows, windows, windows, windows]
        ys = [top_q, median, bottom_q, realized, min, max]
        legends_list = [str(int(quantiles[1] * 100)) + " Prctl", 'Median',
                        str(int(quantiles[0] * 100)) + " Prctl", 'Realized', 'Min', 'Max']
        title = self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')'
        p = figure(title=title, plot_width=700, plot_height=500,toolbar_sticky=False,
                   x_axis_label="Days",y_axis_label="Volatility",toolbar_location="below")
        legend_items = []
        for (colr, leg, x, y, method, line_dash) in zip(colors_list, legends_list, xs, ys, methods_list, line_dash_list):
            # call dynamically the method to plot line, circle etc...
            renderers = []
            if method:
                renderers.append( getattr(p, method)(x, y, color=colr,size=4) )
            renderers.append( p.line(x, y, color=colr, line_dash=line_dash) )
            legend_items.append((leg,renderers))
        # doesnt work: legend = Legend(location=(0, -30), items=legend_items)
        legend = Legend(location=(0, -30), items=legend_items)
        p.add_layout(legend, 'right')

        from bokeh.charts import BoxPlot
        df = pandas.DataFrame({"data": data[0], "group": 0})
        df = df.append(pandas.DataFrame({"data": data[1], "group": 1}))
        df = df.append(pandas.DataFrame({"data": data[2], "group": 2}))
        df = df.append(pandas.DataFrame({"data": data[3], "group": 3}))
        p2 = BoxPlot(df, values='data', label='group', title="Boxplot Summary (" + self._last_date + ") (" + self._symbol + ")" ,toolbar_location="below",
                     legend="bottom_right", plot_width=600, plot_height=400,toolbar_sticky=False)
        from bokeh.models.ranges import DataRange1d
        p2.y_range = DataRange1d(np.min(df['data']-0.01), np.max(df['data']+0.01))

        layout1 = layout([[p, p2]])
        script, div = components(layout1)
        save_graph_to_db(script, div, self._symbol, self._expiry, self._last_date, self._num_days_back, self._resample,
                         self._estimator, name="VOLEST")
        return layout1

    def cones(self, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75]):
        """Plots volatility cones
        Parameters
        ----------
        windows : [int, int, ...]
            List of rolling windows for which to calculate the estimator cones
        quantiles : [lower, upper]
            List of lower and upper quantiles for which to plot the cones
        """
        top_q, median, bottom_q, realized, min, max, f, data  = self.cones_prepare_data(windows, quantiles)

        fig = plt.figure(figsize=(8, 6))
        left, width = 0.07, 0.65
        bottom, height = 0.2, 0.7
        bottom_h = left_h = left + width + 0.02

        rect_cones = [left, bottom, width, height]
        rect_box = [left_h, bottom, 0.17, height]

        cones = plt.axes(rect_cones)
        box = plt.axes(rect_box)

        # set the plots
        cones.plot(windows, max, label="Max")
        cones.plot(windows, top_q, label=str(int(quantiles[1] * 100)) + " Prctl")
        cones.plot(windows, median, label="Median")
        cones.plot(windows, bottom_q, label=str(int(quantiles[0] * 100)) + " Prctl")
        cones.plot(windows, min, label="Min")
        cones.plot(windows, realized, 'r-.', label="Realized")

        # set the x ticks and limits
        cones.set_xticks((windows))
        cones.set_xlim((windows[0] - 5, windows[-1] + 5))

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')

        # set the legend
        pos = cones.get_position()  #
        cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)

        #
        # box plot args
        #

        # set the plots
        box.boxplot(data, notch=1, sym='+')
        box.plot([i for i in range(1, len(windows) + 1)], realized, color='r', marker='*', markeredgecolor='k')

        # set and format the y-axis labels
        locs = box.get_yticks()
        box.set_yticklabels(map(f, locs))

        # move the y-axis ticks on the right side
        box.yaxis.tick_right()

        # turn on the grid
        box.grid(True, axis='y', which='major', alpha=0.5)

        return fig, plt

    def rolling_quantiles(self, window=30, quantiles=[0.25, 0.75]):
        """Plots rolling quantiles of volatility

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        quantiles : [lower, upper]
            List of lower and upper quantiles for which to plot
        """
        if len(quantiles) != 2:
            raise ValueError('A two element list of quantiles is required, lower and upper')
        if quantiles[0] + quantiles[1] != 1.0:
            raise ValueError('The sum of the quantiles must equal 1.0')
        if quantiles[0] > quantiles[1]:
            raise ValueError(
                'The lower quantiles (first element) must be less than the upper quantile (second element)')

        estimator = self._get_estimator(window)
        date = estimator.index

        top_q = estimator.rolling(window=window, center=False).quantile(quantiles[1])
        median = estimator.rolling(window=window, center=False).median()
        bottom_q = estimator.rolling(window=window, center=False).quantile(quantiles[0])
        realized = estimator
        last = estimator[-1]

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # figure args
        #

        fig = plt.figure(figsize=(8, 6))

        left, width = 0.07, 0.65
        bottom, height = 0.2, 0.7
        bottom_h = left_h = left + width + 0.02

        rect_cones = [left, bottom, width, height]
        rect_box = [left_h, bottom, 0.17, height]

        cones = plt.axes(rect_cones)
        box = plt.axes(rect_box)

        #
        # cones plot args
        #

        # set the plots
        cones.plot(date, top_q, label=str(int(quantiles[1] * 100)) + " Prctl")
        cones.plot(date, median, label="Median")
        cones.plot(date, bottom_q, label=str(int(quantiles[0] * 100)) + " Prctl")
        cones.plot(date, realized, 'r-.', label="Realized")

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')

        # set the legend
        cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)

        #
        # box plot args
        #

        # set the plots
        box.boxplot(realized, notch=1, sym='+')
        box.plot(1, last, color='r', marker='*', markeredgecolor='k')

        # set and format the y-axis labels
        locs = box.get_yticks()
        box.set_yticklabels(map(f, locs))

        # move the y-axis ticks on the right side
        box.yaxis.tick_right()

        # turn on the grid
        box.grid(True, axis='y', which='major', alpha=0.5)

        return fig, plt

    def rolling_extremes(self, window=30):
        """Plots rolling max and min of volatility estimator

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        """
        estimator = self._get_estimator(window)
        date = estimator.index
        max = estimator.rolling(window=window, center=False).max()
        min = estimator.rolling(window=window, center=False).min()
        realized = estimator
        last = estimator[-1]

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # Figure args
        #

        fig = plt.figure(figsize=(8, 6))

        left, width = 0.07, 0.65
        bottom, height = 0.2, 0.7
        bottom_h = left_h = left + width + 0.02

        rect_cones = [left, bottom, width, height]
        rect_box = [left_h, bottom, 0.17, height]

        cones = plt.axes(rect_cones)
        box = plt.axes(rect_box)

        #
        # Cones plot args
        #

        # set the plots
        cones.plot(date, max, label="Max")
        cones.plot(date, min, label="Min")
        cones.plot(date, realized, 'r-.', label="Realized")

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')

        # set the legend
        cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)

        #
        # Box plot args
        #

        # set the plots
        box.boxplot(realized, notch=1, sym='+')
        box.plot(1, last, color='r', marker='*', markeredgecolor='k')

        # set and format the y-axis labels
        locs = box.get_yticks()
        box.set_yticklabels(map(f, locs))

        # move the y-axis ticks on the right side
        box.yaxis.tick_right()

        # turn on the grid
        box.grid(True, axis='y', which='major', alpha=0.5)

        return fig, plt

    def rolling_descriptives(self, window=30):
        """Plots rolling first and second moment of volatility estimator

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        """
        estimator = self._get_estimator(window)
        date = estimator.index
        mean = estimator.rolling(window=window, center=False).mean()
        std = estimator.rolling(window=window, center=False).std()
        z_score = (estimator - mean) / std

        realized = estimator
        last = estimator[-1]

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # Figure args
        #

        fig = plt.figure(figsize=(8, 6))

        left, width = 0.07, 0.65
        bottom, height = 0.1, .8
        bottom_h = left_h = left + width + 0.02

        rect_cones = [left, 0.35, width, 0.55]
        rect_box = [left_h, 0.15, 0.17, 0.75]
        rect_z = [left, 0.15, width, 0.15]

        cones = plt.axes(rect_cones)
        box = plt.axes(rect_box)
        z = plt.axes(rect_z)

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # Cones plot args
        #

        # set the plots
        cones.plot(date, mean, label="Mean")
        cones.plot(date, std, label="Std. Dev.")
        cones.plot(date, realized, 'r-.', label="Realized")

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(self._estimator + ' (' + self._symbol + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')

        # shrink the plot up a bit and set the legend
        pos = cones.get_position()  #
        cones.set_position([pos.x0, pos.y0 + pos.height * 0.1, pos.width, pos.height * 0.9])  #
        cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)

        #
        # Box plot args
        #

        # set the plots
        box.boxplot(realized, notch=1, sym='+')
        box.plot(1, last, color='r', marker='*', markeredgecolor='k')

        # set and format the y-axis labels
        locs = box.get_yticks()
        box.set_yticklabels(map(f, locs))

        # move the y-axis ticks on the right side
        box.yaxis.tick_right()

        # turn on the grid
        box.grid(True, axis='y', which='major', alpha=0.5)

        #
        # Z-Score plot args
        #

        # set the plots
        z.plot(date, z_score, 'm-', label="Z-Score")

        # turn on the grid
        z.grid(True, axis='y', which='major', alpha=0.5)

        # create a horizontal line at y=0
        z.axhline(0, 0, 1, linestyle='-', linewidth=1.0, color='black')

        # set the legend
        z.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), ncol=3)

        return fig, plt

    def histogram(self, window=90, bins=100, normed=True):
        """

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        bins : int

        """
        estimator = self._get_estimator(window)
        mean = estimator.mean()
        std = estimator.std()
        last = estimator[-1]

        fig = plt.figure(figsize=(8, 6))

        n, bins, patches = plt.hist(estimator, bins, normed=normed, facecolor='blue', alpha=0.25)

        if normed:
            y = mlab.normpdf(bins, mean, std)
            l = plt.plot(bins, y, 'g--', linewidth=1)

        plt.axvline(last, 0, 1, linestyle='-', linewidth=1.5, color='r')

        plt.grid(True, axis='y', which='major', alpha=0.5)
        plt.title(
            'Distribution of ' + self._estimator + ' estimator values (' + self._symbol +
                ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')

        return fig, plt

    def benchmark_compare(self, window=90, bench='SPX'):
        """

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        bins : int

        """

        y = self._get_estimator(window)
        bench_vol = VolatilityEstimator(db_type=self._db_type, symbol=bench, expiry=None, last_date=self._last_date,
                                        num_days_back=self._num_days_back, resample=self._resample,
                                        estimator=self._estimator, clean=self._clean)

        x = bench_vol._get_estimator(window)
        date = y.index

        ratio = y / x

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # Figure args
        #

        fig = plt.figure(figsize=(8, 6))

        left, width = 0.07, .9

        rect_cones = [left, 0.4, width, .5]
        rect_box = [left, 0.15, width, 0.15]

        cones = plt.axes(rect_cones)
        box = plt.axes(rect_box)

        #
        # Cones plot args
        #

        # set the plots
        self._log.info (("x len",len(x)," y len ", len(y)))
        cones.plot(date, y, label=self._symbol.upper())
        cones.plot(date, x, label=bench.upper())

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(
            self._estimator + ' (' + self._symbol + ' v. ' + bench.upper()
                + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')
        # shrink the plot up a bit and set the legend
        cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)

        #
        # Cones plot args
        #

        # set the plot
        box.plot(date, ratio, label=self._symbol.upper() + '/' + bench.upper())

        # set the y-limits
        box.set_ylim((ratio.min() - 0.05, ratio.max() + 0.05))

        # fill the area
        box.fill_between(date, ratio, 0, color='blue', alpha=0.25)

        # set the legend
        box.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), ncol=3)

        return fig, plt

    def benchmark_correlation(self, window=90, bench='^GSPC'):
        """

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        bins : int

        """

        y = self._get_estimator(window)

        bench_vol = VolatilityEstimator(db_type=self._db_type, symbol=bench, expiry=None, last_date=self._last_date,
                                        num_days_back=self._num_days_back, resample=self._resample,
                                        estimator=self._estimator, clean=self._clean)

        x = bench_vol._get_estimator(window)
        date = y.index

        corr = x.rolling(window=window).corr(other=y)

        if self._estimator is "Skew" or self._estimator is "Kurtosis":
            f = lambda x: "%i" % round(x, 0)
        else:
            f = lambda x: "%i%%" % round(x * 100, 0)

        #
        # Figure args
        #

        fig = plt.figure(figsize=(8, 6))
        cones = plt.axes()

        #
        # Cones plot args
        #

        # set the plots
        cones.plot(date, corr)

        # set the y-limits
        cones.set_ylim((corr.min() - 0.05, corr.max() + 0.05))

        # set and format the y-axis labels
        locs = cones.get_yticks()
        cones.set_yticklabels(map(f, locs))

        # turn on the grid
        cones.grid(True, axis='y', which='major', alpha=0.5)

        # set the title
        cones.set_title(
            self._estimator + ' (Correlation of ' + self._symbol + ' v. ' + bench.upper()
                + ', daily from ' + self._last_date + ' days back ' + str(self._num_days_back) + ')')
        return fig, plt

    def benchmark_regression(self, window=90, bench='^GSPC'):
        """

        Parameters
        ----------
        window : int
            Rolling window for which to calculate the estimator
        bins : int

        """
        y = self._get_estimator(window)

        bench_vol = VolatilityEstimator(db_type=self._db_type, symbol=bench, expiry=None, last_date=self._last_date,
                                        num_days_back=self._num_days_back, resample=self._resample,
                                        estimator=self._estimator, clean=self._clean)

        X = bench_vol._get_estimator(window)

        model = sm.OLS(y, X)
        results = model.fit()

        return results.summary()

    def term_sheet_to_pdf(self, window=30, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75], bins=100, normed=True,
                   bench='SPY'):

        cones_fig, cones_plt = self.cones(windows=windows, quantiles=quantiles)
        rolling_quantiles_fig, rolling_quantiles_plt = self.rolling_quantiles(window=window, quantiles=quantiles)
        rolling_extremes_fig, rolling_extremes_plt = self.rolling_extremes(window=window)
        rolling_descriptives_fig, rolling_descriptives_plt = self.rolling_descriptives(window=window)
        histogram_fig, histogram_plt = self.histogram(window=window, bins=bins, normed=normed)
        benchmark_compare_fig, benchmark_compare_plt = self.benchmark_compare(window=window, bench=bench)
        benchmark_corr_fig, benchmark_corr_plt = self.benchmark_correlation(window=window, bench=bench)
        benchmark_regression = self.benchmark_regression(window=window, bench=bench)

        filename = self._symbol.upper() + '_termsheet_' + datetime.datetime.today().strftime("%Y%m%d") + '.pdf'
        fn = os.path.abspath(os.path.join(filename))
        pp = PdfPages(fn)

        pp.savefig(cones_fig)
        pp.savefig(rolling_quantiles_fig)
        pp.savefig(rolling_extremes_fig)
        pp.savefig(rolling_descriptives_fig)
        pp.savefig(histogram_fig)
        pp.savefig(benchmark_compare_fig)
        pp.savefig(benchmark_corr_fig)

        fig = plt.figure()
        plt.text(0.01, 0.01, benchmark_regression, fontsize=12)
        plt.axis('off')
        pp.savefig(fig)

        pp.close()

        print (filename + ' output complete')

    def define_fig(self,name,ext):
        web_server = self._globalconf.config['paths']['nginx_static_folder']
        filename = self._symbol.upper() + '_'+ name + ext
        fn = os.path.abspath(os.path.join(web_server,"volest", filename))
        return fn

    def term_sheet_to_html(self, window=30, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75], bins=100, normed=True,
                   bench='SPY', open=False):
        ext = '.html'
        cones_fig, cones_plt = self.cones(windows=windows, quantiles=quantiles)
        rolling_quantiles_fig, rolling_quantiles_plt = self.rolling_quantiles(window=window, quantiles=quantiles)
        rolling_extremes_fig, rolling_extremes_plt = self.rolling_extremes(window=window)
        rolling_descriptives_fig, rolling_descriptives_plt = self.rolling_descriptives(window=window)
        histogram_fig, histogram_plt = self.histogram(window=window, bins=bins, normed=normed)
        mpld3.save_html(cones_fig, self.define_fig("cones",ext))
        mpld3.save_html(rolling_quantiles_fig, self.define_fig("rolling_quantiles",ext))
        mpld3.save_html(rolling_extremes_fig, self.define_fig("rolling_extremes",ext))
        mpld3.save_html(rolling_descriptives_fig, self.define_fig("rolling_desc",ext))
        mpld3.save_html(histogram_fig, self.define_fig("histogram",ext))
        pyplt.close(cones_fig)
        pyplt.close(rolling_quantiles_fig)
        pyplt.close(rolling_extremes_fig)
        pyplt.close(rolling_descriptives_fig)
        pyplt.close(histogram_fig)

    def term_sheet_to_db(self, window=30, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75], bins=100, normed=True,
                   bench='SPY', open=False):
        ext = '.json'
        cones_fig, cones_plt = self.cones(windows=windows, quantiles=quantiles)
        rolling_quantiles_fig, rolling_quantiles_plt = self.rolling_quantiles(window=window, quantiles=quantiles)
        rolling_extremes_fig, rolling_extremes_plt = self.rolling_extremes(window=window)
        rolling_descriptives_fig, rolling_descriptives_plt = self.rolling_descriptives(window=window)
        histogram_fig, histogram_plt = self.histogram(window=window, bins=bins, normed=normed)
        cones_fig_dict = mpld3.fig_to_dict(cones_fig)
        rolling_quantiles_fig_dict = mpld3.fig_to_dict(rolling_quantiles_fig)
        rolling_extremes_fig_dict = mpld3.fig_to_dict(rolling_extremes_fig)
        rolling_descriptives_fig_dict = mpld3.fig_to_dict(rolling_descriptives_fig)
        histogram_fig_dict = mpld3.fig_to_dict(histogram_fig)
        pyplt.close(cones_fig)
        pyplt.close(rolling_quantiles_fig)
        pyplt.close(rolling_extremes_fig)
        pyplt.close(rolling_descriptives_fig)
        pyplt.close(histogram_fig)

    def term_sheet_to_json(self, window=30, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75], bins=100, normed=True,
                   bench='SPY', open=False):
        ext = '.json'
        cones_fig, cones_plt = self.cones(windows=windows, quantiles=quantiles)
        rolling_quantiles_fig, rolling_quantiles_plt = self.rolling_quantiles(window=window, quantiles=quantiles)
        rolling_extremes_fig, rolling_extremes_plt = self.rolling_extremes(window=window)
        rolling_descriptives_fig, rolling_descriptives_plt = self.rolling_descriptives(window=window)
        histogram_fig, histogram_plt = self.histogram(window=window, bins=bins, normed=normed)
        mpld3.save_json(cones_fig, self.define_fig("cones",ext))
        mpld3.save_json(rolling_quantiles_fig, self.define_fig("rolling_quantiles",ext))
        mpld3.save_json(rolling_extremes_fig, self.define_fig("rolling_extremes",ext))
        mpld3.save_json(rolling_descriptives_fig, self.define_fig("rolling_desc",ext))
        mpld3.save_json(histogram_fig, self.define_fig("histogram",ext))
        pyplt.close(cones_fig)
        pyplt.close(rolling_quantiles_fig)
        pyplt.close(rolling_extremes_fig)
        pyplt.close(rolling_descriptives_fig)
        pyplt.close(histogram_fig)

    def term_sheet_to_png(self, window=30, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75], bins=100, normed=True):
        ext = '.png'
        cones_fig, cones_plt = self.cones(windows=windows, quantiles=quantiles)
        rolling_quantiles_fig, rolling_quantiles_plt = self.rolling_quantiles(window=window, quantiles=quantiles)
        rolling_extremes_fig, rolling_extremes_plt = self.rolling_extremes(window=window)
        rolling_descriptives_fig, rolling_descriptives_plt = self.rolling_descriptives(window=window)
        histogram_fig, histogram_plt = self.histogram(window=window, bins=bins, normed=normed)

        cones_fig.savefig(self.define_fig("cones",ext), bbox_inches='tight')
        rolling_quantiles_fig.savefig(self.define_fig("rolling_quantiles",ext), bbox_inches='tight')
        rolling_extremes_fig.savefig(self.define_fig("rolling_extremes",ext), bbox_inches='tight')
        rolling_descriptives_fig.savefig(self.define_fig("rolling_desc",ext), bbox_inches='tight')
        histogram_fig.savefig(self.define_fig("histogram",ext), bbox_inches='tight')
        pyplt.close(cones_fig)
        pyplt.close(rolling_quantiles_fig)
        pyplt.close(rolling_extremes_fig)
        pyplt.close(rolling_descriptives_fig)
        pyplt.close(histogram_fig)

if __name__ =="__mainKK__":
    from core import config
    from core.logger import logger
    log = logger("some testing ...")
    globalconf = config.GlobalConfig()

    last_date = "20170706"
    vol = VolatilityEstimator(db_type="underl_ib_hist", symbol="SPY", expiry=None, last_date=last_date,
                              num_days_back=200, resample="1D", estimator="GarmanKlass", clean=True)
    window=30
    windows=[30, 60, 90, 120]
    quantiles=[0.25, 0.75]
    bins=100
    normed=True
    p = vol.cones_bokeh(windows=windows, quantiles=quantiles)
    from bokeh.plotting import show
    show(p)
    from bokeh.plotting import figure, show

if __name__ == "__main__":
    from mainteinance import config
    from core.logger import logger
    log = logger("some testing ...")
    globalconf = config.GlobalConfig()

    last_date = "20170708"


# Uses python3
#import sys
#n = int(input())
#b = sys.stdin.read()
#a = [int(x) for x in b.split()]
#assert(len(a) == n)

import random, sys, time
a = [random.randint(0, 100000) for r in xrange(200000)]
n = len(a)
result = 0

time_start = time.clock()
for i in range(0, n):
    for j in range(i+1, n):
        if a[i]*a[j] > result:
            result = a[i]*a[j]
time_elapsed = (time.clock() - time_start)
print(result)
print(time_elapsed)

time_start = time.clock()
max_1 = a[0]
max_2 = a[1]
for i in range(2, n):
    if a[i] > max_1:
        max_1 = a[i]
    elif a[i] > max_2:
        max_2 = a[i]
time_elapsed = (time.clock() - time_start)
print(max_1 * max_2)
print(time_elapsed)


from methods import *

# plot = input('Įveskite sklypo kadastro Nr.: \n')

avg_price = 0
prep_price = 0

data = Data()

while True:
    try:
        choice = int(input(
            f'1 - Įvesti medžių rūšis ir tūrį \n'
            f'2 - Apskačiuoti likvidinį tūri \n'
            f'3 - Apskaičiuoti medienos kainą \n'
            f'4 - Skaičiuoti vienkartinę kompensaciją \n'
            f'5 - Skaičiuoti kasmetinę kompensaciją \n'
            f'6 - Įrašyti į DB \n'
            f'7 - Peržiūrėti \n'
            f'9 - išeiti iš programos\n'
        ))
    except ValueError:
        print('Neteisingas pasirinkimas')
        continue

    if choice == 1:
        tree_spp = input('Įveskite medžių rūšį: ')
        volume = float(input('Įveskite bendrą medienos tūrį: '))
        tree_volume = add_tree_volume(tree_spp, volume)

    elif choice == 2:
        print(f'Likvidinis tūris {calculate_commercial_volume()}')

    elif choice == 3:
        avg_price = float(input('Įveskite vidutinę medienos kainą: '))
        prep_price = float(input('Įveskite vidutinę medienos ruošos kainą: '))
        timber_price = calculate_timber_price(avg_price, prep_price)
        print(f'Medienos kaina: {timber_price} Eur')

    elif choice == 4:
        single_compensation = calculate_single_compensation(avg_price, prep_price)
        print(f'Vienkartinė kompensacija: {single_compensation} Eur')

    elif choice == 5:
        interest = float(input('Įveskite palūkanų normą: '))
        annual_compensation = calculate_annual_compensation(interest, avg_price, prep_price)
        print(f'Kasmetinė kompensacija: {annual_compensation} Eur')

    elif choice == 6:
        data.add_data(timber_price, single_compensation, annual_compensation)

    elif choice == 9:
        print('Programa baigta')
        break


import matplotlib.pyplot as plt
import numpy as np
from matplotlib.animation import FuncAnimation
right_side_length = 5
down_side_length = 5

n = right_side_length*down_side_length
# Number of particles
m = np.ones(n)  # Particle masses
x = np.zeros((n, 2))  # Particle positions (x and y for ith particle in x[i,0], x[i,1])
v = np.zeros((n, 2))  # Particle velocities
f = np.zeros((n, 2))  # Force accumulator
dt = 0.02  # Time step
x_limit = 10  # limit of x axis
y_limit = 10  # limit of y axis
g = np.array([0, -9.8])  # Acceleration due to gravity

# Initialize
kd = 200
spring_length = 1
y_pos = down_side_length+1
x_pos = 1
for i in range(n):
    rem = i%right_side_length
    if rem ==0:
        y_pos-=1
        x_pos=1
    x[i,:] = np.array([x_pos,y_pos])
    x_pos+=1

# x[0, :] = np.array([1, 2])
# x[1, :] = np.array([2, 2])
# x[2, :] = np.array([1, 1])
# x[3, :] = np.array([2, 1])
for i in range(n):
    v[i, :] = np.array([0,0])
    f[i, :] = np.array([0, 0])

def bound_check(index, pos_array, vel_array):
    x = pos_array[index][0]
    y = pos_array[index][1]
    if x < 0.5:
        x = 0.5
        vel_array[index][0] *= -1
    if y < 0.5:
        y = 0.5
        vel_array[index][1] *= -1
    if x > x_limit - 0.5:
        x = x_limit - 0.5
        vel_array[index][0] *= -1
    if y > y_limit - 0.5:
        y = y_limit - 0.5
        vel_array[index][1] *= -1
    pos_array[index][0] = x
    pos_array[index][1] = y

def change_force(index1, index2, pos_array, force_array, spring_constant, rest_length):
    bw_len = ((pos_array[index1][0] - pos_array[index2][0]) ** 2) + ((pos_array[index1][1] - pos_array[index2][1]) ** 2)
    bw_len = bw_len ** (1 / 2)
    dum1 = spring_constant * (bw_len / rest_length - 1) * ((pos_array[index2][0] - pos_array[index1][0]) / bw_len)
    dum2 = spring_constant * (bw_len / rest_length - 1) * ((pos_array[index2][1] - pos_array[index1][1]) / bw_len)
    force_array[index1][0] += dum1
    force_array[index1][1] += dum2

#def check_boundary(index,right_side_length,down_side_length):

def spring_force(index1, pos_array, force_array, spring_constant, rest_length):
    index2_1 = index1 - right_side_length
    index2_2 = index1 + 1
    index2_3 = index1 + right_side_length
    index2_4 = index1 - 1
    if (index2_1 >= 0):
        change_force(index1, index2_1, pos_array, force_array, spring_constant, rest_length)
    if (index1%right_side_length +1 < right_side_length):
        change_force(index1, index2_2, pos_array, force_array, spring_constant, rest_length)
    if (index2_3 < down_side_length*right_side_length):
        change_force(index1, index2_3, pos_array, force_array, spring_constant, rest_length)
    if (index1%right_side_length-1 >= 0):
        change_force(index1, index2_4, pos_array, force_array, spring_constant, rest_length)
    index2_5 = index1 - right_side_length + 1
    index2_6 = index1 + right_side_length + 1
    index2_7 = index1 + right_side_length - 1
    index2_8 = index1 - right_side_length - 1

    if (index1//right_side_length > 0 and index1%right_side_length +1 < right_side_length):
        change_force(index1,index2_5,pos_array,force_array,spring_constant,rest_length)
    if (index1//right_side_length < down_side_length-1 and index1%right_side_length +1 < right_side_length):
        change_force(index1,index2_6,pos_array,force_array,spring_constant,rest_length)
    if (index1//right_side_length < down_side_length-1 and index1%right_side_length-1 >= 0):
        change_force(index1,index2_7,pos_array,force_array,spring_constant,rest_length)
    if (index1//right_side_length >0 and index1%right_side_length-1 >= 0):
        change_force(index1,index2_8,pos_array,force_array,spring_constant,rest_length)

# Time stepping (this is actually "semi-implicit Euler")
def step():
    # Accumulate forces on each particle
    f.fill(0)
    for i in range(n):
        spring_force(i, x, f, kd, spring_length)
        f[i,:] += m[i]*g
    # Update velocity of each particle
    for i in range(n):
        v[i, :] += f[i, :] / m[i] * dt
    # Update position of each particle
    for i in range(n):
        x[i, :] += v[i, :] * dt
        bound_check(i, x, v)

# Drawing code
fig, ax = plt.subplots()
points, = ax.plot(x[:, 0], x[:, 1], 'o')

def init():
    ax.set_xlim(0, x_limit)
    ax.set_ylim(0, y_limit)
    ax.set_aspect('equal')
    return points,

def animate(frame):
    step()
    points.set_data(x[:, 0], x[:, 1])
    if frame is frames - 1:
        plt.close()
    return points,

totalTime = 10
frames = int(totalTime / dt)
anim = FuncAnimation(fig, animate, frames=range(frames), init_func=init, interval=dt * 1000)
plt.show()

"""
mport matplotlib.pyplot as plt
import numpy as np
import random
from matplotlib.animation import FuncAnimation

n = 4      # Number of particles
m = np.ones(n)      # Particle masses
x = np.zeros((n,2))  # Particle positions (x and y for ith particle in x[i,0], x[i,1])
v = np.zeros((n,2))  # Particle velocities
f = np.zeros((n,2))  # Force accumulator
dt = 0.005           # Time step
frc = np.zeros((n,2))  # Force accumulator

g = np.array([0,0])  # Acceleration due to gravity

# Class for particle

# class particle:
#     def __init__(self, mass, position, velocity, force, number):
#         self.mass = mass
#         self.position = position
#         self.velocity = velocity
#         self.force = force
#         self.number = number
#
# #Creating an array for particles
#
#
# my_objects = []
#
# my_objects.append(particle(0.1, 0.5, 0, 0, 0))
# my_objects.append(particle(0.1, 0, 0, 0, 0))

# Initialize

m[:] = 0.1

# for i in range(n):
#     x[i, :] = np.array([random.randint(0,100)/100, random.randint(0, 100)/100])
#     v[i, :] = np.array([random.randint(1, 5), random.randint(1, 5)])
x[0, :] = np.array([0.5, 0.5])
x[1, :] = np.array([0.6, 0.5])
x[2, :] = np.array([0.6, 0.6])
x[3, :] = np.array([0.5, 0.6])
v[1, :] = np.array([0.0,0.0])

# Time stepping (this is actually "semi-implicit Euler")

def step():
    # Accumulate forces on each particle
    f.fill(0)
    for i in range(n):
        f[i,:] += m[i]*g
    frc = np.zeros((n, 2))
    len = ((x[0][0] - x[1][0])*(x[0][0] - x[1][0]) + (x[0][1] - x[1][1])*(x[0][1] - x[1][1]))
    len = len**(1/2)
    kd = 5
    kl = 1
    frc[0][0] = kl*(len/0.2 - 1)*((x[1][0] - x[0][0]) / len)
    frc[0][1] = kl * (len/0.2 - 1)*((x[1][1] - x[0][1]) / len)
    frc[1][0] = -frc[0][0]
    frc[1][1] = -frc[0][1]
    len = ((x[0][0] - x[2][0]) * (x[0][0] - x[2][0]) + (x[0][1] - x[2][1]) * (x[0][1] - x[2][1]))
    len = len ** (1 / 2)
    frc[0][0] += kl * (len / 0.2 - 1) * ((x[2][0] - x[0][0]) / len)
    frc[0][1] += kl * (len / 0.2 - 1) * ((x[2][1] - x[0][1]) / len)
    frc[2][0] += -(kl * (len / 0.2 - 1) * ((x[2][0] - x[0][0]) / len))
    frc[2][1] += -(kl * (len / 0.2 - 1) * ((x[2][1] - x[0][1]) / len))
    len = ((x[0][0] - x[3][0]) * (x[0][0] - x[3][0]) + (x[0][1] - x[3][1]) * (x[0][1] - x[3][1]))
    len = len ** (1 / 2)
    frc[0][0] += kl * (len / 0.2 - 1) * ((x[3][0] - x[0][0]) / len)
    frc[0][1] += kl * (len / 0.2 - 1) * ((x[3][1] - x[0][1]) / len)
    frc[3][0] += -(kl * (len / 0.2 - 1) * ((x[3][0] - x[0][0]) / len))
    frc[3][1] += -(kl * (len / 0.2 - 1) * ((x[3][1] - x[0][1]) / len))
    len = ((x[1][0] - x[2][0]) * (x[1][0] - x[2][0]) + (x[1][1] - x[2][1]) * (x[1][1] - x[2][1]))
    len = len ** (1 / 2)
    frc[1][0] += kl * (len / 0.2 - 1) * ((x[2][0] - x[1][0]) / len)
    frc[1][1] += kl * (len / 0.2 - 1) * ((x[2][1] - x[1][1]) / len)
    frc[2][0] += -(kl * (len / 0.2 - 1) * ((x[2][0] - x[1][0]) / len))
    frc[2][1] += -(kl * (len / 0.2 - 1) * ((x[2][1] - x[1][1]) / len))
    len = ((x[1][0] - x[3][0]) * (x[1][0] - x[3][0]) + (x[1][1] - x[3][1]) * (x[1][1] - x[3][1]))
    len = len ** (1 / 2)
    frc[1][0] += kl * (len / 0.2 - 1) * ((x[3][0] - x[1][0]) / len)
    frc[1][1] += kl * (len / 0.2 - 1) * ((x[3][1] - x[1][1]) / len)
    frc[3][0] += -(kl * (len / 0.2 - 1) * ((x[3][0] - x[1][0]) / len))
    frc[3][1] += -(kl * (len / 0.2 - 1) * ((x[3][1] - x[1][1]) / len))
    len = ((x[2][0] - x[3][0]) * (x[2][0] - x[3][0]) + (x[2][1] - x[3][1]) * (x[2][1] - x[3][1]))
    len = len ** (1 / 2)
    frc[2][0] += kl * (len / 0.2 - 1) * ((x[3][0] - x[2][0]) / len)
    frc[2][1] += kl * (len / 0.2 - 1) * ((x[3][1] - x[2][1]) / len)
    frc[3][0] += -(kl * (len / 0.2 - 1) * ((x[3][0] - x[2][0]) / len))
    frc[3][1] += -(kl * (len / 0.2 - 1) * ((x[3][1] - x[2][1]) / len))
    for i in range(n):
        f[i,0] += frc[i][0]
        f[i,1] += frc[i][1]

    for i in range(n):
        if x[i][0] >= 1 or x[i][0] <= 0:
            v[i][0] = - 0.8*v[i][0]

    for i in range(n):
        if x[i][1] >= 1 or x[i][1] <= 0:
            v[i][1] = - 0.8*v[i][1]

    for i in range(n):
        v[i,:] += f[i,:]/m[i] * dt
    # Update position of each particle
    for i in range(n):
        x[i,:] += v[i,:] * dt

# Drawing code

fig, ax = plt.subplots()
points, = ax.plot(x[:,0], x[:,1], 'o')

def init():
    ax.set_xlim(0,1)
    ax.set_ylim(0,1)
    ax.set_aspect('equal')
    return points,

def animate(frame):
    step()
    points.set_data(x[:,0], x[:,1])
    if frame is frames-1:
        plt.close()
    return points,

totalTime = 2
frames = int(totalTime/dt)
anim = FuncAnimation(fig, animate, frames=range(frames), init_func=init, interval=dt*1000)
plt.show()
"""


import ply.lex as lex
import sys

class MyLexer(object):

    # Dict of reserved words
    reserved = {
            'array' : 'ARRAY',
            'by' : 'BY',
            'chars' : 'CHARS',
            'dcl' : 'DCL',
            'do' : 'DO',
            'down' : 'DOWN',
            'else' : 'ELSE',
            'elsif' : 'ELSIF',
            'end' : 'END',
            'exit' : 'EXIT',
            'fi' : 'FI',
            'for' : 'FOR',
            'if' : 'IF',
            'in' : 'IN',
            'loc' : 'LOC',
            'od' : 'OD',
            'print' : 'PRINT',
            'proc' : 'PROC',
            'ref' : 'REF',
            'result' : 'RESULT',
            'return' : 'RETURN',
            'returns' : 'RETURNS',
            'syn' : 'SYN',
            'then' : 'THEN',
            'to' : 'TO',
            'type' : 'TYPE',
            'while' : 'WHILE',
            'abs' : 'ABS',
            'asc' : 'ASC',
            'bool' : 'BOOL',
            'char' : 'CHAR',
            'false' : 'FALSE',
            'int' : 'INT',
            'length' : 'LENGTH',
            'lower' : 'LOWER',
            'null' : 'NULL',
            'num' : 'NUM',
            'print' : 'PRINT',
            'read' : 'READ',
            'true' : 'TRUE',
            'upper' : 'UPPER'
    }

    # List of token names.   This is always required
    tokens = [
            'ID',
            'ICONST',
            'CCONST',
            'SCONST',
            'PLUS',
            'MINUS',
            'TIMES',
            'DIVIDE',
            'MODULO',
            'EQUALS',
            'SEMI',
            'ARROW',
            'LPAREN',
            'RPAREN',
            'LBRACKET',
            'RBRACKET',
            'AND',
            'LAND',
            'OR',
            'EQ',
            'NE',
            'GT',
            'GEQ',
            'LT',
            'LEQ',
            'COLON',
            'COMMA',
            'NOT'
            ] + list(reserved.values())

    # Regular expression rules for simple tokens
    t_PLUS = r'\+'
    t_MINUS = r'-'
    t_TIMES = r'\*'
    t_DIVIDE = r'/'
    t_MODULO = r'%'
    t_EQUALS = r'='
    t_SEMI = r';'
    t_ARROW = r'->'
    t_LPAREN  = r'\('
    t_RPAREN  = r'\)'
    t_LBRACKET = r'\['
    t_RBRACKET = r'\]'
    t_AND = r'&&'
    t_LAND = r'&'
    t_OR = r'\|\|'
    t_EQ = r'=='
    t_NE = r'!='
    t_GT = r'>'
    t_GEQ = r'>='
    t_LT = r'<'
    t_LEQ = r'<='
    t_COLON = r':'
    t_COMMA = r','
    t_NOT = r'!'

    # A regular expression rule with some action code
    # Note addition of self parameter since we're in a class
    def t_ICONST(self,t):
        r'\d+'
        return t

    def t_CCONST(self,t):
        r'\'[^\\]\'|\'\^\(\d+\)\'|\'\\[nt\\]\''
        if "^(" in t.value:
            t.value = int(t.value[3:-2])
        elif '\\' in t.value:
            if t.value[2] == 'n':
                t.value = 10
            elif t.value[2] == 't':
                t.value = 11
            else:
                t.value = 92
        else:
            t.value = ord(t.value[1])
        return t

    def t_ID(self,t):
        r'[A-Za-z_][\w_]*'
        t.type = MyLexer.reserved.get(t.value, 'ID')
        return t

    def t_SCOMMENT(self, t):
        r'\/\/.*\n'
        pass

    def t_COMMENT(self, t):
        #r'\/\*(^(\*\/)|\n)*\*\/'
        r'/\*[^*]*\*+(?:[^/*][^*]*\*+)*/'
        pass

    def t_SCONST(self, t):
        #r'\"([^\"]*)\"'
        r'\"(\\.|[^\\"])*\"'
        return t

    # Define a rule so we can track line numbers
    def t_newline(self,t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    # A string containing ignored characters (spaces and tabs)
    t_ignore  = ' \t'

    # Error handling rule
    def t_error(self,t):
        print("Illegal character '%s'" % t.value[0])
        t.lexer.skip(1)

    def t_commenterror(self,t):
        r'\/\*.*'
        print ('{}: Unterminated comment'.format(t.lexer.lineno))
        pass

    def t_stringerror(self,t):
        r'\".*\n'
        print ('{}: Unterminated string'.format(t.lexer.lineno))
        t.lexer.skip(1)

    def t_badescape(self,t):
        r'\a|\r|\f|\v'
        print ("{}: Bad string escape code '{}'".format(t.lexer.lineno, t.value))
        t.lexer.skip(1)

    # Build the lexer
    def build(self,**kwargs):
        self.lexer = lex.lex(module=self, **kwargs)

    # Test it output
    def input(self,data):
        self.lexer.input(data)

    def token(self):
        tok = self.lexer.token()
        #print (tok)
        return tok


class excel():
    config_json={"button":{},"check box":{},"icon":{},"link":{},"tab":{}}
    def __init__(self,file_name,json_file):
        from openpyxl import load_workbook
        work_book=load_workbook(file_name)
        self.allow=work_book['Allow']
        self.column=work_book['Column Name']
        self.table=work_book['Table Name']
        self.show_filter=work_book['Show filter']
        self.ag_grid=work_book['Ag grid']
        self.json_valuejson.load(open(json_file))

    def scanfile(self):
        self.button('button_name')

    def button(self,button_name):
        i=2
        while self.sheet['A'+str(i)].value!=None:
            if self.sheet['A'+str(i)].value==button_name:
                return self.sheet['B'+str(i)].value
            i+=1
    def conf(self,popup_name):
        i=2
        while self.sheet['Q'+str(i)].value!=None:
            if self.sheet['Q'+str(i)].value==popup_name:
                return self.sheet['Q'+str(i)].value
            i+=1
    def export(self):
        print("sucess")

p=excel('config.xlsx','testrun.json')
p.scanfile()
p.scanjson()
p.export()


import numpy as np

def calculate_cumulative_energies(gd, data, node):
    """Calculate cumulative energies as a function of radius in a given node

    Parameters
    ----------
    gd : Dendrogram
        : Dendrogram object.
    data : dict
        Dictionary containing fluid variables.
    node : int
        Selected GRID-dendro node.

    Returns
    -------
    array
        Effective radii.
    dict
        thermal, kinetic, magnetic, gravitational, and total energies
        integrated up to each radius.
    """

    # Create 1-D flattened primitive variables of this node
    req_var = {'rho', 'vel1', 'vel2', 'vel3', 'prs', 'phi', 'dvol'}
    if not req_var.issubset(data):
        msg = "data must contain the following variables: {}".format(req_var)
        raise ValueError(msg)

    # Filter data by node
    prims = {k: gd.filter_data(v, node, drop=True)
             for k, v in data.items() if k != 'dvol'}
    dvol = data['dvol']

    # Sort variables in ascending order of potential
    idx_ordered = prims['phi'].argsort()
    prims = {k: v[idx_ordered] for k, v in prims.items()}

    # Gravitational potential at the HBP boundary
    phimax = prims['phi'][-1]

    # Calculate the center of momentum frame
    rho_cumsum = prims['rho'].cumsum()
    vel1_com = (prims['rho']*prims['vel1']).cumsum() / rho_cumsum
    vel2_com = (prims['rho']*prims['vel2']).cumsum() / rho_cumsum
    vel3_com = (prims['rho']*prims['vel3']).cumsum() / rho_cumsum

    # Kinetic energy
    # \int 0.5 \rho | v - v_com |^2 dV
    # = \int 0.5 \rho |v|^2 dV - (\int 0.5\rho dV) |v_com|^2
    # Note that v_com depends on the limit of the volume integral.
    ekin = (0.5*prims['rho']*(prims['vel1']**2
                              + prims['vel2']**2
                              + prims['vel3']**2)*dvol).cumsum()
    ekin -= (0.5*prims['rho']*dvol).cumsum()*(vel1_com**2 + vel2_com**2
                                              + vel3_com**2)

    # Thermal energy
    gm1 = 5./3. - 1
    ethm = (prims['prs']/gm1*dvol).cumsum()

    # Gravitational energy
    egrv = (prims['rho']*(prims['phi'] - phimax)*dvol).cumsum()

    # Magnetic energy
    if 'bprs' in prims:
        emag = (prims['bprs']*dvol).cumsum()
    else:
        emag = np.zeros(len(egrv))

    # Total energy
    etot = ekin + ethm + emag + egrv

    # Effective radius
    Ncells = len(prims['rho'])
    vol = np.full(Ncells, dvol).cumsum()
    reff = (vol / (4.*np.pi/3.))**(1./3.)

    energies = dict(ekin=ekin, ethm=ethm, emag=emag, egrv=egrv, etot=etot)
    return reff, energies

def find_bound_objects(gd, data, hbp=None, hbr=None, node=None):
    """Finds HBP and HBR

    Parameters
    ----------
    gd : Dendrogram
        GRID-dendro dendrogram object.
    data : dict
        Dictionary containing 1-D flattend fluid variables.
        Must contain ['rho', 'vel1', 'vel2', 'vel3', 'prs', 'phi']
    hbp : dict, optional
        Dictionary containing HBPs.
        Only used internally for recursive purpose.
    hbr : dict, optional
        Dictionary containing HBRs.
        Only used internally for recursive purpose.
    node : int, optional
        ID of the selected node.
        Only used internally for recursive purpose.

    Returns
    -------
    hbp : dict
        Dictionary containing HBPs.
    hbr : dict
        Dictionary containing HBRs.

    Examples
    --------
    >>> import pyathena as pa
    >>> s = pa.LoadSim("/scratch/smoon/M5J2P0N512")
    >>> ds = s.load_hdf5(50, load_method='pyathena')
    >>> cs = 1.0 # isothermal sound speed
    >>> gd = dendrogram.Dendrogram(ds.phigas.data)
    >>> gd.construct()
    >>> gd.prune()  # Remove buds
    >>> data = dict(rho=ds.dens.data.flatten(),
                    vel1=(ds.mom1/ds.dens).data.flatten(),
                    vel2=(ds.mom2/ds.dens).data.flatten(),
                    vel3=(ds.mom3/ds.dens).data.flatten(),
                    prs=(cs**2*ds.dens).data.flatten(),
                    phi=ds.phigas.data.flatten())
    >>> hbp, hbr = find_bound_object(gd, data)
    """
    if node is None:
        node = gd.trunk
    if hbp is None:
        hbp = {}
    if hbr is None:
        hbr = {}

    phi_node = data['phi'][node]
    cells = gd.get_all_descendant_cells(node)
    cells_bound = find_bound_cells(gd, data, node)
    if cells_bound is not None:
        phi_hbr = data['phi'][cells_bound[-1]]
        if phi_hbr > phi_node:
            hbp[node] = cells
            hbr[node] = cells_bound
        else:
            for nd in gd.children[node]:
                hbp, hbr = find_bound_objects(gd, data, hbp, hbr, nd)
    else:
        for nd in gd.children[node]:
            hbp, hbr = find_bound_objects(gd, data, hbp, hbr, nd)
    return hbp, hbr

def find_bound_cells(gd, data, node):
    if 'dvol' not in data:
        data['dvol'] = 1

    reff, engs = calculate_cumulative_energies(gd, data, node)
    idx_bound = (engs['etot'] < 0).nonzero()[0]
    if len(idx_bound) < 1:
        return None
    else:
        idx_bound = idx_bound[-1]

    phi = gd.filter_data(data['phi'], node)
    idx_ordered = phi.argsort()
    cells = gd.get_all_descendant_cells(node)
    cells_bound = cells[idx_ordered][:idx_bound + 1]
    return cells_bound

# def calculate_cum_energies_old(ds, nodes, node, mode='HBR',
#                                boundary_flag='periodic'):
#     """WARNING: Deprecated function. Do not use this.
#
#     Do not remove this function for future extension of
#     calculate_cumulative_energies.
#     """
#     # Create 1-D flattened primitive variables of this node
#     cells = np.array(nodes[node])
#     ds = ds.transpose('z', 'y', 'x')
#     dat1d = dict(cells=cells,
#                  x=np.broadcast_to(ds.x.data, ds.dens.shape
#                                    ).flatten()[cells],
#                  y=np.broadcast_to(ds.y.data, ds.dens.shape
#                                    ).transpose(1, 2, 0).flatten()[cells],
#                  z=np.broadcast_to(ds.z.data, ds.dens.shape
#                                    ).transpose(2, 0, 1).flatten()[cells],
#                  rho=ds.dens.data.flatten()[cells],
#                  vx=ds.vel1.data.flatten()[cells],
#                  vy=ds.vel2.data.flatten()[cells],
#                  vz=ds.vel3.data.flatten()[cells],
#                  prs=ds.prs.data.flatten()[cells],
#                  phi=ds.phi.data.flatten()[cells])
#     # Assume uniform grid
#     dV = ds.dx1*ds.dx2*ds.dx3
#     gm1 = (5./3. - 1)
#     Ncells = len(cells)
#
#     # Sort variables in ascending order of potential
#     cells_sorted = dat1d['phi'].argsort()
#     dat1d = {key: value[cells_sorted] for key, value in dat1d.items()}
#     cells = dat1d['cells']
#
#     # Gravitational potential at the HBP boundary
#     phi0 = dat1d['phi'][-1]
#
#     # Calculate the center of momentum frame
#     # note: dV factor is omitted
#     M = (dat1d['rho']).cumsum()
#     vx0 = (dat1d['rho']*dat1d['vx']).cumsum() / M
#     vy0 = (dat1d['rho']*dat1d['vy']).cumsum() / M
#     vz0 = (dat1d['rho']*dat1d['vz']).cumsum() / M
#     # Potential minimum
#     phi_hpr = dendrogram.filter_by_node(ds.phi, cells=cells)
#     x0, y0, z0 = get_coords_minimum(phi_hpr)
#
#     # Kinetic energy
#     # \int 0.5 \rho | v - v_com |^2 dV
#     # = \int 0.5 \rho |v|^2 dV - (\int 0.5\rho dV) |v_com|^2
#     # Note that v_com depends on the limit of the volume integral.
#     Ekin = (0.5*dat1d['rho']*(dat1d['vx']**2
#                               + dat1d['vy']**2
#                               + dat1d['vz']**2)*dV).cumsum()
#     Ekin -= (0.5*dat1d['rho']*dV).cumsum()*(vx0**2 + vy0**2 + vz0**2)
#
#     # Thermal energy
#     Eth = (dat1d['prs']/gm1*dV).cumsum()
#
#     # Gravitational energy
#     if mode == 'HBR' or mode == 'HBR+1' or mode == 'HBR-1':
#         Egrav = (dat1d['rho']*(dat1d['phi'] - phi0)*dV).cumsum()
#     elif mode == 'virial':
#         dat1d['gx'] = -ds.phi.differentiate('x').data.flatten()[cells]
#         dat1d['gy'] = -ds.phi.differentiate('y').data.flatten()[cells]
#         dat1d['gz'] = -ds.phi.differentiate('z').data.flatten()[cells]
#         Egrav = (dat1d['rho']*((dat1d['x'] - x0)*dat1d['gx']
#                                + (dat1d['y'] - y0)*dat1d['gy']
#                                + (dat1d['z'] - z0)*dat1d['gz'])*dV).cumsum()
#     else:
#         raise ValueError("Unknown mode; select (HBR | HBR+1 | HBR-1 | virial)")
#
#     # Surface terms
#     if mode == 'HBR':
#         Ekin0 = Eth0 = np.zeros(Ncells)
#     elif mode == 'HBR+1' or mode == 'HBR-1':
#         pcn = boundary.precompute_neighbor(ds.phi.shape, boundary_flag)
#         edge = boundary.get_edge_cells(cells, pcn)
#         edg1d = dict(rho=ds.dens.data.flatten()[edge],
#                      vx=ds.vel1.data.flatten()[edge],
#                      vy=ds.vel2.data.flatten()[edge],
#                      vz=ds.vel3.data.flatten()[edge],
#                      prs=ds.prs.data.flatten()[edge])
#         # COM velocity of edge cells
#         M = (edg1d['rho']).sum()
#         vx0 = (edg1d['rho']*edg1d['vx']).sum() / M
#         vy0 = (edg1d['rho']*edg1d['vy']).sum() / M
#         vz0 = (edg1d['rho']*edg1d['vz']).sum() / M
#         # Mean surface energies
#         Ekin0 = (0.5*edg1d['rho']*((edg1d['vx'] - vx0)**2
#                                    + (edg1d['vy'] - vy0)**2
#                                    + (edg1d['vz'] - vz0)**2)).mean()
#         Eth0 = (edg1d['prs']/gm1).mean()
#         # Integrated surface energy to compare with volume energies.
#         # Note that the excess energy is given by \int (E - E_0) dV
#         Ekin0 = (Ekin0*np.ones(Ncells)*dV).cumsum()
#         Eth0 = (Eth0*np.ones(Ncells)*dV).cumsum()
#     elif mode == 'virial':
#         divPx = ((ds.prs*(ds.x - x0)).differentiate('x')
#                  + (ds.prs*(ds.y - y0)).differentiate('y')
#                  + (ds.prs*(ds.z - z0)).differentiate('z')
#                  ).data.flatten()[cells]
#         Eth0 = ((1./3.)*divPx/gm1*dV).cumsum()
#         # Kinetic energy surface term
#         v0 = np.array([vx0, vy0, vz0])
#         # A1
#         rho_rdotv = ds.dens*((ds.x - x0)*ds.vel1
#                              + (ds.y - y0)*ds.vel2
#                              + (ds.z - z0)*ds.vel3)
#         A1 = ((rho_rdotv*ds.vel1).differentiate('x')
#               + (rho_rdotv*ds.vel2).differentiate('y')
#               + (rho_rdotv*ds.vel3).differentiate('z'))
#         A1 = (A1.data.flatten()[cells]*dV).cumsum()
#         # A2
#         grad_rho_r = np.empty((3, 3), dtype=xr.DataArray)
#         for i, crd_i in enumerate(['x', 'y', 'z']):
#             for j, (crd_j, pos0_j) in enumerate(zip(['x', 'y', 'z'], [x0, y0, z0])):
#                 grad_rho_r[i, j] = (ds.dens*(ds[crd_j] - pos0_j)
#                                     ).differentiate(crd_i)
#         A2 = np.empty((3, 3, Ncells))
#         for i, crd_i in enumerate(['x', 'y', 'z']):
#             for j, (crd_j, pos0_j) in enumerate(zip(['x', 'y', 'z'], [x0, y0, z0])):
#                 A2[i, j, :] = (grad_rho_r[i, j].data.flatten()[cells]*dV).cumsum()
#         A2 = np.einsum('i..., ij..., j...', v0, A2, v0)
#         # A3
#         grad_rho_rdotv = np.empty(3, dtype=xr.DataArray)
#         for i, crd_i in enumerate(['x', 'y', 'z']):
#             grad_rho_rdotv[i] = rho_rdotv.differentiate(crd_i)
#         A3 = np.empty((3, Ncells))
#         for i, crd_i in enumerate(['x', 'y', 'z']):
#             A3[i, :] = (grad_rho_rdotv[i].data.flatten()[cells]*dV).cumsum()
#         A3 = np.einsum('i...,i...', v0, A3)
#         # A4
#         div_rhorv = np.empty(3, dtype=xr.DataArray)
#         for i, (crd_i, pos0_i) in enumerate(zip(['x', 'y', 'z'],
#                                                 [x0, y0, z0])):
#             div_rhorv[i] = ((ds.dens*(ds[crd_i] - pos0_i)*ds.vel1).differentiate('x')
#                             + (ds.dens*(ds[crd_i] - pos0_i)*ds.vel2).differentiate('y')
#                             + (ds.dens*(ds[crd_i] - pos0_i)*ds.vel3).differentiate('z'))
#         A4 = np.empty((3, Ncells))
#         for i, crd_i in enumerate(['x', 'y', 'z']):
#             A4[i, :] = (div_rhorv[i].data.flatten()[cells]*dV).cumsum()
#         A4 = np.einsum('i...,i...', v0, A4)
#         Ekin0 = 0.5*(A1 + A2 - A3 - A4)
#
#     Reff = ((np.ones(Ncells)*dV).cumsum() / (4.*np.pi/3.))**(1./3.)
#     if mode == 'HBR':
#         Etot = Ekin + Eth + Egrav
#     elif mode == 'HBR+1':
#         Etot = (Ekin - Ekin0) + (Eth - Eth0) + Egrav
#     elif mode == 'HBR-1':
#         Etot = (Ekin + Ekin0) + (Eth + Eth0) + Egrav
#     elif mode == 'virial':
#         Etot = 2*(Ekin - Ekin0) + 3*gm1*(Eth - Eth0) + Egrav
#
#     energies = dict(Reff=Reff, Ekin=Ekin, Eth=Eth, Ekin0=Ekin0, Eth0=Eth0,
#                     Egrav=Egrav, Etot=Etot)
#     return energies


#!/usr/bin/env python

import rdflib
from collections import OrderedDict
from pprint import pprint
import os
import sys

def get_license_ids():
    graph = rdflib.Graph()
    graph.parse('http://spdx.org/licenses/index.html', 'rdfa')
    ref = rdflib.URIRef('http://spdx.org/rdf/terms#licenseId')
    objs = graph.subject_objects(ref)
    return map(lambda x: x[1].value, objs)

def lit_to_str(lit):
    s = lit.nodeValue.encode('utf-8') if lit.nodeValue else ''
    return s + xml_to_str(lit)

def xml_to_str(parent):
    return ''.join(lit_to_str(x) for x in parent.childNodes)

def license_text_to_str(literal):
    if isinstance(literal.value, unicode):
        return literal.value.encode('utf-8')

    # Assume the literal contains a list of <p> elements under a top level node.
    return xml_to_str(literal.value.firstChild) + '\n'

def get_license_text(licenseId):
    graph = rdflib.Graph()

    try:
        graph.parse('http://spdx.org/licenses/' + licenseId + '.html')
    except:
        return None

    ref = rdflib.URIRef("http://spdx.org/rdf/terms#licenseText")
    objs = graph.subject_objects(ref)
    return license_text_to_str(objs.next()[1])

def write_licenses_dir(ids):
    licenseDir = 'codebom/licenses'
    if not os.path.exists(licenseDir):
        os.makedirs(licenseDir)

    for licenseId in ids:
        sys.stdout.write("Updating license text for '{}'\n".format(licenseId))
        text = get_license_text(licenseId)
        if text is None:
            continue

        with open('{}/{}.txt'.format(licenseDir, licenseId), 'w') as hdl:
            hdl.write(text)

with open('codebom/licenses.py', 'w') as out:
    ids = get_license_ids()
    ids.sort()
    out.write('license_ids = ')
    pprint(ids, indent=4, stream=out)

    write_licenses_dir(ids)


import os
import sys
import platform
INDEX = "https://pypi.org/simple"
PIP = "pip3"
MORE_ARGS = ""
VERSION = ""
PACK = ""
__version__ = '1.0.1-Pre1'

OS_MSG = """
    OS: {}
    Node (网络名称) : {}
    Release (内核版本号) : {}
    Version (详细版本信息) : {}
    Machine (机器类型) : {}
    Architecture (架构) : {}
    CPU Edition (CPU 类型) : {}
"""

class NotInAShell(Exception):
    def __init__(self, msg):
        super().__init__(msg)
        pass

def getAllPackFormat(ppl = 5):
    packagesList = os.popen(PIP + " freeze").read().split("\n")
    lenMax = 0
    for i in packagesList:
        if len(i) > lenMax:
            lenMax = len(i)

    packs = ""
    idx = 0
    for i in packagesList:
        idx += 1
        addSpace = lenMax - len(i)
        pack = i + " " * addSpace
        packs += pack + ' '
        if idx % ppl == 0:
            packs += "\n"
    return packs

def index():
    CLS()
    global INDEX
    MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Index 索引

Now Index: {}
现在的索引: {}

1) Set TUNA Mirror 设置 清华源

2) Set BFSU Mirror设置 北京外国语源

3) Set PyPI Mirror 设置 官方源

4) Set Douban Mirror 设置 豆瓣源

5) Back 返回
====================================

Your choice 你的选择:
""".format(INDEX, INDEX)

    mirror_list = ["https://mirrors.tuna.edu.cn/pypi/web/simple", "https://mirrors.bfsu.edu.cn/pypi/web/simple",
                   "https://pypi.org/simple",
                   "https://pypi.douban.com/simple/"]
    choice = int(input(MSG))
    if choice == 5:
        return True
    elif 0 < choice < 5:
        INDEX = mirror_list[choice - 1]
        return True
    else:
        raise Exception

def help_msg():
    CLS()
    print("""
    用法：
        pip3＜命令＞〔选项〕

    命令：
        install       安装软件包。
        download      下载软件包。
        uninstall     卸载程序包。
        freeze        以需求格式输出已安装的程序包。
        inspect       检查python环境。
        list          列出已安装的程序包。
        show          显示有关已安装程序包的信息。
        check         验证已安装的程序包是否具有兼容的依赖项。
        config        管理本地和全局配置。
        search        搜索PyPI以查找包。
        cache         检查和管理pip的wheel缓存。
        index         检查包索引中的可用信息。
        wheel         根据您的要求制造wheel。
        hash          计算包存档的哈希。
        completion    用于命令完成的辅助命令。
        debug         显示对调试有用的信息。
        help          显示命令的帮助。

    常规选项：
        -h、 --help 显示帮助。
        --debug 让未处理的异常在主子程序之外传播，而不是记录将它们发送到stderr。
        --isolated 在隔离模式下运行pip，忽略环境变量和用户配置。
        --require-virtualenv 允许pip仅在虚拟环境中运行；否则将退出并返回错误。
        --python＜python＞ 使用指定的python解释器运行pip。
        -v、 --verbose 提供更多输出。选项是可添加的，最多可使用3次。
        -V、 --version 显示版本并退出。
        -q、 --quiet 输出更少。选项是可添加的，最多可使用3次（对应至“警告”、“错误”和“关键”日志记录级别）。
        --log＜path＞ 详细附加日志的路径。
        --no-input  禁用输入提示。
        --proxy＜proxy＞  以scheme://[user:passwd@]proxy.server:port的形式指定代理。
        --retries＜retries＞  每个连接应尝试的最大重试次数（默认为5次）。
        --timeout＜sec＞  设置套接字超时（默认为15秒）。
        --exists action＜action＞   当路径已经存在时的默认操作：(s)witch，(i)gnore，(w)ipe，(b)ackup (a)bort。
        --trusted-host＜hostname＞ 将此主机或host:port对标记为可信，即使它没有有效的或任何HTTPS。
        --cert<path> PEM编码的CA证书捆绑包的路径。如果提供，则覆盖默认值。看见有关详细信息，请参阅pip文档中的“SSL证书验证”。
        --client-cert＜path＞ SSL客户端证书的路径，一个包含私钥和PEM格式的证书。
        --cache-dir＜dir＞ 将缓存数据存储在＜dir＞中。
        --no-cache-dir 禁用缓存。
        --disable-pip-version-check  不要定期检查PyPI以确定新版本的pip是否可供下载。隐含--no-index。
        --no-color 抑制彩色输出。
        --no-python-version-warning 对即将推出的不受支持的Python发出无声的弃用警告。
        --use-feature＜feature＞启用可能向后不兼容的新功能。
        --use-deprecated <feature>启用弃用的功能，这些功能将来将被删除。
    """)
    input()
    return

def install():
    CLS()
    global INDEX, MORE_ARGS, VERSION, PACK

    def install_packages():
        CLS()
        global VERSION, PACK, MORE_ARGS

        def Specify_version(pack):
            CLS()
            MSG_SV = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包 -> Specify Version 指定版本

{}

输入版本:
====================================
""".format(os.popen(PIP + " index versions " + pack).read())
            return input(MSG_SV)

        def Select_install():
            CLS()
            MSG_SP = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包 -> Select Package 选择包

输入包:
====================================
"""
            return input(MSG_SP)

        def execute():
            CLS()
            if VERSION == "":
                cmd = PIP + " install " + PACK + " -i " + INDEX + " " + MORE_ARGS
            else:
                cmd = PIP + " install " + PACK + "==" + VERSION + " -i " + INDEX + " " + MORE_ARGS
            MSG_E = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包 -> Execute 执行

Will Execute Command 欲执行命令: {}

====================================
Enter To Execute 回车执行:
""".format(cmd)
            input(MSG_E)
            os.system(cmd)
            input()
            return

        MSG_I = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包

Now Index: {}
现在的索引: {}

1) Select Package 选择包 {}

2) Specify Version 指定版本 {}

3) Execute 执行

4) Back 返回
====================================

Your choice 你的选择:
        """.format(INDEX, INDEX, PACK, VERSION)
        choice_I = int(input(MSG_I))
        if choice_I == 2 and PACK != "":
            VERSION = Specify_version(PACK)
            return install_packages()
        if choice_I == 1:
            PACK = Select_install()
            return install_packages()
        if choice_I == 3:
            execute()
            return install_packages()
        if choice_I == 4:
            return

    def more_args():
        CLS()
        global MORE_ARGS
        MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包-> Add Arguments 额外参数

Arguments 参数:{}

输入参数:
====================================
""".format(MORE_ARGS)
        MORE_ARGS = input(MSG)
        return

    MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装

Now Index: {}
现在的索引: {}

1) Install Package 安装包

2) Add Arguments 额外参数 {}

3) Back 返回
====================================

Your choice 你的选择:
""".format(INDEX, INDEX, MORE_ARGS)

    choice = int(input(MSG))
    if choice == 1:
        install_packages()
        return install()
    if choice == 2:
        more_args()
        return install()
    if choice == 3:
        return

def os_msgF():
    OS = platform.system()
    if OS == "Darwin":
        OS = "MacOS " + platform.mac_ver()[0]
    elif OS == "win32":
        OS = "Windows " + platform.win32_edition() + " " + ".".join(platform.win32_ver())
    elif OS == "linux":
        OS = "Linux " + " ".join(platform.linux_distribution())
    NODE = platform.node()
    RELEASE = platform.release()
    VERSION = platform.version()
    MACHINE = platform.machine()
    ARCH = platform.architecture()[0]
    CPU = platform.processor()
    return OS_MSG.format(OS, NODE, RELEASE, VERSION, MACHINE, ARCH, CPU)

def info():
    CLS()
    MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Info 详细信息

输入包名:
====================================
"""
    cmd = PIP + " show " + input(MSG)
    print(os.popen(cmd).read())
    return input()

def download():
    CLS()
    global INDEX, MORE_ARGS, VERSION, PACK

    def download_packages():
        CLS()
        global VERSION, PACK, MORE_ARGS

        def Specify_version(pack):
            CLS()
            MSG_SV = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载 -> Download Packages 下载包 -> Specify Version 指定版本

{}

输入版本:
====================================
""".format(os.popen(PIP + " index versions " + pack).read())
            return input(MSG_SV)

        def Select_packages():
            CLS()
            MSG_SP = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载 -> Download Packages 下载包 -> Select Package 选择包

输入包:
====================================
"""
            return input(MSG_SP)

        def execute():
            CLS()
            if VERSION == "":
                cmd = PIP + " download " + PACK + " -i " + INDEX + " " + MORE_ARGS
            else:
                cmd = PIP + " download " + PACK + "==" + VERSION + " -i " + INDEX + " " + MORE_ARGS
            MSG_E = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载 -> Download Packages 下载包 -> Execute 执行

Will Execute Command 欲执行命令: {}

====================================
Enter To Execute 回车执行:
""".format(cmd)
            input(MSG_E)
            os.system(cmd)
            input()
            return

        MSG_I = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载 -> Download Packages 下载包

Now Index: {}
现在的索引: {}

1) Select Package 选择包 {}

2) Specify Version 指定版本 {}

3) Execute 执行

4) Back 返回
====================================

Your choice 你的选择:
""".format(INDEX, INDEX, PACK, VERSION)
        choice_I = int(input(MSG_I))
        if choice_I == 2 and PACK != "":
            VERSION = Specify_version(PACK)
            return download_packages()
        if choice_I == 1:
            PACK = Select_packages()
            return download_packages()
        if choice_I == 3:
            execute()
            return download_packages()
        if choice_I == 4:
            return

    def more_args():
        CLS()
        global MORE_ARGS
        MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包-> Add Arguments 额外参数

Arguments 参数:{}

输入参数:
====================================
""".format(MORE_ARGS)
        MORE_ARGS = input(MSG)
        return

    MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载

Now Index: {}
现在的索引: {}

1) Download Package 下载包

2) Add Arguments 额外参数 {}

3) Back 返回
====================================

Your choice 你的选择:
""".format(INDEX, INDEX, MORE_ARGS)

    choice = int(input(MSG))
    if choice == 1:
        download_packages()
        return download()
    if choice == 2:
        more_args()
        return download()
    if choice == 3:
        return

def wheel():
    CLS()

    global INDEX, MORE_ARGS, VERSION, PACK

    def wheel_packages():
        CLS()
        global VERSION, PACK, MORE_ARGS

        def Specify_version(pack):
            CLS()
            MSG_SV = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件 -> Wheel Packages 生成Whl -> Specify Version 指定版本

{}

输入版本:
""".format(os.popen(PIP + " index versions " + pack).read())
            return input(MSG_SV)

        def Select_packages():
            CLS()
            MSG_SP = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件 -> Wheel Packages 生成Whl -> Select Package 选择包

输入包:
====================================
"""
            return input(MSG_SP)

        def execute():
            CLS()
            if VERSION == "":
                cmd = PIP + " wheel " + PACK + " -i " + INDEX + " " + MORE_ARGS
            else:
                cmd = PIP + " wheel " + PACK + "==" + VERSION + " -i " + INDEX + " " + MORE_ARGS
            MSG_E = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件 -> Wheel Packages 生成Whl -> Execute 执行

Will Execute Command 欲执行命令: {}

====================================
Enter To Execute 回车执行:
""".format(cmd)
            input(MSG_E)
            os.system(cmd)
            input()
            return

        MSG_I = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件 -> Wheel Packages 生成Whl

Now Index: {}
现在的索引: {}

1) Select Package 选择包 {}

2) Specify Version 指定版本 {}

3) Execute 执行

4) Back 返回
====================================

Your choice 你的选择:
""".format(INDEX, INDEX, PACK, VERSION)
        choice_I = int(input(MSG_I))
        if choice_I == 2 and PACK != "":
            VERSION = Specify_version(PACK)
            return wheel_packages()
        if choice_I == 1:
            PACK = Select_packages()
            return wheel_packages()
        if choice_I == 3:
            execute()
            return wheel_packages()
        if choice_I == 4:
            return

    def more_args():
        CLS()
        global MORE_ARGS
        MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件 -> Wheel Packages 生成Whl-> Add Arguments 额外参数

Arguments 参数:{}

输入参数:
====================================
                        """.format(MORE_ARGS)
        MORE_ARGS = input(MSG)
        return

    MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Wheel 生成Whl文件

Now Index: {}
现在的索引: {}

1) Wheel Package 生成Whl文件

2) Add Arguments 额外参数 {}

3) Back 返回
====================================

Your choice 你的选择:
        """.format(INDEX, INDEX, MORE_ARGS)

    choice = int(input(MSG))
    if choice == 1:
        wheel_packages()
        return wheel()
    if choice == 2:
        more_args()
        return wheel()
    if choice == 3:
        return

def remove():
    CLS()
    global INDEX, MORE_ARGS, VERSION, PACK

    def remove_packages():
        CLS()
        global VERSION, PACK, MORE_ARGS

        def Select_packages():
            CLS()
            MSG_SP = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Remove 卸载 -> Remove Packages 卸载包 -> Select Package 选择包

输入包:
====================================
                            """
            return input(MSG_SP)

        def execute():
            CLS()
            cmd = PIP + " uninstall " + PACK + " " + MORE_ARGS
            MSG_E = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Remove 卸载 -> Remove Packages 卸载包 -> Execute 执行

Will Execute Command 欲执行命令: {}

====================================
Enter To Execute 回车执行:
                            """.format(cmd)
            input(MSG_E)
            os.system(cmd)
            input()
            return

        PACKAGES = getAllPackFormat(3)
        MSG_I = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Download 下载 -> Download Packages 下载包

Now Index: {}
现在的索引: {}

Packages 安装了的包 :
{}

1) Select Package 选择包 {}

2) Execute 执行

3) Back 返回
====================================

Your choice 你的选择:
        """.format(INDEX, INDEX, PACKAGES, PACK, VERSION)
        choice_I = int(input(MSG_I))
        if choice_I == 1:
            PACK = Select_packages()
            return remove_packages()
        if choice_I == 2:
            execute()
            return remove_packages()
        if choice_I == 3:
            return

    def more_args():
        CLS()
        global MORE_ARGS
        MSG = """
====================================
Python pip for PyPI Tool

MainMenu 主界面 -> Install 安装 -> Install Packages 安装包-> Add Arguments 额外参数

Arguments 参数:{}

输入参数:
====================================
                        """.format(MORE_ARGS)
        MORE_ARGS = input(MSG)
        return

    MSG = """
====================================
Python pip for PyPI Tool

-> Remove 卸载

Now Index: {}
现在的索引: {}

1) Remove Package 卸载包

2) Add Arguments 额外参数 {}

3) Back 返回
====================================

Your choice 你的选择:
        """.format(INDEX, INDEX, MORE_ARGS)

    choice = int(input(MSG))
    if choice == 1:
        remove_packages()
        return remove()
    if choice == 2:
        more_args()
        return remove()
    if choice == 3:
        return

def settings():
    global PIP
    MSG_I = """
    ====================================
    Python pip for PyPI Tool

    MainMenu 主界面 -> Settings 设置

    使用pip : {}
    ====================================

    输入更改的命令主体:
    """.format(PIP)
    PIP = input(MSG_I)

def mainMenu(main=False):
    CLS()
    if main:
        shell_()
    MSG = """
=============================================================================================
Python pip for PyPI Tool

Info: Run by Python {}

信息: 运行在Python {}

{}

Working dir 工作目录 : {}

1) Install 安装

2) Remove 卸载

3) Info 详细信息

4) Upgrade 升级

5) Index 索引 当前是:{}

6) Wheel 生成Whl文件

7) Download 下载

8) Help 帮助

9) Setting 设置

0) Exit 退出
=============================================================================================

Your choice 你的选择:
""".format(sys.version, sys.version, os_msgF(), os.getcwd(), INDEX)
    # TODO 4(update)\9(setting) MODE
    choice = int(input(MSG))
    if choice == 4:
        print("未开放 Not Supported")
        input()
        return mainMenu()
    if choice == 9:
        settings()
        return mainMenu()
    if choice == 5:
        print("\n\n\n")
        index()
        return mainMenu()
    if choice == 8:
        print("\n\n\n")
        help_msg()
        return mainMenu()
    if choice == 0:
        return
    if choice == 1:
        print("\n\n\n")
        install()
        return mainMenu()
    if choice == 3:
        print("\n\n\n")
        info()
        return mainMenu()
    if choice == 7:
        print("\n\n\n")
        download()
        return mainMenu()
    if choice == 2:
        print("\n\n\n")
        remove()
        return mainMenu()
    if choice == 6:
        print("\n\n\n")
        wheel()
        return mainMenu()

def shell_():
    CLS()
    if sys.stdout.isatty():
        return True
    else:
        MSG = """
================================================================
Shell Check Process

This Program  Isn't Run on Shell (e.g. zsh bash cmd powershell)
================================================================

Still Execute(Y/n)?"""
        se = input(MSG)
        if se == 'y' or se == 'Y':
            return False
        else:
            raise NotInAShell("Not run in shell!")

def CLS():
    if sys.stdout.isatty():
        if os.name == "nt":
            os.system("cls")
        else:
            os.system("clear")

CLS()
def mstart():
    try:
        mainMenu(True)
    except:
        print("an error!")
        mstart()

mstart()


import pandas as pd
import time

from tqdm import tqdm

from sentence_transformers import SentenceTransformer
import umap

from langdetect import detect

import textacy
import textacy.tm

from ..cleaning.text import htmlTags

def createTopicEmbedding(
    dataframePath: str, year: str, textColumn: str = 'title', idColumn: str = "nodeID", yearColumn: str = 'year', targetLanguage: str = "en",
    device: str = "cuda", nNei: int = 20, nComp: int = 2, metric: str = "cosine", useStopList: bool = False, denseMap: bool = False, debug: bool = True
):
    starttime = time.time()
    model = SentenceTransformer('all-MiniLM-L6-v2', device=device)
    dfTemp = pd.read_json(dataframePath, lines=True)
    dfTemp.insert(0, 'year', year)
    texts = []
    for idx, row in tqdm(dfTemp.iterrows()):
        if type(row[textColumn])==list:
            textelem = row[textColumn][0]
            try:
                lang = detect(textelem)
            except:
                continue
            if type(textelem) == str:
                if lang == targetLanguage:
                    outtext = htmlTags(textelem)
                    texts.append(
                        (row[idColumn], row[yearColumn], outtext)
                    )
    dfTexts = pd.DataFrame(texts, columns=['idx', 'year', 'text'])
    basedata = dfTexts.drop_duplicates(subset="text", keep="last")
    embedded = model.encode(
        basedata.text.values,
        convert_to_numpy=True,
        device=device,
        normalize_embeddings=True
    )
    time2 = time.time()
    if debug is True:
        print(f"Done embedding in {time2-starttime} sec.")
    embedded_2D = umap.UMAP(
        densmap=denseMap,
        n_neighbors=nNei,
        n_components=nComp,
        metric=metric
    ).fit_transform(embedded)
    time3 = time.time()
    if debug is True:
        print(f"Done mapping to 2D in {time3-time2} sec.")
    basedata.insert(0, 'x', embedded_2D[:, 0])
    basedata.insert(0, 'y', embedded_2D[:, 1])

    records = []
    for idx, row in tqdm(basedata.iterrows()):
        doc = textacy.make_spacy_doc((row["text"], {"idx": row["idx"]}), 'en_core_web_lg')
        records.append(doc)
    corpus = textacy.Corpus("en_core_web_lg", data=records)
    time4 = time.time()
    if debug is True:
        print(f"Done building corpus in {time4-time3} sec.")
    if isinstance(useStopList, list):
        STOPLIST = useStopList
    else:
        STOPLIST = []

    vectorizer = textacy.representations.vectorizers.Vectorizer(
        tf_type="linear",
        idf_type="smooth",
        norm="l2",
        min_df=20,
        max_df=0.9
    )
    tokenized_docs = (
        (
            term.lemma_ for term in textacy.extract.terms(doc, ngs=3, ents=True, ncs=True) if term.lemma_ not in STOPLIST
        ) for doc in corpus
    )
    doc_term_matrix = vectorizer.fit_transform(tokenized_docs)

    topicmodel = textacy.tm.TopicModel("nmf", 15)
    topicmodel.fit(doc_term_matrix)
    time5 = time.time()
    if debug is True:
        print(f"Done fitting topics in {time5-time4} sec.")

    topics = {}
    for topic_idx, top_terms in topicmodel.top_topic_terms(
        vectorizer.id_to_term, top_n=20
    ):
        topics[str(topic_idx)] = " ".join(top_terms)

    topTopic = [(x[0], x[1][0]) for x in topicmodel.top_doc_topics(topicmodel.get_doc_topic_matrix(doc_term_matrix), top_n=1)]

    dfTopic = pd.DataFrame(topTopic)
    basedata.insert(3, 'topic', dfTopic[1].values)
    sizeDict = {key: 0.5 + 100 / val for key, val in basedata.topic.value_counts().to_dict().items()}
    sizes = basedata.topic.apply(lambda x: sizeDict[x])
    basedata.insert(4, 'sizes', sizes)
    if debug is True:
        print(f"Done in {time5-starttime} sec total.")
    return topics, basedata