import matplotlib.pyplot as plt
import setup_problem
from sklearn.base import BaseEstimator, RegressorMixin
import numpy as np
import nodes
import graph
import plot_utils
from sklearn.datasets import make_blobs
import pandas as pd
import os
import patsy
from patsy import dmatrices
import statsmodels.api as sm

class PoissonRegression(BaseEstimator, RegressorMixin):
    ''' Poisson regression with computation graph   '''
    def __init__(self, step_size=0.0005, max_num_epochs= 500, init_param_scale=0.01):
        self.max_num_epochs = max_num_epochs
        self.step_size = step_size
        self.init_param_scale = init_param_scale

        # Build computation graph
        self.x = nodes.ValueNode(node_name="x") # to hold a vector input
        self.y = nodes.ValueNode(node_name="y") # to hold a scalar response
        self.w = nodes.ValueNode(node_name="w") # to hold the parameter vector
        self.b = nodes.ValueNode(node_name="b") # to hold the bias parameter (scalar)

        self.affine = nodes.VectorScalarAffineNode(self.w, self.x, self.b, node_name="affine") # to hold a affine transform
        #self.active = nodes.ExpNode(self.affine, node_name="active") # to hold a activation function node using exp
        self.prediction = nodes.ExpNode(self.affine, node_name="predict") # to hold a prediction node
        self.objective = nodes.PoissonLikehoodNode(self.prediction, self.y,
                                                    node_name="objective") # to hold a negative log likehood node
        # Group nodes into types to construct computation graph function
        self.inputs = [self.x]
        self.outcomes = [self.y]
        self.parameters = [self.w, self.b]

        self.graph = graph.ComputationGraphFunction(self.inputs, self.outcomes,
                                                          self.parameters, self.prediction,
                                                          self.objective)

    def fit(self, X, y):
        num_instances, num_ftrs = X.shape
        y = y.reshape(-1)

        ## TODO: Initialize parameters (small random numbers -- not all 0, to break symmetry )
        s = self.init_param_scale
        #init_values = None
        # ## TODO

        #init_values = {"w": np.zeros(num_ftrs), "b": np.array(0.0)}
        init_values = {"w": np.random.randn(num_ftrs), "b": np.random.randn(1)}

        self.graph.set_parameters(init_values)

        for epoch in range(self.max_num_epochs):
            shuffle = np.random.permutation(num_instances)
            epoch_obj_tot = 0.0
            for j in shuffle:
                obj, grads = self.graph.get_gradients(input_values = {"x": X[j]},
                                                    outcome_values = {"y": y[j]})
                #print(obj)
                epoch_obj_tot += obj
                # Take step in negative gradient direction
                steps = {}
                for param_name in grads:
                    steps[param_name] = -self.step_size * grads[param_name]
                    self.graph.increment_parameters(steps)
                #pdb.set_trace()

            if epoch % 5 == 0:
                print("Epoch ",epoch,": average objective value=",epoch_obj_tot/num_instances)

    def Poisson(self, lamda, k):
        ''' computing Poisson value
            parameters:
            lamda: Poisson parameter
            k: class
        '''
        pro = lamda**k/np.math.factorial(k)*np.exp(-lamda)
        return pro

    def predict(self,X, y=None):
        try:
            getattr(self, "graph")
        except AttributeError:
            raise RuntimeError("You must train classifer before predicting data!")

        num_instances = X.shape[0]
        k_space = range(4000)
        preds = np.zeros(num_instances)
        probabilities = np.zeros((num_instances, len(k_space)))
        for j in range(num_instances):
            preds[j] = self.graph.get_prediction(input_values={"x":X[j]})
            for k in range(len(k_space)):
                probabilities[j][k] = self.Poisson(preds[j], k)

        return preds, probabilities

def main():
    '''
    # Create the  training data
    np.random.seed(2)
    X, y = make_blobs(n_samples=300,cluster_std=.25, centers=np.array([(3,5),(7,1),(10,8)]))
    plt.scatter(X[:, 0], X[:, 1], c=y, s=50)
    plt.show()

    print(X.shape, y.shape)
    print(X[1],y[1])
    '''
    #Create a pandas DataFrame for the counts data set.
    cur_dir = os.path.split(os.path.realpath(__file__))[0]
    file_name = "%s\\nyc_bb_bicyclist_counts.csv"%(cur_dir)
    print(file_name)
    df = pd.read_csv(file_name, header=0, infer_datetime_format=True, parse_dates=[0], index_col=[0])

    #Add a few derived regression variables.
    ds = df.index.to_series()
    df['MONTH'] = ds.dt.month
    df['DAY_OF_WEEK'] = ds.dt.dayofweek
    df['DAY'] = ds.dt.day

    #Create the training and testing data sets.
    mask = np.random.rand(len(df)) < 0.8
    df_train = df[mask]
    df_test = df[~mask]
    print('Training data set length='+str(len(df_train)))
    print('Testing data set length='+str(len(df_test)))

    #Setup the regression expression in patsy notation. We are telling patsy that BB_COUNT is our dependent variable and
    # it depends on the regression variables: DAY, DAY_OF_WEEK, MONTH, HIGH_T, LOW_T and PRECIP.
    expr = """BB_COUNT ~ DAY  + DAY_OF_WEEK + MONTH + HIGH_T + LOW_T + PRECIP"""

    #Set up the X and y matrices
    y_train, X_train = dmatrices(expr, df_train, return_type='dataframe')
    y_test, X_test = dmatrices(expr, df_test, return_type='dataframe')

    print(type(y_train),type(X_train))
    X = np.array(X_train)
    y = np.array(y_train)
    print(type(y),type(X))

    estimator = PoissonRegression(step_size=0.005, max_num_epochs=100)
    estimator.fit(X, y)
    lamda, probability = estimator.predict(X)
    #print(estimator.w.out, estimator.b.out)
    #print(y,lamda,probability)
    print(np.argmax(probability,1))

    '''
    plt.figure()
    line = plt.plot(range(5))[0]                # plot函数返回的是一个列表,因为可以同时画多条线的哦;
    line.set_color('r')
    line.set_linewidth(2.0)
    plt.show()
    '''
if __name__ == '__main__':
    main()


from django.shortcuts import reverse

from rest_framework import serializers

from .models import User, CommonParticipant

def validate_ntub_email(email: str):
    if not email.endswith('@ntub.edu.tw'):
        raise serializers.ValidationError('This is not a ntub email.')

class LoginSerializer(serializers.Serializer):
    access_token = serializers.CharField(help_text='Google OAuth2 AccessToken')

class UserSerializer(serializers.ModelSerializer):
    url = serializers.SerializerMethodField()

    class Meta:
        model = User
        fields = ('id', 'username', 'email', 'code', 'groups', 'url', 'role')

    def get_url(self, user):
        return self.context['request'] \
            .build_absolute_uri(reverse('ical', kwargs=dict(code=user.code)))

class CommonParticipantSerializer(serializers.ModelSerializer):
    creator = serializers.CharField(source='creator.email', read_only=True)
    participant = serializers.SerializerMethodField()
    emails = serializers.ListField(
        child=serializers.EmailField(validators=[validate_ntub_email]),
        write_only=True,
        required=True,
    )

    class Meta:
        model = CommonParticipant
        fields = ('id', 'title', 'creator', 'participant', 'emails')
        read_only_fields = ('id', 'creator', 'participant')

    def get_participant(self, common):
        return common.participant.values_list('email', flat=True)

    def create_participant_from_common(self, common_participant, emails):
        in_db_emails = User.objects \
            .filter(email__in=emails) \
            .values_list('email', flat=True)

        new_emails = set(emails) - set(in_db_emails)
        User.objects.bulk_create([User(email=email) for email in new_emails])
        common_participant.participant.add(
            *User.objects.filter(email__in=emails),
        )

    def create(self, validated_data):
        emails = validated_data.pop('emails')
        common_participant = CommonParticipant.objects.create(**validated_data)
        self.create_participant_from_common(common_participant, emails)

        return common_participant

    def update(self, instance, validated_data):
        emails = validated_data.pop('emails', None)
        common_participant = super().update(instance, validated_data)

        if emails is None:
            return common_participant

        CommonParticipant.participant.through \
            .objects \
            .filter(commonparticipant=instance) \
            .exclude(user__email__in=emails) \
            .delete()

        self.create_participant_from_common(common_participant, emails)

        return common_participant


# -*- coding: utf-8 -*-

import random
from . import utils
from .utils import format_percent, format_2float, format_100p, get_period_name
import numpy as np
import pandas as pd
from pandas.core.base import PandasObject
from tabulate import tabulate
from matplotlib import pyplot as plt
import sklearn.manifold
import sklearn.cluster
import sklearn.covariance
from scipy.optimize import minimize
import scipy.stats
from scipy.stats import t

def set_riskfree_rate(rf, update_all=False):

    """
    设置年化无风险利率，供PerformanceStats使用。
    影响所有PerformanceStats实例，除非其属性已被覆盖
    参数：
    rf (float): 年化利率
    update_all (bool): 更新所以实例的值
    """
    PerformanceStats._yearly_rf = rf
    PerformanceStats._monthly_rf = (np.power(1+rf, 1./12.) - 1.) * 12
    PerformanceStats._daily_rf = (np.power(1+rf, 1./252.) - 1.) * 252

    if update_all:
        from gc import get_objects
        for obj in get_objects():
            if isinstance(obj, PerformanceStats):
                obj.set_riskfree_rate(rf)

class PerformanceStats(object):

    """
    价格序列表现评估的类，提供绘图和统计的方法
    参数：
    price (Series): 价格Series

    属性:
    name (str): 即price series的name
    return_table (DataFrame): 月度和年度收益
    lookback_returns (Series): 不同时间周期的收益(1m, 3m, 6m, ytd...)，ytd: year to date
    stats (Series): 包含所有统计量的Series
    """

    # 用于计算夏普率的无风险收益
    _yearly_rf = 0.
    _monthly_rf = 0.
    _daily_rf = 0.

    def __init__(self, prices):
        super(PerformanceStats, self).__init__()
        self.prices = prices
        self.name = self.prices.name
        self._start = self.prices.index[0]
        self._end = self.prices.index[-1]

        self._update(self.prices)

    def set_riskfree_rate(self, rf):

        """
        设置年化无风险利率，更新全部统计计算结果
        参数：
        rf (float): 年化利率
        """

        self._yearly_rf = rf
        self._monthly_rf = (np.power(1+rf, 1./12.) - 1.) * 12
        self._daily_rf = (np.power(1+rf, 1./252.) - 1.) * 252

        self._update(self.prices)

    def _update(self, obj):
        self._calculate(obj)
        self.return_table = pd.DataFrame(self.return_table).T
        if len(self.return_table.columns) == 13:
            self.return_table.columns = ['Jan', 'Feb', 'Mar', 'Apr', 'May',
                                         'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
                                         'Nov', 'Dec', 'YTD']

        self.lookback_returns = pd.Series(
            [self.mtd, self.three_month, self.six_month, self.ytd,
             self.one_year, self.three_year, self.five_year,
             self.ten_year, self.cagr],
            ['mtd', '3m', '6m', 'ytd', '1y', '3y', '5y', '10y', 'incep'])
        self.lookback_returns.name = self.name

        st = self._stats()
        self.stats = pd.Series(
            [getattr(self, x[0]) for x in st if x[0] is not None],
            [x[0] for x in st if x[0] is not None]).drop_duplicates()

    def _calculate(self, obj):
        # default values
        self.daily_mean = np.nan
        self.daily_vol = np.nan
        self.daily_sharpe = np.nan
        self.best_day = np.nan
        self.worst_day = np.nan
        self.total_return = np.nan
        self.cagr = np.nan
        self.incep = np.nan
        self.drawdown = np.nan
        self.max_drawdown = np.nan
        self.drawdown_details = np.nan
        self.daily_skew = np.nan
        self.daily_kurt = np.nan
        self.monthly_returns = np.nan
        self.avg_drawdown = np.nan
        self.avg_drawdown_days = np.nan
        self.monthly_mean = np.nan
        self.monthly_vol = np.nan
        self.monthly_sharpe = np.nan
        self.best_month = np.nan
        self.worst_month = np.nan
        self.mtd = np.nan
        self.three_month = np.nan
        self.pos_month_perc = np.nan
        self.avg_up_month = np.nan
        self.avg_down_month = np.nan
        self.monthly_skew = np.nan
        self.monthly_kurt = np.nan
        self.six_month = np.nan
        self.yearly_returns = np.nan
        self.ytd = np.nan
        self.one_year = np.nan
        self.yearly_mean = np.nan
        self.yearly_vol = np.nan
        self.yearly_sharpe = np.nan
        self.best_year = np.nan
        self.worst_year = np.nan
        self.three_year = np.nan
        self.win_year_perc = np.nan
        self.twelve_month_win_perc = np.nan
        self.yearly_skew = np.nan
        self.yearly_kurt = np.nan
        self.five_year = np.nan
        self.ten_year = np.nan

        self.return_table = {}

        if len(obj) is 0:
            return

        self.start = obj.index[0]
        self.end = obj.index[-1]

        self.daily_prices = obj
        self.monthly_prices = obj.resample('M').last()
        self.yearly_prices = obj.resample('A').last()

        p = obj
        mp = self.monthly_prices
        yp = self.yearly_prices

        if len(p) is 1:
            return

        self.returns = p.to_returns()
        self.log_returns = p.to_log_returns()
        r = self.returns

        if len(r) < 2:
            return

        self.daily_mean = r.mean() * 252
        self.daily_vol = r.std() * np.sqrt(252)
        self.daily_sharpe = (self.daily_mean - self._daily_rf) / self.daily_vol
        self.best_day = r.max()
        self.worst_day = r.min()

        self.total_return = obj[-1] / obj[0] - 1
        self.ytd = self.total_return
        self.cagr = calc_cagr(p)
        self.incep = self.cagr

        self.drawdown = p.to_drawdown_series()
        self.max_drawdown = self.drawdown.min()
        self.drawdown_details = drawdown_details(self.drawdown)
        if self.drawdown_details is not None:
            self.avg_drawdown = self.drawdown_details['drawdown'].mean()
            self.avg_drawdown_days = self.drawdown_details['days'].mean()

        if len(r) < 4:
            return

        self.daily_skew = r.skew()

        if len(r[(~np.isnan(r)) & (r != 0)]) > 0:
            self.daily_kurt = r.kurt()

        self.monthly_returns = self.monthly_prices.to_returns()
        mr = self.monthly_returns

        if len(mr) < 2:
            return

        self.monthly_mean = mr.mean() * 12
        self.monthly_vol = mr.std() * np.sqrt(12)
        self.monthly_sharpe = ((self.monthly_mean - self._monthly_rf) /
                               self.monthly_vol)
        self.best_month = mr.max()
        self.worst_month = mr.min()

        self.mtd = p[-1] / mp[-2] - 1

        self.pos_month_perc = len(mr[mr > 0]) / float(len(mr) - 1)
        self.avg_up_month = mr[mr > 0].mean()
        self.avg_down_month = mr[mr <= 0].mean()

        for idx in mr.index:
            if idx.year not in self.return_table:
                self.return_table[idx.year] = {1: 0, 2: 0, 3: 0,
                                               4: 0, 5: 0, 6: 0,
                                               7: 0, 8: 0, 9: 0,
                                               10: 0, 11: 0, 12: 0}
            if not np.isnan(mr[idx]):
                self.return_table[idx.year][idx.month] = mr[idx]

        fidx = mr.index[0]
        try:
            self.return_table[fidx.year][fidx.month] = float(mp[0]) / p[0] - 1
        except ZeroDivisionError:
            self.return_table[fidx.year][fidx.month] = 0

        for idx in self.return_table:
            arr = np.array(list(self.return_table[idx].values()))
            self.return_table[idx][13] = np.prod(arr + 1) - 1

        if len(mr) < 3:
            return

        denom = p[:p.index[-1] - pd.DateOffset(months=3)]
        if len(denom) > 0:
            self.three_month = p[-1] / denom[-1] - 1

        if len(mr) < 4:
            return

        self.monthly_skew = mr.skew()

        if len(mr[(~np.isnan(mr)) & (mr != 0)]) > 0:
            self.monthly_kurt = mr.kurt()

        denom = p[:p.index[-1] - pd.DateOffset(months=6)]
        if len(denom) > 0:
            self.six_month = p[-1] / denom[-1] - 1

        self.yearly_returns = self.yearly_prices.to_returns()
        yr = self.yearly_returns

        if len(yr) < 2:
            return

        self.ytd = p[-1] / yp[-2] - 1

        denom = p[:p.index[-1] - pd.DateOffset(years=1)]
        if len(denom) > 0:
            self.one_year = p[-1] / denom[-1] - 1

        self.yearly_mean = yr.mean()
        self.yearly_vol = yr.std()
        self.yearly_sharpe = ((self.yearly_mean - self._yearly_rf) /
                              self.yearly_vol)
        self.best_year = yr.max()
        self.worst_year = yr.min()

        self.three_year = calc_cagr(p[p.index[-1] - pd.DateOffset(years=3):])

        self.win_year_perc = len(yr[yr > 0]) / float(len(yr) - 1)

        tot = 0
        win = 0
        for i in range(11, len(mr)):
            tot += 1
            if mp[i] / mp[i - 11] > 1:
                win += 1
        self.twelve_month_win_perc = float(win) / tot

        if len(yr) < 4:
            return

        self.yearly_skew = yr.skew()

        if len(yr[(~np.isnan(yr)) & (yr != 0)]) > 0:
            self.yearly_kurt = yr.kurt()

        self.five_year = calc_cagr(p[p.index[-1] - pd.DateOffset(years=5):])
        self.ten_year = calc_cagr(p[p.index[-1] - pd.DateOffset(years=10):])

        return

    def _stats(self):
        stats = [('start', 'Start', 'dt'),
                 ('end', 'End', 'dt'),
                 ('_yearly_rf', 'Risk-free rate', 'p'),
                 (None, None, None),
                 ('total_return', 'Total Return', 'p'),
                 ('daily_sharpe', 'Daily Sharpe', 'n'),
                 ('cagr', 'CAGR', 'p'),
                 ('max_drawdown', 'Max Drawdown', 'p'),
                 (None, None, None),
                 ('mtd', 'MTD', 'p'),
                 ('three_month', '3m', 'p'),
                 ('six_month', '6m', 'p'),
                 ('ytd', 'YTD', 'p'),
                 ('one_year', '1Y', 'p'),
                 ('three_year', '3Y (ann.)', 'p'),
                 ('five_year', '5Y (ann.)', 'p'),
                 ('ten_year', '10Y (ann.)', 'p'),
                 ('incep', 'Since Incep. (ann.)', 'p'),
                 (None, None, None),
                 ('daily_sharpe', 'Daily Sharpe', 'n'),
                 ('daily_mean', 'Daily Mean (ann.)', 'p'),
                 ('daily_vol', 'Daily Vol (ann.)', 'p'),
                 ('daily_skew', 'Daily Skew', 'n'),
                 ('daily_kurt', 'Daily Kurt', 'n'),
                 ('best_day', 'Best Day', 'p'),
                 ('worst_day', 'Worst Day', 'p'),
                 (None, None, None),
                 ('monthly_sharpe', 'Monthly Sharpe', 'n'),
                 ('monthly_mean', 'Monthly Mean (ann.)', 'p'),
                 ('monthly_vol', 'Monthly Vol (ann.)', 'p'),
                 ('monthly_skew', 'Monthly Skew', 'n'),
                 ('monthly_kurt', 'Monthly Kurt', 'n'),
                 ('best_month', 'Best Month', 'p'),
                 ('worst_month', 'Worst Month', 'p'),
                 (None, None, None),
                 ('yearly_sharpe', 'Yearly Sharpe', 'n'),
                 ('yearly_mean', 'Yearly Mean', 'p'),
                 ('yearly_vol', 'Yearly Vol', 'p'),
                 ('yearly_skew', 'Yearly Skew', 'n'),
                 ('yearly_kurt', 'Yearly Kurt', 'n'),
                 ('best_year', 'Best Year', 'p'),
                 ('worst_year', 'Worst Year', 'p'),
                 (None, None, None),
                 ('avg_drawdown', 'Avg. Drawdown', 'p'),
                 ('avg_drawdown_days', 'Avg. Drawdown Days', 'n'),
                 ('avg_up_month', 'Avg. Up Month', 'p'),
                 ('avg_down_month', 'Avg. Down Month', 'p'),
                 ('win_year_perc', 'Win Year %', 'p'),
                 ('twelve_month_win_perc', 'Win 12m %', 'p')]

        return stats

    def set_date_range(self, start=None, end=None):
        """
        更新统计、绘图等的日期范围。如果start和end都是None则重置到原始日期

        参数:
        start (date): 起始日期
        end (end): 终止日期
        """
        if start is None:
            start = self._start
        else:
            start = pd.to_datetime(start)

        if end is None:
            end = self._end
        else:
            end = pd.to_datetime(end)

        self._update(self.prices.ix[start:end])

    def display(self):
        """
        展示统计结果概览
        """
        print('Stats for %s from %s - %s' % (self.name, self.start, self.end))
        print('Annual risk-free rate considered: %s' % (format_percent(self._yearly_rf)))
        print('Summary:')
        data = [[format_percent(self.total_return), format_2float(self.daily_sharpe),
                 format_percent(self.cagr), format_percent(self.max_drawdown)]]
        print(tabulate(data, headers=['Total Return', 'Sharpe',
                                      'CAGR', 'Max Drawdown']))

        print('\nAnnualized Returns:')
        data = [[format_percent(self.mtd), format_percent(self.three_month), format_percent(self.six_month),
                 format_percent(self.ytd), format_percent(self.one_year), format_percent(self.three_year),
                 format_percent(self.five_year), format_percent(self.ten_year),
                 format_percent(self.incep)]]
        print(tabulate(data,
                       headers=['mtd', '3m', '6m', 'ytd', '1y',
                                '3y', '5y', '10y', 'incep.']))

        print('\nPeriodic:')
        data = [
            ['sharpe', format_2float(self.daily_sharpe), format_2float(self.monthly_sharpe),
             format_2float(self.yearly_sharpe)],
            ['mean', format_percent(self.daily_mean), format_percent(self.monthly_mean),
             format_percent(self.yearly_mean)],
            ['vol', format_percent(self.daily_vol), format_percent(self.monthly_vol),
             format_percent(self.yearly_vol)],
            ['skew', format_2float(self.daily_skew), format_2float(self.monthly_skew),
             format_2float(self.yearly_skew)],
            ['kurt', format_2float(self.daily_kurt), format_2float(self.monthly_kurt),
             format_2float(self.yearly_kurt)],
            ['best', format_percent(self.best_day), format_percent(self.best_month),
             format_percent(self.best_year)],
            ['worst', format_percent(self.worst_day), format_percent(self.worst_month),
             format_percent(self.worst_year)]]
        print(tabulate(data, headers=['daily', 'monthly', 'yearly']))

        print('\nDrawdowns:')
        data = [
            [format_percent(self.max_drawdown), format_percent(self.avg_drawdown),
             format_2float(self.avg_drawdown_days)]]
        print(tabulate(data, headers=['max', 'avg', '# days']))

        print('\nMisc:')
        data = [['avg. up month', format_percent(self.avg_up_month)],
                ['avg. down month', format_percent(self.avg_down_month)],
                ['up year %', format_percent(self.win_year_perc)],
                ['12m up %', format_percent(self.twelve_month_win_perc)]]
        print(tabulate(data))

    def display_monthly_returns(self):
        """
        展示月度收益率和过去一年收益（YTD）
        """
        data = [['Year', 'Jan', 'Feb', 'Mar', 'Apr', 'May',
                 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'YTD']]
        for k in self.return_table.index:
            r = self.return_table.ix[k].values
            data.append([k] + [format_100p(x) for x in r])
        print(tabulate(data, headers='firstrow'))

    def display_lookback_returns(self):
        """
        展示过去一段时间的收益
        """
        return self.lookback_returns.map('{:,.2%}'.format)

    def plot(self, period='m', figsize=(15, 5), title=None,
             logy=False, **kwargs):
        """
        绘图

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        logy (bool): y周对数坐标
        kwargs: 其他pandas可用的绘图参数
        """
        if title is None:
            title = '%s %s price series' % (self.name, get_period_name(period))

        ser = self._get_series(period)
        ser.plot(figsize=figsize, title=title, logy=logy, **kwargs)

    def plot_histogram(self, period='m', figsize=(15, 5), title=None,
                       bins=20, **kwargs):
        """
        给定周期，绘制直方图

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        bins (int): 直方图条柱数目
        kwargs: 其他pandas hist方法的参数
        """
        if title is None:
            title = '%s %s return histogram' % (
                self.name, get_period_name(period))

        ser = self._get_series(period).to_returns().dropna()

        plt.figure(figsize=figsize)
        ax = ser.hist(bins=bins, figsize=figsize, normed=True, **kwargs)
        ax.set_title(title)
        plt.axvline(0, linewidth=4)
        ser.plot(kind='kde')

    def _get_series(self, per):
        if per == 'y':
            per = 'a'
        return self.daily_prices.asfreq(per, 'ffill')

    def to_csv(self, sep=',', path=None):
        """
        返回csv字符串，如果path不为None，则保存至该路径

        参数:
        sep (char): 分隔符
        path (str): 如果None, 返回CSV字符串，否则保存至该路径
        """
        stats = self._stats()

        data = []
        first_row = ['Stat', self.name]
        data.append(sep.join(first_row))

        for stat in stats:
            k, n, f = stat

            if k is None:
                row = [''] * len(data[0])
                data.append(sep.join(row))
                continue

            row = [n]
            raw = getattr(self, k)
            if f is None:
                row.append(raw)
            elif f == 'p':
                row.append(format_percent(raw))
            elif f == 'n':
                row.append(format_2float(raw))
            elif f == 'dt':
                row.append(raw.strftime('%Y-%m-%d'))
            else:
                raise NotImplementedError('unsupported format %s' % f)

            data.append(sep.join(row))

        res = '\n'.join(data)

        if path is not None:
            with open(path, 'w') as fl:
                fl.write(res)
        else:
            return res

class GroupStats(dict):

    """
    GroupStats用于比较多个序列
    基于{price.name: PerformanceStats}的字典结构，并提供一些便捷方法

    序列次序保持，单个PerformanceStats对象可以通过index为止或者[]获取

    参数:
    prices (Series): 用于比较的多个序列

    属性:
    stats (DataFrame): 不同序列各一列，统计量在不同行
    lookback_returns (DataFrame): 不同回顾周期的收益率 (1m, 3m, 6m, ytd...)
    prices (DataFrame): 合并调整后的价格
    """

    def __init__(self, *prices):
        names = []
        for p in prices:
            if isinstance(p, pd.DataFrame):
                names.extend(p.columns)
            elif isinstance(p, pd.Series):
                names.append(p.name)
            else:
                print('else')
                names.append(getattr(p, 'name', 'n/a'))
        self._names = names

        self._prices = merge(*prices).dropna()

        self._prices = self._prices[self._names]

        if len(self._prices.columns) != len(set(self._prices.columns)):
            raise ValueError('One or more data series provided',
                             'have same name! Please provide unique names')

        self._start = self._prices.index[0]
        self._end = self._prices.index[-1]

        self._update(self._prices)

    def __getitem__(self, key):
        if type(key) == int:
            return self[self._names[key]]
        else:
            return self.get(key)

    def _update(self, data):
        self._calculate(data)

        self.lookback_returns = pd.DataFrame(
            {x.lookback_returns.name: x.lookback_returns for x in
             self.values()})

        self.stats = pd.DataFrame(
            {x.name: x.stats for x in self.values()})

    def _calculate(self, data):
        self.prices = data
        for c in data.columns:
            prc = data[c]
            self[c] = PerformanceStats(prc)

    def _stats(self):
        stats = [('start', 'Start', 'dt'),
                 ('end', 'End', 'dt'),
                 ('_yearly_rf', 'Risk-free rate', 'p'),
                 (None, None, None),
                 ('total_return', 'Total Return', 'p'),
                 ('daily_sharpe', 'Daily Sharpe', 'n'),
                 ('cagr', 'CAGR', 'p'),
                 ('max_drawdown', 'Max Drawdown', 'p'),
                 (None, None, None),
                 ('mtd', 'MTD', 'p'),
                 ('three_month', '3m', 'p'),
                 ('six_month', '6m', 'p'),
                 ('ytd', 'YTD', 'p'),
                 ('one_year', '1Y', 'p'),
                 ('three_year', '3Y (ann.)', 'p'),
                 ('five_year', '5Y (ann.)', 'p'),
                 ('ten_year', '10Y (ann.)', 'p'),
                 ('incep', 'Since Incep. (ann.)', 'p'),
                 (None, None, None),
                 ('daily_sharpe', 'Daily Sharpe', 'n'),
                 ('daily_mean', 'Daily Mean (ann.)', 'p'),
                 ('daily_vol', 'Daily Vol (ann.)', 'p'),
                 ('daily_skew', 'Daily Skew', 'n'),
                 ('daily_kurt', 'Daily Kurt', 'n'),
                 ('best_day', 'Best Day', 'p'),
                 ('worst_day', 'Worst Day', 'p'),
                 (None, None, None),
                 ('monthly_sharpe', 'Monthly Sharpe', 'n'),
                 ('monthly_mean', 'Monthly Mean (ann.)', 'p'),
                 ('monthly_vol', 'Monthly Vol (ann.)', 'p'),
                 ('monthly_skew', 'Monthly Skew', 'n'),
                 ('monthly_kurt', 'Monthly Kurt', 'n'),
                 ('best_month', 'Best Month', 'p'),
                 ('worst_month', 'Worst Month', 'p'),
                 (None, None, None),
                 ('yearly_sharpe', 'Yearly Sharpe', 'n'),
                 ('yearly_mean', 'Yearly Mean', 'p'),
                 ('yearly_vol', 'Yearly Vol', 'p'),
                 ('yearly_skew', 'Yearly Skew', 'n'),
                 ('yearly_kurt', 'Yearly Kurt', 'n'),
                 ('best_year', 'Best Year', 'p'),
                 ('worst_year', 'Worst Year', 'p'),
                 (None, None, None),
                 ('avg_drawdown', 'Avg. Drawdown', 'p'),
                 ('avg_drawdown_days', 'Avg. Drawdown Days', 'n'),
                 ('avg_up_month', 'Avg. Up Month', 'p'),
                 ('avg_down_month', 'Avg. Down Month', 'p'),
                 ('win_year_perc', 'Win Year %', 'p'),
                 ('twelve_month_win_perc', 'Win 12m %', 'p')]

        return stats

    def set_riskfree_rate(self, rf):

        """
        设置年化无风险收益率，计算年度、月度、天收益率，衡量表现得统计量也重新计算
        影响GroupStats对象包含的那些PerformanceStats实例

        参数:
        rf (float): 年化无风险利率
        """
        for key in self._names:
            self[key].set_riskfree_rate(rf)

    def set_date_range(self, start=None, end=None):
        """
        更新统计、绘图等的日期范围。如果start和end都是None则重置到原始日期

        参数:
        start (date): 起始日期
        end (end): 终止日期
        """
        if start is None:
            start = self._start
        else:
            start = pd.to_datetime(start)

        if end is None:
            end = self._end
        else:
            end = pd.to_datetime(end)

        self._update(self._prices.ix[start:end])

    def display(self):
        """
        展示统计结果
        """
        data = []
        first_row = ['Stat']
        first_row.extend(self._names)
        data.append(first_row)

        stats = self._stats()

        for stat in stats:
            k, n, f = stat

            if k is None:
                row = [''] * len(data[0])
                data.append(row)
                continue

            row = [n]
            for key in self._names:
                raw = getattr(self[key], k)
                if f is None:
                    row.append(raw)
                elif f == 'p':
                    row.append(format_percent(raw))
                elif f == 'n':
                    row.append(format_2float(raw))
                elif f == 'dt':
                    row.append(raw.strftime('%Y-%m-%d'))
                else:
                    raise NotImplementedError('unsupported format %s' % f)
            data.append(row)

        print(tabulate(data, headers='firstrow'))

    def display_lookback_returns(self):
        """
        展示当前回顾周期下各价格序列的收益率
        """
        return self.lookback_returns.apply(
            lambda x: x.map('{:,.2%}'.format), axis=1)

    def plot(self, period='m', figsize=(15, 5), title=None,
             logy=False, **kwargs):
        """
        绘图

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        logy (bool): y周对数坐标
        kwargs: 其他pandas可用的绘图参数
        """

        if title is None:
            title = '%s equity progression' % get_period_name(period)
        ser = self._get_series(period).rebase()
        ser.plot(figsize=figsize, logy=logy,
                 title=title, **kwargs)

    def plot_scatter_matrix(self, period='m', title=None,
                            figsize=(10, 10), **kwargs):
        """
        绘图，封装pandas的scatter_matrix

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        kwargs: 其他pandas scatter_matrix可用的绘图参数
        """
        if title is None:
            title = '%s return scatter matrix' % get_period_name(period)

        plt.figure()
        ser = self._get_series(period).to_returns().dropna()
        pd.scatter_matrix(ser, figsize=figsize, **kwargs)
        plt.suptitle(title)

    def plot_histograms(self, period='m', title=None,
                        figsize=(10, 10), **kwargs):
        """
        绘图，封装pandas的hist

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        kwargs: 其他pandas hist可用的绘图参数
        """
        if title is None:
            title = '%s return histogram matrix' % get_period_name(period)

        plt.figure()
        ser = self._get_series(period).to_returns().dropna()
        ser.hist(figsize=figsize, **kwargs)
        plt.suptitle(title)

    def plot_correlation(self, period='m', title=None,
                         figsize=(12, 6), **kwargs):
        """
        绘制相关系数的热度图

        参数:
        period (str): 绘图的时间周期，沿用pandas的记号
        figsize ((x,y)): 图片尺寸
        title (str): 图题
        kwargs: 其他pandas plot_corr_heatmap可用的绘图参数
        """
        if title is None:
            title = '%s return correlation matrix' % get_period_name(period)

        rets = self._get_series(period).to_returns().dropna()
        return rets.plot_corr_heatmap(title=title, figsize=figsize, **kwargs)

    def _get_series(self, per):
        if per == 'y':
            per = 'a'
        return self.prices.asfreq(per, 'ffill')

    def to_csv(self, sep=',', path=None):
        """
        返回csv字符串，如果path不为None，则保存至该路径

        参数:
        sep (char): 分隔符
        path (str): 如果None, 返回CSV字符串，否则保存至该路径
        """
        data = []
        first_row = ['Stat']
        first_row.extend(self._names)
        data.append(sep.join(first_row))

        stats = self._stats()

        for stat in stats:
            k, n, f = stat

            if k is None:
                row = [''] * len(data[0])
                data.append(sep.join(row))
                continue

            row = [n]
            for key in self._names:
                raw = getattr(self[key], k)
                if f is None:
                    row.append(raw)
                elif f == 'p':
                    row.append(format_percent(raw))
                elif f == 'n':
                    row.append(format_2float(raw))
                elif f == 'dt':
                    row.append(raw.strftime('%Y-%m-%d'))
                else:
                    raise NotImplementedError('unsupported format %s' % f)
            data.append(sep.join(row))

        res = '\n'.join(data)

        if path is not None:
            with open(path, 'w') as fl:
                fl.write(res)
        else:
            return res

def to_returns(prices):
    """
    计算价格序列的算术收益率，(p1 / p0) - 1

    参数:
    prices: price series
    """
    return prices / prices.shift(1) - 1

def to_log_returns(prices):
    """
    计算价格的对数收益率，ln(p1/p0)

    参数:
    prices: price series
    """
    return np.log(prices / prices.shift(1))

def to_price_index(returns, start=100):
    """
    由收益率序列计算价格指数，cumprod(1+r)

    参数:
    returns (Series): 算术收益率series
    start (number): 起始价格
    """
    return (returns.replace(to_replace=np.nan, value=0) + 1).cumprod() * start

def rebase(prices, value=100):
    """
    调整所有series到相同的起始价格，便于比较和绘图不同价格序列

    参数:
    prices: price series
    value (number): 起始价格
    """
    return prices / prices.ix[0] * value

def calc_perf_stats(obj):
    """
    计算对象的表现，返回PerformanceStats对象，其包含所有统计信息

    参数:
    obj: prices series
    """
    return PerformanceStats(obj)

def calc_stats(obj):
    """
    计算对象的表现，可以接受Series或DataFrame
    """
    if isinstance(obj, pd.Series):
        return PerformanceStats(obj)
    elif isinstance(obj, pd.DataFrame):
        return GroupStats(*[obj[x] for x in obj.columns])
    else:
        raise NotImplementedError('Unsupported type')

def to_drawdown_series(prices):
    """
    计算回撤序列，drawdown = current / hwm - 1

    参数:
    prices (Series or DataFrame): Series of prices.
    """
    drawdown = prices.copy()
    drawdown = drawdown.fillna(method='ffill')

    drawdown[np.isnan(drawdown)] = -np.Inf

    roll_max = np.maximum.accumulate(drawdown)
    drawdown = drawdown / roll_max - 1.
    return drawdown

def calc_max_drawdown(prices):
    """
    计算最大回撤
    """
    return (prices / prices.expanding(min_periods=1).max()).min() - 1

def drawdown_details(drawdown):
    """
    回撤详情，返回的DataFrame包括列：start, end, days (duration) and drawdown
    注意，其中days是实际的时间 (calendar days)，而非交易日 (trading days)

    参数:
    drawdown (pandas.Series): drawdown Series
    """
    # TODO: 此函数需要改进

    is_zero = drawdown == 0
    start = ~is_zero & is_zero.shift(1)
    start = list(start[start == True].index)

    end = is_zero & (~is_zero).shift(1)
    end = list(end[end == True].index)

    if len(start) is 0:
        return None
    if len(end) is 0:
        end.append(drawdown.index[-1])
    if start[0] > end[0]:
        start.insert(0, drawdown.index[0])
    if start[-1] > end[-1]:
        end.append(drawdown.index[-1])

    result = pd.DataFrame(columns=('start', 'end', 'days', 'drawdown'),
                          index=range(0, len(start)))

    for i in range(0, len(start)):
        dd = drawdown[start[i]:end[i]].min()
        result.ix[i] = (start[i], end[i], (end[i] - start[i]).days, dd)

    return result

def calc_cagr(prices):
    """
    计算价格序列的复合年化收益率CAGR (compound annual growth rate)

    参数:
    prices (pandas.Series): prices Series

    返回:
    CAGR (float)
    """
    start = prices.index[0]
    end = prices.index[-1]
    return (prices.ix[-1] / prices.ix[0]) ** (1 / year_frac(start, end)) - 1

def calc_risk_return_ratio(price):
    """
    计算return/risk，基本上等于未经无风险利率调整的夏普率
    """
    return price.mean() / price.std()

def calc_information_ratio(returns, benchmark_returns):
    """
    信息率：IR = E(r_p - r_b) / std(r_p - r_b)
    http://en.wikipedia.org/wiki/Information_ratio
    """
    diff_rets = returns - benchmark_returns
    diff_std = diff_rets.std()

    if np.isnan(diff_std) or diff_std == 0:
        return 0.0

    return diff_rets.mean() / diff_std

def calc_prob_mom(returns, other_returns):
    """
    Probabilistic momentum
    一项资产表现优于另一资产的概率或置信程度

    Source:
    http://cssanalytics.wordpress.com/2014/01/28/are-simple-momentum-strategies-too-dumb-introducing-probabilistic-momentum/
    """
    return t.cdf(returns.calc_information_ratio(other_returns), len(returns) - 1)

def calc_total_return(prices):
    """
    计算series总收益，last / first - 1
    """
    return (prices.ix[-1] / prices.ix[0]) - 1

def year_frac(start, end):
    """
    计算两个日期间的时间长度，以年为单位

    参数:
    start (datetime): start date
    end (datetime): end date

    """
    if start > end:
        raise ValueError('start cannot be larger than end')

    return (end - start).total_seconds() / 31557600

def merge(*series):
    """
    Merge Series and/or DataFrames
    """
    dfs = []
    for s in series:
        if isinstance(s, pd.DataFrame):
            dfs.append(s)
        elif isinstance(s, pd.Series):
            tmpdf = pd.DataFrame({s.name: s})
            dfs.append(tmpdf)
        else:
            raise NotImplementedError('Unsupported merge type')

    return pd.concat(dfs, axis=1)

def drop_duplicate_cols(df):
    """
    移除冗余列
    """
    names = set(df.columns)
    for n in names:
        if len(df[n].shape) > 1:
            sub = df[n]
            sub.columns = ['%s-%s' % (n, x) for x in range(sub.shape[1])]
            keep = sub.count().idxmax()
            del df[n]
            df[n] = sub[keep]

    return df

def to_monthly(series, method='ffill', how='end'):
    """
    将时间序列转换成月频，使用asfreq_actual
    """
    return series.asfreq_actual('M', method=method, how=how)

def asfreq_actual(series, freq, method='ffill', how='end', normalize=False):
    """
    将时间序列转换成某频率，类似于pandas的asfreq函数，但是保证真实的日期
    例如，如果最后一个数据点在一月为29号，则用29号，而不是31号
    """
    orig = series
    is_series = False
    if isinstance(series, pd.Series):
        is_series = True
        name = series.name if series.name else 'data'
        orig = pd.DataFrame({name: series})

    t = pd.concat([orig, pd.DataFrame({'dt': orig.index.values},
                                      index=orig.index.values)], axis=1)
    dts = t.asfreq(freq=freq, method=method, how=how,
                   normalize=normalize)['dt']

    res = orig.ix[dts.values]

    if is_series:
        return res[name]
    else:
        return res

def calc_inv_vol_weights(returns):
    """
    计算每列波动率的倒数作为权重，这样通过控制每个品种的头寸可以使之波动率都在同一水平

    已剔除全零或NaN的列

    Returns:
        Series {col_name: weight}
    """
    vol = 1.0 / returns.std()
    vol[np.isinf(vol)] = np.NaN
    vols = vol.sum()
    return vol / vols

def calc_mean_var_weights(returns, weight_bounds=(0., 1.),
                          rf=0.,
                          covar_method='ledoit-wolf'):
    """
    计算DataFrame每列收益的均值-方差权重

    参数:
    returns (DataFrame): 多资产的收益率
    weight_bounds ((low, high)): 用于优化的权重界限
    rf (float): 无风险收益率
    covar_method (str): 协方差矩阵的估计方法，目前支持ledoit-wolf, standard

    返回:
        Series {col_name: weight}

    """
    def fitness(weights, exp_rets, covar, rf):
        mean = sum(exp_rets * weights)
        var = np.dot(np.dot(weights, covar), weights)
        util = (mean - rf) / np.sqrt(var)  # Sharpe ratio
        return -util

    n = len(returns.columns)

    exp_rets = returns.mean()

    if covar_method == 'ledoit-wolf':
        covar = sklearn.covariance.ledoit_wolf(returns)[0]
    elif covar_method == 'standard':
        covar = returns.cov()
    else:
        raise NotImplementedError('covar_method not implemented')

    weights = np.ones([n]) / n
    bounds = [weight_bounds for i in range(n)]
    constraints = ({'type': 'eq', 'fun': lambda W: sum(W) - 1.})
    optimized = minimize(fitness, weights, (exp_rets, covar, rf),
                         method='SLSQP', constraints=constraints,
                         bounds=bounds)

    if not optimized.success:
        raise Exception(optimized.message)

    return pd.Series({returns.columns[i]: optimized.x[i] for i in range(n)})

def get_num_days_required(offset, period='d', perc_required=0.90):
    """
    估计需要的交易日数，用以检查给定周期内数据集是否正常

    参数:
    offset (DateOffset): Offset (lookback) period.
    period (str): Period string, 'd', 'm', 'y'
    perc_required (float): percentage of number of days expected required.
    """
    x = pd.to_datetime('2010-01-01')
    delta = x - (x - offset)
    # 粗略估计交易日数
    days = delta.days * 0.69

    if period == 'd':
        req = days * perc_required
    elif period == 'm':
        req = (days / 20) * perc_required
    elif period == 'y':
        req = (days / 252) * perc_required
    else:
        raise NotImplementedError(
            'Period not supported. Supported periods are d, m, y')

    return req

def calc_clusters(returns, n=None, plot=False):
    """
    基于k-means聚类

    参数:
    returns (pd.DataFrame): DataFrame of returns
    n (int): 聚类数，如果没有指定则自动识别
    plot (bool): 是否绘图

    返回:
    dict: {cluster No.: [col names]}
    """
    corr = returns.corr()
    diss = 1 - corr # dissimilarity matrix

    # scale down to 2 dimensions using MDS (multi-dimensional scaling)
    mds = sklearn.manifold.MDS(dissimilarity='precomputed')
    xy = mds.fit_transform(diss)

    def routine(k):
        km = sklearn.cluster.KMeans(n_clusters=k)
        km_fit = km.fit(xy)
        labels = km_fit.labels_
        centers = km_fit.cluster_centers_
        mappings = dict(zip(returns.columns, labels))

        totss = 0
        withinss = 0
        avg = np.array([np.mean(xy[:, 0]), np.mean(xy[:, 1])])
        for idx, lbl in enumerate(labels):
            withinss += sum((xy[idx] - centers[lbl]) ** 2)
            totss += sum((xy[idx] - avg) ** 2)
        pvar_expl = 1.0 - withinss / totss

        return mappings, pvar_expl, labels

    if n:
        result = routine(n)
    else:
        n = len(returns.columns)
        n1 = int(np.ceil(n * 0.6666666666))
        for i in range(2, n1 + 1):
            result = routine(i)
            if result[1] > 0.9:
                break

    if plot:
        fig, ax = plt.subplots()
        ax.scatter(xy[:, 0], xy[:, 1], c=result[2], s=90)
        for i, txt in enumerate(returns.columns):
            ax.annotate(txt, (xy[i, 0], xy[i, 1]), size=14)

    # sanitize return value
    tmp = result[0]
    # {cluster: [list of symbols], cluster2: [...]}
    inv_map = {}
    for k, v in tmp.iteritems():
        inv_map[v] = inv_map.get(v, [])
        inv_map[v].append(k)

    return inv_map

def calc_ftca(returns, threshold=0.5):
    """
    Implementation of David Varadi's Fast Threshold Clustering Algorithm (FTCA).

    http://cssanalytics.wordpress.com/2013/11/26/fast-threshold-clustering-algorithm-ftca/

    比使用k-means聚类更稳定，如果需要得到更多类，threshold设置大些

    参数:
    returns (DataFrame)：每列对应一种资产
    threshold (float): 用于控制序列相关程度的阈值，从而获得不同数量的聚类

    返回:
    dict: {cluster No.: [col names]}

    """
    # cluster index (name)
    i = 0
    # correlation matrix
    corr = returns.corr()
    # remaining securities to cluster
    remain = list(corr.index.copy())
    n = len(remain)
    res = {}

    while n > 0:
        # if only one left then create cluster and finish
        if n == 1:
            i += 1
            res[i] = remain
            n = 0
        # if not then we have some work to do
        else:
            # filter down correlation matrix to current remain
            cur_corr = corr[remain].ix[remain]
            # get mean correlations, ordered
            mc = cur_corr.mean().order()
            # get lowest and highest mean correlation
            low = mc.index[0]
            high = mc.index[-1]

            # case if corr(high,low) > threshold
            if corr[high][low] > threshold:
                i += 1

                # new cluster for high and low
                res[i] = [low, high]
                remain.remove(low)
                remain.remove(high)

                rmv = []
                for x in remain:
                    avg_corr = (corr[x][high] + corr[x][low]) / 2.0
                    if avg_corr > threshold:
                        res[i].append(x)
                        rmv.append(x)
                [remain.remove(x) for x in rmv]

                n = len(remain)

            # otherwise we are creating two clusters - one for high
            # and one for low
            else:
                # add cluster with HC
                i += 1
                res[i] = [high]
                remain.remove(high)
                remain.remove(low)

                rmv = []
                for x in remain:
                    if corr[x][high] > threshold:
                        res[i].append(x)
                        rmv.append(x)
                [remain.remove(x) for x in rmv]

                i += 1
                res[i] = [low]

                rmv = []
                for x in remain:
                    if corr[x][low] > threshold:
                        res[i].append(x)
                        rmv.append(x)
                [remain.remove(x) for x in rmv]

                n = len(remain)

    return res

def limit_weights(weights, limit=0.1):
    """
    根据权重上限，重新按比例分配超限资产的头寸至其他资产

    例:
    weights = {a: 0.7, b: 0.2, c: 0.1}
    limit = 0.5
    a (0.7-0.5) -> b, c => {a: 0.5, b: 0.33, c: 0.167}

    参数:
    weights (Series): 描述权重的序列
    limit (float): 权重上限
    """
    if 1.0 / limit > len(weights):
        raise ValueError('invalid limit -> 1 / limit must be <= len(weights)')

    if isinstance(weights, dict):
        weights = pd.Series(weights)

    if np.round(weights.sum(), 1) != 1.0:
        raise ValueError('Expecting weights (that sum to 1) - sum is %s'
                         % weights.sum())

    res = np.round(weights.copy(), 4)
    to_rebalance = (res[res > limit] - limit).sum()

    ok = res[res < limit]
    ok += (ok / ok.sum()) * to_rebalance

    res[res > limit] = limit
    res[res < limit] = ok

    if not np.all([x <= limit for x in res]):
        return limit_weights(res, limit=limit)

    return res

def random_weights(n, bounds=(0., 1.), total=1.0):
    """
    生成伪随机权重，用于benchmark

    参数:
    n (int): 随机权重的数目
    bounds ((low, high)): 每个权重的边界
    total (float): 权重加和

    """
    low = bounds[0]
    high = bounds[1]

    if high < low:
        raise ValueError('Higher bound must be greater or equal to lower bound')

    if n * high < total or n * low > total:
        raise ValueError('solution not possible with given n and bounds')

    w = [0] * n
    tgt = -float(total)

    for i in range(n):
        rn = n - i - 1
        rhigh = rn * high
        rlow = rn * low

        lowb = max(-rhigh - tgt, low)
        highb = min(-rlow - tgt, high)

        rw = random.uniform(lowb, highb)
        w[i] = rw

        tgt += rw

    random.shuffle(w)
    return w

def plot_heatmap(data, title='Heatmap', show_legend=True,
                 show_labels=True, label_fmt='.2f',
                 vmin=None, vmax=None,
                 figsize=None,
                 cmap='RdBu', **kwargs):
    """
    绘制热度图，基于matplotlib的pcolor.

    参数:
    data (DataFrame): 如相关矩阵
    title (string): 图题
    show_legend (bool): color bar
    show_labels (bool): value labels
    label_fmt (str): Label format string
    vmin (float): Min value for scale
    vmax (float): Max value for scale
    cmap (string): Color map
    kwargs: matplotlib pcolor的其他参数
    """
    fig, ax = plt.subplots(figsize=figsize)

    heatmap = ax.pcolor(data, vmin=vmin, vmax=vmax, cmap=cmap)
    ax.invert_yaxis()

    plt.title(title)

    if show_legend:
        fig.colorbar(heatmap)

    if show_labels:
        vals = data.values
        for x in range(data.shape[0]):
            for y in range(data.shape[1]):
                plt.text(x + 0.5, y + 0.5, format(vals[y, x], label_fmt),
                         horizontalalignment='center',
                         verticalalignment='center',
                         color='w')

    plt.yticks(np.arange(0.5, len(data.index), 1), data.index)
    plt.xticks(np.arange(0.5, len(data.columns), 1), data.columns)

    plt.show()

def plot_corr_heatmap(data, **kwargs):
    """
    相关系数的热度图
    """
    return plot_heatmap(data.corr(), vmin=-1, vmax=1, **kwargs)

def rollapply(data, window, fn):
    """
    对滚动窗口window内的data应用函数fn计算

    参数:
    data (Series/DataFrame): Series或DataFrame
    window (int): 滚动窗口大小
    fn (function): 应用于窗口的函数

    返回：
    依据data类型返回计算结果
    """
    # TODO：pandas自身也有几种实现，看是否有必要
    res = data.copy()
    res[:] = np.nan
    n = len(data)

    if window > n:
        return res

    for i in range(window - 1, n):
        res.iloc[i] = fn(data.iloc[i - window + 1:i + 1])

    return res

def _winsorize_wrapper(x, limits):
    """
    封装scipy winsorize函数以剔除NaN
    """
    if hasattr(x, 'dropna'):
        if len(x.dropna()) == 0:
            return x

        x[~np.isnan(x)] = scipy.stats.mstats.winsorize(x[~np.isnan(x)],
                                                       limits=limits)
        return x
    else:
        return scipy.stats.mstats.winsorize(x, limits=limits)

def winsorize(x, axis=0, limits=0.01):
    """
    Winsorize values based on limits
    """
    x = x.copy()

    if isinstance(x, pd.DataFrame):
        return x.apply(_winsorize_wrapper, axis=axis, args=(limits, ))
    else:
        return pd.Series(_winsorize_wrapper(x, limits).values,
                         index=x.index)

def rescale(x, min=0., max=1., axis=0):
    """
    调整值到[min, max]范围
    """
    def innerfn(x, min, max):
        return np.interp(x, [np.min(x), np.max(x)], [min, max])

    if isinstance(x, pd.DataFrame):
        return x.apply(innerfn, axis=axis, args=(min, max,))
    else:
        return pd.Series(innerfn(x, min, max), index=x.index)

def annualize(returns, durations, one_year=365.):
    """
    年化收益率，(1 + returns) ** (1 / (durations / one_year)) - 1
    """
    return (1. + returns) ** (1. / (durations / one_year)) - 1.

def extend_pandas():
    """
    扩展pandas的PandasObject (Series, DataFrame)，用上述定义的函数

    例如:
        prices_df.to_returns().dropna().calc_clusters()
    """
    PandasObject.to_returns = to_returns
    PandasObject.to_log_returns = to_log_returns
    PandasObject.to_price_index = to_price_index
    PandasObject.rebase = rebase
    PandasObject.calc_perf_stats = calc_perf_stats
    PandasObject.to_drawdown_series = to_drawdown_series
    PandasObject.calc_max_drawdown = calc_max_drawdown
    PandasObject.calc_cagr = calc_cagr
    PandasObject.calc_total_return = calc_total_return
    PandasObject.as_percent = utils.as_percent
    PandasObject.as_format = utils.as_format
    PandasObject.to_monthly = to_monthly
    PandasObject.asfreq_actual = asfreq_actual
    PandasObject.drop_duplicate_cols = drop_duplicate_cols
    PandasObject.calc_information_ratio = calc_information_ratio
    PandasObject.calc_prob_mom = calc_prob_mom
    PandasObject.calc_risk_return_ratio = calc_risk_return_ratio
    PandasObject.calc_inv_vol_weights = calc_inv_vol_weights
    PandasObject.calc_mean_var_weights = calc_mean_var_weights
    PandasObject.calc_clusters = calc_clusters
    PandasObject.calc_ftca = calc_ftca
    PandasObject.calc_stats = calc_stats
    PandasObject.plot_heatmap = plot_heatmap
    PandasObject.plot_corr_heatmap = plot_corr_heatmap
    PandasObject.rollapply = rollapply
    PandasObject.winsorize = winsorize
    PandasObject.rescale = rescale


class Solution(object):
    def subarraySum(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: int
        """
        ret = 0
        sums = 0
        s = dict()
        count = 0
        s[0] = 1
        for num in nums:
            sums += num
            if sums-k in s:
                count += s[sums-k]
            s[sums] = s.get(sums, 0) + 1
        return count


import unittest
from pable import Table, Row, Cell, Separator, Style, InvalidOptionError

class SeparatorTest(unittest.TestCase):

    def test_separator_render_up(self):
        table = Table([['a', 'b']])
        render = Separator(table).render_up()
        self.assertEqual(render, '+---+---+\n')

    def test_separator_render_down(self):
        table = Table([['a', 'b']])
        render = Separator(table).render_down()
        self.assertEqual(render, '+---+---+')

if __name__ == '__main__':
    unittest.main()


from flask import Flask, render_template, redirect, request
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
db = SQLAlchemy(app)

class Todo(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    task = db.Column(db.String(200), nullable=False)
    dateTime = db.Column(db.DateTime, default=datetime.now)

    def __repr__(self):
        return '<Task %r>' % self.id

@app.route('/', methods=['POST', 'GET'])
def index():
    if request.method == 'POST':
        task_item = request.form['task']
        new_task = Todo(task = task_item)

        try:
            db.session.add(new_task)
            db.session.commit()
            return redirect('/')
        except:
            return 'The task could not be added. Please retry.'
    else:
        tasks = Todo.query.order_by(Todo.id).all()
        return render_template("index.html", tasks=tasks)

@app.route('/delete/<int:id>')
def delete(id):
    delTask = Todo.query.get_or_404(id)

    try:
        db.session.delete(delTask)
        db.session.commit()
        return redirect('/')
    except:
        return 'Task could not be deleted. Please retry.'

@app.route('/update/<int:id>', methods = ['GET','POST'])
def update(id):
    utask = Todo.query.get_or_404(id)
    if request.method == 'POST':
        if utask.task != request.form['task']:
            utask.task = request.form['task']
            utask.dateTime = datetime.now()

            try:
                db.session.commit()
                return redirect('/')
            except:
                return 'Task could not be updated. Please retry.'
        else:
            return redirect('/')
    else:
        return render_template('update.html', task = utask)

if __name__=="__main__":
    app.run(debug=True)


import pygame as pg
from levels import Level
from resources import ResourcesContainer
from params import *
from npcs import Collider

def collided(sprite1, sprite2):
	is_collided = False
	collider1 = None
	collider2 = None
	try:
		collider1 = sprite1.collider
	except AttributeError:
		collider1 = sprite1.rect
	try:
		collider2 = sprite2.collider
	except AttributeError:
		collider2 = sprite2.rect

	if isinstance(collider1, Collider):
		return collider1.collide(collider2)
	elif isinstance(collider2, Collider):
		return collider2.collide(collider1)
	else:
		return pg.sprite.collide_rect(sprite1, sprite2)

def main():
	pg.init()
	screen = pg.display.set_mode(DISPLAY_SIZE)
	screen_rect = pg.display.get_surface().get_rect()
	resources = ResourcesContainer()
	resources.load_all()

	#start main theme
	pg.mixer.music.play(loops=-1)

	clock = pg.time.Clock()

	level = Level(resources)
	level.start()

	#==========main game loop========
	running = True
	while running:
		clock.tick(60)

		#events
		for e in pg.event.get():
			if e.type == pg.QUIT:
				running = False
			elif e.type == pg.KEYDOWN and e.unicode == 'r':
				level.start()

		#collide
		#rigid with ground
		for r in level.Grigid:
			r.rect.inflate_ip((2 * abs(r.speed), 2 * abs(r.speed_y)))
		for r, gl in pg.sprite.groupcollide(level.Grigid, level.Gground, False, False).items():
			for g in gl:
				r.collide_static_tile(g)

		for r in level.Grigid:
			r.rect.inflate_ip((-2 * abs(r.speed), -2 * abs(r.speed_y)))

		#update stuff
		level.Gdynamic.update()

		#check if we collide any enemy
		for npc, enemies in pg.sprite.groupcollide(level.GmainNpc, level.Genemies, False, False, collided).items():
			npc.die(enemies)

		#check if any item gone outside the border
		for o in level.Gdynamic:
			if not screen_rect.colliderect(o.rect):
				o.out_of_gamefield()

		#rigid with ground
		for r, gl in pg.sprite.groupcollide(level.Grigid, level.Gground, False, False).items():
			for g in gl:
				r.collide_static_tile(g)

		#draw stuff
		bg = pg.Surface(DISPLAY_SIZE)
		if resources.bg is not None:
			bg.blit(resources.bg, (0, 0))
		else:
			bg.fill(pg.Color(BG_COLOR))

		font = pg.font.SysFont('verdana', size = 14)
		fps_text = font.render('FPS: {}'.format(int(clock.get_fps())), True, pg.Color('#111111'))
		bg.blit(fps_text, (10, 10))

		screen.blit(bg, dest = (0,0))
		level.Gdrawables.draw(screen)
		pg.display.update()

if __name__ == '__main__':
	main()


fname = input('enter a file name: ')
try:
    fhand = open(fname)
except:
    print('Not a file name!')
    quit()

counts = dict()

for line in fhand:
    if line[:4] == 'From':
        words = line.split()
        if words[1] not in counts:
            counts[words[1]] = 1
        else:
            counts[words[1]] += 1

for i in counts:
    counts[i] = int(counts[i]/2)

largest = None

for interval in counts:
    if largest == None or interval > largest:
        largest = interval

print(f'{largest} {counts[largest]}')


'''
Copyright (C) 2016 Travis DeWolf

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
import numpy as np
import vrep

try:
    # close any open connections
    vrep.simxFinish(-1)
    # Connect to the V-REP continuous server
    clientID = vrep.simxStart('127.0.0.1', 19997, True, True, 500, 5)

    if clientID != -1: # if we connected successfully
        print ('Connected to remote API server')

        # --------------------- Setup the simulation

        vrep.simxSynchronous(clientID,True)

        joint_names = ['shoulder', 'elbow']
        # joint target velocities discussed below
        joint_target_velocities = np.ones(len(joint_names)) * 10000.0

        # get the handles for each joint and set up streaming
        joint_handles = [vrep.simxGetObjectHandle(clientID,
            name, vrep.simx_opmode_blocking)[1] for name in joint_names]

        # get handle for target and set up streaming
        _, target_handle = vrep.simxGetObjectHandle(clientID,
                        'target', vrep.simx_opmode_blocking)

        dt = .001
        vrep.simxSetFloatingParameter(clientID,
                vrep.sim_floatparam_simulation_time_step,
                dt, # specify a simulation time step
                vrep.simx_opmode_oneshot)

        # --------------------- Start the simulation

        # start our simulation in lockstep with our code
        vrep.simxStartSimulation(clientID,
                vrep.simx_opmode_blocking)

        count = 0
        track_hand = []
        track_target = []
        while count < 1: # run for 1 simulated second

            # get the (x,y,z) position of the target
            _, target_xyz = vrep.simxGetObjectPosition(clientID,
                    target_handle,
                    -1, # retrieve absolute, not relative, position
                    vrep.simx_opmode_blocking)
            if _ !=0 : raise Exception()
            track_target.append(np.copy(target_xyz)) # store for plotting
            target_xyz = np.asarray(target_xyz)

            q = np.zeros(len(joint_handles))
            dq = np.zeros(len(joint_handles))
            for ii,joint_handle in enumerate(joint_handles):
                # get the joint angles
                _, q[ii] = vrep.simxGetJointPosition(clientID,
                        joint_handle,
                        vrep.simx_opmode_blocking)
                if _ !=0 : raise Exception()
                # get the joint velocity
                _, dq[ii] = vrep.simxGetObjectFloatParameter(clientID,
                        joint_handle,
                        2012, # parameter ID for angular velocity of the joint
                        vrep.simx_opmode_blocking)
                if _ !=0 : raise Exception()

            L = np.array([.42, .225]) # arm segment lengths

            xyz = np.array([L[0] * np.cos(q[0]) + L[1] * np.cos(q[0]+q[1]),
                            0,
                            # have to add .1 offset to z position
                            L[0] * np.sin(q[0]) + L[1] * np.sin(q[0]+q[1]) + .15])
            track_hand.append(np.copy(xyz)) # store for plotting
            # calculate the Jacobian for the hand
            JEE = np.zeros((3,2))
            JEE[0,1] = L[1] * -np.sin(q[0]+q[1])
            JEE[2,1] = L[1] * np.cos(q[0]+q[1])
            JEE[0,0] = L[0] * -np.sin(q[0]) + JEE[0,1]
            JEE[2,0] = L[0] * np.cos(q[0]) + JEE[2,1]

            # get the Jacobians for the centres-of-mass for the arm segments
            JCOM1 = np.zeros((6,2))
            JCOM1[0,0] = L[0] * -np.sin(q[0]) # COM is in a weird place
            JCOM1[2,0] = L[0] * np.cos(q[0])  # because of offset
            JCOM1[4,0] = 1.0

            JCOM2 = np.zeros((6,2))
            JCOM2[0,1] = L[1] * -np.sin(q[0]+q[1]) # COM is in a weird place
            JCOM2[2,1] = L[1] * np.cos(q[0]+q[1])  # because of offset
            JCOM2[4,1] = 1.0
            JCOM2[0,0] = L[0] * -np.sin(q[0]) + JCOM2[0,1]
            JCOM2[2,0] = L[0] * np.cos(q[0]) + JCOM2[2,1]
            JCOM2[4,0] = 1.0

            m1 = 1.171 # from VREP
            M1 = np.diag([m1, m1, m1, .008, .008, .001])
            m2 = .329 # from VREP
            M2 = np.diag([m2, m2, m2, 5e-4, 5e-4, 1e-4])

            # generate the mass matrix in joint space
            Mq = np.dot(JCOM1.T, np.dot(M1, JCOM1)) + \
                 np.dot(JCOM2.T, np.dot(M2, JCOM2))

            # compensate for gravity
            gravity = np.array([0, 0, -9.81, 0, 0, 0,])
            Mq_g = np.dot(JCOM1.T, np.dot(M1, gravity)) + \
                    np.dot(JCOM2.T, np.dot(M2, gravity))

            Mx_inv = np.dot(JEE, np.dot(np.linalg.inv(Mq), JEE.T))
            Mu,Ms,Mv = np.linalg.svd(Mx_inv)
            # cut off any singular values that could cause control problems
            for i in range(len(Ms)):
                Ms[i] = 0 if Ms[i] < 1e-5 else 1./float(Ms[i])
            # numpy returns U,S,V.T, so have to transpose both here
            Mx = np.dot(Mv.T, np.dot(np.diag(Ms), Mu.T))

            # calculate desired movement in operational (hand) space
            kp = 100
            kv = np.sqrt(kp)
            u_xyz = np.dot(Mx, kp * (target_xyz - xyz))

            u = np.dot(JEE.T, u_xyz) - np.dot(Mq, kv * dq) - Mq_g
            u *= -1 # because the joints on the arm are backwards

            for ii,joint_handle in enumerate(joint_handles):
                # the way we're going to do force control is by setting
                # the target velocities of each joint super high and then
                # controlling the max torque allowed (yeah, i know)

                # get the current joint torque
                _, torque = \
                    vrep.simxGetJointForce(clientID,
                            joint_handle,
                            vrep.simx_opmode_blocking)
                if _ !=0 : raise Exception()

                # if force has changed signs,
                # we need to change the target velocity sign
                if np.sign(torque) * np.sign(u[ii]) < 0:
                    joint_target_velocities[ii] = \
                            joint_target_velocities[ii] * -1
                    vrep.simxSetJointTargetVelocity(clientID,
                            joint_handle,
                            joint_target_velocities[ii], # target velocity
                            vrep.simx_opmode_blocking)
                if _ !=0 : raise Exception()

                # and now modulate the force
                vrep.simxSetJointForce(clientID,
                        joint_handle,
                        abs(u[ii]), # force to apply
                        vrep.simx_opmode_blocking)
                if _ !=0 : raise Exception()

            # move simulation ahead one time step
            vrep.simxSynchronousTrigger(clientID)
            count += dt

        # stop the simulation
        vrep.simxStopSimulation(clientID, vrep.simx_opmode_blocking)

        # Before closing the connection to V-REP,
        #make sure that the last command sent out had time to arrive.
        vrep.simxGetPingTime(clientID)

        # Now close the connection to V-REP:
        vrep.simxFinish(clientID)
    else:
        raise Exception('Failed connecting to remote API server')

finally:

    # stop the simulation
    vrep.simxStopSimulation(clientID, vrep.simx_opmode_blocking)

    # Before closing the connection to V-REP,
    # make sure that the last command sent out had time to arrive.
    vrep.simxGetPingTime(clientID)

    # Now close the connection to V-REP:
    vrep.simxFinish(clientID)
    print('connection closed...')

    import matplotlib as mpl
    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt

    track_hand = np.array(track_hand)
    track_target = np.array(track_target)

    fig = plt.figure()
    ax = fig.gca(projection='3d')
    # plot start point of hand
    ax.plot([track_hand[0,0]], [track_hand[0,1]], [track_hand[0,2]], 'bx', mew=10)
    # plot trajectory of hand
    ax.plot(track_hand[:,0], track_hand[:,1], track_hand[:,2])
    # plot trajectory of target
    ax.plot(track_target[:,0], track_target[:,1], track_target[:,2], 'rx', mew=10)

    ax.set_xlim([-1, 1])
    ax.set_ylim([-.5, .5])
    ax.set_zlim([0, 1])
    ax.legend()

    plt.show()


a, c = 0, 0
b = 1
f = open('fibonacci.txt', 'a')

for i in range(1000000):
        'print(a)'
        a = b + c
        c = b
        b = a

f.write(str(a))
f.write("\n")
f.close()


import imp
import logging
import os
from Qt.QtWidgets import *
import pipeGlobal
import ui
reload(ui)
from ui import TaskUI
from miraLibs.pipeLibs import pipeFile, Step, get_up_step_tasks
from miraLibs.dccLibs import get_engine
from miraLibs.pipeLibs.pipeDb import task_from_db_path
from miraLibs.pyLibs import join_path, copy

class TaskInit(TaskUI):
    def __init__(self, parent=None):
        super(TaskInit, self).__init__(parent)
        self.set_signals()
        self.selected = None
        self.__engine = get_engine.get_engine()
        self.__db = self.my_task_widget.db
        self.__logger = logging.getLogger("Task Init")

    def set_signals(self):
        self.my_task_widget.task_view.pressed.connect(self.on_task_pressed)
        self.init_btn.clicked.connect(self.init_task)
        self.work_stack.list_widget.copy_to_local_action.triggered.connect(self.copy_to_local)

    def on_task_pressed(self, index):
        self.selected = index.data()
        self.set_dir()
        self.show_up_step_task_status()

    def show_up_step_task_status(self):
        self.up_step_table.clearContents()
        self.up_step_table.setRowCount(0)
        up_step_tasks = get_up_step_tasks.get_up_step_tasks(self.selected.project, self.selected.entity_type,
                                                            self.selected.asset_type_sequence,
                                                            self.selected.asset_name_shot, self.selected.step)
        if up_step_tasks:
            for task_info in up_step_tasks:
                step = task_info.get("step").get("name")
                task = task_info.get("code")
                status = task_info.get("status").get("name")
                status_color = task_info.get("status").get("color")
                self.up_step_table.append_row(step, task, status, status_color)

    def init_task(self):
        work_file = pipeFile.get_task_work_file(self.selected.project, self.selected.entity_type,
                                                self.selected.asset_type_sequence, self.selected.asset_name_shot,
                                                self.selected.step, self.selected.task, version="001")
        context = pipeFile.PathDetails.parse_path(work_file)
        local_file = context.local_work_path
        if os.path.isfile(local_file):
            msg = QMessageBox.information(self, "Warming Tip", "%s already exist\nDo you want to replace it?" % local_file,
                                          QMessageBox.Yes | QMessageBox.No)
            if msg.name == "No":
                return
            else:
                self.do_init_task(self.selected.step, local_file)
        else:
            self.do_init_task(self.selected.step, local_file)
        self.set_dir()

    def do_init_task(self, step, local_file):
        custom_dir = pipeGlobal.custom_dir
        start_dir = os.path.join(custom_dir, self.selected.project, "start").replace("\\", "/")
        if not os.path.isdir(start_dir):
            start_dir = os.path.join(custom_dir, "defaultProject", "start").replace("\\", "/")
        fn_, path, desc = imp.find_module(step, [start_dir])
        mod = imp.load_module(step, fn_, path, desc)
        mod.main(local_file, True)
        self.update_task_status(local_file)

    def update_task_status(self, file_path):
        task = task_from_db_path.task_from_db_path(self.__db, file_path)
        self.__db.update_task_status(task, "In progress")
        self.__logger.info("Change task status to In progress.")
        from datetime import datetime
        now_time = datetime.now().strftime('%Y-%m-%d')
        self.__db.update_task(task, sub_date=now_time)
        self.__logger.info("Change task sub date: %s" % now_time)

    def set_dir(self):
        local_file = pipeFile.get_task_work_file(self.selected.project, self.selected.entity_type,
                                                 self.selected.asset_type_sequence, self.selected.asset_name_shot,
                                                 self.selected.step, self.selected.task, "000", local=True)
        work_file = pipeFile.get_task_work_file(self.selected.project, self.selected.entity_type,
                                                self.selected.asset_type_sequence, self.selected.asset_name_shot,
                                                self.selected.step, self.selected.task, "000")
        publish_file = pipeFile.get_task_publish_file(self.selected.project, self.selected.entity_type,
                                                      self.selected.asset_type_sequence, self.selected.asset_name_shot,
                                                      self.selected.step, self.selected.task, "000")
        self.set_widget_dir(self.local_stack, local_file)
        self.set_widget_dir(self.work_stack, work_file)
        self.set_widget_dir(self.publish_stack, publish_file)

    def set_widget_dir(self, stack_widget, file_path):
        if file_path:
            file_dir = os.path.dirname(os.path.dirname(file_path))
            if self.__engine != "python":
                engine = Step(self.selected.project, self.selected.step).engine
                file_dir = join_path.join_path2(file_dir, engine)
            stack_widget.set_dir(file_dir)

    def copy_to_local(self):
        file_paths = self.work_stack.list_widget.get_selected()
        if not file_paths:
            return
        file_path = file_paths[0]
        if not os.path.isfile(file_path):
            return
        try:
            temp_context = pipeFile.PathDetails.parse_path(file_path)
            next_version_file = temp_context.next_version_file
            context = pipeFile.PathDetails.parse_path(next_version_file)
            engine = Step(context.project, context.step).engine
            local_path = context.local_work_path
            copy.copy(file_path, local_path)
            work_dir = os.path.dirname(os.path.dirname(local_path))
            work_engine_dir = join_path.join_path2(work_dir, engine)
            self.local_stack.set_dir(work_engine_dir)
            self.update_task_status(file_path)
            self.file_widget.setCurrentIndex(0)
        except RuntimeError as e:
            logging.error(str(e))

def main():
    from miraLibs.qtLibs import render_ui
    render_ui.render(TaskInit)

if __name__ == "__main__":
    main()


# coding:utf-8
import os
ne_dir = '../../output/named_entity'
ne_list_path = '../../output/named_entity_list'
'''
GPE 釜山
ORG 基督教神學院
GPE 釜山
GPE 釜山
MISC 忠清南道天安
PERSON 大衛·當臣
'''

def separate():
  ne_type_list = ['GPE', 'ORG', 'MISC', 'PERSON', 'LOC']
  ne_files = eval('{{{0}}}'.format(','.join(["'{0}': open(os.path.join(ne_dir, '{0}'), 'w')".format(t) for t in ne_type_list])))

  ne_list_f = open(ne_list_path, 'r')
  while True:
    line = ne_list_f.readline()
    if not line:
      break
    line = line.strip()
    sep_p = line.find(' ')
    ne_type = line[:sep_p]
    entity = line[sep_p+1:]
    ne_files[ne_type].write(entity+'\n')

if __name__ == '__main__':
  separate()


#Import required libraries
import numpy as np

class ModelAnalyser:
    def benchmarkModel(self, model, x_test, y_test):
        '''Given a trained model and testing data produces the following performance metrics: Accuracy, Precision, Recall, F1.'''
        predictions = model.predict(([x_test]))
        predictions = [np.argmax(i) for i in predictions] #Takes the argmax of the maximum value in the array revealing the actual predicted classes.
        accuracy = self.arrayAccuracy(predictions, y_test)
        precision = self.arrayPrecision(predictions, y_test)
        recall = self.arrayRecall(predictions, y_test)
        f1 = (2*precision*recall)/(precision+recall)
        outputMetrics = {'Accuracy':accuracy, 'Precision': precision, 'Recall': recall, 'F1':f1}
        return outputMetrics

    def confusionMatrix(self, predictions:list, evidence:list, numClasses:int) -> np.array:
        '''Computes confusion matrix given a set of predictions, true labels, and number of classes.'''
        confusionMatrix = np.zeros((numClasses, numClasses)) #Intialise confusion matrix.

        #Populate confusion matrix where rows represent predictions and columns represent true values/
        for i in range(len(predictions)):
            confusionMatrix[predictions[i]][evidence[i]] = confusionMatrix[predictions[i]][evidence[i]] + 1

        #Display confusion matrix.
        for i in range(numClasses):
            for j in range(numClasses):
                print(str(int(confusionMatrix[i][j])) + " ", end = "")
            print()

        return

    def arrayAccuracy(self, pred:list, actual:list):
        '''Computes model accuracy given a list of predictions and actual values, returns as percentage.'''
        count = 0
        for i in range(len(pred)):
            if(pred[i] == actual[i]):
                count = count + 1

        dataSize = len(pred)
        accuracy = (count/dataSize)*100
        return accuracy

    def arrayPrecision(self, pred:list, actual:list):
        """Computes model precision given a list of predictions and actual values, returns as percentage."""
        tp = 0
        fp = 0
        numClasses = self._unique(pred)
        precisionList = []

        for _class in numClasses:
            for i in range(len(pred)):
                if pred[i]==_class and actual[i]==pred[i]:
                    tp+=1
                if pred[i]==_class and actual[i]!=pred[i]:
                    fp+=1

            precision = tp/(tp+fp)
            precisionList.append(precision)
            tp = 0
            fp = 0

        return sum(precisionList)/len(precisionList)

    def arrayRecall(self, pred, actual):
        """Computes model recall given a list of predictions and actual values, returns as percentage."""
        tp = 0
        fn = 0
        numClasses = self._unique(pred)
        recallList = []

        for _class in numClasses:
            for i in range(len(pred)):
                if pred[i]==_class and actual[i]==pred[i]:
                    tp+=1
                if pred[i]!=_class and actual[i]==_class:
                    fn+=1

            recall = tp/(tp+fn)
            recallList.append(recall)
            tp = 0
            fn = 0

        return sum(recallList)/len(recallList)

    def _unique(self, _list):
        """Given a list returns list of unique values within that list."""
        arr = np.array(_list)
        uniqueArr = np.unique(arr)
        return uniqueArr.tolist()


#!/usr/bin/env python3

import os
import subprocess

# TODO: handle ctrl-c during check run and during this script execution

welcome_message = """Welcome to Pre-HAP Validation!

This tool runs multiple checks verifying that the environment is ready to start the \
Hardware Automation Package (HAP) phase.
"""

def print_summary(checks):
    print("\nTest results summary:")
    for check in checks:
        print(f"{check['filename'] + ' ':.<50} {check['result']}")

def run_check(check) -> str:
    check_path = os.path.join(check["path"], check["filename"])

    try:
        subprocess.check_call(check_path)
    except subprocess.CalledProcessError as e:
        return "Failed"

    return "Success"

def create_desktop_shortcut():
    # Copy launcher to the desktop if it does not exist yet
    if not os.path.isfile("/home/ubuntu/Desktop/pre-hap-validation-launcher.desktop"):
        subprocess.run(
            [
                "cp",
                "/etc/xdg/autostart/pre-hap-validation-launcher.desktop",
                "/home/ubuntu/Desktop",
            ]
        )
        subprocess.run(
            ["chown", "ubuntu:ubuntu", "/home/ubuntu/Desktop/pre-hap-validation-launcher.desktop"]
        )
        subprocess.run(["chmod", "+x", "/home/ubuntu/Desktop/pre-hap-validation-launcher.desktop"])

        # Trust the launcher
        subprocess.run(
            [
                "sudo",
                "-H",
                "-u",
                "ubuntu",
                "bash",
                "-c",
                "dbus-launch gio set /home/ubuntu/Desktop/pre-hap-validation-launcher.desktop 'metadata::trusted' yes &> /dev/null",
            ]
        )

def main():
    create_desktop_shortcut()

    print(welcome_message)

    #
    # Create the list of checks to be executed
    #

    checks_enabled_path = "checks-enabled"

    checks = []

    # Search path for executable files
    for root, directories, files in os.walk(checks_enabled_path):
        for file in files:

            # Check if the file is executable
            executable = os.access(os.path.join(root, file), os.X_OK)

            # Add the file to the list only if it is executable
            if executable:
                item = {"path": root, "filename": file, "result": "Not run"}
                checks.append(item)

    # Sort by filename
    checks_sorted = sorted(checks, key=lambda k: k["filename"])

    #
    # List checks to be run
    #

    if 0 == len(checks_sorted):
        print("No checks found in 'checks-enabled' directory. Exiting.")
        exit(1)

    print("The following checks will be run:\n")

    for check in checks_sorted:
        print(f"  {check['filename']}")

    #
    # Run checks
    #

    for check in checks_sorted:
        print(f"\nRun check '{check['filename']}'?")

        valid_answers = ["r", "s", "q", ""]
        answer = "invalid initial answer"

        while answer not in valid_answers:
            answer = input("(R)un, (s)kip, (q)uit: ").lower()

        if answer in ["", "r"]:
            print(f"Running check '{check['filename']}'...\n")
            check["result"] = run_check(check)
        elif answer == "s":
            print(f"Skipping check '{check['filename']}'.")
            check["result"] = "Skipped"
        elif answer == "q":
            print(f"Quitting.")
            break
        else:
            print(f"ERROR: got unexpected answer, exiting.")
            break

    # Print test results
    print_summary(checks_sorted)

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib

def img_to_np_array(path_to_img, dimX, dimY, dimZ):
    f = open(path_to_img, 'rb')
    img_str = f.read()
    img_arr = np.fromstring(img_str, np.float32)
    img = np.reshape(img_arr, (dimX, dimY, dimZ))
    f.close()
    return img

# NEMA standard - FWHM is determined by linear interpolation between adjacent
# pixels at half the maximum value of the response function.

def fwhm_from_vector(vector, vxl_size=1):
    x = range(np.size(vector))
    ## Find maximum value
    x_poly2 = np.array([np.argmax(vector) - 1, np.argmax(vector), np.argmax(vector) + 1])
    y_poly2 = np.array([vector[np.argmax(vector) - 1], vector[np.argmax(vector)], vector[np.argmax(vector) + 1]])
    coeff_poly2 = np.polyfit(x_poly2, y_poly2, 2)
    poly2 = np.poly1d(coeff_poly2)
    x_max_val = -coeff_poly2[1] / (2 * coeff_poly2[0]) # p = -b/2a
    y_max_val = -(coeff_poly2[1] * coeff_poly2[1] - 4 * coeff_poly2[2] * coeff_poly2[0]) / (4 * coeff_poly2[0]) # q = -D/4a
    # print(x_max_val, y_max_val)
    ## Find FWHM
    for ii in x[:-1]:
        if  vector[ii] < y_max_val/2 and y_max_val/2 < vector[ii + 1]:
            x1 = np.array([ii, ii + 1])
            xp1 = np.linspace(ii, ii + 1, 100)
        if y_max_val/2 < vector[ii]  and y_max_val/2 > vector[ii + 1]:
            x2 = np.array([ii, ii + 1])
            xp2 = np.linspace(ii, ii + 1, 100)
    coeff_line1 = np.polyfit(x1, vector[x1], 1)
    line1 = np.poly1d(coeff_line1)
    coeff_line2 = np.polyfit(x2, vector[x2], 1)
    line2 = np.poly1d(coeff_line2)
    xFWHMl = (y_max_val/2 - coeff_line1[1])/coeff_line1[0]
    xFWHMr = (y_max_val/2 - coeff_line2[1])/coeff_line2[0]
    ## Visualization
    # plt.bar(x, vector)
    # xp = np.linspace(np.argmax(vector) - 1, np.argmax(vector) + 1, 100)
    # plt.plot(xp, poly2(xp), 'g--')
    # plt.plot(xp1, line1(xp1), 'r--')
    # plt.plot(xp2, line2(xp2), 'r--')
    # plt.plot([xFWHMl, xFWHMr], [y_max_val/2, y_max_val/2], 'r', linewidth=3.0)
    # plt.show()
    fwhm = (xFWHMr - xFWHMl) * vxl_size
    # print(fwhm)
    return fwhm


def countingSort(array):
    # create count array based on max + 1
    count = [0] * (max(array) + 1)
    # fill count array
    for i in range(len(array)):
        count[array[i]] += 1
    # update count array with cumulative sum
    for i in range(len(count) - 1):
        count[i+1] += count[i]

    # shift count array to right
    for i in range(len(count) - 1, 0, -1):
        count[i] = count[i-1]
    # create output array
    output = [0] * len(array)
    # iterate array and fill output array using count array values as indices
    for i in range(len(array)):
        output[count[array[i]]] = array[i]
        count[array[i]] += 1

    # return array
    return output

array = [1,0,3,1,3,1]
print(countingSort(array))


# ********************************************************************** <====

from artmgr.transport.basew import BaseWTransport

# ********************************************************************** ====>

import os
import sys
import errno
import stat
import re

# chunksize for reading/writing local files
CHUNK = 8192

# ---------------------------------------------------------------------

def mkpath_recursive(path):
    """Test a local path and, if it does not exist, create it recursively"""
    try:
        mode = os.stat( path ).st_mode
        if not stat.S_ISDIR(mode):
            raise InvalidArgumentError("parent path '"+str(path)+"' not a dir")
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise
        (head,tail) = os.path.split( path )
        if head:
            mkpath_recursive( head )
        os.mkdir( path )

# ---------------------------------------------------------------------

class LocalTransport( BaseWTransport ):
    """
    A full R/W transport instance that uses a locally visible directory to
    store and read all artifact data
    """

    def __init__( self, basedir, subrepo ):
        """
        Constructor
          @param basedir (str): local folder to use
          @param subrepo (str): name of the repository we are dealing with
        """
        if not basedir:
            raise InvalidArgumentError("Empty basedir in local transport")
        if not subrepo:
            raise InvalidArgumentError("Empty subrepo in local transport")
        self._basedir = os.path.join(basedir,subrepo)
        super(LocalTransport,self).__init__()

    def init_base( self ):
        """Ensure the base path for the repository exists"""
        mkpath_recursive( self._basedir )

    def get( self, sourcename, dest ):
        """
        Read a file into a file-like destination.
        @param sourcename (str): name of the file in remote repo
        @param dest (file): an object with a write() method
        @return (bool): \c True if ok, \c False if the file does not exist
        """
        name = os.path.join(self._basedir,sourcename)
        try:
            with open(name, 'rb') as f:
                while True:
                    bytes = f.read( CHUNK )
                    if not bytes:
                        break
                    dest.write( bytes )
            return True
        except IOError as e:
            if e.errno == errno.ENOENT:
                return False
            raise

    def otype( self, path ):
        """
        Given the path of am object, return:
          * 'F' for a file,
          * 'D' for a directory,
          * \c None if the path does not exist
        """
        oldname = os.path.join(self._basedir,path)
        try:
            mode = os.stat( oldname ).st_mode
        except OSError as e:
            if e.errno == errno.ENOENT:
                return None
            raise
        return 'D' if stat.S_ISDIR(mode) else 'F' if stat.S_ISREG(mode) else '?'

    def put( self, source, destname ):
        """
        Store a file. If a file with the same name existed, it is overwritten
        @param source (file): an object with a read() method
        @param destname (str): name of the destination file,
        relative to repo base directory
        """
        name = os.path.join(self._basedir,destname)
        with open(name, 'wb') as f:
            while True:
                bytes = source.read( CHUNK )
                if not bytes:
                    break
                f.write( bytes )

    def delete( self, filename ):
        """
        Delete a file
        """
        name = os.path.join(self._basedir,filename)
        os.unlink( name )

    def rename( self, oldname, newname ):
        """
        Rename a file into a new name and/or folder
        """
        oldname = os.path.join(self._basedir,oldname)
        newname = os.path.join(self._basedir,newname)
        os.rename( oldname, newname )

    def folder_create( self, path ):
        """
        Make a folder in the repository, assuming all parent folders exist
        """
        os.mkdir( os.path.join(self._basedir,path) )

    def folder_list( self, path ):
        """
        Return the list of all components (files & folders) in a folder
        *This method is optional*
        """
        return os.listdir( os.path.join(self._basedir,path) )


import tensorflow as tf
import  os
import random
import cv2

# def _int64_feature(value):
#     if not isinstance(value,list):
#         value=[value]
#     return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def _byte_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _convet_to_example(file_name,image):
    image=image.tobytes()
    example=tf.train.Example(features=tf.train.Features(feature={
        'image_name':_byte_feature(tf.compat.as_bytes(file_name)),
        'image':_byte_feature(image)
         }))
    return example

def get_name_list(source_path):
    name_list=os.listdir(source_path)
    random.seed(1)
    random.shuffle(name_list)
    return name_list[0:-100],name_list[-100:]

def write_data_tfreocord(source_path,name_list,tfrecord_dir):
    writer=tf.python_io.TFRecordWriter(tfrecord_dir)
    for image_name in  name_list:
        image_path=os.path.join(source_path,image_name)
        image=cv2.imread(image_path)
        example=_convet_to_example(image_name,image)
        writer.write(example.SerializeToString())

if __name__=='__main__':
    man_data_dir='../data/cycle_gan_data/a_resized'
    woman_data_dir = '../data/cycle_gan_data/b_resized'
    man_train_tfrecord='data/man_train_tfrecord.tfrecords'
    man_test_tfrecord = 'data/man_test_tfrecord.tfrecords'
    woman_train_tfrecord = 'data/woman_train_tfrecord.tfrecords'
    woman_test_tfrecord = 'data/woman_test_tfrecord.tfrecords'
    man_train,man_test=get_name_list(man_data_dir)
    print(len(man_train),len(man_test))
    woman_train, woman_test = get_name_list(woman_data_dir)
    print(len(woman_train), len(woman_test))
    print('write man train')
    write_data_tfreocord(source_path=man_data_dir,name_list=man_train,tfrecord_dir=man_train_tfrecord)
    print('write man test')
    write_data_tfreocord(source_path=man_data_dir,name_list=man_test,tfrecord_dir=man_test_tfrecord)
    print('write woman train')
    write_data_tfreocord(source_path=woman_data_dir,name_list=woman_train,tfrecord_dir=woman_train_tfrecord)
    print('write woman test')
    write_data_tfreocord(source_path=woman_data_dir,name_list=woman_test,tfrecord_dir=woman_test_tfrecord)
    print('done')


weight = int(input("What is your weight in KG?: "))
height = float(input("What is your height in Meters?: "))
BMI = weight/(height**2)

if BMI<18.5:
    print("underweight")
elif (BMI>=18.5) and (BMI<25):
    print("normal")
elif (BMI>=25) and (BMI<30):
    print("overweight")
else: print("obese")


import gameConst as gc
import random
import arcade

class Obstacle(arcade.Sprite):

    def __init__(self, x, y):
        super().__init__("images/obstext.png")
        self.center_x = x + gc.OBSTACLE_W / 2
        self.bottom = y
        self.width = gc.OBSTACLE_W
        if y == 0:
            self.top = random.randrange(start=gc.MIN_HEIGHT, stop=(gc.GAME_HEIGHT - gc.GAP_SIZE - gc.MIN_HEIGHT))
        else:
            self.angle = 180

    def update(self):
        self.center_x -= gc.WORLD_SPEED
        if self.right <= 0:
            super().kill()

    def inBound(self):
        return self.center_x + self.width / 2 >= 0


cantantes = ['2pac','Drake','Bad Bunny',"Reik"]

numeros = [1,2,5,8,3,4]

# Ordenar una lista
print(numeros)
numeros.sort()
print(numeros)

#Agregar elementos

cantantes.append("Alex Ubago")
cantantes.insert(1,"David Bisbal")#Hay que elegir en que indice quieres agregar algo

#Eliminar Elementos
cantantes.pop(1) #Elimina por indice

cantantes.remove("Reik") # Elimina por elementos

#Dar la vuelta la lista

numeros.reverse()

#Buscar dentro de una lista

print('Drake' in cantantes)#Devuelve un booleano

#Contar Elementos

print(len(cantantes))

# Cuantas veces aparece un elemento en una lista

print(numeros.count(8))

#Conseguir indice
print(cantantes.index("Drake"))

#Unir listas
cantantes.extend(numeros)


"""
Input cpdomains is a list
output to be a list of elements with counter adding subdomains and providing output
"""
from nose.tools import assert_equal
import collections

def subdomain_counter(cpdomains):
    counter = collections.Counter()

    for domain in cpdomains:
        count, domain = domain.split()
        count = int(count)
        subdomains = domain.split(".")

        for i in range(0, len(subdomains)):
            counter[".".join(subdomains[i:])] += count
    return ["{} {}".format(k, v) for k, v in counter.items()]

# assert_equal(subdomain_counter(["9001 discuss.leetcode.com"]),["9001 discuss.leetcode.com", "9001 leetcode.com", "9001 com"])
print(subdomain_counter(["9001 discuss.leetcode.com"]))


#!/usr/bin/python3
"""Fetch to a page module """
import requests
import sys

if __name__ == "__main__":
    url = sys.argv[1]
    req = requests.get(url)
    headers = req.headers
    print(headers.get('X-Request-Id'))


from typing import DefaultDict
import sqlalchemy

from mojap_metadata import Metadata
import mojap_metadata.converters.postgres_converter.postgres_functions as pg
from mojap_metadata.converters import BaseConverter

_default_type_converter = {
    "int8": "int8",
    "int16": "int16",
    "int32": "int32",
    "int64": "int64",
    "bigint": "int64",
    "int2": "int32",
    "int4": "int32",
    "integer": "int32",
    "smallint": "int32",
    "numeric": "float64",
    "double precision": "float64",
    "text": "string",
    "uuid": "string",
    "character": "string",
    "tsvector": "string",
    "jsonb": "string",
    "varchar": "string",
    "bpchar": "string",
    "date": "date64",
    "boolean": "bool",
    "timestamptz": "timestamp(ms)",
    "timestamp": "timestamp(ms)",
    "datetime": "timestamp(ms)",
    "bool": "bool",
}

class PostgresConverter(BaseConverter):
    def __init__(self):
        """
        Extracts and converts metadata to Metadata format
        """

        super().__init__()
        self._default_type_converter = _default_type_converter

    def convert_to_mojap_type(self, col_type: str) -> str:
        """Converts our postgress datatypes to mojap-metadata types

        Args:
            ct (str): str representation of postgres column types

        Returns:
            str: String representation of our metadata column types
        """

        output = (
            "string"
            if self._default_type_converter.get(col_type) is None
            else self._default_type_converter.get(col_type)
        )
        return output

    def get_object_meta(
        self, connection: sqlalchemy.engine.Engine, table: str, schema: str
    ) -> Metadata:
        """Extracts metadata from table and converts to mojap metadata format

        Args:
            connection: Database connection
            table: table name
            schema: schema name

        Returns:
            Metadata: Metadata object
        """

        rows = pg.list_meta_data(connection, table, schema)
        columns = []

        for col in rows[0]:

            column_type = self.convert_to_mojap_type(str(col[1]))
            columns.append(
                {
                    "name": col[0].lower(),
                    "type": column_type,
                    "description": None if str(col[3]) is None else str(col[3]),
                    "nullable": True if col[2] == "YES" else False,
                }
            )

        d = {"name": table, "columns": columns}

        meta_output = Metadata.from_dict(d)
        return meta_output

    def generate_from_meta(self, connection: sqlalchemy.engine.Engine) -> dict():
        """Extracts metadata for all the schema and tables and returns a list
        of Metadata

        Args:
            connection: Database connection with database details

        Returns:
            Metadata: Metadata object
        """
        meta_list_per_schema = DefaultDict(list)

        schema_names = pg.list_schemas(
            connection
        )  # database name will be passed on in the connection

        for schema in sorted(schema_names):
            table_names = pg.list_tables(connection, schema)

            for table in table_names:
                meta_output = self.get_object_meta(connection, table, schema)
                meta_list_per_schema[f"schema: {schema}"].append(meta_output)

        return meta_list_per_schema