# -*- coding: utf-8 -*-
"""
python -c "import doctest, ibeis; print(doctest.testmod(ibeis.model.hots.hots_nn_index))"
python -m doctest -v ibeis/model/hots/hots_nn_index.py
python -m doctest ibeis/model/hots/hots_nn_index.py
"""
from __future__ import absolute_import, division, print_function
# Standard
from six.moves import zip, map, range
#from itertools import chain
import sys
# Science
import numpy as np
# UTool
import utool
# VTool
from ibeis.other import ibsfuncs
import vtool.nearest_neighbors as nntool
(print, print_, printDBG, rrr, profile) = utool.inject(__name__, '[nnindex]', DEBUG=False)

NOCACHE_FLANN = '--nocache-flann' in sys.argv

def get_indexed_cfgstr(ibs, aid_list):
    """
    Creates a config string for the input into the nearest neighbors index
    It is based off of the features which were computed for it and the indexes
    of the input annotations.

    TODO: We should probably use the Annotation UUIDS rather than the ROWIDs
    to compute this configstr

    """
    feat_cfgstr = ibs.cfg.feat_cfg.get_cfgstr()
    # returns something like: _daids((6)qbm6uaegu7gv!ut!)_FEAT(params)
    daid_cfgstr = utool.hashstr_arr(aid_list, 'daids')  # todo change to uuids
    new_cfgstr = '_' + daid_cfgstr + feat_cfgstr
    return new_cfgstr

def build_ibs_inverted_descriptor_index(ibs, aid_list):
    """
    Aggregates descriptors of input annotations and returns inverted information
    """
    try:
        if len(aid_list) == 0:
            msg = ('len(aid_list) == 0\n'
                    'Cannot build inverted index without features!')
            raise AssertionError(msg)
        desc_list = ibs.get_annot_desc(aid_list)
        dx2_desc, dx2_aid, dx2_fx = _try_build_inverted_descriptor_index(aid_list, desc_list)
        return dx2_desc, dx2_aid, dx2_fx
    except Exception as ex:
        intostr = ibs.get_infostr()
        print(intostr)
        utool.printex(ex, 'cannot build inverted index', key_list=list(locals().keys()))
        raise

def _try_build_inverted_descriptor_index(aid_list, desc_list):
    """
    Wrapper which performs logging and error checking
    """
    if utool.NOT_QUIET:
        print('[agg_desc] stacking descriptors from %d annotations' % len(aid_list))
    try:
        dx2_desc, dx2_aid, dx2_fx = _build_inverted_descriptor_index(aid_list, desc_list)
    except MemoryError as ex:
        utool.printex(ex, 'cannot build inverted index', '[!memerror]')
        raise
    if utool.NOT_QUIET:
        print('[agg_desc] stacked %d descriptors from %d annotations'
                % (len(dx2_desc), len(aid_list)))
    return dx2_desc, dx2_aid, dx2_fx

def _build_inverted_descriptor_index(aid_list, desc_list):
    """
    Stacks descriptors into a flat structure and returns inverse mapping from
    flat database descriptor indexes (dx) to annotation ids (aid) and feature
    indexes (fx). Feature indexes are w.r.t. annotation indexes.

    Output:
        dx2_desc - flat descriptor stack
        dx2_aid  - inverted index into annotations
        dx2_fx   - inverted index into features

    # Example with 2D Descriptors
    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> from ibeis.model.hots.hots_nn_index import _build_inverted_descriptor_index
    >>> DESC_TYPE = np.uint8
    >>> aid_list  = [1, 2, 3, 4, 5]
    >>> desc_list = [
    ...     np.array([[0, 0], [0, 1]], dtype=DESC_TYPE),
    ...     np.array([[5, 3], [2, 30], [1, 1]], dtype=DESC_TYPE),
    ...     np.empty((0, 2), dtype=DESC_TYPE),
    ...     np.array([[5, 3], [2, 30], [1, 1]], dtype=DESC_TYPE),
    ...     np.array([[3, 3], [42, 42], [2, 6]], dtype=DESC_TYPE),
    ...     ]
    >>> dx2_desc, dx2_aid, dx2_fx = _build_inverted_descriptor_index(aid_list, desc_list)
    >>> print(repr(dx2_desc.T))
    array([[ 0,  0,  5,  2,  1,  5,  2,  1,  3, 42,  2],
           [ 0,  1,  3, 30,  1,  3, 30,  1,  3, 42,  6]], dtype=uint8)
    >>> print(repr(dx2_aid))
    array([1, 1, 2, 2, 2, 4, 4, 4, 5, 5, 5])
    >>> print(repr(dx2_fx))
    array([0, 1, 0, 1, 2, 0, 1, 2, 0, 1, 2])

    cdef:
        list aid_list, desc_list
        long nFeat, aid
        iter aid_nFeat_iter, nFeat_iter, _ax2_aid, _ax2_fx
        np.ndarray dx2_aid, dx2_fx, dx2_desc

    """
    # Build inverted index of (aid, fx) pairs
    aid_nFeat_iter = zip(aid_list, map(len, desc_list))
    nFeat_iter = map(len, desc_list)
    # generate aid inverted index for each feature in each annotation
    _ax2_aid = ([aid] * nFeat for (aid, nFeat) in aid_nFeat_iter)
    # Avi: please test the timing of the lines neighboring this statement.
    #_ax2_aid = ([aid] * nFeat for (aid, nFeat) in aid_nFeat_iter)
    # generate featx inverted index for each feature in each annotation
    _ax2_fx  = (range(nFeat) for nFeat in nFeat_iter)
    # Flatten generators into the inverted index
    #dx2_aid = np.array(list(chain.from_iterable(_ax2_aid)))
    #dx2_fx  = np.array(list(chain.from_iterable(_ax2_fx)))
    dx2_aid = np.array(utool.flatten(_ax2_aid))
    dx2_fx  = np.array(utool.flatten(_ax2_fx))
    # Stack descriptors into numpy array corresponding to inverted inexed
    # This might throw a MemoryError
    dx2_desc = np.vstack(desc_list)
    return dx2_desc, dx2_aid, dx2_fx

#@utool.indent_func('[build_invx]')
def build_flann_inverted_index(ibs, aid_list, **kwargs):
    """
    Build a inverted index (using FLANN)
    """
    # Aggregate descriptors
    dx2_desc, dx2_aid, dx2_fx = build_ibs_inverted_descriptor_index(ibs, aid_list)
    # hash which annotations are input
    indexed_cfgstr = get_indexed_cfgstr(ibs, aid_list)
    flann_params = {'algorithm': 'kdtree', 'trees': 4}
    flann_cachedir = ibs.get_flann_cachedir()
    precomp_kwargs = {'cache_dir': flann_cachedir,
                      'cfgstr': indexed_cfgstr,
                      'flann_params': flann_params,
                      'use_cache': kwargs.get('use_cache', not NOCACHE_FLANN)}
    # Build/Load the flann index
    flann = nntool.flann_cache(dx2_desc, **precomp_kwargs)
    return dx2_desc, dx2_aid, dx2_fx, flann

class HOTSIndex(object):
    """ HotSpotter Nearest Neighbor (FLANN) Index Class
    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> import ibeis
    >>> ibs = ibeis.test_main(db='testdb1')  #doctest: +ELLIPSIS
    <BLANKLINE>
    ...
    >>> daid_list = [1, 2, 3, 4]
    >>> hsindex = HOTSIndex(ibs, daid_list)  #doctest: +ELLIPSIS
    [nnindex...
    >>> print(hsindex) #doctest: +ELLIPSIS
    <ibeis.model.hots.hots_nn_index.HOTSIndex object at ...>

    """
    def __init__(hsindex, ibs, daid_list, **kwargs):
        print('[nnindex] building HOTSIndex object')
        dx2_desc, dx2_aid, dx2_fx, flann = build_flann_inverted_index(
            ibs, daid_list, **kwargs)
        # Agg Data
        hsindex.dx2_aid  = dx2_aid
        hsindex.dx2_fx   = dx2_fx
        hsindex.dx2_data = dx2_desc
        # Grab the keypoints names and image ids before query time
        #hsindex.rx2_kpts = ibs.get_annot_kpts(daid_list)
        #hsindex.rx2_gid  = ibs.get_annot_gids(daid_list)
        #hsindex.rx2_nid  = ibs.get_annot_nids(daid_list)
        hsindex.flann = flann

    def __getstate__(hsindex):
        """ This class it not pickleable """
        #printDBG('get state HOTSIndex')
        return None

    #def __del__(hsindex):
    #    """ Ensure flann is propertly removed """
    #    printDBG('deleting HOTSIndex')
    #    if getattr(hsindex, 'flann', None) is not None:
    #        nn_selfindex.flann.delete_index()
    #        #del hsindex.flann
    #    hsindex.flann = None

    def nn_index(hsindex, qfx2_desc, K, checks):
        (qfx2_dx, qfx2_dist) = hsindex.flann.nn_index(qfx2_desc, K, checks=checks)
        return (qfx2_dx, qfx2_dist)

    def nn_index2(hsindex, qreq, qfx2_desc):
        """ return nearest neighbors from this data_index's flann object """
        flann   = hsindex.flann
        K       = qreq.cfg.nn_cfg.K
        Knorm   = qreq.cfg.nn_cfg.Knorm
        checks  = qreq.cfg.nn_cfg.checks

        (qfx2_dx, qfx2_dist) = flann.nn_index(qfx2_desc, K + Knorm, checks=checks)
        qfx2_aid = hsindex.dx2_aid[qfx2_dx]
        qfx2_fx  = hsindex.dx2_fx[qfx2_dx]
        return qfx2_aid, qfx2_fx, qfx2_dist, K, Knorm

class HOTSMultiIndex(object):
    """
    Generalization of a HOTSNNIndex

    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> import ibeis
    >>> daid_list = [1, 2, 3, 4]
    >>> num_forests = 8
    >>> ibs = ibeis.test_main(db='testdb1')  #doctest: +ELLIPSIS
    <BLANKLINE>
    ...
    >>> split_index = HOTSMultiIndex(ibs, daid_list, num_forests)  #doctest: +ELLIPSIS
    [nnsindex...
    >>> print(split_index) #doctest: +ELLIPSIS
    <ibeis.model.hots.hots_nn_index.HOTSMultiIndex object at ...>

    """

    def __init__(split_index, ibs, daid_list, num_forests=8):
        print('[nnsindex] make HOTSMultiIndex over %d annots' % (len(daid_list),))
        # Remove unknown names
        aid_list = daid_list
        known_aids_list, unknown_aids = ibsfuncs.group_annots_by_known_names(ibs, aid_list)

        num_bins = min(max(map(len, known_aids_list)), num_forests)

        # Put one name per forest
        forest_aids, overflow_aids = utool.sample_zip(
            known_aids_list, num_bins, allow_overflow=True, per_bin=1)

        forest_indexes = []
        extra_indexes = []
        for tx, aids in enumerate(forest_aids):
            print('[nnsindex] building forest %d/%d with %d aids' %
                  (tx + 1, num_bins, len(aids)))
            if len(aids) > 0:
                hsindex = HOTSIndex(ibs, aids)
                forest_indexes.append(hsindex)

        if len(overflow_aids) > 0:
            print('[nnsindex] building overflow forest')
            overflow_index = HOTSIndex(ibs, overflow_aids)
            extra_indexes.append(overflow_index)
        if len(unknown_aids) > 0:
            print('[nnsindex] building unknown forest')
            unknown_index = HOTSIndex(ibs, unknown_aids)
            extra_indexes.append(unknown_index)
        #print('[nnsindex] building normalizer forest')  # TODO

        split_index.forest_indexes = forest_indexes
        split_index.extra_indexes = extra_indexes
        #split_index.overflow_index = overflow_index
        #split_index.unknown_index = unknown_index

#@utool.classmember(HOTSMultiIndex)
def nn_index(split_index, qfx2_desc, num_neighbors):
    qfx2_dx_list   = []
    qfx2_dist_list = []
    qfx2_aid_list  = []
    qfx2_fx_list   = []
    qfx2_rankx_list = []  # ranks index
    qfx2_treex_list = []  # tree index
    for tx, hsindex in enumerate(split_index.forest_indexes):
        flann = hsindex.flann
        # Returns distances in ascending order for each query descriptor
        (qfx2_dx, qfx2_dist) = flann.nn_index(qfx2_desc, num_neighbors, checks=1024)
        qfx2_dx_list.append(qfx2_dx)
        qfx2_dist_list.append(qfx2_dist)
        qfx2_fx = hsindex.dx2_fx[qfx2_dx]
        qfx2_aid = hsindex.dx2_aid[qfx2_dx]
        qfx2_fx_list.append(qfx2_fx)
        qfx2_aid_list.append(qfx2_aid)
        qfx2_rankx_list.append(np.array([[rankx for rankx in range(qfx2_dx.shape[1])]] * len(qfx2_dx)))
        qfx2_treex_list.append(np.array([[tx for rankx in range(qfx2_dx.shape[1])]] * len(qfx2_dx)))
    # Combine results from each tree
    (qfx2_dist_, qfx2_aid_,  qfx2_fx_, qfx2_dx_, qfx2_rankx_, qfx2_treex_,) = \
            join_split_nn(qfx2_dist_list, qfx2_dist_list, qfx2_rankx_list, qfx2_treex_list)

def join_split_nn(qfx2_dx_list, qfx2_dist_list, qfx2_aid_list, qfx2_fx_list, qfx2_rankx_list, qfx2_treex_list):
    qfx2_dx    = np.hstack(qfx2_dx_list)
    qfx2_dist  = np.hstack(qfx2_dist_list)
    qfx2_rankx = np.hstack(qfx2_rankx_list)
    qfx2_treex = np.hstack(qfx2_treex_list)
    qfx2_aid   = np.hstack(qfx2_aid_list)
    qfx2_fx    = np.hstack(qfx2_fx_list)

    # Sort over all tree result distances
    qfx2_sortx = qfx2_dist.argsort(axis=1)
    # Apply sorting to concatenated results
    qfx2_dist_  = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_dist)]
    qfx2_aid_   = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_dx)]
    qfx2_fx_    = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_aid)]
    qfx2_dx_    = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_fx)]
    qfx2_rankx_ = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_rankx)]
    qfx2_treex_ = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_treex)]
    return (qfx2_dist_, qfx2_aid_,  qfx2_fx_, qfx2_dx_, qfx2_rankx_, qfx2_treex_,)

#@utool.classmember(HOTSMultiIndex)
def split_index_daids(split_index):
    for hsindex in split_index.forest_indexes:
        pass

#if __name__ == '__main__':
#    #python -m doctest -v ibeis/model/hots/hots_nn_index.py
#    import doctest
#    doctest.testmod()


'''
Load embedding, create dictionary, convert text to index
'''

import io
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
import argparse
#import json
import os
import numpy as np
import pickle
import pdb

def text2index(text, vocab, analyzer):
    # 1 is unk
    doc_toks = [vocab[y] if y in vocab else 1 for y in analyzer(text) ]
    return doc_toks

def load_vectors(fname):
    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
    n, d = map(int, fin.readline().split())
    data = {}
    for line in fin:
        tokens = line.rstrip().split(' ')
        data[tokens[0]] = np.fromiter(map(float, tokens[1:]), dtype=np.float)
    return data

def build_vocab(text, emb, emb_dim=300, max_df=.7, max_features=20000, stop_words= 'english'):
    '''
    Fit vocabulary
    :param text: list of documents for creating vocabulary
    :return: vectorizer
    '''
    vect = CountVectorizer(stop_words=stop_words, max_df=max_df, max_features=max_features,
                           token_pattern=r"(?u)[!\"#\$\%&\'()\*\+,-./:;<=>\?@\[\\\]\^_`{|}~\w]+")
    vect.fit(text)

    no_embedding = [k for k in vect.vocabulary_.keys() if k not in emb]
    print("No Embeddings for: ")
    print(len(no_embedding))

    vocab = [k for i, k in enumerate(vect.vocabulary_.keys()) if k in emb]
    new_vocab = dict([(k, i + 2) for i, k in enumerate(vocab)])
    # Set 0 to be the padding index, 1 to be unk
    vect.vocabulary_ = new_vocab
    print('Vocabulary size: ', len(new_vocab))

    embedding = np.zeros(shape=(len(new_vocab) + 2, emb_dim))
    for k,i in new_vocab.items():
        embedding[i] = emb[k]

    return vect, embedding

def df2List(df, vocab, analyzer, label_dict, ismnli = False):
    out = []
    for i, row in df.iterrows():
        set1 = text2index(row['sentence1'], vocab, analyzer)
        set2 = text2index(row['sentence2'], vocab, analyzer)
        label = label_dict[row['label']]
        if ismnli:
            genre = row['genre']
        else:
            genre = 'snli'
        out.append([set1, set2, label, i, genre])
    return out

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--inputPath", default = '../hw2_data/') # Should have train/val in this directory
    parser.add_argument("--embPath", default='../hw2_data/wiki-news-300d-1M.vec')  # embedding vector path
    parser.add_argument("--emb_dim", type=int, default = 300)
    parser.add_argument("--outPath")  # Output Path
    parser.add_argument("--max_df", type=float, default = 0.7)
    parser.add_argument("--max_features", type=int, default=20000)
    parser.add_argument("--stop_words", default = 'english')

    args = parser.parse_args()

    if not os.path.isdir(args.outPath):
        os.mkdir(args.outPath)

    print("Data processing parameters: ", args)

    print("Loading Data")
    train = pd.read_csv(args.inputPath + 'snli_train.tsv', header = 0, sep = '\t')
    test = pd.read_csv(args.inputPath + 'snli_val.tsv', header=0, sep='\t')
    train_mnli = pd.read_csv(args.inputPath + 'mnli_train.tsv', header=0, sep='\t')
    test_mnli = pd.read_csv(args.inputPath + 'mnli_val.tsv', header=0, sep='\t')

    emb = load_vectors(args.embPath)

    print("Fitting Vocabulary")
    vect, embedding = build_vocab(train['sentence1'] + ' ' + train['sentence2'], emb, emb_dim = args.emb_dim,
                                   max_df = args.max_df, max_features = args.max_features, stop_words=args.stop_words)

    #vect = pickle.load(open(args.outPath + 'vect.p', 'rb'))

    vocab = vect.vocabulary_
    analyzer = vect.build_analyzer()

    print('Transform data frame')
    label_dict = {'entailment': 0, 'neutral': 1, 'contradiction': 2}

    train2 = df2List(train, vocab, analyzer, label_dict)
    test2 = df2List(test, vocab, analyzer, label_dict)
    train_mnli2 = df2List(train_mnli, vocab, analyzer, label_dict, ismnli = True)
    test_mnli2 = df2List(test_mnli, vocab, analyzer, label_dict, ismnli = True)

    pickle.dump(train2, open(args.outPath + 'train.p', 'wb'))
    pickle.dump(test2, open(args.outPath + 'test.p', 'wb'))
    pickle.dump(train_mnli2, open(args.outPath + 'train_mnli.p', 'wb'))
    pickle.dump(test_mnli2, open(args.outPath + 'test_mnli.p', 'wb'))
    pickle.dump(vect, open(args.outPath + 'vect.p', 'wb'))
    pickle.dump(embedding, open(args.outPath + 'embedding.p', 'wb'))

    # Document length:

    lsLen = [max(len(x[0]), len(x[1])) for x in train2]
    print('Median doc size: ', np.percentile(lsLen, 50))
    print('95 percentile: ', np.percentile(lsLen, 95))
    print('Max: ', max(lsLen))

    lsLen = [max(len(x[0]), len(x[1])) for x in train_mnli2]
    print('Median mnli_doc size: ', np.percentile(lsLen, 50))
    print('95 percentile: ', np.percentile(lsLen, 95))
    print('Max: ', max(lsLen))


import requests

city = input('enter the city... ')
api_address = 'https://samples.openweathermap.org/data/2.5/weather?q={},uk&appid=b6907d289e10d714a6e88b30761fae22'.format(
    city)

url = api_address + city
data = requests.get(url).json()
# print(data)

weather = data['weather']
print(weather[0]['description'])


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as stats

# Change the display options
pd.options.display.max_columns = None
pd.options.display.max_rows = None

species_df = pd.read_csv('species_info.csv')
observations_df = pd.read_csv('observations.csv')

# print(species_df.head())
# print(observations_df.head())

# Describe data of species

# print(species_df.dtypes)

species = species_df.astype({'category': 'string',
                             'scientific_name': 'string',
                             'common_names': 'string',
                             'conservation_status': 'string'})  # Change our types of columns

# print(species_df.info())
print(species_df.describe())
print(species_df.category.value_counts())
# print(species_df.conservation_status.value_counts(normalize=True))

# Pie and bar of category
"""
sub_category = species_df.category.value_counts()

plt.figure(figsize=(10, 8))

plt.pie(species_df.category.value_counts().values, labels=species_df.category.value_counts().index, autopct='%1.1f%%')

plt.suptitle('Category of species', fontweight='bold')
plt.savefig('pie_category.png')
plt.show()
"""
# Describe data of observations

# print(observations_df.dtypes)

observations_df = observations_df.astype({'scientific_name': 'string',
                                          'park_name': 'string'})

# print(observations_df.info())
print(observations_df.describe())
# print(observations_df.observations.median())
# print(observations_df.observations.mode())
# print(observations_df.observations.mad())

# The distribution of conservation_status for animals
"""
status_counts = species_df.conservation_status.value_counts()
plt.figure(figsize=(10, 8))

plt.subplot(1, 2, 1)
sns.countplot(x='conservation_status', data=species_df)

plt.xlabel('Conservation status')
plt.ylabel('Count of status')

plt.xticks(rotation=15)

plt.subplot(1, 2, 2)
plt.pie(status_counts, labels=status_counts.index, autopct='%1.1f%%')

plt.axis('equal')

plt.suptitle('Distribution of conservation status for animals', fontweight='bold')
plt.subplots_adjust(wspace=0.5)
plt.savefig('dis_con_status.png')
plt.show()
plt.clf()
"""
# Certain types of species more likely to be endangered

influence = pd.crosstab(species_df.category, species_df.conservation_status)
influence_prop = influence / len(species_df)

print(influence)
print(influence_prop)

influence_marginals = influence_prop.sum(axis=0)
influence_marginals_1 = influence_prop.sum(axis=1)

print(influence_marginals)
print(influence_marginals_1)

chi2, pval, dof, expected = stats.chi2_contingency(influence)
print(expected)
print(chi2)

# Species were spotted the most at each park
"""
merged_df = species_df.merge(observations_df)

grouped_df = merged_df.groupby('category')['observations'].count()
print(grouped_df)
plt.figure(figsize=(15, 8))

plt.subplot(1, 2, 1)
sns.boxplot(x='category', y='observations', data=merged_df)

plt.xlabel('Species')
plt.ylabel('Number of observations')
plt.xticks(rotation=15)

plt.subplot(1, 2, 2)
plt.pie(grouped_df, labels=grouped_df.index, autopct='%1.1f%%')

plt.suptitle('Species were spotted the most at each park', fontweight='bold')

plt.savefig('species_observ.png')
plt.show()
plt.clf()
"""

# sns.histplot(x='observations', data=observations_df)
# plt.show()
print(species_df.scientific_name.mode())


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os

# sigmoid函数
def sigmoid(z):
    return 1 / (1 + np.exp(-z))

# 定义回归模型
def model(X, theta):
    return sigmoid(np.dot(X, theta.T))

# 计算梯度
def gradient(X, y, theta):
    grad = np.zeros(theta.shape)  # 初始化梯度，维度与参数向量的维度相同
    error = (model(X, theta) - y).ravel()  # 计算偏差
    for j in range(len(theta.ravel())):  # 计算n个偏导数（梯度）
        term = np.multiply(error, X[:, j])
        grad[0, j] = np.sum(term) / len(X)
    return grad

# 定义损失函数
def cost(X, y, theta):
    return np.sum((np.multiply(-y, np.log(model(X, theta)))) - (np.multiply(1 - y, np.log(1 - model(X, theta)))))/(len(X))

path = 'datas' + os.sep + 'iris.csv'
irisData = pd.read_csv(path, header=None, names=['petal_len', 'petal_width', 'sepal_len', 'sepal_width', 'class'],
                       dtype={'petal_len': float, 'petal_width': float, 'sepal_len': float, 'sepal_width': float,
                              'class': str})
irisData.loc[irisData['class'] == 'setosa', 'class'] = 0  # 将setosa置为0
irisData.loc[irisData['class'] == 'versicolor', 'class'] = 1  # 将versicolor置为1
irisData.loc[irisData['class'] == 'virginica', 'class'] = 2  # 将virginica置为2

print("---------------打印数据信息------------------ #")
print(irisData.head())  # 打印前两行
print(irisData.shape)  # 打印数据维度
print(irisData.describe())  # 打印描述信息
print()

# 绘制数据分布图像
positive = irisData[irisData['class'] == 0]  # 设置正类
negative = irisData[irisData['class'] == 1]  # 设置负类

# fig, ax = plt.subplots(figsize=(8, 6))
fig, figer1 = plt.subplots(figsize=(10, 5))  # 设置图像大小
figer1.scatter(positive['sepal_len'], positive['sepal_width'], s=30, c='b', marker='o', label='setosa')  # 绘制setosa花的散点图
figer1.scatter(negative['sepal_len'], negative['sepal_width'], s=30, c='r', marker='x',
               label='versicolor')  # 绘制versicolor花的散点图
figer1.legend(loc=2)  # 标题放在左上角
figer1.set_xlabel('sepal_len')  # 设置x标签
figer1.set_ylabel('sepal_width')  # 设置y标签
plt.show()  # 显示初始图像

irisData.insert(2, 'Ones', 1)  # 在第3列插入一列数据,值为1

print("----------打印初始数据的前五行------------ ")
print(irisData.head())

orig_data = irisData.as_matrix()  # 构造一个矩阵
print(orig_data.dtype)
print("----------------初始打印矩阵-----------------")
print(orig_data[:5, :])

cols = orig_data.shape[1]  # 得到矩阵的列数
orig_data = orig_data[:100, :]  # 取矩阵的前100行数据

scaled_data1 = orig_data[:50, 2:cols]  # 第一类数据矩阵，选择花瓣属性
scaled_data2 = orig_data[50:100, 2:cols]  # 第二类数据矩阵
np.random.shuffle(scaled_data1)  # 打乱第一类数据的顺序
np.random.shuffle(scaled_data2)  # 打乱第二类数据的顺序
np.random.shuffle(orig_data)
# 从两个矩阵中分别取固定个数的数据作为测试集
# scaled_data = orig_data[4:100, 2:cols]
#scaled_data = np.vstack((scaled_data1[:25, :], scaled_data2[:25, :]))  # 50%
#scaled_data = np.vstack((scaled_data1[:15, :], scaled_data2[:15, :]))  # 30%
scaled_data = np.vstack((scaled_data1[:5, :], scaled_data2[:5, :]))  # 10%

np.random.shuffle(scaled_data)  # 打乱测试集数据的顺序

print("-------打印测试集-------")
print(scaled_data)
print("------测试集的属性-------")
print(scaled_data.shape)

# 从两个矩阵中分别取相同个数的数据作为训练集
# orig_data = orig_data[:4, 2:cols]
#orig_data = np.vstack((scaled_data1[25:50, :], scaled_data2[25:50, :]))  # 50%
#orig_data = np.vstack((scaled_data1[15:50, :], scaled_data2[15:50, :]))  # 70%
orig_data = np.vstack((scaled_data1[5:50, :], scaled_data2[5:50, :]))   # 90%

np.random.shuffle(orig_data)  # 打乱训练集数据的顺序

print("---------打印训练集--------")
print(orig_data)

X = orig_data[:100, 1:cols - 1]  # 选择前三列
y = orig_data[:100, cols - 1:cols]  # 选择最后一列结果
print("-------打印X的值-------")
print(X)
print("---------打印y的值----------")
print(y)

# 构造参数向量
theta = np.zeros([1, 3])

# 打印矩阵的维度
print("----------打印训练数据的信息----------")
print("参数值为：")
print(theta)
print("X的维度为：")
print(X.shape)
print("y的维度为")
print(y.shape)
print("参数的维度为")
print(theta.shape)

c = cost(X, y, theta)  # 求初始损失函数的值
print("--------初始损失值为-------")
print(X.dtype)
print(c)

#  刷新数据,打乱数据的顺序
def shuffleData(data):
    np.random.shuffle(data)
    cols = data.shape[1]
    X = data[:100, 0:cols - 1]
    y = data[:100, cols - 1:]
    return X, y

import time

# 定义梯度下降求解函数
def descent(data, theta, batchSize, threshold, alpha):
    init_time = time.time()  # 设置初始时间
    i = 0  # 设置迭代次数
    k = 0  # batch
    X, y = shuffleData(data)  # 打乱数据
    grad = np.zeros(theta.shape)  # 计算初始的梯度
    costs = [cost(X, y, theta)]  # 计算初始损失函数值

    # 开始迭代
    while True:
        grad = gradient(X[k:k + batchSize], y[k:k + batchSize], theta)  # 求解梯度值
        k += batchSize  # 取batch个数据
        if k >= n:  # 如果数据取完
            k = 0
            X, y = shuffleData(data)  # 对数据进行重新洗牌
        theta = theta - alpha * grad  # 对参数进行更新
        print(theta)
        cost_new = cost(X, y, theta)  # 计算新的损失值
        print(cost_new)
        costs.append(cost_new)  # 将新的损失之追加到列表末尾
        i += 1  # 更新循环变量

        value = costs  # cost为损失值
        if abs(value[-1] - value[-2]) < threshold:
            break
    return theta, i - 1, costs, grad, time.time() - init_time

# 绘制图像
def Run(data, theta, batchSize, thresh, alpha):
    theta, iter, costs, grad, dur = descent(data, theta, batchSize, thresh, alpha)  # 开始执行梯度下降
    name = "Original" if (data[:, 1] > 2).sum() > 1 else "Scaled"
    name += " data - learning rate: {} -".format(alpha)
    # 选择梯度下降策略和停止方案
    if batchSize == n:
        strDescType = "Gradient"
    elif batchSize == 1:
        strDescType = "Stochastic"
    else:
        strDescType = "Mini-batch({})".format(batchSize)
    name += strDescType + " descent - stop: "
    strStop = "costs change < {}".format(thresh)
    name += strStop
    print("***{}\nTheta: {} - Iter: {} - Last cost: {:03.2f} - Duration: {:03.2f}s".format(name, theta, iter, costs[-1],
                                                                                dur))
    fig, ax = plt.subplots(figsize=(12, 4))
    ax.plot(np.arange(len(costs)), costs, 'r')
    ax.set_xlabel('Iterations')
    ax.set_ylabel('Cost')
    ax.set_title(name)
    plt.show()
    return theta

# 开始训练模型
n = 100  # 一次读入100个数据进行训练
print("打印矩阵")
print(orig_data)

theta = Run(orig_data, theta, n, thresh=0.000001, alpha=0.1)  # 两次迭代损失函数变化非常小时停止(1e-6)

# 对结果进行测试
# 设定阈值 大于0.5则为1，小于0.5为0
def predict(X, theta):
    return [1 if x >= 0.5 else 0 for x in model(X, theta)]

scaled_X = scaled_data[:, :3]  # 设置测试集输入
y = scaled_data[:, 3]  # 正确值

print("--------打印测试的数据---------")
print(scaled_X)

print("----------theta的值为-----------")
print(theta)
predictions = predict(scaled_X, theta)
print("-----------打印预测值-----------")
print(predictions)
print("-------------打印真实值-----------")
print(y)

correct = [1 if ((a == 1 and b == 1) or (a == 0 and b == 0)) else 0 for (a, b) in zip(predictions, y)]
accuracy = (sum(map(int, correct)) / len(correct)) * 100
print('正确率 = {0}%'.format(accuracy))

# 设置分割曲线函数
def y1(x2, theta):
    # y = theta[0] + theta[1]* x1 + theta[2] * x2
    x1 = (-(theta[0, 0] + theta[0, 2] * x2)) / theta[0, 1]
    return x1

x2 = np.linspace(0, 5, 1000)
x1 = y1(x2, theta)

fig, figer1 = plt.subplots(figsize=(10, 5))  # 设置图像大小
figer1.scatter(positive['sepal_len'], positive['sepal_width'], s=30, c='b', marker='o', label='setosa')  # 绘制setosa花的散点图
figer1.scatter(negative['sepal_len'], negative['sepal_width'], s=30, c='r', marker='x',
               label='versicolor')  # 绘制versicolor花的散点图
figer1.legend(loc=2)  # 标题放在左上角
figer1.set_xlabel('sepal_len')  # 设置x标签
figer1.set_ylabel('sepal_width')  # 设置y标签
plt.plot(x1, x2, 'r-', linewidth=1)
plt.show()  # 显示结果图像


"""
Roll adjusted and multiple prices for a given contract, after checking that we do not have positions

NOTE: this does not update the roll calendar .csv files stored elsewhere. Under DRY the sole source of production
  roll info is the multiple prices series
"""
from dataclasses import dataclass
import numpy as np

from syscore.interactive import print_menu_of_values_and_get_response, get_and_convert
from syscore.objects import success, failure, status, named_object
from syscore.text import landing_strip, print_with_landing_strips_around

from sysdata.data_blob import dataBlob

from sysobjects.contracts import futuresContract
from sysobjects.production.roll_state import (
    default_state,
    roll_adj_state,
    explain_roll_state_str,
    allowable_roll_state_from_current_and_position,
    RollState,
    no_roll_state,
)

from sysproduction.reporting.report_configs import roll_report_config
from sysproduction.reporting.reporting_functions import run_report_with_data_blob

from sysproduction.data.positions import diagPositions, updatePositions
from sysproduction.data.contracts import dataContracts
from sysproduction.data.prices import diagPrices, get_valid_instrument_code_from_user

from sysproduction.reporting.data.rolls import (
    rollingAdjustedAndMultiplePrices,
    relative_volume_in_forward_contract_versus_price,
)

no_change_required = named_object("No roll required")
EXIT_CODE = "EXIT"

def interactive_update_roll_status():

    with dataBlob(log_name="Interactive_Update-Roll-Status") as data:
        function_to_call = get_rolling_master_function()
        function_to_call(data)

def get_rolling_master_function():
    MANUAL_INPUT = "Manually input instrument codes and manually decide when to roll"
    MENU_OPTIONS = [
        MANUAL_INPUT,
        "Cycle through instrument codes automatically, but manually decide when to roll",
        "Cycle through instrument codes automatically, auto decide when to roll, manually confirm rolls",
        "Cycle through instrument codes automatically, auto decide when to roll, automatically roll",
    ]

    function_list = [
        update_roll_status_manual_cycle,
        update_roll_status_auto_cycle_manual_decide,
        update_roll_status_auto_cycle_manual_confirm,
        update_roll_status_full_auto,
    ]

    print("How do you want to do your rolls today?")
    selection = print_menu_of_values_and_get_response(
        MENU_OPTIONS, default_str=MANUAL_INPUT
    )
    selection_idx = MENU_OPTIONS.index(selection)

    function_to_call = function_list[selection_idx]

    return function_to_call

@dataclass
class RollDataWithStateReporting(object):
    instrument_code: str
    original_roll_status: RollState
    position_priced_contract: int
    allowable_roll_states_as_list_of_str: list
    days_until_roll: int
    relative_volume: float

    @property
    def original_roll_status_as_string(self):
        return self.original_roll_status.name

    def display_roll_query_banner(self):

        print(landing_strip(80))
        print("Current State: %s" % self.original_roll_status)
        print(
            "Current position in priced contract %d (if zero can Roll Adjusted prices)"
            % self.position_priced_contract
        )
        print("")
        print("These are your options:")
        print("")

        for state_number, state in enumerate(self.allowable_roll_states_as_list_of_str):
            print("%s: %s" % (state, explain_roll_state_str(state)))

        print("")

def update_roll_status_manual_cycle(data: dataBlob):

    do_another = True
    while do_another:
        instrument_code = get_valid_instrument_code_from_user(
            data=data, allow_exit=True, exit_code=EXIT_CODE
        )
        if instrument_code is EXIT_CODE:
            # belt and braces
            do_another = False
        else:
            manually_report_and_update_roll_state_for_code(data, instrument_code)

    return success

def update_roll_status_auto_cycle_manual_decide(data: dataBlob):
    days_ahead = get_days_ahead_to_consider_when_auto_cycling()
    instrument_list = get_list_of_instruments_to_auto_cycle(data, days_ahead=days_ahead)
    for instrument_code in instrument_list:
        manually_report_and_update_roll_state_for_code(
            data=data, instrument_code=instrument_code
        )

    return success

def update_roll_status_auto_cycle_manual_confirm(data: dataBlob):
    days_ahead = get_days_ahead_to_consider_when_auto_cycling()
    auto_parameters = get_auto_roll_parameters()
    instrument_list = get_list_of_instruments_to_auto_cycle(data, days_ahead=days_ahead)

    for instrument_code in instrument_list:
        roll_data = setup_roll_data_with_state_reporting(data, instrument_code)
        roll_state_required = auto_selected_roll_state_instrument(
            data=data, roll_data=roll_data, auto_parameters=auto_parameters
        )

        if roll_state_required is no_change_required:
            warn_not_rolling(instrument_code, auto_parameters)
        else:
            modify_roll_state(
                data=data,
                instrument_code=instrument_code,
                original_roll_state=roll_data.original_roll_status,
                roll_state_required=roll_state_required,
                confirm_adjusted_price_change=True,
            )

def update_roll_status_full_auto(data: dataBlob):
    days_ahead = get_days_ahead_to_consider_when_auto_cycling()
    instrument_list = get_list_of_instruments_to_auto_cycle(data, days_ahead=days_ahead)
    auto_parameters = get_auto_roll_parameters()

    for instrument_code in instrument_list:
        roll_data = setup_roll_data_with_state_reporting(data, instrument_code)
        roll_state_required = auto_selected_roll_state_instrument(
            data=data, roll_data=roll_data, auto_parameters=auto_parameters
        )

        if roll_state_required is no_change_required:
            warn_not_rolling(instrument_code, auto_parameters)
        else:

            modify_roll_state(
                data=data,
                instrument_code=instrument_code,
                original_roll_state=roll_data.original_roll_status,
                roll_state_required=roll_state_required,
                confirm_adjusted_price_change=False,
            )

def get_days_ahead_to_consider_when_auto_cycling() -> int:
    days_ahead = get_and_convert(
        "How many days ahead should I look for expiries?",
        type_expected=int,
        allow_default=True,
        default_value=10,
    )

    return days_ahead

def get_list_of_instruments_to_auto_cycle(data: dataBlob, days_ahead: int = 10) -> list:

    diag_prices = diagPrices()
    list_of_potential_instruments = (
        diag_prices.get_list_of_instruments_in_multiple_prices()
    )
    instrument_list = [
        instrument_code
        for instrument_code in list_of_potential_instruments
        if include_instrument_in_auto_cycle(
            data=data, instrument_code=instrument_code, days_ahead=days_ahead
        )
    ]

    print_with_landing_strips_around(
        "Identified following instruments that are near expiry %s"
        % str(instrument_list)
    )

    return instrument_list

def include_instrument_in_auto_cycle(
    data: dataBlob, instrument_code: str, days_ahead: int = 10
) -> bool:

    days_until_expiry = days_until_earliest_expiry(data, instrument_code)
    return days_until_expiry <= days_ahead

def days_until_earliest_expiry(data: dataBlob, instrument_code: str) -> int:

    data_contracts = dataContracts(data)
    carry_days = data_contracts.days_until_carry_expiry(instrument_code)
    roll_days = data_contracts.days_until_roll(instrument_code)
    price_days = data_contracts.days_until_price_expiry(instrument_code)

    return min([carry_days, roll_days, price_days])

@dataclass
class autoRollParameters:
    min_volume: float
    manual_prompt_for_position: bool
    state_when_position_held: RollState

def get_auto_roll_parameters() -> autoRollParameters:
    min_volume = get_and_convert(
        "Minimum relative volume before rolling",
        type_expected=float,
        allow_default=True,
        default_value=0.1,
    )

    manual_prompt_for_position_str = input(
        "Manually prompt for state if have position? (n / *anything for yes*)"
    )
    if manual_prompt_for_position_str == "n":
        manual_prompt_for_position = False
    else:
        manual_prompt_for_position = True

    if manual_prompt_for_position:
        state_when_position_held = no_change_required
    else:
        state_when_position_held = get_state_to_use_for_held_position()

    auto_parameters = autoRollParameters(
        min_volume=min_volume,
        manual_prompt_for_position=manual_prompt_for_position,
        state_when_position_held=state_when_position_held,
    )

    return auto_parameters

STATE_OPTIONS = [RollState.Passive, RollState.Force, RollState.Force_Outright]
STATE_OPTIONS_AS_STR = [str(state) for state in STATE_OPTIONS]

def get_state_to_use_for_held_position() -> RollState:

    print(
        "Choose state to automatically assume if we have a position in priced contract AND roll state is currently NO ROLL"
    )

    select_state_for_position_held = print_menu_of_values_and_get_response(
        STATE_OPTIONS_AS_STR, default_str=STATE_OPTIONS_AS_STR[0]
    )

    state_when_position_held = STATE_OPTIONS[
        STATE_OPTIONS_AS_STR.index(select_state_for_position_held)
    ]

    return state_when_position_held

def auto_selected_roll_state_instrument(
    data: dataBlob,
    roll_data: RollDataWithStateReporting,
    auto_parameters: autoRollParameters,
) -> RollState:

    if roll_data.relative_volume < auto_parameters.min_volume:

        print_with_landing_strips_around(
            "For %s relative volume of %f is less than minimum of %s : NOT AUTO ROLLING"
            % (
                roll_data.instrument_code,
                roll_data.relative_volume,
                auto_parameters.min_volume,
            )
        )
        return no_change_required

    no_position_held = roll_data.position_priced_contract == 0

    if no_position_held:
        print_with_landing_strips_around(
            "No position held, auto rolling adjusted price for %s"
            % roll_data.instrument_code
        )
        return roll_adj_state

    if auto_parameters.manual_prompt_for_position:
        run_roll_report(data, roll_data.instrument_code)
        roll_state_required = get_roll_state_required(roll_data)
        return roll_state_required

    original_roll_status = roll_data.original_roll_status
    if original_roll_status is no_roll_state:
        roll_state_required = auto_parameters.state_when_position_held

        print_with_landing_strips_around(
            "Automatically changing state from %s to %s for %s"
            % (original_roll_status, roll_state_required, roll_data.instrument_code)
        )
    else:
        print_with_landing_strips_around(
            "Roll status already set to %s for %s: not changing"
            % (original_roll_status, roll_data.instrument_code)
        )
        return no_change_required

    return roll_state_required

def warn_not_rolling(instrument_code: str, auto_parameters: autoRollParameters):

    print_with_landing_strips_around(
        "\n NOT rolling %s as doesn't meet auto parameters %s\n"
        % (instrument_code, str(auto_parameters))
    )

def manually_report_and_update_roll_state_for_code(
    data: dataBlob, instrument_code: str
):
    run_roll_report(data, instrument_code)
    manually_update_roll_state_for_code(data, instrument_code)

def manually_update_roll_state_for_code(data: dataBlob, instrument_code: str):
    # First get the roll info
    # This will also update to console

    data.log.setup(instrument_code=instrument_code)
    roll_data = setup_roll_data_with_state_reporting(data, instrument_code)

    roll_state_required = get_roll_state_required(roll_data)

    modify_roll_state(
        data=data,
        instrument_code=instrument_code,
        original_roll_state=roll_data.original_roll_status,
        roll_state_required=roll_state_required,
        confirm_adjusted_price_change=True,
    )

    return success

def run_roll_report(data: dataBlob, instrument_code: str):
    config = roll_report_config.new_config_with_modified_output("console")
    config.modify_kwargs(instrument_code=instrument_code)
    report_results = run_report_with_data_blob(config, data)
    if report_results is failure:
        raise Exception("Can't run roll report, so can't change status")

def get_roll_state_required(roll_data: RollDataWithStateReporting) -> RollState:
    invalid_input = True
    while invalid_input:
        roll_data.display_roll_query_banner()
        roll_state_required_as_str = print_menu_of_values_and_get_response(
            roll_data.allowable_roll_states_as_list_of_str
        )

        if roll_state_required_as_str != roll_data.original_roll_status_as_string:
            # check if changing
            print("")
            check = input(
                "Changing roll state for %s from %s to %s, are you sure y/n to try again/<RETURN> to exit: "
                % (
                    roll_data.instrument_code,
                    roll_data.original_roll_status_as_string,
                    roll_state_required_as_str,
                )
            )
            print("")
            if check == "y":
                # happy
                return RollState[roll_state_required_as_str]

            elif check == "":
                print("Okay, we're done")
                return no_change_required

            else:
                print("OK. Choose again.")
                # back to top of loop
                continue
        else:
            print("No change")
            return no_change_required

def setup_roll_data_with_state_reporting(
    data: dataBlob, instrument_code: str
) -> RollDataWithStateReporting:
    diag_positions = diagPositions(data)
    diag_contracts = dataContracts(data)

    original_roll_status = diag_positions.get_roll_state(instrument_code)
    priced_contract_date = diag_contracts.get_priced_contract_id(instrument_code)

    contract = futuresContract(instrument_code, priced_contract_date)

    position_priced_contract = int(diag_positions.get_position_for_contract(contract))

    allowable_roll_states = allowable_roll_state_from_current_and_position(
        original_roll_status, position_priced_contract
    )

    days_until_roll = diag_contracts.days_until_roll(instrument_code)

    relative_volume = relative_volume_in_forward_contract_versus_price(
        data=data, instrument_code=instrument_code
    )
    if np.isnan(relative_volume):
        relative_volume = 0.0

    roll_data_with_state = RollDataWithStateReporting(
        instrument_code=instrument_code,
        original_roll_status=original_roll_status,
        position_priced_contract=position_priced_contract,
        allowable_roll_states_as_list_of_str=allowable_roll_states,
        days_until_roll=days_until_roll,
        relative_volume=relative_volume,
    )

    return roll_data_with_state

def modify_roll_state(
    data: dataBlob,
    instrument_code: str,
    original_roll_state: RollState,
    roll_state_required: RollState,
    confirm_adjusted_price_change: bool = True,
):

    if roll_state_required is no_change_required:
        return

    if roll_state_required is original_roll_state:
        return

    update_positions = updatePositions(data)
    update_positions.set_roll_state(instrument_code, roll_state_required)
    if roll_state_required is roll_adj_state:
        state_change_to_roll_adjusted_prices(
            data=data,
            instrument_code=instrument_code,
            original_roll_state=original_roll_state,
            confirm_adjusted_price_change=confirm_adjusted_price_change,
        )

def state_change_to_roll_adjusted_prices(
    data: dataBlob,
    instrument_code: str,
    original_roll_state: RollState,
    confirm_adjusted_price_change: bool = True,
):
    # Going to roll adjusted prices
    update_positions = updatePositions(data)

    roll_result = roll_adjusted_and_multiple_prices(
        data=data,
        instrument_code=instrument_code,
        confirm_adjusted_price_change=confirm_adjusted_price_change,
    )

    if roll_result is success:
        # Return the state back to default (no roll) state
        data.log.msg(
            "Successful roll! Returning roll state of %s to %s"
            % (instrument_code, default_state)
        )

        update_positions.set_roll_state(instrument_code, default_state)
    else:
        data.log.msg(
            "Something has gone wrong with rolling adjusted of %s! Returning roll state to previous state of %s"
            % (instrument_code, original_roll_state)
        )
        update_positions.set_roll_state(instrument_code, original_roll_state)

def roll_adjusted_and_multiple_prices(
    data: dataBlob, instrument_code: str, confirm_adjusted_price_change: bool = True
) -> status:
    """
    Roll multiple and adjusted prices

    THE POSITION MUST BE ZERO IN THE PRICED CONTRACT! WE DON'T CHECK THIS HERE

    :param data: dataBlob
    :param instrument_code: str
    :return:
    """
    print(landing_strip(80))
    print("")
    print("Rolling adjusted prices!")
    print("")
    try:
        rolling_adj_and_mult_object = rollingAdjustedAndMultiplePrices(
            data, instrument_code
        )

        # this will also do the roll calculations
        rolling_adj_and_mult_object.compare_old_and_new_prices()
    except Exception as e:
        print("Error %s when trying to calculate roll prices" % str(e))
        return failure

    if confirm_adjusted_price_change:
        confirm_roll = input(
            "Confirm roll adjusted prices for %s are you sure y/n:" % instrument_code
        )
        if confirm_roll != "y":
            print(
                "\nUSER DID NOT WANT TO ROLL: Setting roll status back to previous state"
            )
            return failure
    else:
        print_with_landing_strips_around("AUTO ROLLING - NO USER CONFIRMATION REQUIRED")

    try:
        rolling_adj_and_mult_object.write_new_rolled_data()
    except Exception as e:
        data.log.warn(
            "%s went wrong when rolling: Going to roll-back to original multiple/adjusted prices"
            % e
        )
        rolling_adj_and_mult_object.rollback()
        return failure

    return success


import time
import aiohttp
import discord
import importlib
import os
import sys
import requests
import asyncio
from io import BytesIO

from discord.ext import commands
from my_utils import permissions, default, dataIO
from my_utils.guildstate import state_instance

class admin(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.config = default.get("config.json")
        self._last_result = None

    @commands.command()
    @commands.check(permissions.is_owner)
    async def load(self, ctx, name: str):
        """ Loads an extension. """
        try:
            self.bot.load_extension(f"cogs.{name}")
        except Exception as e:
            return await ctx.send(default.traceback_maker(e))
        await ctx.send(f"Loaded extension **{name}.py**")

    @commands.command()
    @commands.check(permissions.is_owner)
    async def unload(self, ctx, name: str):
        """ Unloads an extension. """
        try:
            self.bot.unload_extension(f"cogs.{name}")
        except Exception as e:
            return await ctx.send(default.traceback_maker(e))
        await ctx.send(f"Unloaded extension **{name}.py**")

    @commands.command()
    @commands.check(permissions.is_owner)
    async def reload(self, ctx, name: str):
        """ Reloads an extension. """
        try:
            self.bot.reload_extension(f"cogs.{name}")
        except Exception as e:
            return await ctx.send(default.traceback_maker(e))
        await ctx.send(f"Reloaded extension **{name}.py**")

    @commands.command()
    @commands.check(permissions.is_owner)
    async def reloadall(self, ctx):
        """ Reloads all extensions. """
        error_collection = []
        for file in os.listdir("cogs"):
            if file.endswith(".py"):
                name = file[:-3]
                try:
                    self.bot.reload_extension(f"cogs.{name}")
                except Exception as e:
                    error_collection.append(
                        [file, default.traceback_maker(e, advance=False)]
                    )

        if error_collection:
            output = "\n".join([f"**{g[0]}** ```diff\n- {g[1]}```" for g in error_collection])
            return await ctx.send(
                f"Attempted to reload all extensions, was able to reload, "
                f"however the following failed: \n\n{output}"
            )

        await ctx.send("Successfully reloaded all extensions")

    @commands.command()
    @commands.check(permissions.is_owner)
    async def reloadutils(self, ctx, name: str):
        """ Reloads a utils module. """
        name_maker = f"utils_folder/{name}.py"
        try:
            module_name = importlib.import_module(f"utils_folder.{name}")
            importlib.reload(module_name)
        except ModuleNotFoundError:
            return await ctx.send(f"Couldn't find module named **{name_maker}**")
        except Exception as e:
            error = default.traceback_maker(e)
            return await ctx.send(f"Module **{name_maker}** returned error and was not reloaded...\n{error}")
        await ctx.send(f"Reloaded module **{name_maker}**")

    @commands.command()
    @commands.check(permissions.is_owner)
    async def reboot(self, ctx):
        """ Reboot the bot """
        await ctx.send('Rebooting now...')
        time.sleep(1)
        dataIO.backup_states(state_instance)
        await self.bot.close()
        sys.exit()

    @commands.command()
    @commands.check(permissions.is_owner)
    async def dm(self, ctx, user_id: int, *, message: str):
        """ DM the user of your choice """
        user = self.bot.get_user(user_id)
        if not user:
            return await ctx.send(f"Could not find any UserID matching **{user_id}**")

        try:
            await user.send(message)
            await ctx.send(f"✉ Sent a DM to **{user_id}**")
        except discord.Forbidden:
            await ctx.send("This user might be having DMs blocked or it's a bot account...")

    @commands.group()
    @commands.check(permissions.is_owner)
    async def change(self, ctx):
        if ctx.invoked_subcommand is None:
            await ctx.send_help(str(ctx.command))

    @change.command(name="playing")
    @commands.check(permissions.is_owner)
    async def change_playing(self, ctx, *, playing: str):
        """ Change playing status. """
        if self.config.status_type == "idle":
            status_type = discord.Status.idle
        elif self.config.status_type == "dnd":
            status_type = discord.Status.dnd
        else:
            status_type = discord.Status.online

        if self.config.playing_type == "listening":
            playing_type = 2
        elif self.config.playing_type == "watching":
            playing_type = 3
        else:
            playing_type = 0

        try:
            await self.bot.change_presence(
                activity=discord.Activity(type=playing_type, name=playing),
                status=status_type
            )
            dataIO.change_value("config.json", "playing", playing)
            await ctx.send(f"Successfully changed playing status to **{playing}**")
        except discord.InvalidArgument as err:
            await ctx.send(err)
        except Exception as e:
            await ctx.send(e)

    @change.command(name="username")
    @commands.check(permissions.is_owner)
    async def change_username(self, ctx, *, name: str):
        """ Change username. """
        try:
            await self.bot.user.edit(username=name)
            await ctx.send(f"Successfully changed username to **{name}**")
        except discord.HTTPException as err:
            await ctx.send(err)

    @change.command(name="nickname")
    @commands.check(permissions.is_owner)
    async def change_nickname(self, ctx, *, name: str = None):
        """ Change nickname. """
        try:
            await ctx.guild.me.edit(nick=name)
            if name:
                await ctx.send(f"Successfully changed nickname to **{name}**")
            else:
                await ctx.send("Successfully removed nickname")
        except Exception as err:
            await ctx.send(err)

    @change.command(name="avatar")
    @commands.check(permissions.is_owner)
    async def change_avatar(self, ctx, url: str = None):
        """ Change avatar. """
        if url is None and len(ctx.message.attachments) == 1:
            url = ctx.message.attachments[0].url
        else:
            url = url.strip('<>') if url else None

        try:
            bio = requests.get(url).content
            await self.bot.user.edit(avatar=bio)
            await ctx.send(f"Successfully changed the avatar. Currently using:\n{url}")
        except aiohttp.InvalidURL:
            await ctx.send("The URL is invalid...")
        except discord.InvalidArgument:
            await ctx.send("This URL does not contain a useable image")
        except discord.HTTPException as err:
            await ctx.send(err)
        except TypeError:
            await ctx.send("You need to either provide an image URL or upload one with the command")

    @change.command(name="def_prefix")
    @commands.check(permissions.is_owner)
    async def change_default_prefix(self, ctx, prefix):
        """Changes the default premanent prefix"""
        dataIO.change_value("config.json", "prefix", prefix)
        await ctx.send(f"Successfully changed default prefix to **{prefix}**")

    @commands.command(aliases = ["api_for", "api"])
    @commands.check(permissions.is_owner)
    async def search_api(self, ctx, category = ""):
        """ Search for some apis """
        if category != "":
            your_api = requests.get(f"https://api.publicapis.org/entries?category={category.lower()}&https=true").json()
        elif category.lower() == "categories":
            your_api = requests.get(f"https://api.publicapis.org/categories").json()
        else:
            your_api = requests.get("https://api.publicapis.org/random?auth=null").json()
        if your_api['count'] == 0:
            return await ctx.send("No APIs found")
        apis = f"{your_api['entries'][0]['Category']} apis\n"
        def auth(index):
            if your_api['entries'][i]['Auth'] != None:
                return your_api['entries'][i]['Auth']
            return "None"

        for i in range(your_api["count"]):
            apis += f"**{i+1}**. {your_api['entries'][i]['API']} - {your_api['entries'][i]['Description']} | Auth: {auth(i)} | Cors: {your_api['entries'][i]['Cors']} | Link: {your_api['entries'][i]['Link']}\n"
        if len(str(apis)) > 1999:
            apis = apis[:2000][::-1]
            arr = apis.index(".")
            apis = apis[arr:][::-1]

        return await ctx.send(apis)

    @commands.group(aliases = ["file"])
    @commands.check(permissions.is_owner)
    async def fil(self, ctx):
        if ctx.invoked_subcommand is None:
            await ctx.send_help(str(ctx.command))

    @fil.group()
    @commands.check(permissions.is_owner)
    async def add(self, ctx, location = ""):
        if len(ctx.message.attachments) == 1 and location != "":
            try:
                await ctx.message.attachments[0].save(f"{location}\{ctx.message.attachments[0].filename}")
            except FileNotFoundError:
                await ctx.send("Directory not found. Creating directory...")
                os.makedirs(location)
                await ctx.message.attachments[0].save(f"{location}\{ctx.message.attachments[0].filename}")
        elif len(ctx.message.attachments) == 1 and location == "":
            await ctx.message.attachments[0].save(f"{ctx.message.attachments[0].filename}")
        else:
            return await ctx.send("Provide a file as an attachment")
        await ctx.message.delete(delay=1)
        return await ctx.send(f"The {ctx.message.attachments[0].filename} has been added")

    @fil.group()
    @commands.check(permissions.is_owner)
    async def remove(self, ctx, file_name_with_path):
        await ctx.send("Are you sure you want to remove the file. Please remember to unload if the file is and existing cog.\n(y/n)")
        def mcheck(message):
            if message.author == ctx.author:
                return True
            return False
        try:
            answer = await self.bot.wait_for('message', timeout=20, check=mcheck)
        except asyncio.TimeoutError:
            return await ctx.send("You didn't respond in time")
        if answer.content == "y":
            pass
        else:
            return await ctx.send("As you wish, the file will not be removed")
        try:
            default.delete(file_name_with_path)
            await ctx.send(f"Removed {file_name_with_path}")
        except Exception as e:
            await ctx.send(e)
        await ctx.message.delete(delay=1)

def setup(bot):
    bot.add_cog(admin(bot))


import torch
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
from math import exp
import math

def gaussian(window_size, sigma):
	gauss = torch.Tensor([exp(-(x - window_size/2)**2/float(2*sigma**2)) for x in range(window_size)])
	return gauss/gauss.sum()

def create_window(window_size, channel):
	_1D_window = gaussian(window_size, 1.5).unsqueeze(1)
	_2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
	window = Variable(_2D_window.expand(channel, 1, window_size, window_size))
	return window

def SSIM1(img1, img2):
	(_, channel, _, _) = img1.size()
	window_size = 11
	pad = int(window_size/11)
	window = create_window(window_size, channel).to(img1.device)
	mu1 = F.conv2d(img1, window, padding = pad, groups = channel)
	mu2 = F.conv2d(img2, window, padding = pad, groups = channel)

	mu1_sq = mu1.pow(2)
	mu2_sq = mu2.pow(2)
	mu1_mu2 = mu1*mu2

	sigma1_sq = F.conv2d(img1*img1, window, padding = pad, groups = channel) - mu1_sq
	sigma2_sq = F.conv2d(img2*img2, window, padding = pad, groups = channel) - mu2_sq
	sigma12 = F.conv2d(img1*img2, window, padding = pad, groups = channel) - mu1_mu2

	C1 = 0.01**2
	C2 = 0.03**2

	ssim_map = ((2*mu1_mu2 + C1)*(2*sigma12 + C2))/((mu1_sq + mu2_sq + C1)*(sigma1_sq + sigma2_sq + C2))
	return ssim_map.mean()

def SSIM(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
	# Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
	if val_range is None:
		if torch.max(img1) > 128:
			max_val = 255
		else:
			max_val = 1

		if torch.min(img1) < -0.5:
			min_val = -1
		else:
			min_val = 0
		L = max_val - min_val
	else:
		L = val_range

	padd = 0
	(_, channel, height, width) = img1.size()
	if window is None:
		real_size = min(window_size, height, width)
		window = create_window(real_size, channel=channel).to(img1.device)

	mu1 = F.conv2d(img1, window, padding=padd, groups=channel)
	mu2 = F.conv2d(img2, window, padding=padd, groups=channel)

	mu1_sq = mu1.pow(2)
	mu2_sq = mu2.pow(2)
	mu1_mu2 = mu1 * mu2

	sigma1_sq = F.conv2d(img1 * img1, window, padding=padd, groups=channel) - mu1_sq
	sigma2_sq = F.conv2d(img2 * img2, window, padding=padd, groups=channel) - mu2_sq
	sigma12 = F.conv2d(img1 * img2, window, padding=padd, groups=channel) - mu1_mu2

	C1 = (0.01 * L) ** 2
	C2 = (0.03 * L) ** 2

	v1 = 2.0 * sigma12 + C2
	v2 = sigma1_sq + sigma2_sq + C2
	cs = torch.mean(v1 / v2)  # contrast sensitivity

	ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)

	if size_average:
		ret = ssim_map.mean()
	else:
		ret = ssim_map.mean(1).mean(1).mean(1)

	if full:
		return ret, cs
	return ret

def PSNR(img1, img2):
	mse = np.mean( (img1/255. - img2/255.) ** 2 )
	if mse == 0:
		return 100
	PIXEL_MAX = 1
	return 20 * math.log10(PIXEL_MAX / math.sqrt(mse))

def CIEDE2000(Lab_1, Lab_2):
	'''Calculates CIEDE2000 color distance between two CIE L*a*b* colors'''
	C_25_7 = 6103515625  # 25**7

	L1, a1, b1 = Lab_1[0], Lab_1[1], Lab_1[2]
	L2, a2, b2 = Lab_2[0], Lab_2[1], Lab_2[2]
	C1 = math.sqrt(a1 ** 2 + b1 ** 2)
	C2 = math.sqrt(a2 ** 2 + b2 ** 2)
	C_ave = (C1 + C2) / 2
	G = 0.5 * (1 - math.sqrt(C_ave ** 7 / (C_ave ** 7 + C_25_7)))

	L1_, L2_ = L1, L2
	a1_, a2_ = (1 + G) * a1, (1 + G) * a2
	b1_, b2_ = b1, b2

	C1_ = math.sqrt(a1_ ** 2 + b1_ ** 2)
	C2_ = math.sqrt(a2_ ** 2 + b2_ ** 2)

	if b1_ == 0 and a1_ == 0:
		h1_ = 0
	elif a1_ >= 0:
		h1_ = math.atan2(b1_, a1_)
	else:
		h1_ = math.atan2(b1_, a1_) + 2 * math.pi

	if b2_ == 0 and a2_ == 0:
		h2_ = 0
	elif a2_ >= 0:
		h2_ = math.atan2(b2_, a2_)
	else:
		h2_ = math.atan2(b2_, a2_) + 2 * math.pi

	dL_ = L2_ - L1_
	dC_ = C2_ - C1_
	dh_ = h2_ - h1_
	if C1_ * C2_ == 0:
		dh_ = 0
	elif dh_ > math.pi:
		dh_ -= 2 * math.pi
	elif dh_ < -math.pi:
		dh_ += 2 * math.pi
	dH_ = 2 * math.sqrt(C1_ * C2_) * math.sin(dh_ / 2)

	L_ave = (L1_ + L2_) / 2
	C_ave = (C1_ + C2_) / 2

	_dh = abs(h1_ - h2_)
	_sh = h1_ + h2_
	C1C2 = C1_ * C2_

	if _dh <= math.pi and C1C2 != 0:
		h_ave = (h1_ + h2_) / 2
	elif _dh > math.pi and _sh < 2 * math.pi and C1C2 != 0:
		h_ave = (h1_ + h2_) / 2 + math.pi
	elif _dh > math.pi and _sh >= 2 * math.pi and C1C2 != 0:
		h_ave = (h1_ + h2_) / 2 - math.pi
	else:
		h_ave = h1_ + h2_

	T = 1 - 0.17 * math.cos(h_ave - math.pi / 6) + 0.24 * math.cos(2 * h_ave) + 0.32 * math.cos(
		3 * h_ave + math.pi / 30) - 0.2 * math.cos(4 * h_ave - 63 * math.pi / 180)

	h_ave_deg = h_ave * 180 / math.pi
	if h_ave_deg < 0:
		h_ave_deg += 360
	elif h_ave_deg > 360:
		h_ave_deg -= 360
	dTheta = 30 * math.exp(-(((h_ave_deg - 275) / 25) ** 2))

	R_C = 2 * math.sqrt(C_ave ** 7 / (C_ave ** 7 + C_25_7))
	S_C = 1 + 0.045 * C_ave
	S_H = 1 + 0.015 * C_ave * T

	Lm50s = (L_ave - 50) ** 2
	S_L = 1 + 0.015 * Lm50s / math.sqrt(20 + Lm50s)
	R_T = -math.sin(dTheta * math.pi / 90) * R_C

	k_L, k_C, k_H = 1, 1, 1

	f_L = dL_ / k_L / S_L
	f_C = dC_ / k_C / S_C
	f_H = dH_ / k_H / S_H

	dE_00 = math.sqrt(f_L ** 2 + f_C ** 2 + f_H ** 2 + R_T * f_C * f_H)
	return dE_00

def rgb2xyz(rgb):
    def format(c):
        c = c / 255.
        if c > 0.04045: c = ((c + 0.055) / 1.055) ** 2.4
        else: c = c / 12.92
        return c * 100
    rgb = list(map(format, rgb))
    xyz = [None, None, None]
    xyz[0] = rgb[0] * 0.4124 + rgb[1] * 0.3576 + rgb[2] * 0.1805
    xyz[1] = rgb[0] * 0.2126 + rgb[1] * 0.7152 + rgb[2] * 0.0722
    xyz[2] = rgb[0] * 0.0193 + rgb[1] * 0.1192 + rgb[2] * 0.9505
    return xyz

def xyz2lab(xyz):
    def format(c):
        if c > 0.008856: c = c ** (1. / 3.)
        else: c = (7.787 * c) + (16. / 116.)
        return c
    xyz[0] = xyz[0] / 95.047
    xyz[1] = xyz[1] / 100.00
    xyz[2] = xyz[2] / 108.883
    xyz = list(map(format, xyz))
    lab = [None, None, None]
    lab[0] = (116. * xyz[1]) - 16.
    lab[1] = 500. * (xyz[0] - xyz[1])
    lab[2] = 200. * (xyz[1] - xyz[2])
    return lab