#day42

from selenium import webdriver

# Inicialize o driver do navegador
driver = webdriver.Chrome()

# Acesse o site de música
driver.get("https://www.youtube.com/")

# Localize a caixa de pesquisa no site
search_box = driver.find_element_by_xpath('//input[@id="search"]')

# Digite a música que você deseja reproduzir
search_box.send_keys("music name")
search_box.submit()

# Localize o primeiro vídeo na lista de resultados de pesquisa
video = driver.find_element_by_xpath('//a[@id="video-title"]')

# Clique no vídeo para reproduzí-lo
video.click()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 16 12:05:18 2022

@author: cbone
"""

#fichier refaisant la procédure de Gilett avec nos données pour comparaison

#import extraction_data as extr
import numpy as np
import detatt_mk as da
import torch
import matplotlib.pyplot as plt
from scipy import stats
def mkdir_p(mypath):
    '''Creates a directory. equivalent to using mkdir -p on the command line'''

    from errno import EEXIST
    from os import makedirs,path

    try:
        makedirs(mypath)
    except OSError as exc: # Python >2.5
        if exc.errno == EEXIST and path.isdir(mypath):
            pass
        else: raise

obs = torch.load('figures/Europe/rand_hist_IPSL-CM6A-LR.pt').detach().numpy()[0:115] * 1.06
print(f"obs shape : {obs.shape}")
#print(f"obs : {obs}")

liste_models = ['ACCESS', 'BCC', 'CESM2', 'CNRM', 'CanESM5', 'FGOALS',
                'GISS', 'HadGEM3', 'IPSL', 'MIRO', 'ESM2', 'NorESM2']
#liste_models = ['ACCESS-ESM1-5', 'BCC-CSM2-MR', 'CESM2',
 #               'CNRM-CM6-1', 'CanESM5', 'FGOALS-g3', 'GISS-E2-1-G',
  #              'HadGEM3-GC31-LL', 'IPSL-CM6A-LR', 'MIROC6', 'MRI-ESM2-0',
   #             'NorESM2-LM']

variab = np.array([0.1,0.13,0.15,0.11,0.17,0.10,0.11,0.13,0.1,0.15,0.13,0.15])

Mean_orig = np.load('figures/Europe/concatenated_IPSL-CM6A-LR.npy')
print(f"Mean_orig[3] : {Mean_orig[3]}")
print(f"Mean_orig shape : {Mean_orig.shape}")
mean_tot = np.mean(np.load('figures/Europe/combined_data.npy')[:,1],axis=0)
print(f"mean_tot shape : {mean_tot.shape}")
aer = torch.load('figures/Europe/aer.pt')
print(f"aer : {aer}")

for tensor in aer:
    print(tensor.shape)

mean = np.copy(Mean_orig)

#obs = obs - np.mean(obs)
# =============================================================================
# mean[0] -= np.mean(mean,axis=1)[0]
# mean[1] -= np.mean(mean,axis=1)[1]
# mean[2] -= np.mean(mean,axis=1)[2]
# mean[3] -= np.mean(mean,axis=1)[3]
# =============================================================================

#mean = np.subtract(mean - np.mean(mean,axis=1))
count = np.load("figures/Europe/count.npy")
print(f"count shape : {count.shape}")
print(f"count : {count}")
noise = np.load("figures/Europe/noise.npy")
print(f"noise shape : {noise.shape}")

mean = np.transpose(mean)
anom_index = noise.shape[0]
noise = np.transpose(noise)
model = 'IPSL-CM6A-LR'

(xr,yr,cn1,cn2)=da.reduce_dim(mean,obs[:,None],noise[:,list(range(1,anom_index,2))],noise[:,list(range(0,anom_index,2))])
result=da.tls(xr[:,0:3],yr,cn1,ne=count[0:3],cn2=cn2,rof_flag=1,RCT_flag=0)
name_exp = 'ROF/'+model+'/'
path = '/usr/home/habbar/Bureau/data_nc/'
mkdir_p(path+name_exp)
print(f"result Beta : {result['beta']}")
print(f"result Beta shape : {result['beta'].shape}")

mean_variab = np.mean(variab)
sum_combined = np.zeros((115))
for forc in range(3):
    ecart = (result['betaCI'][forc,1] - result['betaCI'][forc,0]) / 2

    ecart = ecart / 1.64
    print(ecart)
    x_inf=np.zeros((115))
    x_sup=np.zeros((115))
    nn=np.arange(115)
    for i in nn:
        Xn=stats.norm.rvs(size=10000,loc = Mean_orig[forc][i], scale = mean_variab/np.sqrt(count[forc]))
        betan=stats.norm.rvs(size=10000,loc = result['beta'][forc], scale = ecart)
        mult=Xn*betan
        x_inf[i]=np.percentile(mult,5)
        x_sup[i]=np.percentile(mult,95)

    plt.fill_between(nn,x_inf,x_sup,alpha=0.5)
    np.save(path+name_exp+str(forc)+'_result_mean',Mean_orig[forc]*result['beta'][forc])
    np.save(path+name_exp+str(forc)+'_result_min',x_inf)
    np.save(path+name_exp+str(forc)+'_result_max',x_sup)
    plt.plot(nn,Mean_orig[forc]*result['beta'][forc])
    plt.grid()
    plt.savefig(path+name_exp+str(forc)+'_result.png')

    sum_product = np.sum(result['beta'][forc] * Mean_orig[forc])

    print(f"Somme des produits pour le forçage {forc}: {sum_product}")

    # Ajouter la courbe calculée à la somme globale
    sum_combined += Mean_orig[forc] * result['beta'][forc]

# Plot de la somme des courbes combinées
plt.plot(nn, sum_combined, label='Somme des courbes combinées', color='black')
plt.legend()

# Plot de l'observation historique cible
plt.plot(nn, obs, label='Observation historique cible', color='red')
plt.legend()
# sauvegarde de la figure
plt.savefig(path+name_exp+'sum_combined.png')

plt.show()


import base64
import csv
import json
import typing
from datetime import datetime

from Crypto.Cipher import AES
from Crypto.Util import Padding
from starlette.responses import Response

class PrettyJSONResponse(Response):
    media_type = "application/json"

    def render(self, content: typing.Any) -> bytes:
        return json.dumps(
            content,
            ensure_ascii=False,
            allow_nan=False,
            indent=4,
            separators=(", ", ": "),
        ).encode("utf-8")

def int_to_datestr(value):
    s = str(value)
    fecnac = datetime(year=int(s[0:4]), month=int(s[4:6]), day=int(s[6:8]))
    return fecnac.strftime("%d/%m/%Y")

def str_to_datestr(value):
    return datetime.strptime(value, "%Y-%m-%d").strftime("%d/%m/%Y")

def encrypt_param(param, variant="ruc"):
    encryption_key = base64.b64decode("aCIbjMuVGtwF8nlSKoPydE==")
    text = json.dumps({variant: param}).encode()
    text_padded = Padding.pad(text, AES.block_size)
    iv = base64.b64decode("JAwlt7SNbYLycmPRqeDFou==")
    cipher = AES.new(encryption_key, AES.MODE_CBC, iv)
    cipher_enc = cipher.encrypt(text_padded)
    return base64.b64encode(cipher_enc).decode()

def get_delimitter(buffer):
    try:
        dialect = csv.Sniffer().sniff(buffer.readline(), delimiters=";,")
        return dialect.delimiter
    except csv.Error:
        return "\t"


"""
Page for utilities
Çeşitli fonksiyonların tanımlandığı yer
"""

import json
import os
import tempfile
import uuid
import xml.etree.ElementTree as et
import zipfile

import ee
import folium
import geemap.foliumap as geemap
import numpy as np
import pandas as pd
import streamlit as st
from lxml import etree
import plotly.graph_objects as go
from matplotlib import colors

delta_nbr_colors = {
    "Veri Yok": "ffffff",
    "Yüksek yeniden büyüme": "7a8737",
    "Düşük yeniden büyüme": "acbe4d",
    "Yanmamış": "0ae042",
    "Düşük Tahribat": "fff70b",
    "Orta-Düşük tahribat": "ffaf38",
    "Orta-yüksek tahribat": "ff641b",
    "Yüksek tahribat": "a41fd6",
}

ndvi_colors = {
    "min":-1, "max":1,
    "palette": [
        "#d73027",
        "#f46d43",
        "#fdae61",
        "#fee08b",
        "#d9ef8b",
        "#a6d96a",
        "#66bd63",
        "#1a9850",
    ]
}

ndvi_colors2 = {
    "min":-1, "max":1,
    "palette": [
        "#af002d", # -1, -0.9
        "#af002d", # -0.9, -0.8
        "#af002d", # -0.8, -0.7
        "#af002d", # -0.7, -0.6
        "#af002d", # -0.6, -0.5
        "#af002d", # -0.5, -0.4
        "#af002d", # -0.4, -0.3
        "#af002d", # -0.3, -0.2
        "#af002d", # -0.2, -0.1
        "#af002d", # -0.1, 0
        "#af002d", # 0, 0.1
        "#ee4c41", # 0.1, 0.2
        "#fd8958", # 0.2, 0.3
        "#ffbf7b", # 0.3, 0.4
        "#ffedab", # 0.4, 0.5
        "#e6f8a6", # 0.5, 0.6
        "#b9e284", # 0.6, 0.7
        "#66cc66", # 0.7, 0.8
        "#009966", # 0.8, 0.9
        "#006633", # 0.9, 1
    ]
}

ndvi_colors2_legend = {
    "0.9-1.0": "006633",
    "0.8-0.9": "009966",
    "0.7-0.8": "66cc66",
    "0.6-0.7": "b9e284",
    "0.5-0.6": "e6f8a6",
    "0.4-0.5": "ffedab",
    "0.3-0.4": "ffbf7b",
    "0.2-0.3": "fd8958",
    "0.1-0.2": "ee4c41",
    "-1.0-0.1 ": "af002d",
}

ndvi_colors2_hist = {
    "0.90": "#006633",
    "0.80": "#009966",
    "0.70": "#66cc66",
    "0.60": "#b9e284",
    "0.50": "#e6f8a6",
    "0.40": "#ffedab",
    "0.30": "#ffbf7b",
    "0.20": "#fd8958",
    "0.10": "#ee4c41",
    "0.00": "#af002d"
}

ndvi_colors2_imshow = colors.ListedColormap([
    "#af002d",
    "#ee4c41",
    "#fd8958",
    "#ffbf7b",
    "#ffedab",
    "#e6f8a6",
    "#b9e284",
    "#66cc66",
    "#009966",
    "#006633"
    ])

scale = [-1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
ndvi_colors2_norm=colors.BoundaryNorm(scale, 10)

mask_seq_imshow = colors.ListedColormap([
    "white",
    "darkred",
    "red",
    "orange"
    ])

mask_seq_colors = {
    "min":0, "max": 3,
    "palette": [
        "#ffffff", # 0
        "#8b0000", # 0
        "#ff0000", # 1
        "#ffa500", # 2
    ]
}
mask_scale = [0, 1, 2, 3]
mask_colors_norm=colors.BoundaryNorm(mask_scale, 3)

evi_colors = {
    # "min":0.1, "max":0.8,
    "min":0, "max":1,
    "palette": [
        'FFFFFF', 'CE7E45', 'DF923D', 'F1B555', 'FCD163', '99B718',
        '74A901', '66A000', '529400', '3E8601', '207401', '056201',
        '004C00', '023B01', '012E01', '011D01', '011301'
    ]
}

evi_scale = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
evi_colors_norm=colors.BoundaryNorm(evi_scale, 8)

evi_colors_imshow = colors.ListedColormap([
    "#FFFFFF",
    "#CE7E45",
    "#DF923D",
    "#F1B555",
    "#FCD163",
    "#99B718",
    "#74A901",
    "#66A000",
    "#529400",
    "#3E8601",
    "#207401",
    "#056201",
    "#004C00",
    "#023B01",
    "#012E01",
    "#011D01",
    "#011301"
    ])

mbi_colors = {
    "min":-0.3, "max":0.6,
    "palette": [
        'FFFFFF', 'CE7E45', 'DF923D', 'F1B555', 'FCD163', '99B718',
        '74A901', '66A000', '529400', '3E8601', '207401', '056201',
        '004C00', '023B01', '012E01', '011D01', '011301'
    ]
}

mbi_scale = [-0.3, -0.2, -0.1, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
mbi_colors_norm=colors.BoundaryNorm(mbi_scale, 9)

mbi_colors_imshow = colors.ListedColormap([
    "#FFFFFF",
    "#CE7E45",
    "#DF923D",
    "#F1B555",
    "#FCD163",
    "#99B718",
    "#74A901",
    "#66A000",
    "#529400",
    "#3E8601",
    "#207401",
    "#056201",
    "#004C00",
    "#023B01",
    "#012E01",
    "#011D01",
    "#011301"
    ])

NAMES = [
    "Veri Yok",
    "Yüksek tahribat",
    "Orta-yüksek tahribat",
    "Orta-Düşük tahribat",
    "Düşük Tahribat",
    "Yanmamış",
    "Düşük yeniden büyüme",
    "Yüksek yeniden büyüme",
]

def calculate_dnbr_dataframe(number_of_pixels):
    """
    The function to calculate the DNBR dataframe.
    """
    names = NAMES
    values = np.array(number_of_pixels)  # pixel numbers
    hectares = values * 900 / 10000  # convert to hectares
    percenteges = hectares / np.sum(hectares) * 100  # calculate percenteges

    dataframe = pd.DataFrame(
        {"dNBR sınıfları": names, "hektar": hectares, "yüzde": percenteges}
    )
    dataframe = dataframe.style.format(precision=2)
    return dataframe.to_html()

def get_plotly_charts(number_of_pixels):
    """
    The function to generate the plotly charts.
    """
    fig = go.Figure(
        data=[
            go.Pie(
                labels=NAMES,
                values=list(number_of_pixels),
                sort=False,
                marker=dict(
                    colors=[
                        "ffffff",
                        "a41fd6",
                        "ff641b",
                        "ffaf38",
                        "fff70b",
                        "0ae042",
                    ]
                ),
            )
        ],
    )

    return fig

def get_pixel_counts(image, geometry):
    """
    The function to get the pixel counts of classes in an dNBR image.
    """
    # pylint: disable=no-member
    thresholds = ee.Image([-1000, -251, -101, 99, 269, 439, 659, 2000])
    classified = image.lt(thresholds).reduce("sum").toInt()
    allpix = classified.updateMask(classified)
    pixstats = allpix.reduceRegion(
        maxPixels= 10000000000,
        reducer=ee.Reducer.count(),  # count pixels in a single class
        geometry=geometry,
        scale=30,
    )

    allpixels = ee.Number(pixstats.get("sum"))  # extract pixel count as a number
    allpixels.getInfo()
    results = []

    results = []
    for i in range(8):
        single_mask = classified.updateMask(classified.eq(i))  # mask a single class
        stats = single_mask.reduceRegion(
            maxPixels= 10000000000,
            reducer=ee.Reducer.count(),  # count pixels in a single class
            geometry=geometry,
            scale=30,
        )
        pix = ee.Number(stats.get("sum"))

        results.append(pix.getInfo())
    return results

def map_search(folium_map: geemap.Map) -> None:  # sourcery skip: use-named-expression
    """
    The function to generate the search box above the map.
    """
    keyword = st.text_input("Bölge arayın:", "")
    if keyword:
        locations = geemap.geocode(keyword)
        if locations is not None and len(locations) > 0:
            str_locations = [str(g)[1:-1] for g in locations]
            location = st.selectbox("Bölge seçin:", str_locations)
            loc_index = str_locations.index(location)
            selected_loc = locations[loc_index]
            lat, lng = selected_loc.lat, selected_loc.lng
            folium.Marker(location=[lat, lng], popup=location).add_to(folium_map)
            folium_map.set_center(lng, lat, 12)
            st.session_state["zoom_level"] = 12

def kml_geometry_export(file_path):
    """
    The function to export the geometry of a KML file.
    """
    # pylint: disable=c-extension-no-member
    root = etree.parse(file_path)

    for i in root.iter():
        path = root.getelementpath(i).split("}")[0] + "}"

    tree = et.parse(file_path)
    root = tree.getroot()

    name = root.find(f".//*{path}coordinates")
    geolist = name.text.strip().split(" ")

    geometry = []

    for i in geolist:
        current = i.split(",")
        # en az 3 point içermesi lazım yoksa EEException error veriyor.
        geometry.append([float(current[0]), float(current[1])])

    return ee.Geometry.Polygon(geometry)

def uploaded_file_to_gdf(data):
    """
    The function to convert uploaded file to geodataframe.
    """
    _, file_extension = os.path.splitext(data.name)
    file_id = str(uuid.uuid4())
    file_path = os.path.join(tempfile.gettempdir(), f"{file_id}{file_extension}")

    with open(file_path, "wb") as file:
        file.write(data.getbuffer())

    # gpd.io.file.fiona.drvsupport.supported_drivers["KML"] = "rw"

    if file_path.lower().endswith(".kml"):

        return kml_geometry_export(file_path)

    if file_path.lower().endswith(".kmz"):
        # unzip it to get kml file
        in_kmz = os.path.abspath(file_path)
        out_dir = os.path.dirname(in_kmz)
        out_kml = os.path.join(out_dir, "doc.kml")
        with zipfile.ZipFile(in_kmz, "r") as zip_ref:
            zip_ref.extractall(out_dir)

        return kml_geometry_export(out_kml)

    if file_path.lower().endswith(".geojson"):
        with open(file_path, "r", encoding="utf-8") as file:
            data = json.loads(file.read())

        return ee.Geometry.Polygon(data["features"][0]["geometry"]["coordinates"][0])


from django.conf.urls import url, include
from django.views.generic import TemplateView
from . import django_urls

from allauth.socialaccount.providers.facebook.views import FacebookOAuth2Adapter
from allauth.socialaccount.providers.twitter.views import TwitterOAuthAdapter

from rest_framework.decorators import api_view

from rest_auth.urls import urlpatterns
from rest_auth.registration.views import SocialLoginView
from rest_auth.social_serializers import TwitterLoginSerializer

class FacebookLogin(SocialLoginView):
    adapter_class = FacebookOAuth2Adapter

class TwitterLogin(SocialLoginView):
    adapter_class = TwitterOAuthAdapter
    serializer_class = TwitterLoginSerializer

class TwitterLoginSerializerFoo(TwitterLoginSerializer):
    pass

@api_view(['POST'])
def twitter_login_view(request):
    serializer = TwitterLoginSerializerFoo(
        data={'access_token': '11223344', 'token_secret': '55667788'},
        context={'request': request}
    )
    serializer.is_valid(raise_exception=True)

class TwitterLoginNoAdapter(SocialLoginView):
    serializer_class = TwitterLoginSerializer

urlpatterns += [
    url(r'^rest-registration/', include('rest_auth.registration.urls')),
    url(r'^test-admin/', include(django_urls)),
    url(r'^account-email-verification-sent/$', TemplateView.as_view(),
        name='account_email_verification_sent'),
    url(r'^account-confirm-email/(?P<key>[-:\w]+)/$', TemplateView.as_view(),
        name='account_confirm_email'),
    url(r'^social-login/facebook/$', FacebookLogin.as_view(), name='fb_login'),
    url(r'^social-login/twitter/$', TwitterLogin.as_view(), name='tw_login'),
    url(r'^social-login/twitter-no-view/$', twitter_login_view, name='tw_login_no_view'),
    url(r'^social-login/twitter-no-adapter/$', TwitterLoginNoAdapter.as_view(), name='tw_login_no_adapter'),
    url(r'^accounts/', include('allauth.socialaccount.urls'))
]


#8/13/18
#cnn classification

#neural network transforms an input by putting it through a series of hidden layers
#layers made-up of neurons-connected neurons to neurons

#cnn-layers built in 3 dimensions (width,height,depth)
#for cnn, neurons in one layer don't connect to all neurons in the next layer (only a small fraction of other neurons)
#cnn-i.hidden layers/feature extraction ii. classification (probability)
import numpy as np
from keras.layers import Conv2D, Activation, MaxPool2D, Flatten, Dense
from keras.models import Sequential

#filter/kernels produce a feature map
#relu-to make the output non-linear
#stide-the size of the step that the convolutional filter moves each time (filter slides pixel by pixel)

#feed the image into the model (512 x 512 pixels with 3 channels)
img_shape=(28,28,1)

#hidden layers/feature extraction
model=Sequential()
#add a convolutional layer with 3x3 by 3 filters and a stride of one
#set padding so that the input size equals the output size
model.add(Conv2D(6,2,input_shape=img_shape))
#add a relu activation to the layer
model.add(Activation('relu'))
#Pooling
model.add(MaxPool2D(2))

#fully connected layers
#use flatten to convert 3d to 1d
model.add(Flatten())
#add dense layer with ten neurons
model.add(Dense(10))
#using the softmax activation function for the last layer
model.add(Activation('softmax'))
#model summary
model.summary

#classification (optimizer, loss, and metrics)
from keras.datasets import mnist

(x_train,y_train),(x_test,y_test)=mnist.load_data()
x_train=np.expand_dims(x_train,-1)
x_test=np.expand_dims(x_test,-1)

model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',
metrics=['acc'])

#train the model, iterating on the data in batches of 32 samples with 5 epochs
model.fit(x_train, y_train, batch_size=32, epochs=5, validation_data=(x_test,y_test))


import numpy as np
from numpy.testing import assert_array_equal
import pytest

from tompy.core import argsort_like, rankdata

def test_argsort_like():
    # confirm sorting
    a = np.random.randint(0, 100, 42)
    b = a[np.random.permutation(len(a))]

    assert_array_equal(a[argsort_like(a, b)], b)

    # confirm raising value error if different number of elements
    with pytest.raises(ValueError):
        argsort_like(a, b[1:])

    # confirm raising value error if different elements
    b[0] = -1  # impossible value
    with pytest.raises(ValueError):
        argsort_like(a, b)

def test_rankdata():
    # check vectors
    a = [0.5, 0.2, 4, -3]
    target = [2, 1, 3, 0]
    assert_array_equal(rankdata(a), target)

    # check matrices without axis definition
    a = [[0.5, 0.2, 4, -3], [2, 6, 1, 5]]
    target = [[2, 1, 3, 0], [1, 3, 0, 2]]
    assert_array_equal(rankdata(a), target)

    # check matrices with axis=1 (same as above)
    assert_array_equal(rankdata(a, axis=1), target)

    # check matrices with axis=0
    a = [[0.5, 0.2, 4, -3], [2, 6, 1, 5], [-3, 5, 2, 0]]
    target = [[1, 0, 2, 0], [2, 2, 0, 2], [0, 1, 1, 1]]
    assert_array_equal(rankdata(a, axis=0), target)

    # check matrices with axis=None
    target = [4,  3,  8,  0,  6, 11,  5,  9,  1, 10,  7,  2]
    assert_array_equal(rankdata(a, axis=None), target)


import numpy as np
import matplotlib.pyplot as plt # For verbose
import scipy.optimize
import sys          # For error handling
from copy import copy

#######################################
### 1: Frequency sampling functions ###
#######################################

# 1.0: dithering
def drawDithering(m,bounds = None):
    '''Draws m samples a <= x < b, with bounds=(a,b) (default: (0,2*pi)).'''
    if bounds is None:
        (lowb,highb) = (0,2*np.pi)
    else:
        (lowb,highb) = bounds
    return np.random.uniform(low=lowb,high=highb,size=m)

# 1.1: frequency sampling functions
# 1.1.1: gaussian sampling
def drawFrequencies_Gaussian(d,m,Sigma = None):
    '''draws frequencies according to some sampling pattern'''
    if Sigma is None:
        Sigma = np.identity(d)
    Om = np.random.multivariate_normal(np.zeros(d), np.linalg.inv(Sigma), m).T # inverse of sigma
    return Om

# 1.1.2: folded gaussian sampling
def drawFrequencies_FoldedGaussian(d,m,Sigma = None):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, for R from folded Gaussian with variance 1, phi uniform'''
    if Sigma is None:
        Sigma = np.identity(d)
    R = np.abs(np.random.randn(m)) # folded standard normal distribution radii
    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere
    SigFact = np.linalg.inv(np.linalg.cholesky(Sigma))

    Om = SigFact@phi*R

    return Om

# 1.1.3: adapted radius sampling
def sampleFromPDF(pdf,x,nsamples=1):
    '''x is a vector (the support of the pdf), pdf is the values of pdf eval at x'''
    # Note that this can be more general than just the adapted radius distribution

    pdf = pdf/np.sum(pdf) # ensure pdf is normalized

    cdf = np.cumsum(pdf)

    # necessary?
    cdf[-1] = 1.

    sampleCdf = np.random.uniform(0,1,nsamples)

    sampleX = np.interp(sampleCdf, cdf, x)

    return sampleX

def pdfAdaptedRadius(r,KMeans=False):
    '''up to a constant'''
    if KMeans:
        return r*np.exp(-(r**2)/2)  # Dont take the gradient according to sigma into account
    else:
        return np.sqrt(r**2 + (r**4)/4)*np.exp(-(r**2)/2)

def drawFrequencies_AdaptedRadius(d,m,Sigma = None,KMeans=False):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, for R from adapted with variance 1, phi uniform'''
    if Sigma is None:
        Sigma = np.identity(d)

    # Sample the radii
    r = np.linspace(0,5,2001)
    R = sampleFromPDF(pdfAdaptedRadius(r,KMeans),r,nsamples=m)

    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere
    SigFact = np.linalg.inv(np.linalg.cholesky(Sigma))

    Om = SigFact@phi*R

    return Om

def pdf_diffOfGaussians(r,GMM_upper=None,GMM_lower=None):
    """Here, GMM is given in terms of SD and not variance"""
    if isinstance(GMM_upper,tuple):
        (weights_upper,sigmas_upper) = GMM_upper
    elif GMM_upper is None:
        weights_upper = np.array([]) # Empty array
    else:
        (weights_upper,sigmas_upper) = (np.array([1.]),np.array([GMM_upper]))

    if isinstance(GMM_lower,tuple):
        (weights_lower,sigmas_lower) = GMM_lower
    elif GMM_lower is None:
        weights_lower = np.array([])
    else:
        (weights_lower,sigmas_lower) = (np.array([1.]),np.array([GMM_lower]))

    res = np.zeros(r.shape)
    # Add
    for k in range(weights_upper.size):
        res += weights_upper[k]*np.exp(-0.5*(r**2)/(sigmas_upper[k]**2))
    # Substract
    for k in range(weights_lower.size):
        res -= weights_lower[k]*np.exp(-0.5*(r**2)/(sigmas_lower[k]**2))

    # Ensure pdf is positive
    pdf_is_negative = res < 0
    if any(pdf_is_negative):
        print(res[:5])
        # Print a warning if the negative pdf values are significant (not due to rounding errors)
        tol = 1e-8
        if np.max(np.abs(res[np.where(pdf_is_negative)[0]])) > tol:
            print("WARNING: negative pdf values detected and replaced by zero, check the validity of your input")
        # Correct the negative values
        res[np.where(pdf_is_negative)[0]] = 0.

    return res

def drawFrequencies_diffOfGaussians(d,m,GMM_upper,GMM_lower=None,verbose=0):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, TODO, phi uniform'''

    # reasonable sampling
    n_Rs = 1001
    if isinstance(GMM_upper,tuple):
        R_max = 4*np.max(GMM_upper[1]) # GMM_upper is (weights, cov)-type tuple
    else:
        R_max = 4*GMM_upper
    r = np.linspace(0,R_max,n_Rs)

    if verbose > 0:
        plt.plot(r,pdf_diffOfGaussians(r,GMM_upper,GMM_lower))
        plt.xlabel('frequency norm r')
        plt.ylabel('pdf(r)')
        plt.show()

    # sample from the diff of gaussians pdf
    R = sampleFromPDF(pdf_diffOfGaussians(r,GMM_upper,GMM_lower),r,nsamples=m)

    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere

    Om = phi*R

    return Om

# General function for convenience
def drawFrequencies(drawType,d,m,Sigma = None,nb_cat_per_dim=None):
    """Draw the 'frequencies' or projection matrix Omega for sketching.

    Arguments:
        - drawType: a string indicating the sampling pattern (Lambda) to use, one of the following:
            -- "gaussian"       or "G"  : Gaussian sampling > Lambda = N(0,Sigma^{-1})
            -- "foldedGaussian" or "FG" : Folded Gaussian sampling (i.e., the radius is Gaussian)
            -- "adaptedRadius"  or "AR" : Adapted Radius heuristic
        - d: int, dimension of the data to sketch
        - m: int, number of 'frequencies' to draw (the target sketch dimension)
        - Sigma: is either:
            -- (d,d)-numpy array, the covariance of the data (note that we typically use Sigma^{-1} in the frequency domain).
            -- a tuple (w,cov) describing a scale mixture of Gaussians where,
                -- w:  (K,)-numpy array, the weights of the scale mixture
                -- cov: (K,d,d)-numpy array, the K different covariances in the mixture
            -- None: same as Sigma = identity matrix (belongs to (d,d)-numpy array case)
                 If Sigma is None (default), we assume that data was normalized s.t. Sigma = identity.
        - nb_cat_per_dim: (d,)-array of ints, the number of categories per dimension for integer data,
                    if its i-th entry = 0 (resp. > 0), dimension i is assumed to be continuous (resp. int.).
                    By default all entries are assumed to be continuous. Frequencies for int data is drawn as follows:
                    1. Chose one dimension among the categorical ones, set Omega along all others to zero
                    2. For the chosen dimension with C categories, we draw its component omega ~ U({0,...,C-1}) * 2*pi/C

    Returns:
        - Omega: (d,m)-numpy array containing the 'frequency' projection matrix
    """
    # Parse drawType input
    if drawType.lower() in ["drawfrequencies_gaussian","gaussian","g"]:
        drawFunc = drawFrequencies_Gaussian
    elif drawType.lower() in ["drawfrequencies_foldedgaussian","foldedgaussian","folded_gaussian","fg"]:
        drawFunc = drawFrequencies_FoldedGaussian
    elif drawType.lower() in ["drawfrequencies_adapted","adaptedradius","adapted_radius","ar"]:
        drawFunc = drawFrequencies_AdaptedRadius
    elif drawType.lower() in ["drawfrequencies_adapted_kmeans","adaptedradius_kmeans","adapted_radius_kmeans","arkm","ar-km"]:
        drawFunc = lambda _a,_b,_c : drawFrequencies_AdaptedRadius(_a,_b,_c,KMeans=True)
    else:
        raise ValueError("drawType not recognized")

    # Handle no input
    if Sigma is None:
        Sigma = np.identity(d)

    # Handle
    if isinstance(Sigma,np.ndarray):
        Omega = drawFunc(d,m,Sigma)

    # Handle mixture-type input
    elif isinstance(Sigma,tuple):
        (w,cov) = Sigma # unpack
        K = w.size
        # Assign the frequencies to the mixture components
        assignations = np.random.choice(K,m,p=w)
        Omega = np.zeros((d,m))
        for k in range(K):
            active_index = (assignations == k)
            if any(active_index):
                Omega[:,np.where(active_index)[0]] = drawFunc(d,active_index.sum(),cov[k])

    elif (isinstance(Sigma,float) or isinstance(Sigma,int)) and Sigma > 0:
        Omega = drawFunc(d,m,Sigma*np.eye(d))

    else:
        raise ValueError("Sigma not recognized")

    # If needed, overwrite the integer entries
    if nb_cat_per_dim is not None:
        intg_index = np.nonzero(nb_cat_per_dim)[0]
        d_intg = np.size(intg_index)

        Omega_intg = np.zeros((d_intg,m))
        for intgdim_localindex,intg_globalindex in enumerate(intg_index):
            C = nb_cat_per_dim[intg_globalindex]
            Omega_intg[intgdim_localindex,:] = (2*np.pi/C)*np.random.randint(0,C,(1,m))
        # Mask
        masks_pool = np.eye(d_intg)
        masks = masks_pool[np.random.choice(d_intg,m)].T
        Omega_intg = Omega_intg*masks

        Omega[intg_index] = Omega_intg

    return Omega

# The following funtions allows to estimate Sigma (some utils first, main function is "estimate_Sigma")
# Optimization problem to fit a GMM curve to the data
def _fun_grad_fit_sigmas(p,R,z):
    """
    Function and gradient to solve the optimization problem
        min_{w,sigs2} sum_{i = 1}^n ( z[i] - sum_{k=1}^K w[k]*exp(-R[i]^2*sig2[k]/2) )^2
    Arguments:
        - p, a (2K,) numpy array obtained by stacking
            - w : (K,) numpy array
            - sigs2 : (K,) numpy array
        - R: (n,) numpy array, data to fit (x label)
        - z: (n,) numpy array, data to fit (y label)
    Returns:
        - The function evaluation
        - The gradient
    """

    K = p.size//2
    w = p[:K]
    sigs2 = p[K:]
    n = R.size
    # Naive implementation, TODO better?
    fun = 0
    grad = np.zeros(2*K)
    for i in range(n):
        fun += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.))**2
        grad[:K] += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.)) * (- np.exp(-(sigs2*R[i]**2)/2.)) # grad of w
        grad[K:] += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.)) * (- w * np.exp(-(sigs2*R[i]**2)/2.)) * (-0.5*R[i]**2) # grad of sigma2
    return (fun,grad)

# For normalization in the optimization problem
def _callback_fit_sigmas(p):
    K = p.size//2
    p[:K] /= np.sum(p[:K])

def estimate_Sigma_from_sketch(z,Phi,K=1,c=20,mode='max',sigma2_bar=None,weights_bar=None,should_plot=False):
    # Parse
    if mode == 'max':
        mode_criterion = np.argmax
    elif mode == 'min':
        mode_criterion = np.argmin
    else:
        raise ValueError("Unrecocgnized mode ({})".format(mode))

    # sort the freqs by norm
    Rs = np.linalg.norm(Phi.Omega,axis=0)
    i_sort = np.argsort(Rs)
    Rs = Rs[i_sort]
    z_sort = z[i_sort]/Phi.c_norm # sort and normalize individual entries to 1

    # Initialization
    # number of freqs per box
    s = Phi.m//c
    # init point
    if sigma2_bar is None:
        sigma2_bar = np.random.uniform(0.3,1.6,K)
    if weights_bar is None:
        weights_bar = np.ones(K)/K

    # find the indices of the max of each block
    jqs = np.empty(c)
    for ic in range(c):
        j_max = mode_criterion(np.abs(z_sort)[ic*s:(ic+1)*s]) + ic*s
        jqs[ic] = j_max
    jqs = jqs.astype(int)
    R_tofit = Rs[jqs]
    z_tofit = np.abs(z_sort)[jqs]

    # Set up the fitting opt. problem
    f = lambda p: _fun_grad_fit_sigmas(p,R_tofit,z_tofit) # cost

    p0 = np.zeros(2*K) # initial point
    p0[:K] = weights_bar # w
    p0[K:] = sigma2_bar

    # Bounds of the optimization problem
    bounds = []
    for k in range(K): bounds.append([1e-5,1]) # bounds for the weigths
    for k in range(K): bounds.append([5e-4*sigma2_bar[k],2e3*sigma2_bar[k]]) # bounds for the sigmas -> cant cange too much

    # Solve the sigma^2 optimization problem
    sol = scipy.optimize.minimize(f, p0,jac = True, bounds = bounds,callback=_callback_fit_sigmas)
    p = sol.x
    weights_bar = np.array(p[:K])/np.sum(p[:K])
    sigma2_bar = np.array(p[K:])

    # Plot if required
    if should_plot:
        plt.figure(figsize=(10,5))
        rfit = np.linspace(0,Rs.max(),100)
        zfit = np.zeros(rfit.shape)
        for k in range(K):
            zfit += weights_bar[k]*np.exp(-(sigma2_bar[k]*rfit**2)/2.)
        plt.plot(Rs,np.abs(z_sort),'.')
        plt.plot(R_tofit,z_tofit,'.')
        plt.plot(rfit,zfit)
        plt.xlabel('R')
        plt.ylabel('|z|')
        plt.show()

    return sigma2_bar

def estimate_Sigma(dataset,m0,K=None,c=20,n0=None,drawFreq_type = "AR",nIterations=5,mode='max',verbose=0):
    """Automatically estimates the "Sigma" parameter(s) (the scale of data clusters) for generating the sketch operator.

    We assume here that Sigma = sigma2_bar * identity matrix.
    To estimate sigma2_bar, lightweight sketches of size m0 are generated from (a small subset of) the dataset
    with candidate values for sigma2_bar. Then, sigma2_bar is updated by fitting a Gaussian
    to the absolute values of the obtained sketch. Cfr. https://arxiv.org/pdf/1606.02838.pdf, sec 3.3.3.

    Arguments:
        - dataset: (n,d) numpy array, the dataset X: n examples in dimension d
        - m0: int, number of candidate 'frequencies' to draw (can be typically smaller than m).
        - K:  int (default 1), number of scales to fit (if > 1 we fit a scale mixture)
        - c:  int (default 20), number of 'boxes' (i.e. number of maxima of sketch absolute values to fit)
        - n0: int or None, if given, n0 samples from the dataset are subsampled to be used for Sigma estimation
        - drawType: a string indicating the sampling pattern (Lambda) to use in the pre-sketches, either:
            -- "gaussian"       or "G"  : Gaussian sampling > Lambda = N(0,Sigma^{-1})
            -- "foldedGaussian" or "FG" : Folded Gaussian sampling (i.e., the radius is Gaussian)
            -- "adaptedRadius"  or "AR" : Adapted Radius heuristic
        - nIterations: int (default 5), the maximum number of iteration (typically stable after 2 iterations)
        - mode: 'max' (default) or 'min', describe which sketch entries per block to fit
        - verbose: 0,1 or 2, amount of information to print (default: 0, no info printed). Useful for debugging.

    Returns: If K = 1:
                - Sigma: (d,d)-numpy array, the (diagonal) estimated covariance of the clusters in the dataset;
             If K > 1: a tuple (w,Sigma) representing the scale mixture model, where:
                - w:     (K,)-numpy array, the weigths of the scale mixture (sum to 1)
                - Sigma: (K,d,d)-numpy array, the dxd covariances in the scale mixture
    """

    return_format_is_matrix = K is None
    K = 1 if K is None else K

    (n,d) = dataset.shape
    # X is the subsampled dataset containing only n0 examples
    if n0 is not None and n0<n:
        X = dataset[np.random.choice(n,n0,replace=False)]
    else:
        X = dataset

    # Check if we dont overfit the empirical Fourier measurements
    if (m0 < (K * 2)*c):
        print("WARNING: overfitting regime detected for frequency sampling fitting")

    # Initialization
    #maxNorm = np.max(np.linalg.norm(X,axis=1))
    sigma2_bar = np.random.uniform(0.3,1.6,K)
    weights_bar = np.ones(K)/K

    # Actual algorithm
    for i in range(nIterations):
        # Draw frequencies according to current estimate
        sigma2_bar_matrix = np.outer(sigma2_bar,np.eye(d)).reshape(K,d,d)  # covariances in (K,d,d) format
        Omega0 = drawFrequencies(drawFreq_type,d,m0,Sigma = (weights_bar,sigma2_bar_matrix))

        # Compute unnormalized complex exponential sketch
        Phi0 = SimpleFeatureMap("ComplexExponential",Omega0)
        z0 = computeSketch(X,Phi0)

        should_plot = verbose > 1 or (verbose > 0 and i >= nIterations - 1)
        sigma2_bar = estimate_Sigma_from_sketch(z0,Phi0,K,c,mode,sigma2_bar,weights_bar,should_plot)

    if return_format_is_matrix:
        Sigma = sigma2_bar[0]*np.eye(d)
    else:
        sigma2_bar_matrix = np.outer(sigma2_bar,np.eye(d)).reshape(K,d,d)  # covariances in (K,d,d) format
        Sigma = (weights_bar,sigma2_bar_matrix)

    return Sigma

#######################################
###    2: Feature map functions     ###
#######################################

# 2.1: Common sketch nonlinearities and derivatives
def _complexExponential(t,T=2*np.pi):
    return np.exp(1j*(2*np.pi)*t/T)
def _complexExponential_grad(t,T=2*np.pi):
    return ((1j*2*np.pi)/T)*np.exp(1j*(2*np.pi)*t/T)

def _universalQuantization(t,Delta=np.pi,centering=True):
    if centering:
        return ( (t // Delta) % 2 )*2-1 # // stands for "int division
    else:
        return ( (t // Delta) % 2 ) # centering=false => quantization is between 0 and +1

def _universalQuantization_complex(t,Delta=np.pi,centering=True):
    return _universalQuantization(t-Delta/2,Delta=Delta,centering=centering) + 1j*_universalQuantization(t-Delta,Delta=Delta,centering=centering)

def _sawtoothWave(t,T=2*np.pi,centering=True):
    if centering:
        return ( t % T )/T*2-1
    else:
        return ( t % T )/T # centering=false => output is between 0 and +1

def _sawtoothWave_complex(t,T=2*np.pi,centering=True):
    return _sawtoothWave(t-T/4,T=T,centering=centering) + 1j*_sawtoothWave(t-T/2,T=T,centering=centering)

def _triangleWave(t,T=2*np.pi):
    return (2*(t % T)/T ) - (4*(t % T)/T - 2)*( (t // T) % 2 ) - 1

def _fourierSeriesEvaluate(t,coefficients,T=2*np.pi):
    """T = period
    coefficients = F_{-K}, ... , F_{-1}, F_{0}, F_{1}, ... F_{+K}"""
    K = (coefficients.shape[0]-1)/2
    ks = np.arange(-K,K+1)
    # Pre-alloc
    ft = np.zeros(t.shape) + 0j
    for i in range(2*int(K)+1):
        ft += coefficients[i]*np.exp(1j*(2*np.pi)*ks[i]*t/T)
    return ft

# dict of nonlinearities and their gradient returned as a tuple
_dico_nonlinearities = {
    "complexexponential":(_complexExponential,_complexExponential_grad),
    "universalquantization":(_universalQuantization,None),
    "universalquantization_complex":(_universalQuantization_complex,None),
    "sawtooth":(_sawtoothWave,None),
    "sawtooth_complex":(_sawtoothWave_complex,None),
    "cosine": (lambda x: np.cos(x),lambda x: -np.sin(x))
 }

# 2.2 FeatureMap objects
# Abstract feature map class
class FeatureMap:
    """Template for a generic Feature Map. Useful to check if an object is an instance of FeatureMap."""
    def __init__(self):
        pass
    def __call__(self):
        raise NotImplementedError("The way to compute the feature map is not specified.")
    def grad(self):
        raise NotImplementedError("The way to compute the gradient of the feature map is not specified.")

# TODO find a better name
class SimpleFeatureMap(FeatureMap):
    """Feature map the type Phi(x) = c_norm*f(Omega^T*x + xi)."""
    def __init__(self, f, Omega, xi = None, c_norm = 1.):
        """
        - f can be one of the following:
            -- a string for one of the predefined feature maps:
                -- "complexExponential"
                -- "universalQuantization"
                -- "cosine"
            -- a callable function
            -- a tuple of function (specify the derivative too)

        """
        # 1) extract the feature map
        self.name = None
        if isinstance(f, str):
            try:
                (self.f,self.f_grad) = _dico_nonlinearities[f.lower()]
                self.name = f # Keep the feature function name in memory so that we know we have a specific fct
            except KeyError:
                raise NotImplementedError("The provided feature map name f is not implemented.")
        elif callable(f):
            (self.f,self.f_grad) = (f,None)
        elif (isinstance(f,tuple)) and (len(f) == 2) and (callable(f[0]) and callable(f[1])):
            (self.f,self.f_grad) = f
        else:
            raise ValueError("The provided feature map f does not match any of the supported types.")

        # 2) extract Omega the projection matrix TODO allow callable Omega for fast transform
        if (isinstance(Omega,np.ndarray) and Omega.ndim == 2):
            self.Omega = Omega
            (self.d,self.m) = Omega.shape
        else:
            raise ValueError("The provided projection matrix Omega should be a (d,m) numpy array.")
        # 3) extract the dithering
        if xi is None:
            self.xi = np.zeros(self.m)
        else:
            self.xi = xi
        # 4) extract the normalization constant
        if isinstance(c_norm, str):
            if c_norm.lower() in ['unit','normalized']:
                self.c_norm = 1./np.sqrt(self.m)
            else:
                raise NotImplementedError("The provided c_norm name is not implemented.")
        else:
            self.c_norm = c_norm

    # magic operator to be able to call the FeatureMap object as a function
    def __call__(self,x):
        return self.c_norm*self.f(np.matmul(self.Omega.T,x.T).T + self.xi) # Evaluate the feature map at x

    def grad(self,x):
        """Gradient (Jacobian matrix) of Phi, as a (d,m)-numpy array"""
        return self.c_norm*self.f_grad(np.matmul(self.Omega.T,x.T).T + self.xi)*self.Omega

#######################################
### 3: Actual sketching functions   ###
#######################################

#################################
# 3.1 GENERAL SKETCHING ROUTINE #
#################################
def computeSketch(dataset, featureMap, datasetWeights = None, batch_size = None):
    """
    Computes the sketch of a dataset given a generic feature map.

    More precisely, evaluates
        z = sum_{x_i in X} w_i * Phi(x_i)
    where X is the dataset, Phi is the sketch feature map, w_i are weights assigned to the samples (typically 1/n).

    Arguments:
        - dataset        : (n,d) numpy array, the dataset X: n examples in dimension d
        - featureMap     : the feature map Phi, given as one of the following:
            -- a function, z_x_i = featureMap(x_i), where x_i and z_x_i are (n,)- and (m,)-numpy arrays, respectively
            -- a FeatureMap instance (e.g., constructed as featureMap = SimpleFeatureMap("complexExponential",Omega) )
        - datasetWeights : (n,) numpy array, optional weigths w_i in the sketch (default: None, corresponds to w_i = 1/n)

    Returns:
        - sketch : (m,) numpy array, the sketch as defined above
    """
    # TODOs:
    # - add possibility to specify classes and return one sketch per class
    # - defensive programming
    # - efficient implementation, take advantage of parallelism

    (n,d) = dataset.shape # number of samples, dimension

    # Determine the sketch dimension and sanity check: the dataset is nonempty and the map works
    if isinstance(featureMap,FeatureMap): # featureMap is the argument, FeatureMap is the class
        m = featureMap.m
    else:
        try:
            m = featureMap(dataset[0]).shape[0]
        except:
            raise ValueError("Unexpected error while calling the sketch feature map:", sys.exc_info()[0])

    sketch = np.zeros(m)

    # Split the batches
    if batch_size is None:
        batch_size = int(1e6/m) # Rough heuristic, best batch size will vary on different machines
    nb_batches = int(np.ceil(n/batch_size))

    if datasetWeights is None:
        for b in range(nb_batches):
            sketch = sketch + featureMap(dataset[b*batch_size:(b+1)*batch_size]).sum(axis=0)
        sketch /= n
    else:
        sketch = datasetWeights@featureMap(X)
    return sketch

#################################
# 3.2 PRIVATE SKETCHING METHODS #
#################################
def sensisitivty_sketch(featureMap,n = 1,DPdef = 'UDP',sensitivity_type = 1):
    """
    Computes the sensitity of a provided sketching function.

    The noisy sketch operator A(X) is given by
        A(X) := (1/n)*[sum_{x_i in X} featureMap(x_i)] + w
    where w is Laplacian or Gaussian noise.

    Arguments:
        - featureMap, the sketch the sketch featureMap (Phi), provided as either:
            -- a FeatureMap object with a known sensitivity (i.e., complex exponential or universal quantization periodic map)
            -- (m,featureMapName,c_normalization): tuple (deprectated, only useful for code not supporting FeatureMap objects),
                that should contain:
                -- m: int, the sketch dimension
                -- featureMapName: string, name of sketch feature function f, values supported:
                    -- 'complexExponential' (f(t) = exp(i*t))
                    -- 'universalQuantization_complex' (f(t) = sign(exp(i*t)))
                -- c_normalization: real, the constant before the sketch feature function (e.g., 1. (default), 1./sqrt(m),...)
        - n: int, number of sketch contributions being averaged (default = 1, useful to add noise on n independently)
        - DPdef: string, name of the Differential Privacy variant considered, i.e. the neighbouring relation ~:
            -- 'remove', 'add', 'remove/add', 'UDP' or 'standard': D~D' iff D' = D U {x'} (or vice versa)
            -- 'replace', 'BDP': D~D' iff D' = D \ {x} U {x'} (or vice versa)
        - sensitivity_type: int, 1 (default) for L1 sensitivity, 2 for L2 sensitivity.

    Returns: a positive real, the L1 or L2 sensitivity of the sketching operator defined above.

    Cfr: Differentially Private Compressive K-means, https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8682829.
    """

    # TODO include real cases (cosine, real universal quantization)

    # The sensitivity is of the type: c_feat*c_
    if isinstance(featureMap,FeatureMap):
        m = featureMap.m
        featureMapName = featureMap.name
        c_normalization = featureMap.c_norm
    elif (isinstance(featureMap,tuple)) and (len(featureMap) == 3):
        (m,featureMapName,c_normalization) = featureMap
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    # Sensitivity is given by S = c_featureMap * c_sensitivity_type * c_DPdef, check all three conditions (ughh)
    if featureMapName.lower() == 'complexexponential':
        if sensitivity_type == 1:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return m*np.sqrt(2)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return 2*m*np.sqrt(2)*(c_normalization/n)
        elif sensitivity_type == 2:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return np.sqrt(m)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
    elif featureMapName.lower() == 'universalquantization_complex': # Assuming normalized in [-1,+1], TODO check real/complex case?
        if sensitivity_type == 1:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return m*2*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return 2*m*2*(c_normalization/n)
        elif sensitivity_type == 2:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return np.sqrt(2)*np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
    print(sensitivity_type)
    raise Exception('You provided ({},{});\nThe sensitivity for this (feature map,DP definition) combination is not implemented.'.format(featureMapName.lower(),DPdef.lower()))
    return None

def computeSketch_DP(dataset, featureMap, epsilon, delta = 0,DPdef = 'UDP',useImproveGaussMechanism=True,budget_split_num = None):
    """
    Computes the Differentially Private sketch of a dataset given a generic feature map.

    More precisely, evaluates the DP sketching mechanism:
        z = ( sum_{x_i in X} Phi(x_i) + w_num )/( |X| + w_den )
    where X is the dataset, Phi is the sketch feature map, w_num and w_den are Laplacian or Gaussian random noise.

    Arguments:
        - dataset        : (n,d) numpy array, the dataset X: n examples in dimension d
        - featureMap, the sketch the sketch featureMap (Phi), provided as either:
            -- a FeatureMap object with a known sensitivity (i.e., complex exponential or universal quantization periodic map)
            -- (featureMap(x_i),m,featureMapName,c_normalization): tuple (deprectated, only useful for old code),
                that should contain:
                -- featMap: a function, z_x_i = featMap(x_i), where x_i and z_x_i are (n,)- and (m,)-numpy arrays, respectively
                -- m: int, the sketch dimension
                -- featureMapName: string, name of sketch feature function f, values supported:
                    -- 'complexExponential' (f(t) = exp(i*t))
                    -- 'universalQuantization' (f(t) = sign(exp(i*t)))
                -- c_normalization: real, the constant before the sketch feature function (e.g., 1. (default), 1./sqrt(m),...)
        - epsilon: real > 0, the privacy parameter epsilon
        - delta:  real >= 0, the privacy parameter delta in approximate DP; if delta=0 (default), we have "pure" DP.
        - DPdef: string, name of the Differential Privacy variant considered, i.e. the neighbouring relation ~:
            -- 'remove', 'add', 'remove/add', 'UDP' or 'standard' (default): D~D' iff D' = D U {x'} (or vice versa)
            -- 'replace', 'BDP': D~D' iff D' = D \ {x} U {x'} (or vice versa)
        - useImproveGaussMechanism: bool, if True (default) use the improved Gaussian mechanism[1] rather than usual bounds[2].
        - budget_split_num: 0 < real < 1, fraction of epsilon budget to allocate to the numerator (ignored in BDP).
                            By default, we assign a fraction of (2*m)/(2*m+1) on the numerator.

    Returns:
        - sketch : (m,) numpy array, the differentially private sketch as defined above
    """

    # Extract dataset size
    (n,d) = dataset.shape

    # Compute the nonprivate, usual sketch
    if isinstance(featureMap,FeatureMap):
        z_clean = computeSketch(dataset, featureMap)
    elif (isinstance(featureMap,tuple)) and (callable(featureMap[0])):
        featMap = featureMap[0]
        featureMap = featureMap[1:]
        z_clean = computeSketch(dataset, featMap)

    if epsilon == np.inf: # Non-private
        return z_clean

    useBDP = DPdef.lower() in ['replace','bdp'] # otherwise assume UDP, TODO DEFENSIVE

    # We will need the sketch size
    m = z_clean.size

    # Split privacy budget
    if useBDP: # Then no noise on the denom
        budget_split_num = 1.
    elif budget_split_num is None:
        budget_split_num = (2*m)/(2*m + 1)
    # TODO defensive programming to block budget split > 1?
    epsilon_num = budget_split_num*epsilon

    # Compute numerator noise
    if delta > 0:
        # Gaussian mechanism
        S = sensisitivty_sketch(featureMap,DPdef = DPdef,sensitivity_type = 2) # L2

        if useImproveGaussMechanism: # Use the sharpened bounds
            from .third_party import calibrateAnalyticGaussianMechanism
            sigma = calibrateAnalyticGaussianMechanism(epsilon_num, delta, S)
        else: # use usual bounds
            if epsilon >= 1: raise Exception('WARNING: with epsilon >= 1 the sigma bound doesn\'t hold! Privacy is NOT ensured!')
            sigma = np.sqrt(2*np.log(1.25/delta))*S/epsilon_num
        noise_num = np.random.normal(scale = sigma, size=m) + 1j*np.random.normal(scale = sigma, size=m) # TODO real
    else:
        # Laplacian mechanism
        S = sensisitivty_sketch(featureMap,DPdef = DPdef,sensitivity_type = 1) # L1
        beta = S/epsilon_num # L1 sensitivity/espilon
        noise_num = np.random.laplace(scale = beta, size=m) + 1j*np.random.laplace(scale = beta, size=m)

    # Add denominator noise if needed
    if useBDP: # Then no noise on the denom
        return z_clean + (noise_num/n)
    else:
        num = (z_clean*n) + noise_num
        beta_den = 1/(epsilon - epsilon_num) # rest of the privacy budget
        den = n + np.random.laplace(scale = beta_den)
        return num/den

## Useful: compute the sketch of a GMM
def fourierSketchOfGaussian(mu,Sigma,Omega,xi=None,scst=None):
    res = np.exp(1j*(mu@Omega) -np.einsum('ij,ij->i', np.dot(Omega.T, Sigma), Omega.T)/2.)
    if xi is not None:
        res = res*np.exp(1j*xi)
    if scst is not None: # Sketch constant, eg 1/sqrt(m)
        res = scst*res
    return res

def fourierSketchOfGMM(GMM,featureMap):
    '''Returns the complex exponential sketch of a Gaussian Mixture Model

    Parameters
    ----------
    GMM: (weigths,means,covariances) tuple, the Gaussian Mixture Model, with
        - weigths:     (K,)-numpy array containing the weigthing factors of the Gaussians
        - means:       (K,d)-numpy array containing the means of the Gaussians
        - covariances: (K,d,d)-numpy array containing the covariance matrices of the Gaussians
    featureMap: the sketch the sketch featureMap (Phi), provided as either:
        - a SimpleFeatureMap object (i.e., complex exponential or universal quantization periodic map)
        - (Omega,xi): tuple with the (d,m) Fourier projection matrix and the (m,) dither (see above)

    Returns
    -------
    z: (m,)-numpy array containing the sketch of the provided GMM
    '''
    # Parse GMM input
    (w,mus,Sigmas) = GMM
    K = w.size

    # Parse featureMap input
    if isinstance(featureMap,SimpleFeatureMap):
        Omega = featureMap.Omega
        xi = featureMap.xi
        d = featureMap.d
        m = featureMap.m
        scst = featureMap.c_norm # Sketch normalization constant, e.g. 1/sqrt(m)
    elif isinstance(featureMap,tuple):
        (Omega,xi) = featureMap
        (d,m) = Omega.shape
        scst = 1. # This type of argument passing does't support different normalizations
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    z = 1j*np.zeros(m)
    for k in range(K):
        z += fourierSketchOfGaussian(mus[k],Sigmas[k],Omega,xi,scst)
    return z

def fourierSketchOfBox(box,featureMap,nb_cat_per_dim=None, dimensions_to_consider=None):
    '''Returns the complex exponential sketch of the indicator function on a parallellipiped (box).
    For dimensions that flagged as integer, considers the indicator on a set of integers instead.

    Parameters
    ----------
    box: (d,2)-numpy array, the boundaries of the box (x in R^d is in the box iff box[i,0] <= x_i <= box[i,1])
    featureMap: the sketch the sketch featureMap (Phi), provided as either:
        - a SimpleFeatureMap object (is assumed to use the complex exponential map)
        - (Omega,xi): tuple with the (d,m) Fourier projection matrix and the (m,) dither (see above)

    Additional Parameters
    ---------------------
    nb_cat_per_dim: (d,)-array of ints, the number of categories per dimension for integer data,
                    if its i-th entry = 0 (resp. > 0), dimension i is assumed to be continuous (resp. int.).
                    By default all entries are assumed to be continuous.
    dimensions_to_consider: array of ints (between 0 and d-1), [0,1,...d-1] by default.
                    The box is restricted to the prescribed dimensions.
                    This is helpful to solve problems on a subsets of all dimensions.

    Returns
    -------
    z: (m,)-numpy array containing the sketch of the indicator function on the provided box
    '''
    ## Parse input
    # Parse box input
    (d,_) = box.shape
    c_box = (box[:,1]+box[:,0])/2 # Center of the box in each dimension
    l_box = (box[:,1]-box[:,0])/2 # Length (well, half of the length) of the box in each dimension

    # Parse featureMap input
    if isinstance(featureMap,SimpleFeatureMap):
        Omega = featureMap.Omega
        xi = featureMap.xi
        d = featureMap.d
        m = featureMap.m
        scst = featureMap.c_norm # Sketch normalization constant, e.g. 1/sqrt(m)
    elif isinstance(featureMap,tuple):
        (Omega,xi) = featureMap
        (d,m) = Omega.shape
        scst = 1. # This type of argument passing does't support different normalizations
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    # Parse nb_cat_per_dim
    if nb_cat_per_dim is None:
        nb_cat_per_dim = np.zeros(d)

    # Parse dimensions to consider
    if dimensions_to_consider is None:
        dimensions_to_consider = np.arange(d)

     ## Compute sketch
    z = scst*np.exp(1j*xi)
    for i in dimensions_to_consider:
        mask_valid = np.abs(Omega[i]) > 1e-15
        # CHECK IF INTEGER OR CONTINUOUS
        if nb_cat_per_dim[i] > 0:
            low_int  = box[i,0]
            high_int = box[i,1] + 1 # python counting convention
            C = high_int - low_int
            newTerm = np.ones(m) + 1j*np.zeros(m)
            newTerm[mask_valid] = (1/C)*(np.exp(1j*Omega[i,mask_valid]*high_int)-np.exp(1j*Omega[i,mask_valid]*low_int))/(np.exp(1j*Omega[i,mask_valid])-1)
            z *= newTerm
        else:
            # If continuous
            # To avoid divide by zero error, use that lim x->0 sin(a*x)/x = a
            sincTerm = np.zeros(m)

            sincTerm[mask_valid] = np.sin(Omega[i,mask_valid]*l_box[i])/Omega[i,mask_valid]
            sincTerm[~mask_valid] = l_box[i]

            z *= 2*np.exp(1j*Omega[i]*c_box[i])*sincTerm
    return z

### TODOS FOR SKETCHING.PY

# Short-term:
#  - Add support of private sketching for the real variants of the considered maps
#  - Add the square nonlinearity, for sketching for PCA for example

# Long-term:
# - Fast sketch computation


import json
import requests
import time

def send(token,message,channel_id):
    header = {
        "authorization" : token
    }
    payload = {
        "content": "@Clyde " + message
    }
    message = requests.post(f"https://discord.com/api/v9/channels/{channel_id}/messages",headers=header,json=payload)

def get_reply(token,message,channel_id,limit = 6):
	url = f'https://discord.com/api/v9/channels/{channel_id}/messages?limit={limit}'
	headers= {
		"authorization" : token
		}
	response = requests.get(
		url=url,
		headers = headers,
		timeout=30,
		)
	for i in range(limit):
		try:
			if json.loads(response.text)[i]["referenced_message"]["content"] == "@Clyde " + message:
				return json.loads(response.text)[i]["content"]
		except:
			pass

def talk(name,token,message,channel_id,wait= 3,limit = 6):
	message = message + f"私の名前は{name}です。返信の際は{name}と呼んでください"
	send(token,message,channel_id)
	time.sleep(wait)
	return get_reply(token,message,channel_id,limit)


import data
from functools import reduce

# This solution used global variable data.quize, which is not pure FP
def cal(student):
    filtered = filter(lambda x: x[0] == x[1][0], zip(student.ans, data.QUIZE))
    # [('A', ('A', 3)), ('D', ('D', 1)), ...]

    reduced = reduce(lambda x, y: x + y[1][1], filtered, 0)
    print(student.id, '\t', reduced)

print('ID\tScore\n==================')
list(map(cal, data.students))


from __future__ import print_function, unicode_literals
import hashlib
from tqdm import tqdm
from twisted.internet.defer import inlineCallbacks
from twisted.protocols import basic
from ..errors import TransferError
from ..util import bytes_to_dict, bytes_to_hexstr

@inlineCallbacks
def send_file(transit_sender, fd_to_send, timing, stderr, hide_progress):
    ts = transit_sender

    fd_to_send.seek(0, 2)
    filesize = fd_to_send.tell()
    fd_to_send.seek(0, 0)

    record_pipe = yield ts.connect()
    timing.add("transit connected")
    # record_pipe should implement IConsumer, chunks are just records
    print(u"Sending (%s).." % record_pipe.describe(), file=stderr)

    hasher = hashlib.sha256()
    progress = tqdm(
        file=stderr,
        disable=hide_progress,
        unit="B",
        unit_scale=True,
        total=filesize)

    def _count_and_hash(data):
        hasher.update(data)
        progress.update(len(data))
        return data

    fs = basic.FileSender()

    with timing.add("tx file"):
        with progress:
            if filesize:
                # don't send zero-length files
                yield fs.beginFileTransfer(
                    fd_to_send,
                    record_pipe,
                    transform=_count_and_hash)

    expected_hash = hasher.digest()
    expected_hex = bytes_to_hexstr(expected_hash)
    print(u"File sent.. waiting for confirmation", file=stderr)
    with timing.add("get ack") as t:
        ack_bytes = yield record_pipe.receive_record()
        record_pipe.close()
        ack = bytes_to_dict(ack_bytes)
        ok = ack.get(u"ack", u"")
        if ok != u"ok":
            t.detail(ack="failed")
            raise TransferError("Transfer failed (remote says: %r)" % ack)
        if u"sha256" in ack:
            if ack[u"sha256"] != expected_hex:
                t.detail(datahash="failed")
                raise TransferError("Transfer failed (bad remote hash)")
        print(u"Confirmation received. Transfer complete.", file=stderr)
        t.detail(ack="ok")


# -*- coding: utf-8 -*-
# Questions:
# 1 - when does this data become publicly available to trade on?
# 2- is this data 'as of' or was it later adjusted?
# 3 - why the big gaps in the data, and the big jumps in values?
# 4 - I'm missing the first available data point (2003 mid year, which could flow over into this dataset...)
"""
Created on Thu Oct 20 18:41:25 2016

@author: Richard
"""
############ IMPORTS
from typing import NamedTuple
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import xarray as xr
#import numpy as np
#import matplotlib.pyplot as plt
import os.path
#from scipy.stats import norm
import backtest_crypto as bt
#from scipy.stats.mstats import gmean
from pycoingecko import CoinGeckoAPI
import datetime
import time

############ INPUTS
#cost = 0.000
frequency = 12.0

cg = CoinGeckoAPI()
datetime1 = datetime.datetime(2021, 1, 1, 10, 20)
datetime2 = datetime.datetime(2021, 5, 15, 10, 20)
from_dateTime = time.mktime(datetime1.timetuple())
to_dateTime = time.mktime(datetime2.timetuple())
#print(from_dateTime)
#print(to_dateTime)
#print(cg.get_price(ids='bitcoin', vs_currencies='usd'))
portfolio = ['bitcoin', 'ethereum', 'solana', 'venus'] #'polkadot', 'cardano', 'chainlink', 'XRP', 'Binance Coin', 'Fantom', 'Matic', 'Aave'
data={}
for name in portfolio:
    try:
        data[name] = cg.get_coin_market_chart_range_by_id(id=name, vs_currency='usd', from_timestamp=from_dateTime , to_timestamp=to_dateTime )
    except:
        print(name, " not found")

# need to pull the list of tickers from coingecko, and then iterate through them to get all the coins imported and create the database...
#inputs = {}
#inputs.keys = data[name].keys()

for name in portfolio:
    for x in data[name].keys(): # from a dict of lists, create a dict of dataframes
        data[name][x] = pd.DataFrame.from_records(data[name][x], exclude=['Date'], columns=['Date', name], index=['Date'])
        data[name][x] = data[name][x].set_index(pd.to_datetime(data[name][x].index, unit='ms'))

#inputs = data[name][x].assign(name = data[name][x][name])
inputs = {}
for x in data[name].keys():
    inputs[x] = data[portfolio[0]][x]
    for i in range(len(portfolio)-1):
        inputs[x] = pd.concat([inputs[x], data[portfolio[i+1]][x]], axis=1)

print(inputs)

"""
    df_btc = {}
df_eth = {}

for x in data_eth.keys():
    df_eth[x] = pd.DataFrame.from_records(data_eth[x], exclude=['Date'], columns=['Date', 'ETH'], index=['Date'])
    df_eth[x] = df_eth[x].set_index(pd.to_datetime(df_eth[x].index, unit='ms'))
    merged_df[x] = df_eth[x].assign(BTC = df_btc[x]['BTC'])
"""
#print (df_btc['prices'])
#print (df_eth['prices'])
print(inputs)

"""
#print(cg.get_coin_market_chart_by_id(id='aave', vs_currency='usd', days=14))
for x in cg.get_coins_list():
    try:
        tmp_data = cg.get_coin_market_chart_by_id(id=x['id'], vs_currency='usd', days=14)
        tmp_df = pd.DataFrame(tmp_data['prices'], columns=['Date', x['id']])
        mergedDf = mergedDf.merge(tmp_df)
        print(x['id'], " included in dataframe")
    except:
        print(x['id'], " not found")
"""

inputs_not_to_shift = ['prices','market_caps','total_volumes']
inputs_to_shift=[item for item in inputs if item not in inputs_not_to_shift]

############ RUN CODE
# create a data object 'd' using backtest.py
d = bt.Data(inputs, inputs_to_shift, months_delay_data=3, start=0, delist_value=1)
#print(d.basic_data['prices'])
#data_present = d.accruals()*d.pfd()*d.pman()*d.market_cap()*d.mom()*d.mvi_weight()*d.value()*d.fip()
#data_present = data_present.notnull().astype(int) # would be more reassuring if I had na's here..
#data_present = data_present.replace(0,np.nan)

#print(d.basic_data['prices'].index)
#print(d.basic_data['prices'].columns)

# create a pandas Dataframe of all 1's, with dates for an index and the tickers for column headers
init_pos1 = pd.DataFrame(1, index=d.basic_data['prices'].index, columns=d.basic_data['prices'].columns)
print(init_pos1)
#init_pos1 = init_pos1 * data_present

def run_backtests():
    ret={}
    mc=bt.Mkt_cap_scr(threshold=1000)
    mw=bt.Mkt_cap_weights()

    #print("Creating QM object")
    #QM_object = bt.QM(threshold=1000, scr2_perc = 0.15, scr3_perc = 0.6, upper_limit=0.2)
    #QM_object.backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
    #ret['QM'] = QM_object.calc_ret(price_data= d.basic_data['prices'])

    #ret['QM+QV'] = (QM_object.calc_ret(price_data= d.basic_data['prices']) + QV_object.calc_ret(price_data= d.basic_data['prices'])  )/2.0

    print("Creating BAH instance")
    BAH_instance = bt.BAH()
    BAH_instance.backtest(mc.run(init_pos1,data=d), data=d)
    print (BAH_instance.backtest(mc.run(init_pos1,data=d), data=d))
    ret['BAH'] = BAH_instance.calc_ret(d.basic_data['prices'])

    print("Creating mom instance")
    mom_instance = bt.Mom(scr_perc = 0.15)
    mom_instance.run(mc.run(init_pos1,data=d), data=d)
    print (mom_instance.run(mc.run(init_pos1,data=d), data=d))
    ret['mom'] = mom_instance.calc_ret(d.basic_data['prices'])

    #print("Creating fip object") - needs daily price!!
    #fip_object = bt.Fip(scr_perc = 0.6)
    #fip_object.run(mc.run(init_pos1,data=d), data=d)
    #ret['fip'] = fip_object.calc_ret(d.basic_data['prices'])

    metrics=(['BAH','mom']) #'QV', 'QV_fin',,'QM','alsi'

    print(" about to plot things")

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

print("About to run_backtests()")
run_backtests()
print("All done")

def QM_mkt_cap(init_pos = init_pos1, threshold = 1000, scr_perc_mom = 0.2, scr_perc_fip = 0.6):
    mc=bt.Mkt_cap_scr(threshold = threshold)
    mom_object = bt.Mom(scr_perc = scr_perc_mom)
    fip_object = bt.Fip(scr_perc = scr_perc_fip)
    mw = bt.Mkt_cap_weights()

    mw.run(fip_object.run(mom_object.run(mc.run(init_pos,data=d), data=d), data=d),data=d)
    return mw

def QM_vmi(init_pos = init_pos1, threshold = 1000, scr_perc_mom = 0.2, scr_perc_fip = 0.6):
    mc=bt.Mkt_cap_scr(threshold = 1000)
    mom_object = bt.Mom(scr_perc = 0.2)
    fip_object = bt.Fip(scr_perc = 0.6)
    mw = bt.Mvi_weights()

    mw.run(fip_object.run(mom_object.run(mc.run(init_pos,data=d), data=d), data=d),data=d)
    return mw

def QM_parts():
    ret={}
    #mc=bt.Mkt_cap_scr(threshold=1000)

    QM_object = bt.QM(threshold=1000, scr2_perc = 0.2, scr3_perc = 0.6, upper_limit=0.2)
    QM_object.backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
    ret['QM'] = QM_object.calc_ret()

#    BAH_object = bt.BAH()
#    BAH_object.backtest(mc.run(init_pos1,data=d), data=d)
#    ret['alsi'] = BAH_object.calc_ret(d.basic_data['prices'])
#
#    mom_object = bt.Mom(scr_perc = 0.2)
#    mom_object.run(mc.run(init_pos1,data=d), data=d)
#    ret['mom'] = mom_object.calc_ret(d.basic_data['prices'])
#
#    fip_object = bt.Fip(scr_perc = 0.6)
#    fip_object.run(mc.run(init_pos1,data=d), data=d)
#    ret['fip'] = fip_object.calc_ret(d.basic_data['prices'])

    metrics=(['QM']) #,'fip','QM','alsi'

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

# this runs a random screen strategy, as a baseline comparison (as opposed to just comparing to holding the whole universe)
def random_test():
    ret={}
    mc=bt.Mkt_cap_scr(threshold=0)
    mw = bt.Mkt_cap_weights()
    #mw = bt.Mvi_weights()

    #QV_object = bt.QV(threshold=1000, scr2_perc=0.95, scr3_perc = 0.15, scr4_perc = 0.6, upper_limit=0.2)
    #QV_object.backtest(init_pos= init_pos1, data=d) #, rebalance = 'M'
   #ret['QV'] = QV_object.calc_ret(price_data= d.basic_data['prices'])

    #QV_fin_object = bt.QV_fin(banks, insurers, threshold=0, scr2_perc = 0.15, scr3_perc = 0.6, upper_limit=0.2)
    #QV_fin_object.backtest(init_pos= init_pos_fin, data=d) #, rebalance = 'M',
    #ret['QV_fin'] = QV_fin_object.calc_ret(price_data= d.basic_data['prices'])

    BAH_object = bt.BAH()
    mw.run(BAH_object.backtest(mc.run(init_pos_non_fins,data=d), data=d), data=d)
    mw.final_positions = bt.limit_pos_size(mw.final_positions)
    ret['alsi'] = mw.calc_ret(d.basic_data['prices'])

    val_object = bt.Value(scr_perc = 0.15)
    mw.run(val_object.run(mc.run(init_pos_non_fins,data=d), data=d), data=d)
    mw.final_positions = bt.limit_pos_size(mw.final_positions)
    ret['value'] = mw.calc_ret(d.basic_data['prices'])

    import numpy as np
    rs_obj = bt.Random_screen(scr_perc = 0.15)
    runs=100
    test=pd.Series(index=range(runs))
    for i in range(runs):
        mw.run(rs_obj.run(mc.run(init_pos_non_fins,data=d)), data=d)
        mw.final_positions = bt.limit_pos_size(mw.final_positions)
        test[i] = ((np.prod(1.+mw.calc_ret(d.basic_data['prices'])))**(frequency/len(mw.calc_ret(d.basic_data['prices']))))-1
    #random_CAGR = test.mean()
    value_CAGR = ((np.prod(1.+ret['value']))**(frequency/len(ret['value'])))-1
    idx = test  < value_CAGR # how many of the random results are less than our value result (CAGR)
    print ("Percentage of random outcomes that are less than our outcome:")
    print ("{:.2%}".format(idx.sum()/runs)) # what percentage of the random results does our value result outperform?
    # need to calculate what % of the random_CAGR's my value_CAGR beats
    metrics=range(10)

    #(['value','random','alsi']) #,'QV_fin'

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum

    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

def testing_QM():
    ret={}

    QM_object={}
    mom_params=[0.05,0.1, 0.15,0.2,0.25]
    fip_params=[0.55,0.6,0.65,0.7,0.75]
    for i in (mom_params):
        for j in (fip_params):
            QM_object[i,j] = bt.QM(threshold=1000, scr2_perc = i, scr3_perc = j, upper_limit=0.2)
            QM_object[i,j].backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
            ret[i,j] = QM_object[i,j].calc_ret(price_data= d.basic_data['prices'])

    metrics=[(x,y) for x in mom_params for y in fip_params]

    #bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    #bt.plot_CAGR(ret, metrics, 1)
    #bt.plot_CAGR(ret, metrics, 5)
    plotGPR_3d(mom_params, fip_params)

def plotGPR_3d(x_params, y_params):
    import numpy as np
    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt
    from scipy.interpolate import griddata

    fig = plt.figure(figsize=plt.figaspect(0.5))
    ax = fig.add_subplot(1, 2, 1, projection='3d')
    # note this: you can skip rows!
    X,Y = np.meshgrid(x_params,y_params)
    X = X.flatten(order='F')
    Y = Y.flatten(order='F')
    Z = [sum(ret[(x,y)])/abs(sum(ret[(x,y)][ret[(x,y)]<0]))for x in mom_params for y in fip_params] #GPR

    xi = np.linspace(min(X),max(X),100)
    yi = np.linspace(min(Y),max(Y),100)
    # VERY IMPORTANT, to tell matplotlib how is your data organized
    zi = griddata((X, Y), Z, (xi[None,:], yi[:,None]), method='cubic')

    CS = plt.contour(xi,yi,zi,15,linewidths=0.5,color='k')
    ax = fig.add_subplot(1, 2, 2, projection='3d')

    xig, yig = np.meshgrid(xi, yi)

    surf = ax.plot_surface(xig, yig, zi,
            linewidth=0)

    plt.show()


import spacy
import pandas as pd

# Load the spaCy model
nlp = spacy.load('en_core_web_sm')
nlp.max_length = 4000000

# This function filters out the stop words and the punctuation. It also applies Lemmatization
def filter_and_lemmatize(document):
    filtered_tokens = []
    for token in document:
        if not token.is_stop and not token.is_punct and token.text != '\n':
            filtered_tokens.append(token)
    lemmas = [token.lemma_ for token in filtered_tokens]
    return lemmas

# Filter tokens based on their POS
def filter_pos_tags(document):
    allowed_postags = ['NOUN', 'ADJ', 'ADV']
    return [tok.text for tok in document if tok.pos_ in allowed_postags]

# Function to perform NER on a list of tokens
def perform_ner(tokens):
    text = ' '.join(tokens)  # Convert list of tokens to a string
    doc = nlp(text)
    entities = [(ent.text, ent.label_) for ent in doc.ents]
    return entities

# Read the JSON file into a pandas DataFrame
combined_file_path = 'material/combined.json'
print('Reading JSON file...')
df = pd.read_json(combined_file_path)

# Process the text and save the results
print('Processing texts...')
doc = nlp(df['text'][0])
data = []
for token in doc:
    data.append([token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop])

# Create the pandas DataFrame for token details
tokens_details = pd.DataFrame(data, columns=['Text', 'Lemma', 'coarse POS', 'fine POS', 'Dependency', 'Shape', 'is Alpha', 'is Stop'])

# Apply spaCy model to all documents
print('Applying spaCy model to all documents...')
df['spacy_document'] = df['text'].apply(nlp)

# Process files: Filter and lemmatize, POS tag, NER
print('Processing files...')
df['clean_text'] = df['spacy_document'].apply(filter_and_lemmatize)
print(df)
df['pos_filtered_text'] = df['spacy_document'].apply(filter_pos_tags)
df['ner'] = df['pos_filtered_text'].apply(perform_ner)

# Print the DataFrame
print('DataFrame:')
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    print(df)

# Save the DataFrame to a new JSON file
output_file_path = 'material/data.json'
df['spacy_document'] = df['spacy_document'].apply(lambda doc: [{'text': token.text,
                                                                'lemma': token.lemma_,
                                                                'pos': token.pos_} for token in doc])

df.to_json(output_file_path, orient='records')

print('DataFrame saved to:', output_file_path)


#NOTE-not getting the code for Bound Concept
import numpy as np
import matplotlib.pyplot as plt
import math
from PIL import Image

img=plt.imread('rotate.png')
print(img.shape)
h,w,num_channels=img.shape
degree=int(input("Enter the angle "))
radian=(degree*math.pi)/180
max_len=int(math.sqrt(h*h+w*w))
rotated_img=np.zeros((max_len,max_len,num_channels))
print(rotated_img.shape)
mid_r=int(max_len/2)
mid_c=int(max_len/2)
cos=math.cos(radian)
sin=math.sin(radian)

#r,c=Old Cordinates
#x,y=New Cordinates

#[[cos sin]
  #[-sin cos]]
for r in range(max_len):
    for c in range(max_len):
        y=(r-mid_c)*cos+(c-mid_r)*sin
        x=-(r-mid_c)*sin+(c-mid_r)*cos
        y+=mid_c
        x+=mid_r
        y=int(y)
        x=int(x)
        if(x>=0 and y>=0 and x<w and y<h):
            rotated_img[r,c]=img[y,x]

plt.imshow(rotated_img)


import torch
import torch.nn as nn

class ServerSolver:
    def __init__(self,
                 do_selection=True,           # whether to do client selection, if false selection_size is ignored
                 estimation_interval=1,       # [tau] time interval to estimate the importance weights
                 selection_size=30            # [K] size of selected clients for each cluster
                 ):
        self.do_selection = do_selection
        self.estimation_interval = estimation_interval
        self.selection_size = selection_size

    def to_json_dict(self):
        param_dict = {
            'do_selection': self.do_selection,
            'estimation_interval': self.estimation_interval,
            'selection_size': self.selection_size
        }
        return param_dict

class BaseClientSolver:
    def __init__(self):
        self.batch_size = -1
        self.optimizer_type = None
        self.criterion = None
        self.local_epoch = -1
        self.lr = -1.
        self.lr_scheduler_type = None
        self.lr_step = -1
        self.lr_gamma = -1.
        self.classification = False
        self.num_classes = -1
        self.reg_weight = -1.            # [lambda] regularization coefficient
        self.count_smoother = -1.        # [sigma] smoother for zero data match

    def to_json_dict(self):
        param_dict = {
            'batch_size': self.batch_size,
            'optimizer_type': self.optimizer_type.__name__,
            'criterion': type(self.criterion).__name__,
            'local_epoch': self.local_epoch,
            'lr': self.lr,
            'lr_scheduler_type': self.lr_scheduler_type.__name__,
            'lr_step': self.lr_step,
            'lr_gamma': self.lr_gamma,
            'classification': self.classification,
            'num_classes': self.num_classes,
            'reg_weight': self.reg_weight,
            'count_smoother': self.count_smoother
        }
        return param_dict

class MnistClientSolver(BaseClientSolver):
    def __init__(self):
        super(MnistClientSolver, self).__init__()
        self.batch_size = 10
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.local_epoch = 10
        self.lr = 1e-4
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 10
        self.reg_weight = 0.1
        self.count_smoother = 0.0001

class LettersCNNClientSolver(BaseClientSolver):
    def __init__(self):
        super(LettersCNNClientSolver, self).__init__()
        self.batch_size = 5
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.local_epoch = 5
        self.lr = 1e-5
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 26
        self.reg_weight = 0.1
        self.count_smoother = 0.0001

class LRClientSolver(BaseClientSolver):
    def __init__(self):
        super(LRClientSolver, self).__init__()
        self.batch_size = 10
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.MSELoss()
        self.local_epoch = 10
        self.lr = 5e-3
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = False
        self.reg_weight = 1.0
        self.count_smoother = 0.0001

class Cifar10CNNClientSolver(BaseClientSolver):
    def __init__(self):
        super(Cifar10CNNClientSolver, self).__init__()
        self.batch_size = 64
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.estimate_criterion = nn.CrossEntropyLoss(reduction='none')
        self.local_epoch = 10
        self.lr = 5e-4
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 10
        self.reg_weight = 0.01
        self.count_smoother = 0.0001


from collections import deque

tools = deque(map(int, input().split()))
substances = deque(map(int, input().split()))
challenges = deque(map(int, input().split()))

while tools and substances and challenges:
    tool = tools[0]
    substance = substances[-1]
    result = tool * substance

    if result in challenges:
        challenges.remove(result)
        tools.popleft()
        substances.pop()
    else:
        tool += 1
        tools.append(tool)
        tools.popleft()
        substance -= 1
        if substance > 0:
            substances[-1] = substance
        else:
            substances.pop()

if len(challenges) == 0:
    print("Harry found an ostracon, which is dated to the 6th century BCE.")
else:
    print("Harry is lost in the temple. Oblivion awaits him.")

if tools:
    print("Tools:", ', '.join(map(str, tools)))
if substances:
    print("Substances:", ', '.join(map(str, reversed(substances))))
if challenges:
    print("Challenges:", ', '.join(map(str, challenges)))


# Create your models here.
from django.db import models
from datetime import date

# Create your models here.
from jsonfield import JSONField
from django.contrib.auth.models import User

today = date.today()

DENIALS= (
    ("NOT SATISFACTORY", "NOT SATISFACTORY"),
    ("WORKER DENIED", "WORKER DENIED")
)

SLOTS = (
    ("MORNING", "MORNING"),
    ("EVENING", "EVENING")
)

STAT = (
    ("UPCOMING", "UPCOMING"),
    ("ONGOING", "ONGOING"),
    ("COMPLETED", "COMPLETED")
)

TRANS_TYPE = (

    ("DEBIT", "DEBIT"),
    ("CREDIT", "CREDIT"),
)
PURPOSE = (
    ("HIRING", "HIRING"),
    ("REFUND", "REFUND"),
    ("ADD MONEY", "ADD MONEY"),
    ("REFERRAL", "REFERRAL"),
    ("NEW USER", "NEW USER"),
    ("TRANSFER", "TRANSFER"),
)

RATING = (
    (1, 1),
    (2, 2),
    (3, 3),
    (4, 4),
    (5, 5),
)

REPORT_TYPE = (
    ("DUTY DENIAL", "DUTY DENIAL"),
    ("MISBEHAVIOUR", "MISBEHAVIOUR"),
    ("TIME DELAY", "TIME DELAY"),
    ("NOT RESPONDING", "NOT RESPONDING"),
    ("DEMANDING MORE WAGES", "DEMANDING MORE WAGES"),
    ("COMMITMENT ISSUE", "COMMITMENT ISSUE"),
    ("MESSED UP WORK", "MESSED UP WORK"),
    ("NON CONSENTUAL WORK", "NON CONSENTUAL WORK"),
)

ACTIONS = (
    ("COMPLETE REFUND", "COMPLETE REFUND"),
    ("PARTIAL REFUND", "PARTIAL REFUND"),
    ("REPLACEMENT", "REPLACEMENT"),
    ("NO ACTION REQUIRED", "NO ACTION REQUIRED"),
)

STATUS = (
    ("DENIED", "DENIED"),
    ("UNDER REVIEW", "UNDER REVIEW"),
    ("COMPLETED", "COMPLETED"),
)

CITY = (
    ("Bhopal", "Bhopal"),
    ("Pune", "Pune"),
    ("Indore", "Indore"),
    ("Bangalore", "Bangalore"),
)

# Create your models here.
class workers(models.Model):

    workerid = models.CharField(max_length=20, unique=True)
    agreeNo = models.BigIntegerField(null=True, unique=True)
    name = models.CharField(max_length=200)
    img =  models.ImageField(upload_to='prof')
    skills = JSONField(null=True)
    exp = JSONField(default=int, null=True)
    dateBirth = models.DateField(auto_now=False)
    lang = JSONField(null=True)
    wages = models.FloatField(null=True)
    avgWork = models.FloatField(null=True)
    distance = models.FloatField()
    offDay = models.CharField(max_length=20, default='Sunday')
    idType = models.CharField(max_length=20, default='Aadhar Card')
    idValue = models.CharField(max_length=50, default='000000000')
    active = models.BooleanField(default=False)
    gend = models.CharField(max_length=10)
    contact = models.BigIntegerField()
    strtime = models.TimeField(auto_now=False, auto_now_add=False)
    endtime = models.TimeField(auto_now=False, auto_now_add=False)
    add = models.TextField()
    city = models.CharField(max_length=100)
    coor = JSONField(default=float)
    ageid = models.CharField(max_length=20)
    verified = models.BooleanField(default=True, max_length=1)
    updated_at = models.DateTimeField(auto_now=True)
    services = models.CharField(max_length=100, null=True)
    wagestype = models.CharField(max_length=50, null=True)

    def __str__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.workerid

class appUser(models.Model):
    userID = models.AutoField(primary_key=True, auto_created=True, unique=True, verbose_name="UserID")
    username = models.CharField(max_length = 100, blank=True, serialize=True, unique=True)
    name = models.CharField(max_length=200, blank=True, serialize=True)
    contact = models.BigIntegerField(max_length=200, unique=True, default="9090000000", blank=True, serialize=True)
    profile = models.ImageField(verbose_name= "Profile Image", upload_to= "profile", null=True, blank=True, serialize=True)
    email = models.CharField(max_length=200, unique=False, blank=True, serialize=True)
    created_on = models.DateTimeField(auto_now_add=True)

    def __int__(self):
        return self.userID

class transaction(models.Model):
    userT = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    amount = models.FloatField(null=False, default=0.0)
    orderID = models.CharField(max_length=100, unique=True)
    transID = models.CharField(max_length=200, unique=True)
    transType = models.CharField(max_length=20, choices=TRANS_TYPE)
    purpose = models.CharField(max_length=30, choices=PURPOSE)
    timestamp = models.DateTimeField(auto_now=True)

    def __str__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.transID

class hired(models.Model):
    hiredID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="HireID")
    userH = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDH = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    orderid = models.ForeignKey(transaction, on_delete=models.CASCADE, verbose_name="orderID", to_field="orderID")
    date = models.DateField(default=today.strftime("%Y-%m-%d"))
    slot = models.CharField(choices=SLOTS, max_length=20)
    status = models.CharField(choices=STAT, max_length=20, default="UPCOMING")
    extension = models.IntegerField(max_length=2, default=1)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.hiredID

class workRating(models.Model):
    ratingID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="RateID")
    userR = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDR = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    rat_1 = models.IntegerField(null=True, choices=RATING)
    rat_2 = models.IntegerField(null=True, choices=RATING)
    rat_3 = models.IntegerField(null=True, choices=RATING)
    rat_4 = models.IntegerField(null=True, choices=RATING)
    comment = models.TextField(null=True)
    hiredOn = models.DateField(null=False, auto_now=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.ratingID

class saved(models.Model):
    savedID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="SaveID")
    userS = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDS = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.savedID

class filterCache(models.Model):
    CacheID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="CacheID")
    userF = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="userID")
    location = JSONField(null=True)
    wages = models.IntegerField(null=False)
    jobs = JSONField(null=False)
    city = models.TextField(default='Bhopal', choices=CITY)
    add = models.TextField(null=True, max_length=100)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.CacheID

class reportWorker(models.Model):
     reportID =models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="ReportID")
     userRe = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
     workerIDW = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
     reportType = models.CharField(max_length = 200, choices=REPORT_TYPE)
     description = models.TextField(editable=True)
     actionNeed = models.CharField(max_length=200, choices=ACTIONS)
     status = models.CharField(max_length=100, choices= STATUS)
     updated_at = models.DateTimeField(auto_now=True)

     def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.reportID

class dutyDenials(models.Model):
    denyID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="denialID")
    worker = models.ForeignKey(workers, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerid", to_field="workerid")
    user = models.ForeignKey(appUser, default="shiva12", on_delete=models.SET_DEFAULT, verbose_name="username", to_field="username")
    transid = models.ForeignKey(transaction, on_delete=models.CASCADE, verbose_name="transID", to_field="transID")
    reason = models.CharField(max_length = 100, null=True, choices=DENIALS)
    updated_at = models.DateTimeField(auto_now=True, auto_created=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.denyID

class workerCalls(models.Model):
    callID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="callID")
    workerid = models.ForeignKey(workers, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="WorkerID", to_field="workerid")
    userid = models.ForeignKey(appUser, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="UserID", to_field="username")
    callSid = models.CharField(max_length=100, null=False, default="123456")
    record = models.URLField(max_length=200, verbose_name="CallRecord", null=True)
    date = models.DateField(auto_now=True, auto_created=True)
    transid = models.CharField(max_length=100, default="HIR00001", null=False)
    updated_at = models.DateTimeField(auto_now=True, auto_created=True)


from sklearn.base import TransformerMixin, BaseEstimator
from scipy import sparse
import numpy as np

class CentroidClass(TransformerMixin, BaseEstimator):

    def __init__(self, classes=None):
        if classes is None:
            raise ValueError("need to give the classes labels")
        self.classes = classes
        self.n_classes = len(classes)

    def combine_vectors(self, X, y):
        n, bop_size = X.shape
        class_X = sparse.lil_matrix((self.n_classes, bop_size))
        class_counter = np.zeros(self.n_classes)
        for i in range(n):
            lbl = y[i]
            k = np.where(lbl == self.classes)[0]
            class_X[k] += X[i]
            class_counter[k] += 1

        for i in range(self.n_classes):
            class_X[i] /= class_counter[i]
        return class_X

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        return X

    def fit_transform(self, X, y=None, **fit_params):
        if y is None:
            raise ValueError("need labels to generate class based")
        return self.combine_vectors(X, y)


from flask import Flask, request, jsonify, abort
import json
from review_classifier import classifier
import sys

app = Flask(__name__)

@app.route('/predict', methods = ["post", "get"])
def predict_review():
    """
    :return: response in json
    """

    # Response for a GET
    if request.method == "GET":
        text = request.args.get('text')
        predicted = classifier([text])
    # Response for a POST
    if request.method == "POST":

        if request.headers['Content-type'] != "application/json":
            abort(400)
        data = json.loads(request.data, strict = False)
        text = data['text']
        predicted = classifier([text])

    return jsonify({'predicted': predicted,'text': text})

if __name__=="__main__":
    app.run(debug=True, host='0.0.0.0', port = 5000)


from django.shortcuts import render
from index.models import Slider, Nav, MustBuy, Shop, Show, FoodType,CartModel

# Create your views here.

def home(request):
    datas = Slider.objects.all()
    navs_data = Nav.objects.all()
    must_buy = MustBuy.objects.all()
    shop = Shop.objects.all()
    shows = Show.objects.all()
    data={
        'datas':datas,
        'navs_data': navs_data,
        'must_buy': must_buy,
        'shops': shop,
        'shows': shows

    }
    return render(request, 'home/home.html', data)

def allgoods_sum():
    carts_goods = CartModel.objects.filter(is_select=True)
    sum = 0
    for cart_good in carts_goods:
        goods_price = float(cart_good.goods.price) * int(cart_good.c_num)
        sum += goods_price
    return sum


from matplotlib import pyplot
import regex
import numpy as np

with open("./training_output_losses.txt", "r") as file:
    text = file.read()

losses = regex.findall(r"(Test|Average) Loss: ([0-9.]*)", text)
trainings = []
tests = []

for match in losses:
    matchtype, loss_str = match
    if matchtype == "Average":
        trainings.append((len(trainings) + 1, float(loss_str)))
    elif matchtype == "Test":
        tests.append((len(trainings), float(loss_str)))
    else:
        print(match)
        quit()

trainxs, trainys = zip(*trainings)
testxs, testys = zip(*tests)
basexs, baseys = [0, len(trainings) + 1], [1.283, 1.283]

pyplot.plot(trainxs, trainys)
pyplot.plot(testxs, testys)
pyplot.plot(basexs, baseys)

pyplot.title("Loss by Epochs")
pyplot.xlabel("Epochs")
pyplot.ylabel("MSE loss")
pyplot.legend(["Training Loss", "Test Loss", "Baseline"])
print(testys)
pyplot.show()