from flask import Flask, render_template, request
import sqlite3

app = Flask(__name__)

@app.route('/',methods = ['POST', 'GET'])
def home():
    if request.method == 'GET':
        return render_template('index.html')

@app.route('/thankyou',methods = ['POST', 'GET'])
def thankyou():
    if request.method == 'GET':
        return render_template('thankyou.html')
    elif request.method == 'POST':
        emailid = request.form.get('eid')
        conn = sqlite3.connect("emailist")
        cur=conn.cursor()
        cur.execute("SELECT * from emails_table")
        print(cur.fetchall())
#cur.execute('INSERT INTO emails_table (email) VALUES (?), ("t@iron.com")')
#cur.execute("INSERT INTO movie VALUES(%s,%s)",(movID,Name))
        cur.execute("INSERT INTO emails_table (email) VALUES (?)", (emailid,))
        conn.commit()
        conn.close()
        return render_template('thankyou.html')

if __name__ == '__main__':
   app.run(debug=True)


import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity, linear_kernel

def GetMoviesByDescription(movieName):
    movie_list=[]

    csv_path = "cleaned data/movies.csv"
    df = pd.read_csv(csv_path)
    df['description'] = df['description'].fillna('')
    #create the matrix
    tf = TfidfVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english')
    tfidf_matrix = tf.fit_transform(df['description'])

    #calaculate the  Cosine Similarity Score
    cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)

    md = df.reset_index()
    titles = df['title']
    indices = pd.Series(df.index, index=df['title'])

    try:
        idx = indices[movieName]
        sim_scores = list(enumerate(cosine_sim[idx]))
        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
        sim_scores = sim_scores[1:31]
        movie_indices = [i[0] for i in sim_scores]
        recdf = titles.iloc[movie_indices]
        count = 0
        for index, value in recdf.items():
            count = count + 1
            movie_list.append(value)
            if(count == 8):
                break
    except:
        movie_list.append("No Recommendation available.")
    return movie_list

def GetMoviesByUserRating(movieName):
    movies = pd.read_csv("raw data/movies.csv")
    ratings = pd.read_csv("raw data/ratings.csv")
    movievsuser = ratings.pivot(index='movieId',columns='userId',values='rating')
    movievsuser.fillna(0, inplace=True)
    ratingsByMovie = ratings.groupby('movieId')['rating'].agg('count')
    ratingsByUser = ratings.groupby('userId')['rating'].agg('count')
    movievsuser = movievsuser.loc[ratingsByMovie[ratingsByMovie > 40].index,:]
    csr_data = csr_matrix(movievsuser.values)
    movievsuser.reset_index(inplace=True)

    # Using KNN algorithm to predict similarity with cosine distance
    knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
    knn.fit(csr_data)

    reccomendCount = 8
    listing = []
    movieList = movies[movies['title'].str.contains(movieName)]
    if len(movieList):
        movie_id= movieList.iloc[0]['movieId']
        movie_id = movievsuser[movievsuser['movieId'] == movie_id].index[0]
        distances , indices = knn.kneighbors(csr_data[movie_id],n_neighbors=reccomendCount+1)
        recommendedMovies = sorted(list(zip(indices.squeeze().tolist(),distances.squeeze().tolist())),key=lambda x: x[1])[:0:-1]
        recMoviesList = []
        for val in recommendedMovies:
            movie_id = movievsuser.iloc[val[0]]['movieId']
            idx = movies[movies['movieId'] == movie_id].index
            recMoviesList.append({'Title':movies.iloc[idx]['title'].values[0],'Distance':val[1]})
        df = pd.DataFrame(recMoviesList,index=range(1,reccomendCount+1))
        df['Distance'] = pd.to_numeric(df['Distance'])
        df= df.sort_values('Distance')
        listing = df['Title']

    return listing

def GetMoviesByGenre(movieName):
    file_to_load = "raw data/movies.csv"
    ratings = pd.read_csv("raw data/ratings.csv")

    data = pd.read_csv(file_to_load)
    data['genres'] = data['genres'].str.replace(r'|', ' ')
    genre_data=data[['title','genres']]
    genre_data=genre_data.set_index('title')

    #convert genre column to array
    cv = CountVectorizer()
    X = cv.fit_transform(genre_data["genres"]).toarray()

    similarities = cosine_similarity(X)
    movie_index = data.loc[data['title'].str.contains(movieName)].index[0]

    similarity_values = pd.Series(similarities[movie_index])

    #We converted list into series in order to preserve the actual indexes of dataset even after sorting
    similarity_values.sort_values(ascending=False)
    similar_movie_indexes = list(similarity_values.sort_values(ascending=False).index)

    #Remove the already watched movie from index list
    similar_movie_indexes.remove(movie_index)

    movie_list=[]
    for i in range(8):
        movie_list.append(genre_data.index[similar_movie_indexes[i]])

    return movie_list

def GetPredictionsForMovie(moviename):
    complete_df = pd.read_csv("cleaned data/complete_df_with_predictions.csv")

    mv = complete_df.loc[complete_df['title'].str.contains(moviename),['title']]
    movie = mv.head(1)
    names = movie.to_numpy()
    name = names[0][0]

    #based on all the users in the dataframe and their predictions what is the average rating the movie will get
    movie_rating=round((complete_df.loc[complete_df['title']==name,['predicted rating']].values).mean(),2)

    #from data already available what is the average of the movie
    movie_gavg=round((complete_df.loc[complete_df['title']==name,['MAvg']].values).mean(),2)

    percdiff = round(((movie_rating-movie_gavg)/movie_gavg*100),2)

    summary = {'Predicted Rating': movie_rating, 'Actual Rating': movie_gavg ,"Percentage Difference%":percdiff}
    return summary

def GetPredictions(moviename, userid):
    complete_df = pd.read_csv("cleaned data/complete_df_with_predictions.csv")
    #print(complete_df.head())
    userid=int(userid)
    try:
        mv = complete_df.loc[complete_df['title'].str.contains(moviename),['title']]
        movie = mv.head(1)
        names = movie.to_numpy()
        name = names[0][0]
        #print(name)

        #based on users past ratings what is the prediction for a particular movie
        pred_rating=round(complete_df.loc[(complete_df['user']==userid) & (complete_df['title']==name),['predicted rating']].values[0][0],2)

        #from data already available what is the average of the movie
        user_rating=round(complete_df.loc[(complete_df['user']==userid) & (complete_df['title']==name),['rating']].values[0][0],2)

        percdiff = round(((pred_rating-user_rating)/user_rating*100),2)

        summary = {'Predicted Rating': pred_rating, 'Actual Rating': user_rating ,"Percentage Difference%":percdiff}
        return summary
    except:
        pred_rating=0
        user_rating=0
        percdiff = 0

        #based on all the users in the dataframe and their predictions what is the average rating a user gives
        user_rating=round((complete_df.loc[complete_df['user']==userid,['predicted rating']].values).mean(),2)

        #from data already available what is the average of the movie
        user_uavg=round((complete_df.loc[complete_df['user']==userid,['UAvg']].values).mean(),2)

        percdiff = round(((user_rating-user_uavg)/user_uavg*100),2)

        print ("this user has not rated this movie")
        summary = {'Status': "this user has not rated this movie, showing a prediction of what this user is likely to predict",'Predicted Rating': user_rating, 'Actual Rating': user_uavg ,"Percentage Difference%":percdiff}
    return summary


import requests
import random
from utils.others import get_atitle, get_genre, get_t_from_u, get_urls
from utils.anilist import Anilist
from utils.techzapi import TechZApi

def get_genre_html(li):
    x = """{}"""
    html = ""

    for i in li:
        html += x.format(i.strip())

    return html

def get_eps_html(data=None, api: TechZApi = None, anime=None):
    if not data:
        anime = api.gogo_search(anime)[0].get("id").strip()
        data = api.gogo_anime(anime).get("episodes")

    x = """{}"""
    html = ""
    pos = 1

    for i in data:
        i = i.replace("-episode-", "/")
        html += x.format(f"/episode/{i}", str(pos))
        pos += 1

    if api:
        return html, anime
    return html

def get_eps_html2(data):
    x = """{}"""
    html = ""
    pos = 1
    for i in data:
        i = i.replace("-episode-", "/")
        html += x.format(f"/episode/{i}", str(pos))
        pos += 1
    return html

ANIME_POS = """

{}

{}

{}

 {}  {}

"""

ANIME_POS2 = """

{}

{}

 {}

"""

def animeRecHtml(data):
    if not data:
        return "Not Available"

    if len(data) == 0:
        return "Not Available"

    html = ""

    for i in data.get("recommendations").get("edges"):
        i = i.get("node").get("mediaRecommendation")
        img = i.get("coverImage")
        if img:
            img = img.get("medium").replace("small", "medium")
        else:
            img = i.get("bannerImage")
        title = get_atitle(i.get("title"))
        url = get_urls(title)
        x = ANIME_POS.format(
            url,
            str(i.get("meanScore")).strip() + " / 100",
            "Ep " + str(i.get("episodes")).strip(),
            img,
            title,
            i.get("format"),
            i.get("status"),
        )
        if x not in html:
            html += x

    return html

def animeRecHtml2(data):
    if not data:
        return "Not Available"

    if len(data) == 0:
        return "Not Available"

    html = ""

    for i in data:
        i = i.get("node").get("mediaRecommendation")

        img = i.get("coverImage")
        if img:
            img = img.get("medium").replace("small", "medium")
        else:
            img = i.get("bannerImage")
        title = get_atitle(i.get("title"))
        url = get_urls(title)
        x = ANIME_POS.format(
            url,
            str(i.get("meanScore")).strip() + " / 100",
            "Ep " + str(i.get("episodes")).strip(),
            img,
            title,
            i.get("format"),
            i.get("status"),
        )
        if x not in html:
            html += x

    return html

def get_trending_html(data):
    html = ""
    for id, i in data:
        try:
            img = i[5]
            title = i[0]
            url = get_urls(id)
            x = ANIME_POS.format(url, i[1], "Ep " + str(i[2]), img, title, i[3], i[4])
            html += x
        except:
            pass

    return html

def get_search_html(data):
    html = ""

    for i in data:
        if "dub" in i.get("id").lower():
            d = "DUB"
        else:
            d = "SUB"
        x = ANIME_POS2.format(
            "/anime/" + i.get("id"),
            d,
            i.get("img"),
            i.get("title"),
            "Released: " + i.get("year"),
        )
        html += x

    return html

def get_recent_html(data):
    html = ""

    for i in data:
        url = i.get("id").split("-episode-")[0]
        x = ANIME_POS.format(
            f"/anime/{url}",
            i.get("lang"),
            "Ep " + str(i.get("episode")),
            i.get("img"),
            i.get("title"),
            f"Latest {i.get('lang')}",
            "HD",
        )
        html += x

    return html

def get_selector_btns(url, current, episodes):
    if episodes < 2:
        return ""

    selector = ""

    if current == 1:
        x = """<button class="sbtn inline-flex text-white bg-indigo-500 border-0 py-2 px-6 focus:outline-none hover:bg-indigo-600 rounded text-lg ">Episode NEXT</button>"""

        selector += x.replace("usrl", url + str(current + 1)).replace(
            "NEXT", str(current + 1)
        )

    elif current == episodes:
        x = """<button class="sbtn inline-flex text-white bg-indigo-500 border-0 py-2 px-6 focus:outline-none hover:bg-indigo-600 rounded text-lg ">Episode PREV</button>"""

        selector += x.replace("usrl", url + str(current - 1)).replace(
            "PREV", str(current - 1)
        )

    else:
        x = """<button class="sbtn inline-flex text-white bg-indigo-500 border-0 py-2 px-6 focus:outline-none hover:bg-indigo-600 rounded text-lg ">Episode PREV</button>"""

        selector += x.replace("usrl", url + str(current - 1)).replace(
            "PREV", str(current - 1)
        )

        x = """<button class="sbtn inline-flex text-white bg-indigo-500 border-0 py-2 px-6 focus:outline-none hover:bg-indigo-600 rounded text-lg ">Episode NEXT</button>"""

        selector += x.replace("usrl", url + str(current + 1)).replace(
            "NEXT", str(current + 1)
        )
    return selector

SLIDER_HTML = """

{}

{}

 {} {} {} HD

{}

   Watch Now    Details

"""

def slider_gen():
    data = Anilist().trending()
    random.shuffle(data)
    html = ""
    pos = 1

    for i in data:
        img = i.get("bannerImage")
        if not img:
            img = (
                i.get("coverImage")
                .get("medium")
                .replace("small", "large")
                .replace("medium", "large")
            )
        title = get_atitle(i.get("title"))
        url = get_urls(title)
        temp = SLIDER_HTML.format(
            f"#{pos} Spotlight",
            title,
            i.get("type"),
            i.get("status"),
            get_genre(i.get("genres")),
            i.get("description"),
            url.replace("/anime/", "/episode/") + "/1",
            url,
            url,
            img,
        )
        html += temp
        pos += 1
    return html

def episodeHtml(episode, title, dl=True):
    isSub = episode.get("SUB")
    isDub = episode.get("DUB")
    DL = episode.get("DL")
    sub = dub = dlsub = dldub = ""
    defa = 0
    s, d = 1, 1

    if isSub:
        for i in isSub:
            if defa == 0:
                defa = f"/embed?url={i}&title={title}"
                sub += f"""
 Server{s}
"""
            else:
                sub += f"""
 Server{s}
"""
            s += 1

    if isDub:
        for i in isDub:
            if defa == 0:
                defa = f"/embed?url={i}&title={title}"
                dub += f"""
 Server{d}
"""
            else:
                dub += f"""
 Server{d}
"""
            d += 1

    if DL:
        link = DL.get("SUB")
        if link:
            for n, l in link.items():
                dlsub += f"""
 {n}
"""
        link = DL.get("DUB")
        if link:
            for n, l in link.items():
                dldub += f"""
 {n}
"""

    if sub != "":
        t4 = f"""

 SUB:

{sub}

"""
    else:
        t4 = ""

    if dub != "":
        t5 = f"""

 DUB:

{dub}

"""
    else:
        t5 = ""

    if dlsub != "":
        t6 = f"""

 SUB:

{dlsub}

"""
    else:
        t6 = ""

    if dldub != "":
        t7 = f"""

 DUB:

{dldub}

"""
    else:
        t7 = ""

    t8 = f"""Download

Download Links:
{t6}{t7}
"""

    html = t4 + t5
    if dl:
        html += t8

    return html, defa


import numpy as np
import matplotlib.pyplot as plt
from cartoplot import cartoplot
import imageio
from netCDF4 import Dataset
import pickle

def get(string):
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    path_grid = "/home/robbie/Dropbox/Data/grid.nc"

    if string == 'lon':
        grid_data = Dataset(path_grid)
        lon = np.array(grid_data.variables["lon"])
        return(lon)
    elif string == 'lat':
        grid_data = Dataset(path_grid)
        lat = np.array(grid_data.variables["lat"])
        return(lat)
    elif string == 'mask':
        im = imageio.imread('J_Mask.tif')
        mask = np.flipud(np.array(im))
        return(mask)

def EASE():
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    mask = pickle.load( open( "/home/robbie/Dropbox/Code/mask_348x348.p", "rb" ) )
    return(mask)

def OSISAF():
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    mask = pickle.load( open( "/home/robbie/custom_modules/mask_1120x760.p", "rb" ) )
    return(mask)

def plot(region_string):

    regions_dict = {"Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}

    code = regions_dict[region_string]

    fig = plt.figure(figsize=(10, 8))

    cartoplot(get('lon'), get('lat'), get('mask'),color_scale=(code+1,code-1))

    print(code)

    plt.show()


import tensorflow as tf
import tensorflow_datasets as tfds

config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)

def load_celeba_dataset(args, shuffle_files=False, batch_size=128):
  ds_train, ds_test = tfds.load(name='celeb_a', split=['train', 'test'], data_dir=args.data_dir,
      batch_size=batch_size, download=True, shuffle_files=shuffle_files)
  return ds_train, ds_test


# -*- encoding: UTF-8 -*-
from django.http import Http404
from django.db.models.loading import get_model
from django.contrib.staticfiles.storage import staticfiles_storage
from django.contrib.admin.views.decorators import staff_member_required
from django.core.urlresolvers import reverse
from django.shortcuts import render
from django.forms.widgets import Select
import models

def detail(request, app, cls, slug):
    """ generic view that return direct CMS model rendered content """
    model = get_model(app, cls)
    if model and issubclass(model, models.CMSModel):
        return model.get_response(request, slug)
    raise Http404

@staff_member_required
def imagechooser(request, app, cls):
    model = get_model(app, cls)
    datas = {
        'tinymce_path': staticfiles_storage.url('tiny_mce'),
        'chosen_path': staticfiles_storage.url('chosen'),
        'admin_path': staticfiles_storage.url('admin')
    }
    if model and issubclass(model, models.CMSModel):
        if getattr(model.CMSMeta, 'image_model', None):
            images = [('', '----')]
            for fileitem in model.CMSMeta.image_model.objects.all().order_by('title'):
                if fileitem.file.name.lower().endswith(('.jpg', '.jpeg', '.gif', '.png')):
                    images.append((fileitem.get_absolute_url(), fileitem.title))
            datas['select_files'] = Select(choices=images, attrs={'class': 'chosen-single', 'style': 'width:200px'}).render('file', '')
            #datas['form_upload'] = None
    # gestion upload if any
    # send result back
    return render(request, 'imagechooser.html', datas)

@staff_member_required
def tinymcejs(request, app, cls):
    datas = {
        'tinymce_path': staticfiles_storage.url('tiny_mce'),
        'imagechooser_path': reverse('picocms-imagechooser', args=(app, cls))
    }
    return render(request, 'tiny_mce_src.js', datas, content_type='application/javascript')


from __future__ import absolute_import, unicode_literals

import base64
import json
import random

import warnings
import websocket

from c8 import constants
from c8.api import APIWrapper
from c8.apikeys import APIKeys
from c8.c8ql import C8QL
from c8.collection import StandardCollection
from c8.exceptions import (
    CollectionCreateError,
    CollectionDeleteError,
    CollectionFindError,
    CollectionListError,
    CollectionPropertiesError,
    EventCreateError,
    EventGetError,
    FabricCreateError,
    FabricDeleteError,
    FabricGetMetadataError,
    FabricListError,
    FabricPropertiesError,
    FabricSetMetadataError,
    FabricUpdateMetadataError,
    GetAPIKeys,
    GetDcDetailError,
    GetDcListError,
    GetLocalDcError,
    GraphCreateError,
    GraphDeleteError,
    GraphListError,
    RestqlCreateError,
    RestqlCursorError,
    RestqlDeleteError,
    RestqlExecuteError,
    RestqlImportError,
    RestqlListError,
    RestqlUpdateError,
    RestqlValidationError,
    ServerConnectionError,
    ServerVersionError,
    SpotRegionAssignError,
    SpotRegionUpdateError,
    StreamAppGetSampleError,
    StreamCommunicationError,
    StreamConnectionError,
    StreamCreateError,
    StreamDeleteError,
    StreamListError,
    StreamPermissionError,
)
from c8.executor import AsyncExecutor, BatchExecutor, DefaultExecutor
from c8.graph import Graph
from c8.keyvalue import KV
from c8.request import Request
from c8.search import Search
from c8.stream_apps import StreamApps
from c8.stream_collection import StreamCollection

__all__ = [
    "StandardFabric",
    "AsyncFabric",
    "BatchFabric",
]
ENDPOINT = "/streams"

def raise_timeout(signum, frame):
    raise TimeoutError

class Fabric(APIWrapper):
    """Base class for Fabric API wrappers.

    :param connection: HTTP connection.
    :type connection: c8.connection.Connection
    :param executor: API executor.
    :type executor: c8.executor.Executor
    """

    def enum(**enums):
        return type("Enum", (), enums)

    SPOT_CREATION_TYPES = enum(
        AUTOMATIC="automatic", NONE="none", SPOT_REGION="spot_region"
    )

    def __init__(self, connection, executor):
        self.url = connection.url
        self.header = connection.headers
        self.stream_port = constants.STREAM_PORT
        super(Fabric, self).__init__(connection, executor)

    def __getitem__(self, name):
        """Return the collection API wrapper.

        :param name: Collection name.
        :type name: str | unicode
        :returns: Collection API wrapper.
        :rtype: c8.collection.StandardCollection
        """
        return self.collection(name)

    @property
    def name(self):
        """Return fabric name.

        :returns: Fabric name.
        :rtype: str | unicode
        """
        return self.fabric_name

    @property
    def c8ql(self):
        """Return C8QL (C8Db Query Language) API wrapper.

        :returns: C8QL API wrapper.
        :rtype: c8.c8ql.C8QL
        """
        return C8QL(self._conn, self._executor)

    @property
    def key_value(self):
        """Return KV (Key Value) API wrapper.

        :returns: KV API wrapper.
        :rtype: c8.keyvalue.KV
        """
        return KV(self._conn, self._executor)

    def on_change(self, collection, callback, timeout=60):
        """Execute given input function on receiving a change.

        :param collection: Collection name(s) regex to listen for
        :type collection: str
        :param timeout: timeout value
        :type timeout: int
        :param callback: Function to execute on a change
        :type callback: function
        """
        if not callback:
            raise ValueError("You must specify a callback function")

        if not collection:
            raise ValueError(
                "You must specify a collection on which realtime "
                "data is to be watched!"
            )

        namespace = constants.STREAM_LOCAL_NS_PREFIX + self.fabric_name

        subscription_name = "%s-%s-subscription-%s" % (
            self.tenant_name,
            self.fabric_name,
            str(random.randint(1, 1000)),
        )

        url = self.url.split("//")[1].split(":")[0]

        topic = "wss://{}/_ws/ws/v2/consumer/persistent/{}/{}/{}/{}".format(
            url, self.tenant_name, namespace, collection, subscription_name
        )

        ws = websocket.create_connection(topic, header=self.header, timeout=timeout)

        try:
            # "pyC8 Realtime: Begin monitoring realtime updates for " + topic
            while True:
                msg = json.loads(ws.recv())
                data = base64.b64decode(msg["payload"])
                ws.send(json.dumps({"messageId": msg["messageId"]}))
                callback(data)
        except websocket.WebSocketTimeoutException:
            pass
        except Exception as e:
            raise Exception(e)
        finally:
            ws.close()

    def properties(self):
        """Return fabric properties.

        :returns: Fabric properties.
        :rtype: dict
        :raise c8.exceptions.FabricPropertiesError: If retrieval fails.
        """
        request = Request(
            method="get",
            endpoint="/database/current",
        )

        def response_handler(resp):
            if not resp.is_success:
                raise FabricPropertiesError(resp, request)
            result = resp.body["result"]
            result["system"] = result.pop("isSystem")
            return result

        return self._execute(request, response_handler)

    def update_spot_region(self, tenant, fabric, new_dc):
        """Updates spot primary region for the geo-fabric

        :param tenant: tenant name
        :type tenant: str
        :param fabric: fabric name
        :type fabric: str
        :param new_dc: New spot region
        :type new_dc: str
        :returns: True if request successful,false otherwise
        :rtype: bool
        :raise c8.exceptions.SpotRegionUpdateError: If updation fails.
        """

        request = Request(
            method="put", endpoint="/_fabric/{}/database/{}".format(fabric, new_dc)
        )

        def response_handler(resp):
            if not resp.is_success:
                raise SpotRegionUpdateError(resp, request)
            return True

        return self._execute(request, response_handler)

    def fabrics_detail(self):
        request = Request(method="get", endpoint="/database/user")

        def response_handler(resp):
            if not resp.is_success:
                raise FabricListError(resp, request)
            return [
                {"name": col["name"], "options": col["options"]}
                for col in map(dict, resp.body["result"])
            ]

        return self._execute(request, response_handler)

    def version(self):
        """Return C8Db server version.

        :returns: Server version.
        :rtype: str | unicode
        :raise c8.exceptions.ServerVersionError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/version", params={"details": False})

        def response_handler(resp):
            if not resp.is_success:
                raise ServerVersionError(resp, request)
            return resp.body["version"]

        return self._execute(request, response_handler)

    def ping(self):
        """Ping the C8Db server by sending a test request.

        :returns: Response code from server.
        :rtype: int
        :raise c8.exceptions.ServerConnectionError: If ping fails.
        """
        request = Request(
            method="get",
            endpoint="/collection",
        )

        def response_handler(resp):
            code = resp.status_code
            if code in {401, 403}:
                raise ServerConnectionError("bad username and/or password")
            if not resp.is_success:
                raise ServerConnectionError(resp.error_message or "bad server response")
            return code

        return self._execute(request, response_handler)

    #########################
    # Datacenter Management #
    #########################

    def dclist(self, detail=False):
        """Return the list of names of Datacenters

        :param detail: detail list of DCs if set to true else only DC names
        :type: boolean
        :returns: DC List.
        :rtype: [str | unicode ]
        :raise c8.exceptions.GetDcListError: If retrieval fails.
        """
        properties = self.properties()
        if not detail:
            return properties["options"]["dcList"].split(",")

        tenant_name = properties["options"]["tenant"]

        request = Request(
            method="get", endpoint="/datacenter/_tenant/{}".format(tenant_name)
        )

        def response_handler(resp):
            if not resp.is_success:
                raise GetDcListError(resp, request)
            dc_list = resp.body[0]["dcInfo"]
            for dc in dc_list:
                if dc["name"] not in properties["options"]["dcList"]:
                    dc_list.remove(dc)
            return dc_list

        return self._execute(request, response_handler, custom_prefix="")

    def localdc(self, detail=True):
        """Fetch data for a local/regional the data center

        :param detail: Details of local DC if set to true else only DC name.
        :type: boolean
        :returns: Local DC details.
        :rtype: str | dict
        :raise c8.exceptions.GetLocalDcError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/datacenter/local")

        def response_handler(resp):
            if not resp.is_success:
                raise GetLocalDcError(resp, request)
            if detail:
                return resp.body
            return resp.body["name"]

        return self._execute(request, response_handler, custom_prefix="")

    def get_dc_detail(self, dc):
        """Fetch data for data center, identified by dc-name

        :param dc: DC name
        :type: str
        :returns: DC details.
        :rtype: dict
        :raise c8.exceptions.GetDcDetailError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/datacenter/{}".format(dc))

        def response_handler(resp):
            if not resp.is_success:
                raise GetDcDetailError(resp, request)
            return resp.body

        return self._execute(request, response_handler, custom_prefix="")

    def dclist_all(self):
        """Fetch data about all the data centers

        :returns: DC List.
        :rtype: [str | unicode ]
        :raise c8.exceptions.GetDcListError: If retrieval fails.
        """

        request = Request(method="get", endpoint="/datacenter/all")

        def response_handler(resp):
            if not resp.is_success:
                raise GetDcListError(resp, request)
            return resp.body

        return self._execute(request, response_handler, custom_prefix="")

    def assign_dc_spot(self, dc, spot_region=False):
        """Assigns spot region of a fed

        :param dc: dc name
        :type dc: str
        :param spot_region: If True, makes the region a spot region
        :type spot_region: bool
        :returns: True if request successful, False otherwise
        :rtype: bool
        :raise c8.exceptions.SpotRegionAssignError: If assignment fails.
        """
        data = json.dumps(spot_region)
        request = Request(method="put", endpoint="/datacenter/{}/{}".format(dc, data))

        def response_handler(resp):
            if not resp.is_success:
                raise SpotRegionAssignError(resp, request)
            return True

        return self._execute(request, response_handler, custom_prefix="")

    #######################
    # Fabric Management #
    #######################

    def fabrics(self):
        """Return the names all fabrics.

        :returns: Fabric names.
        :rtype: [str | unicode]
        :raise c8.exceptions.FabricListError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/database")

        def response_handler(resp):
            if not resp.is_success:
                raise FabricListError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def has_fabric(self, name):
        """Check if a fabric exists.

        :param name: Fabric name.
        :type name: str | unicode
        :returns: True if fabric exists, False otherwise.
        :rtype: bool
        """
        return name in self.fabrics()

    def create_fabric(
        self,
        name,
        spot_dc=None,
        users=None,
        dclist=None,
        spot_creation_type=SPOT_CREATION_TYPES.AUTOMATIC,
    ):
        """Create a new fabric.

        :param name: Fabric name.
        :type name: str | unicode
        :param spot_creation_type: Specifying the mode of creating geo-fabric.
                                   If you use AUTOMATIC, a random spot region
                                   will be assigned by the system. If you
                                   specify NONE, a geo-fabric is created
                                   without the spot properties. If you specify
                                   SPOT_REGION,pass the corresponding spot
                                   region in the spot_dc parameter.
        :type name: Enum containing spot region creation types
        :param name: Spot Region name, if spot_creation_type is set to
                     SPOT_REGION
        :type name: str
        :param users: List of users with access to the new fabric
        :type users: [str | unicode]
        :param dclist: list of strings of datacenters
        :type dclist: [str | unicode]
        :returns: True if fabric was created successfully.
        :rtype: bool
        :raise c8.exceptions.FabricCreateError: If create fails.
        """
        data = {"name": name}
        if users is not None:
            data["users"] = users

        options = {}
        dcl = ""
        if dclist:
            # Process dclist param (type list) to build up comma-separated
            # string of DCs
            for dc in dclist:
                if len(dcl) > 0:
                    dcl += ","
                dcl += dc
        options["dcList"] = dcl

        if spot_creation_type == self.SPOT_CREATION_TYPES.NONE:
            options["spotDc"] = ""
        elif spot_creation_type == self.SPOT_CREATION_TYPES.SPOT_REGION and spot_dc:
            options["spotDc"] = spot_dc

        data["options"] = options

        request = Request(method="post", endpoint="/database", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise FabricCreateError(resp, request)
            return True

        return self._execute(request, response_handler)

    def get_fabric_metadata(self):
        """Fetch information about a GeoFabric.

        :returns: Fabric information.
        :rtype: dict
        :raise c8.exceptions.FabricGetMetadataError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/database/metadata")

        def response_handler(resp):
            if not resp.is_success:
                raise FabricGetMetadataError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def set_fabric_metadata(self, metadata):
        """Set the GeoFabric Metadata.

        :param metadata: Fabric metadata.
        :type metadata: dict
        :returns: True if metadata was set successfully.
        :rtype: bool
        :raise c8.exceptions.FabricSetMetadataError: If set fails.
        """

        data = {"metadata": metadata}
        request = Request(method="put", endpoint="/database/metadata", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise FabricSetMetadataError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def update_fabric_metadata(self, metadata):
        """Modfiy the GeoFabric metadata.

        :param metadata: Fabric metadata.
        :type metadata: dict
        :returns: True if metadata was set successfully.
        :rtype: bool
        :raise c8.exceptions.FabricUpdateMetadataError: If update fails.
        """
        data = {"metadata": metadata}
        request = Request(method="patch", endpoint="/database/metadata", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise FabricUpdateMetadataError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def delete_fabric(self, name, ignore_missing=False):
        """Delete the fabric.

        :param name: Fabric name.
        :type name: str | unicode
        :param ignore_missing: Do not raise an exception on missing fabric.
        :type ignore_missing: bool
        :returns: True if fabric was deleted successfully, False if fabric
            was not found and **ignore_missing** was set to True.
        :rtype: bool
        :raise c8.exceptions.FabricDeleteError: If delete fails.
        """
        request = Request(method="delete", endpoint="/database/{}".format(name))

        def response_handler(resp):
            if resp.error_code == 1228 and ignore_missing:
                return False
            if not resp.is_success:
                raise FabricDeleteError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    #########################
    # Collection Management #
    #########################

    def collection(self, name):
        """Return the standard collection API wrapper.

        :param name: Collection name.
        :type name: str | unicode
        :returns: Standard collection API wrapper.
        :rtype: c8.collection.StandardCollection
        """
        if self.has_collection(name):
            return StandardCollection(self._conn, self._executor, name)
        else:
            raise CollectionFindError("Collection not found")

    def has_collection(self, name):
        """Check if collection exists in the fabric.

        :param name: Collection name.
        :type name: str | unicode
        :returns: True if collection exists, False otherwise.
        :rtype: bool
        """
        return any(col["name"] == name for col in self.collections())

    def collections(self, collectionModel=None):
        """Return the collections in the fabric.

        :returns: Collections in the fabric and their details.
        :rtype: [dict]
        :raise c8.exceptions.CollectionListError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/collection")

        def response_handler(resp):
            if not resp.is_success:
                raise CollectionListError(resp, request)
            if collectionModel is not None:
                docs = [
                    col
                    for col in map(dict, resp.body["result"])
                    if col["collectionModel"] == collectionModel
                ]
            else:
                docs = [col for col in map(dict, resp.body["result"])]
            collections = []

            for col in docs:
                c = {
                    "id": col["id"],
                    "name": col["name"],
                    "system": col["isSystem"],
                    "type": StandardCollection.types[col["type"]],
                    "status": StandardCollection.statuses[col["status"]],
                    "collectionModel": col["collectionModel"],
                }
                if "isSpot" in col.keys():
                    c["isSpot"] = col["isSpot"]
                else:
                    c["strongConsistency"] = col.get("strongConsistency", False)
                collections.append(c)
            return collections

        return self._execute(request, response_handler)

    def create_collection(
        self,
        name,
        sync=False,
        edge=False,
        user_keys=True,
        key_increment=None,
        key_offset=None,
        key_generator="traditional",
        shard_fields=None,
        index_bucket_count=None,
        sync_replication=None,
        enforce_replication_factor=None,
        strong_consistency=None,
        local_collection=False,
        is_system=False,
        stream=False,
        *,
        spot_collection=None,
    ):
        """Create a new collection.

        :param name: Collection name.
        :type name: str | unicode
        :param sync: If set to True, document operations via the collection
            will block until synchronized to disk by default.
        :type sync: bool
        :param edge: If set to True, an edge collection is created.
        :type edge: bool
        :param key_generator: Used for generating document keys. Allowed values
            are "traditional" or "autoincrement".
        :type key_generator: str | unicode
        :param user_keys: If set to True, users are allowed to supply document
            keys. If set to False, the key generator is solely responsible for
            supplying the key values.
        :type user_keys: bool
        :param key_increment: Key increment value. Applies only when value of
            **key_generator** is set to "autoincrement".
        :type key_increment: int
        :param key_offset: Key offset value. Applies only when value of
            **key_generator** is set to "autoincrement".
        :type key_offset: int
        :param shard_fields: Field(s) used to determine the target shard.
        :type shard_fields: [str | unicode]
        :param index_bucket_count: Number of buckets into which indexes using
            hash tables are split. The default is 16, and this number has to be
            a power of 2 and less than or equal to 1024. For large collections,
            one should increase this to avoid long pauses when the hash table
            has to be initially built or re-sized, since buckets are re-sized
            individually and can be initially built in parallel. For instance,
            64 may be a sensible value for 100 million documents.
        :type index_bucket_count: int
        :param sync_replication: If set to True, server reports success only
            when collection is created in all replicas. You can set this to
            False for faster server response, and if full replication is not a
            concern.
        :type sync_replication: bool
        :param enforce_replication_factor: Check if there are enough replicas
            available at creation time, or halt the operation.
        :type enforce_replication_factor: bool
        :param strong_consistency: If True, strong consistency is enabled
        :type strong_consistency: bool
        :param is_system: If True, able to create system collections
        :type is_system: bool
        :param stream: If True, create a local stream for collection.
        :type stream: bool
        :param spot_collection: If True, it is a spot collection.
            Deprecated. Use If strong_consistency instead.
        :type spot_collection: bool
        :returns: Standard collection API wrapper.
        :rtype: c8.collection.StandardCollection
        :raise c8.exceptions.CollectionCreateError: If create fails.
        """
        # Newer versions of GDN has renamed the isSpot property to strongConsistency.
        # We are using a keyword only argument to keep the backward compatibility of the SDK
        if spot_collection is not None and strong_consistency is not None:
            raise TypeError("create_collection recieved both spot_collection and strong_consistency")
        elif spot_collection is not None:
            warnings.simplefilter("once", DeprecationWarning)
            warnings.warn("spot_collection is deprecated. Use strong_consistency instead.", DeprecationWarning, 2)
            warnings.simplefilter("default", DeprecationWarning)
            isSpot = spot_collection
        elif strong_consistency is not None:
            isSpot = strong_consistency
        else:
            isSpot = False

        key_options = {"type": key_generator, "allowUserKeys": user_keys}
        if key_increment is not None:
            key_options["increment"] = key_increment
        if key_offset is not None:
            key_options["offset"] = key_offset
        if spot_collection and local_collection:
            return "Collection can either be spot or local"
        else:
            # Both strong_consistency and isSpot is added to the request body.
            # Correct value will be picked by the GDN depending on the supported property.
            data = {
                "name": name,
                "waitForSync": sync,
                "keyOptions": key_options,
                "type": 3 if edge else 2,
                "isSpot": isSpot,
                "strongConsistency": isSpot,
                "isLocal": local_collection,
                "isSystem": is_system,
                "stream": stream,
            }

        if shard_fields is not None:
            data["shardKeys"] = shard_fields
        if index_bucket_count is not None:
            data["indexBuckets"] = index_bucket_count

        params = {}
        if sync_replication is not None:
            params["waitForSyncReplication"] = sync_replication
        if enforce_replication_factor is not None:
            params["enforceReplicationFactor"] = enforce_replication_factor

        request = Request(
            method="post", endpoint="/collection", params=params, data=data
        )

        def response_handler(resp):
            if resp.is_success:
                return self.collection(name)
            raise CollectionCreateError(resp, request)

        return self._execute(request, response_handler)

    def update_collection_properties(
        self, collection_name, has_stream=None, wait_for_sync=None
    ):
        """Changes the properties of a collection.
           Note: except for waitForSync and hasStream, collection properties cannot be changed once a collection is created.
        :param collection_name: Collection name.
        :type collection_name: str | unicode
        :param has_stream: True if creating a live collection stream.
        :type has_stream: bool
        :param wait_for_sync: True if all data must be synced to storage before operation returns.
        :type wait_for_sync: bool
        """

        data = {}
        if has_stream is not None:
            data["hasStream"] = has_stream
        if wait_for_sync is not None:
            data["waitForSync"] = wait_for_sync

        request = Request(
            method="put",
            endpoint="/collection/{}/properties".format(collection_name),
            data=data,
        )

        def response_handler(resp):
            if resp.is_success:
                return resp.body
            raise CollectionPropertiesError(resp, request)

        return self._execute(request, response_handler)

    def delete_collection(self, name, ignore_missing=False, system=None):
        """Delete the collection.

        :param name: Collection name.
        :type name: str | unicode
        :param ignore_missing: Do not raise an exception on missing collection.
        :type ignore_missing: bool
        :param system: Whether the collection is a system collection.
        :type system: bool
        :returns: True if collection was deleted successfully, False if
            collection was not found and **ignore_missing** was set to True.
        :rtype: bool
        :raise c8.exceptions.CollectionDeleteError: If delete fails.
        """
        params = {}
        if system is not None:
            params["isSystem"] = system

        request = Request(
            method="delete", endpoint="/collection/{}".format(name), params=params
        )

        def response_handler(resp):
            if resp.error_code == 1203 and ignore_missing:
                return False
            if not resp.is_success:
                raise CollectionDeleteError(resp, request)
            return True

        return self._execute(request, response_handler)

    ####################
    # Graph Management #
    ####################

    def graph(self, name):
        """Return the graph API wrapper.

        :param name: Graph name.
        :type name: str | unicode
        :returns: Graph API wrapper.
        :rtype: c8.graph.Graph
        """
        return Graph(self._conn, self._executor, name)

    def has_graph(self, name):
        """Check if a graph exists in the fabric.

        :param name: Graph name.
        :type name: str | unicode
        :returns: True if graph exists, False otherwise.
        :rtype: bool
        """
        for graph in self.graphs():
            if graph["name"] == name:
                return True
        return False

    def graphs(self):
        """List all graphs in the fabric.

        :returns: Graphs in the fabric.
        :rtype: [dict]
        :raise c8.exceptions.GraphListError: If retrieval fails.
        """
        request = Request(method="get", endpoint="/graph")

        def response_handler(resp):
            if not resp.is_success:
                raise GraphListError(resp, request)
            return [
                {
                    "id": body["_id"],
                    "name": body["_key"],
                    "revision": body["_rev"],
                    "orphan_collections": body["orphanCollections"],
                    "edge_definitions": [
                        {
                            "edge_collection": definition["collection"],
                            "from_vertex_collections": definition["from"],
                            "to_vertex_collections": definition["to"],
                        }
                        for definition in body["edgeDefinitions"]
                    ],
                    "shard_count": body.get("numberOfShards"),
                    "replication_factor": body.get("replicationFactor"),
                }
                for body in resp.body["graphs"]
            ]

        return self._execute(request, response_handler)

    def create_graph(
        self, name, edge_definitions=None, orphan_collections=None, shard_count=None
    ):
        """Create a new graph.

        :param name: Graph name.
        :type name: str | unicode
        :param edge_definitions: List of edge definitions, where each edge
            definition entry is a dictionary with fields "edge_collection",
            "from_vertex_collections" and "to_vertex_collections" (see below
            for example).
        :type edge_definitions: [dict]
        :param orphan_collections: Names of additional vertex collections that
            are not in edge definitions.
        :type orphan_collections: [str | unicode]
        :param shard_count: Number of shards used for every collection in the
            graph. To use this, parameter **smart** must be set to True and
            every vertex in the graph must have the smart field. This number
            cannot be modified later once set. Applies only to enterprise
            version of C8Db.
        :type shard_count: int
        :returns: Graph API wrapper.
        :rtype: c8.graph.Graph
        :raise c8.exceptions.GraphCreateError: If create fails.

        Here is an example entry for parameter **edge_definitions**:

        .. code-block:: python

            {
                'edge_collection': 'teach',
                'from_vertex_collections': ['teachers'],
                'to_vertex_collections': ['lectures']
            }
        """
        data = {"name": name}
        if edge_definitions is not None:
            data["edgeDefinitions"] = [
                {
                    "collection": definition["edge_collection"],
                    "from": definition["from_vertex_collections"],
                    "to": definition["to_vertex_collections"],
                }
                for definition in edge_definitions
            ]
        if orphan_collections is not None:
            data["orphanCollections"] = orphan_collections
        if shard_count is not None:  # pragma: no cover
            data["numberOfShards"] = shard_count

        request = Request(method="post", endpoint="/graph", data=data)

        def response_handler(resp):
            if resp.is_success:
                return Graph(self._conn, self._executor, name)
            raise GraphCreateError(resp, request)

        return self._execute(request, response_handler)

    def delete_graph(self, name, ignore_missing=False, drop_collections=None):
        """Drop the graph of the given name from the fabric.

        :param name: Graph name.
        :type name: str | unicode
        :param ignore_missing: Do not raise an exception on missing graph.
        :type ignore_missing: bool
        :param drop_collections: Drop the collections of the graph also. This
            is only if they are not in use by other graphs.
        :type drop_collections: bool
        :returns: True if graph was deleted successfully, False if graph was not
            found and **ignore_missing** was set to True.
        :rtype: bool
        :raise c8.exceptions.GraphDeleteError: If delete fails.
        """
        params = {}
        if drop_collections is not None:
            params["dropCollections"] = drop_collections

        request = Request(
            method="delete", endpoint="/graph/{}".format(name), params=params
        )

        def response_handler(resp):
            if resp.error_code == 1924 and ignore_missing:
                return False
            if not resp.is_success:
                raise GraphDeleteError(resp, request)
            return True

        return self._execute(request, response_handler)

    ########################
    # Async Job Management #
    ########################

    # Pratik: APIs not supported in documentation. Waiting for verification
    # def async_jobs(self, status, count=None):
    #     """Return IDs of async jobs with given status.
    #
    #     :param status: Job status (e.g. "pending", "done").
    #     :type status: str | unicode
    #     :param count: Max number of job IDs to return.
    #     :type count: int
    #     :returns: List of job IDs.
    #     :rtype: [str | unicode]
    #     :raise c8.exceptions.AsyncJobListError: If retrieval fails.
    #     """
    #     params = {}
    #     if count is not None:
    #         params['count'] = count
    #
    #     request = Request(
    #         method='get',
    #         endpoint='/job/{}'.format(status),
    #         params=params
    #     )
    #
    #     def response_handler(resp):
    #         if resp.is_success:
    #             return resp.body
    #         raise AsyncJobListError(resp, request)
    #
    #     return self._execute(request, response_handler)
    #
    # def clear_async_jobs(self, threshold=None):
    #     """Clear async job results from the server.
    #
    #     Async jobs that are still queued or running are not stopped.
    #
    #     :param threshold: If specified, only the job results created prior to
    #         the threshold (a unix timestamp) are deleted. Otherwise, all job
    #         results are deleted.
    #     :type threshold: int
    #     :returns: True if job results were cleared successfully.
    #     :rtype: bool
    #     :raise c8.exceptions.AsyncJobClearError: If operation fails.
    #     """
    #     if threshold is None:
    #         url = '/job/all'
    #         params = None
    #     else:
    #         url = '/job/expired'
    #         params = {'stamp': threshold}
    #
    #     request = Request(
    #         method='delete',
    #         endpoint=url,
    #         params=params
    #     )
    #
    #     def response_handler(resp):
    #         if resp.is_success:
    #             return True
    #         raise AsyncJobClearError(resp, request)
    #
    #     return self._execute(request, response_handler)

    ########################
    # Streams Management   #
    ########################

    def stream(self, operation_timeout_seconds=30):
        """Return the stream collection API wrapper.

        :returns: stream collection API wrapper.
        :rtype: c8.stream_collection.StreamCollection
        """
        return StreamCollection(
            self,
            self._conn,
            self._executor,
            self.url,
            self.stream_port,
            operation_timeout_seconds,
        )

    def streams(self, local=None):
        """Get list of all streams under given fabric

        :returns: List of streams under given fabric.
        :rtype: json
        :raise c8.exceptions.StreamListError: If retrieving streams fails.
        """
        if local is False:
            url_endpoint = "/streams?global=true"

        elif local is True:
            url_endpoint = "/streams?global=false"

        elif local is None:
            url_endpoint = "/streams"

        request = Request(method="get", endpoint=url_endpoint)

        def response_handler(resp):
            code = resp.status_code
            if resp.is_success:
                return [
                    {
                        "name": col["topic"],
                        "topic": col["topic"],
                        "local": col["local"],
                        "db": col["db"],
                        "tenant": col["tenant"],
                        "type": StreamCollection.types[col["type"]],
                        "status": "terminated"
                        if "terminated" in col
                        else "active",  # noqa
                    }
                    for col in map(dict, resp.body["result"])
                ]
            elif code == 403:
                raise StreamPermissionError(resp, request)
            raise StreamListError(resp, request)

        return self._execute(request, response_handler)

    def has_stream(self, stream, isCollectionStream=False, local=False):
        """Check if the list of streams has a stream with the given name.

        :param stream: The name of the stream for which to check in the list
                       of all streams.
        :type stream: str | unicode
        :returns: True=stream found; False=stream not found.
        :rtype: bool
        """
        if isCollectionStream is False:
            if local is False and "c8globals" not in stream:
                stream = "c8globals." + stream
            elif local is True and "c8locals" not in stream:
                stream = "c8locals." + stream
        return any(mystream["name"] == stream for mystream in self.streams(local=local))

    def create_stream(self, stream, local=False):
        """
        Create the stream under the given fabric

        :param stream: name of stream
        :param local: Operate on a local stream instead of a global one.
        :returns: 200, OK if operation successful
        :raise: c8.exceptions.StreamCreateError: If creating streams fails.
        """
        if local is True:
            endpoint = "{}/{}?global=False".format(ENDPOINT, stream)
        elif local is False:
            endpoint = "{}/{}?global=True".format(ENDPOINT, stream)

        request = Request(method="post", endpoint=endpoint)

        def response_handler(resp):
            code = resp.status_code
            if resp.is_success:
                return resp.body["result"]
            elif code == 502:
                raise StreamCommunicationError(resp, request)
            raise StreamCreateError(resp, request)

        return self._execute(request, response_handler)

    def delete_stream(self, stream, force=False):
        """
        Delete the streams under the given fabric

        :param stream: name of stream
        :param force:
        :returns: 200, OK if operation successful
        :raise: c8.exceptions.StreamDeleteError: If deleting streams fails.
        """
        endpoint = f"{ENDPOINT}/{stream}"
        if force:
            endpoint = endpoint + "?force=true"

        request = Request(method="delete", endpoint=endpoint)

        def response_handler(resp):
            code = resp.status_code
            if resp.is_success:
                return True
            elif code == 403:
                raise StreamPermissionError(resp, request)
            elif code == 412:
                raise StreamDeleteError(resp, request)
            raise StreamConnectionError(resp, request)

        return self._execute(request, response_handler)

    #####################
    # Restql Management #
    #####################

    def save_restql(self, data):
        """Save restql by name.

        :param data: data to be used for restql POST API
        :type data: dict
        :returns: Results of restql API
        :rtype: dict
        :raise c8.exceptions.RestqlCreateError: if restql operation failed
        """

        query_name = data["query"]["name"]
        if " " in query_name:
            raise RestqlValidationError("White Spaces not allowed in Query " "Name")

        request = Request(method="post", endpoint="/restql", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlCreateError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def import_restql(self, queries, details=False):
        """Import custom queries.

        :param queries: queries to be imported
        :type queries: [dict]
        :param details: Whether to include details
        :type details: bool
        :returns: Results of importing restql
        :rtype: dict
        :raise c8.exceptions.RestqlImportError: if restql operation failed
        """

        data = {"queries": queries, "details": details}

        request = Request(method="post", endpoint="/restql/import", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlImportError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def execute_restql(self, name, data=None):
        """Execute restql by name.

        :param name: restql name
        :type name: str | unicode
        :param data: restql data (optional)
        :type data: dict
        :returns: Results of execute restql
        :rtype: dict
        :raise c8.exceptions.RestqlExecuteError: if restql execution failed
        """

        if data is None or not ("bindVars" in data or "batchSize" in data):
            data = {}

        request = Request(
            method="post", data=data, endpoint="/restql/execute/{}".format(name)
        )

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlExecuteError(resp, request)
            return resp.body

        return self._execute(request, response_handler)

    def read_next_batch_restql(self, id):
        """Read next batch from query worker cursor.

        :param id: the cursor-identifier
        :type id: int
        :returns: Results of execute restql
        :rtype: dict
        :raise c8.exceptions.RestqlCursorError: if fetch next batch failed
        """

        request = Request(method="put", endpoint="/restql/fetch/{}".format(id))

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlCursorError(resp, request)
            return resp.body

        return self._execute(request, response_handler)

    def get_all_restql(self):
        """Get all restql associated for user.

        :returns: Details of all restql
        :rtype: list
        :raise c8.exceptions.RestqlListError: if getting restql failed
        """

        request = Request(method="get", endpoint="/restql/user")

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlListError(resp, request)
            return resp.body["result"]

        return self._execute(request, response_handler)

    def update_restql(self, name, data):
        """Update restql by name.

        :param name: name of restql
        :type name: str | unicode
        :param data: restql data
        :type data: dict
        :returns: True if restql is updated
        :rtype: bool
        :raise c8.exceptions.RestqlUpdateError: if query update failed
        """
        request = Request(method="put", data=data, endpoint="/restql/" + name)

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlUpdateError(resp, request)
            return True

        return self._execute(request, response_handler)

    def delete_restql(self, name):
        """Delete restql by name.

        :param name: restql name
        :type name: str | unicode
        :returns: True if restql is deleted
        :rtype: bool
        :raise c8.exceptions.RestqlDeleteError: if restql deletion failed
        """
        request = Request(method="delete", endpoint="/restql/" + name)

        def response_handler(resp):
            if not resp.is_success:
                raise RestqlDeleteError(resp, request)
            return True

        return self._execute(request, response_handler)

    ########################
    # Events #
    ########################

    def create_event(self, payload):
        """Create an event.

        :param payload: Payload to create event
        :type payload: dict
        :returns:  Dictionary containing the event id
        :rtype: dict
        :raise c8.exceptions.EventCreateError: if event creation failed

        Here is an example entry for parameter **payload**:

        .. code-block:: python

            {
            "action": "string",
            "attributes": {},
            "description": "string",
            "details": "string",
            "entityName": "string",
            "entityType": "string",
            "status": "string"
            }
        """
        request = Request(method="post", endpoint="/events", data=payload)

        def response_handler(resp):
            if not resp.is_success:
                raise EventCreateError(resp, request)
            return resp.body

        return self._execute(request, response_handler)

    def delete_event(self, eventIds):
        """Delete an event/s.

        :param eventIds: The event id for which you want to fetch the event details
        :type eventId: list of strings(event Ids)
        :returns: List containig all the information of existing events
        :rtype: list
        :raise c8.exceptions.EventDeleteError: if event creation failed

        """
        data = json.dumps((eventIds))

        request = Request(method="delete", endpoint="/events", data=data)

        def response_handler(resp):
            if not resp.is_success:
                raise EventGetError(resp, request)
            return True

        return self._execute(request, response_handler)

    def get_all_events(self):
        """Create an event.

        :returns: List containig all the information of existing events
        :rtype: list
        :raise c8.exceptions.EventGetError: if event creation failed

        """
        request = Request(method="get", endpoint="/events/tenant")

        def response_handler(resp):
            if not resp.is_success:
                raise EventGetError(resp, request)
            return resp.body

        return self._execute(request, response_handler)

    def get_event_by_Id(self, eventId):
        """Create an event.

        :param eventId: The event id for which you want to fetch the event details
        :returns: List containig all the information of existing events
        :rtype: list
        :raise c8.exceptions.EventGetError: if event creation failed

        """
        request = Request(method="get", endpoint="/events/" + str(eventId))

        def response_handler(resp):
            if not resp.is_success:
                raise EventGetError(resp, request)
            return resp.body

        return self._execute(request, response_handler)

    ########################
    # Stream Apps #
    ########################

    def stream_app(self, name):
        return StreamApps(self._conn, self._executor, name)

    def validate_stream_app(self, data):
        """validates a stream app by given data

        :param data: stream app defination string
        """
        body = {"definition": data}
        req = Request(
            method="post", endpoint="/streamapps/validate", data=json.dumps(body)
        )

        def response_handler(resp):
            if resp.is_success is True:
                return True
            return False

        return self._execute(req, response_handler)

    def retrieve_stream_app(self):
        """retrieves all the stream apps of a fabric"""
        req = Request(
            method="get",
            endpoint="/streamapps",
        )

        def response_handler(resp):
            if resp.is_success is True:
                return resp.body
            return False

        return self._execute(req, response_handler)

    def get_samples_stream_app(self):
        """gets samples for stream apps"""
        req = Request(
            method="get",
            endpoint="/streamapps/samples",
        )

        def response_handler(resp):
            if resp.is_success is not True:
                raise StreamAppGetSampleError(resp, req)
            return resp.body["streamAppSample"]

        return self._execute(req, response_handler)

    def create_stream_app(self, data, dclist=[]):
        """Creates a stream application by given data

        :param data: stream app definition
        :param dclist: regions where stream app has to be deployed
        """
        # create request body
        req_body = {"definition": data, "regions": dclist}
        # create request
        req = Request(method="post", endpoint="/streamapps", data=json.dumps(req_body))

        # create response handler

        def response_handler(resp):
            if resp.is_success is True:
                return True
            return False

        # call api
        return self._execute(req, response_handler)

    ########################
    # APIKeys #
    ########################

    def api_keys(self, keyid):
        """Return the API keys API wrapper.

        :param keyid: API Key id
        :type kaeyid: string | unicode
        :returns: API keys API wrapper.
        :rtype: c8.stream_collection.StreamCollection
        """
        return APIKeys(self._conn, self._executor, keyid)

    def list_all_api_keys(self):
        """List the API keys.

        :returns:list.
        :raise c8.exceptions.GetAPIKeys: If request fails
        """
        request = Request(
            method="get",
            endpoint="/key",
        )

        # create response handler

        def response_handler(resp):
            if not resp.is_success:
                raise GetAPIKeys(resp, request)
            else:
                return resp.body["result"]

        return self._execute(request, response_handler, custom_prefix="/_api")

    ##############################
    # Search, View and Analyzers #
    ##############################
    def search(self):
        """Returns the Search APIWrapper
        :returns: Search API Wrapper
        :rtype: c8.search.Search
        """
        return Search(self._conn, self._executor)

class StandardFabric(Fabric):
    """Standard fabric API wrapper.

    :param connection: HTTP connection.
    :type connection: c8.connection.Connection
    """

    def __init__(self, connection):
        super(StandardFabric, self).__init__(
            connection=connection, executor=DefaultExecutor(connection)
        )

    def __repr__(self):
        return "<StandardFabric {}>".format(self.name)

    def begin_async_execution(self, return_result=True):
        """Begin async execution.

        :param return_result: If set to True, API executions return instances
            of :class:`c8.job.AsyncJob`, which you can use to retrieve
            results from server once available. If set to False, API executions
            return None and no results are stored on server.
        :type return_result: bool
        :returns: Fabric API wrapper built specifically for async execution.
        :rtype: c8.fabric.AsyncFabric
        """
        return AsyncFabric(self._conn, return_result)

    def begin_batch_execution(self, return_result=True):
        """Begin batch execution.

        :param return_result: If set to True, API executions return instances
            of :class:`c8.job.BatchJob` that are populated with results on
            commit. If set to False, API executions return None and no results
            are tracked client-side.
        :type return_result: bool
        :returns: Fabric API wrapper built specifically for batch execution.
        :rtype: c8.fabric.BatchFabric
        """
        return BatchFabric(self._conn, return_result)

class AsyncFabric(Fabric):
    """Fabric API wrapper tailored specifically for async execution.

    See :func:`c8.fabric.StandardFabric.begin_async_execution`.

    :param connection: HTTP connection.
    :type connection: c8.connection.Connection
    :param return_result: If set to True, API executions return instances of
        :class:`c8.job.AsyncJob`, which you can use to retrieve results
        from server once available. If set to False, API executions return None
        and no results are stored on server.
    :type return_result: bool
    """

    def __init__(self, connection, return_result):
        super(AsyncFabric, self).__init__(
            connection=connection, executor=AsyncExecutor(connection, return_result)
        )

    def __repr__(self):
        return "<AsyncFabric {}>".format(self.name)

class BatchFabric(Fabric):
    """Fabric API wrapper tailored specifically for batch execution.

    See :func:`c8.fabric.StandardFabric.begin_batch_execution`.

    :param connection: HTTP connection.
    :type connection: c8.connection.Connection
    :param return_result: If set to True, API executions return instances of
        :class:`c8.job.BatchJob` that are populated with results on commit.
        If set to False, API executions return None and no results are tracked
        client-side.
    :type return_result: bool
    """

    def __init__(self, connection, return_result):
        super(BatchFabric, self).__init__(
            connection=connection, executor=BatchExecutor(connection, return_result)
        )

    def __repr__(self):
        return "<BatchFabric {}>".format(self.name)

    def __enter__(self):
        return self

    def __exit__(self, exception, *_):
        if exception is None:
            self._executor.commit()

    def queued_jobs(self):
        """Return the queued batch jobs.

        :returns: Queued batch jobs or None if **return_result** parameter was
            set to False during initialization.
        :rtype: [c8.job.BatchJob] | None
        """
        return self._executor.jobs

    def commit(self):
        """Execute the queued requests in a single batch API request.

        If **return_result** parameter was set to True during initialization,
        :class:`c8.job.BatchJob` instances are populated with results.

        :returns: Batch jobs, or None if **return_result** parameter was set to
            False during initialization.
        :rtype: [c8.job.BatchJob] | None
        :raise c8.exceptions.BatchStateError: If batch state is invalid
            (e.g. batch was already committed or the response size did not
            match expected).
        :raise c8.exceptions.BatchExecuteError: If commit fails.
        """
        return self._executor.commit()


from fastapi import APIRouter
from pydantic import BaseModel
from starlette.requests import Request

from ozz_backend import app_logger
from ozz_backend.persistence_layer import User

router = APIRouter(
    prefix="/user",
    tags=["user"],
    # dependencies=[Depends(get_token_header)],
)

class UserOngoingOut(BaseModel):
    user_id: str
    mission_id: int
    quest_id: int

@router.get('/test')
def test_api():
    app_logger.info('test')
    return {'test'}

@router.get('/user-ongoing', response_model=UserOngoingOut)
def get_ongoing_info(request: Request, user_id: int, mission_id: int):
    app_logger.info(f'[{request.method}] {request.url}: {request.client.host}:{request.client.port}')
    result = User.get_user_ongoing_info(user_id, mission_id)
    ongoing = UserOngoingOut(user_id=result.user_id, mission_id=result.mission_id, quest_id=result.quest_id)
    return ongoing


from django.shortcuts import render
from subscribers.models import subscriberForm
from subscribe.settings import EMAIL_HOST_USER
from django.core.mail import send_mail,BadHeaderError
# Create your views here.

def index(request):
    form=subscriberForm()
    return render(request,"index.html",{"form":form})

def subscribe(request):

    if request.method=="GET":
        form=subscriberForm(request.GET)

        if form.is_valid():
            """ saving data in our database """
            form.save()

            """ using this cleaned data must, to get value of email"""

            senderEmail=form.cleaned_data['Email']

            subject="Welcome to "+senderEmail

            """here your message will be, also you send templates etc"""

            message="Thanking you for subscribing me, you get every updates within a seconds..."

            """ receiver email address"""

            recipient=str(senderEmail)

            if subject and message and recipient :
                try:

                    """ send email is function in django to use this to send mail"""
                    """ if you want send bulk email add more email in [recipients]"""

                    send_mail(subject,message,EMAIL_HOST_USER,[recipient],fail_silently=False)
                except BadHeaderError :

                    """ bad header error means prevent from header injection apply by hackers"""

                    return render(request,"subscribe.html",{'message':'Invalid header found.','class':'text-danger'})

                return render(request,"subscribe.html",{'message':'Thanking you For Subscribing..','class':'text-success'})

    return render(request,"subscribe.html",{"message":"Make sure all fields are entered and valid.",'class':'text-info'})


import json
import logging
import os
import random
import time
from datetime import datetime
from uuid import uuid4

import paho.mqtt.client as mqtt

# MQTT broker details
BROKER_ADDRESS = os.getenv("BROKER_HOST")
BROKER_PORT = 1883

# Configuring file handler for logging
log_file = f"{__file__}.log"
# Logging setup
logging.basicConfig(
    filename=log_file,
    filemode="w",
    format="%(asctime)s - %(levelname)s - %(message)s",
    level=logging.INFO,
)
logger = logging.getLogger(__name__)

# Creating unique sensor IDs for each sensor
temp_sensor_id = str(uuid4())
hum_sensor_id = str(uuid4())

# Simulated sensor data generation for temperature
def generate_temperature_data() -> dict:
    """
    Generate random temperature data.

    Returns:
        dict: Generated sensor data.
    """
    temperature = round(20 + (30 * random.random()), 2)
    timestamp = datetime.utcnow().isoformat()  # ISO8601 format

    data = {
        "sensor_id": temp_sensor_id,
        "topic": "temperature",
        "value": temperature,
        "timestamp": timestamp,
    }
    return data

def generate_humidity_data() -> dict:
    """
    Generate random humidity data.

    Returns:
        dict: Generated sensor data.
    """
    humidity = round(40 + (60 * random.random()), 2)
    timestamp = datetime.utcnow().isoformat()

    data = {
        "sensor_id": hum_sensor_id,
        "topic": "humidity",
        "value": humidity,
        "timestamp": timestamp,
    }
    return data

def on_publish(client, userdata, mid):
    """
    MQTT on_publish callback function.

    Args:
        client: The MQTT client instance.
        userdata: User data.
        mid: Message ID.
    """
    logger.info(f"Message Published: {mid}")

def on_connect(client, userdata, flags, rc):
    """
    MQTT on_connect callback function.

    Args:
        client: The MQTT client instance.
        userdata: User data.
        flags: Flags.
        rc: Return code.
    """
    if rc == 0:
        logger.info("Connected to Mosquitto MQTT Broker!")
    else:
        logger.error(f"Failed to connect, return code: {rc}")

# Create MQTT client instance
client = mqtt.Client()
client.on_connect = on_connect
client.on_publish = on_publish

# Connect to broker
client.connect(BROKER_ADDRESS, port=BROKER_PORT)

# Start the MQTT loop
client.loop_start()

try:
    while True:
        sensor_data_temp = generate_temperature_data()
        sensor_data_hum = generate_humidity_data()
        temperature_payload = json.dumps(sensor_data_temp)
        humidity_payload = json.dumps(sensor_data_hum)

        # Publishing the topics
        client.publish("sensors/temperature", temperature_payload)
        client.publish("sensors/humidity", humidity_payload)

        time.sleep(15)  # Publish every 5 seconds
except KeyboardInterrupt:
    logger.info("Publisher stopped.")
    client.loop_stop()
    client.disconnect()


from atelier_4_ex1 import gen_list_random_int

import matplotlib.pyplot as plt
import numpy as np
import time ,random

def extract_elements_list(list_in_which_to_choose,int_nbr_of_element_to_extract=10):
     list_in_which_to_choose_length,mix_length  = len(list_in_which_to_choose),0
     mixList = list()
     while mix_length < int_nbr_of_element_to_extract :
        random_ = gen_list_random_int(0,list_in_which_to_choose_length)
        if random_ not in mixList :
            mixList.append(random_)
            mix_length += 1
        else :
            continue

     return [ list_in_which_to_choose[elem] for elem in mixList ]

# Test de votre code

# def extract_elements_list2(list_in_which_to_choose,int_nbr_of_element_to_extract=10):
#      list_in_which_to_choose_length,mix_length  = len(list_in_which_to_choose),0
#      mixList = list()
#      while mix_length < int_nbr_of_element_to_extract :
#         random_ = gen_list_random_int(0,list_in_which_to_choose_length)
#         mixList.append(random_)
#         mix_length += 1

#      return [ list_in_which_to_choose[elem] for elem in mixList ]

# print(extract_elements_list( [ i for i in range(1,11)],4))

def pref_mix(func1,func2,lst,num=100):

   result = ([],[])

   for elem in lst :
    data1 ,data2= [],[]
    nb_elements = int(elem / 2)
    for index in range(num) :
      lst_elem = list(range(elem))
      # first function
      start = time.perf_counter()
      func1(lst_elem,nb_elements)
      end = time.perf_counter() - start
      data1.append(end)

      start = time.perf_counter()
      func2(lst_elem,nb_elements)
      end = time.perf_counter() - start
      data2.append(end)
    result[0].append(sum(data1)/len(data1))
    result[1].append(sum(data2)/len(data2))
   return result

list_test = [500,1000,2500,5000,7500]

result = pref_mix(extract_elements_list,random.sample, list_test ,100)

print(result)

#Ici on décrit les abscisses
#Entre 0 et 5 en 10 points
fig, ax = plt.subplots()
#Dessin des courbes, le premier paramètre
#correspond aux point d'abscisse le
#deuxième correspond aux points d'ordonnées
#le troisième paramètre, optionnel permet de
#choisir éventuellement la couleur et le marqueur

ax.plot(list_test,result[0], 'bo-',label='extract_elements_list')
ax.plot(list_test,result[1], 'r*-',label='random.sample')

ax.set(xlabel='temps', ylabel='nombre d\'elements',
 title='temps d’exécution moyen pour extract_elements_list et random.sample')
ax.legend(loc='upper center', shadow=True, fontsize='x-large')
#fig.savefig("test.png")
plt.show()


import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import datasets
from sklearn import linear_model
import matplotlib.pyplot as plt
def sigmoid(z):
    return 1/(1+np.exp(-z))
def costfunction(X, y, w):
    cost = 0
    size = y.shape[0]
    for i in range(size):
        if y[i] == 1:
            cost -= np.log(sigmoid(X[i]*w))
        else:
            cost -= np.log(1 - sigmoid(X[i]*w))
    return cost / size
def gradAscent(traindata,label,iter,alpha,step,lamda=0.001):
    dataMat=np.mat(traindata)
    labelMat=np.mat(label)
    m,n=np.shape(dataMat)
    weights=np.ones((n,1))
    weights=np.mat(weights)
    for k in range(iter):
        temp=costfunction(dataMat,labelMat,weights)
        weights=weights-alpha*((dataMat.transpose())*(sigmoid(dataMat*weights)-labelMat)+lamda*weights)
        if k%200==0:
            print("Loss is: ",temp,weights.transpose())
        if (k/step==0 and k!=0):
            alpha=alpha/5
    return weights
def preprocessing(x_train,x_test):
    sc=StandardScaler()
    sc.fit(x_train)
    x_train_scaled=sc.transform(x_train)
    x_test_scaled=sc.transform(x_test)
    return x_train_scaled,x_test_scaled
def split(ratio):
    Data = datasets.load_iris()
    #Data = datasets.load_wine()  #for Dataset wine
    x = Data.data
    y=Data.target
    x_train, x_test, y_train, y_test = train_test_split(x,y,test_size = ratio, random_state = 0)
    return x_train,x_test,y_train,y_test
def plot(X,Y):
    x_min, x_max = X[:, 0].min() - .2, X[:, 0].max() + .2
    y_min, y_max = X[:, 1].min() - .2, X[:, 1].max() + .2
    h = .02
    logreg =linear_model.LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial')
    logreg.fit(X,Y)
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    Z = logreg.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)
    plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors='k', cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.show()
if __name__=='__main__':
    x_train,x_test,y_train,y_test=split(0.3)
    x_train_scaled,x_test_scaled=preprocessing(x_train,x_test)
    #logreg=linear_model.LogisticRegression(C=1e4) #for ovr
    logreg=linear_model.LogisticRegression(C=1e4,multi_class='multinomial',solver='lbfgs') #ovm
    logreg.fit(x_train_scaled,y_train)
    print("Accuracy:",logreg.score(x_test_scaled,y_test))
    plot(x_train_scaled[:,:2],y_train)


# Author: Sirui Feng
'''
This file splits each review on periods and conjuctions.
'''

import re
import json
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
import csv
from word_stemmer import word_stemmer

public_utilities_path = 'data/public_utilities.json'

def split_period(review):
	'''
	Splits sentences on periods.
	'''
	p = re.compile(r'[^\s\.][^\.\n]+')
	sentences = p.findall(review)
	return sentences

def split_conjunctions(sentence):
	'''
	Splits each sentence on conjuctions.
	'''
	conjuctions = [';', 'for', 'and', 'nor', 'but', 'or', 'yet', 'so']
	clause = re.split('; | and | nor | but | or | yet | so | although | despite | though | however | on the other hand | in contrast ', sentence)
	clause = [x.strip() for x in clause]
	clause = [x for x in clause if len(x) != 0]
	return clause

def gen_sentences():
	'''
	Reads in the sentences and splits on periods and conjuctions.
	'''
	with open(public_utilities_path) as datafile:
		with open('data/full_data.csv', 'w') as outfile:
			writer = csv.DictWriter(outfile, fieldnames = ['review_id', \
				'business_id', 'user_id', 'stars', 'blob_polarity', 'review', \
				'label'])
			writer.writeheader()
			i=0

			for line in datafile:
				i += 1
				print(i)
				row = json.loads(line)
				review = row['text']
				review = review.lower()

				#split only on periods
				sentences = split_period(review)
				for s in sentences:
					blob = TextBlob(s, analyzer = NaiveBayesAnalyzer())
					polarity = blob.polarity
					#s = word_stemmer(s)

					writer.writerow({'review_id':row['review_id'], \
						'business_id': row['business_id'], \
						'user_id':row['user_id'], 'stars':row['stars'], \
						'blob_polarity': polarity, 'review': s})

gen_sentences()


import argparse
import json
EXAMPLE_USAGE = """
Example Usage via RLlib CLI:
    rllib rollout /tmp/ray/checkpoint_dir/checkpoint-0 --run DQN
    --env CartPole-v0 --steps 1000000 --out rollouts.pkl

Example Usage via executable:
    ./rollout.py /tmp/ray/checkpoint_dir/checkpoint-0 --run DQN
    --env CartPole-v0 --steps 1000000 --out rollouts.pkl
"""

def create_parser(parser_creator = None):
    #parser = argparse.ArgumentParser("Ray training with custom IG environment")

    ## parser for rollouts
    parser_creator = parser_creator or argparse.ArgumentParser
    parser = parser_creator(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Roll out a reinforcement learning agent "
                    "given a checkpoint.",
        epilog=EXAMPLE_USAGE)

    parser.add_argument(
        "--checkpoint", default='' ,type=str, help="Checkpoint from which to roll out.")
    required_named = parser.add_argument_group("required named arguments")
    required_named.add_argument(
        "--run",
        type=str,
        required=True,
        help="The algorithm or model to train. This may refer to the name "
             "of a built-on algorithm (e.g. RLLib's DQN or PPO), or a "
             "user-defined trainable function or class registered in the "
             "tune registry.")
    required_named.add_argument(
        "--env", type=str, help="The gym environment to use.")
    parser.add_argument(
        "--no-render",
        default=False,
        action="store_const",
        const=True,
        help="Suppress rendering of the environment.")
    parser.add_argument(
        "--monitor",
        default=False,
        action="store_true",
        help="Wrap environment in gym Monitor to record video. NOTE: This "
             "option is deprecated: Use `--video-dir [some dir]` instead.")
    parser.add_argument(
        "--video-dir",
        type=str,
        default=None,
        help="Specifies the directory into which videos of all episode "
             "rollouts will be stored.")
    parser.add_argument(
        "--steps",
        default=20000,
        help="Number of timesteps to roll out (overwritten by --episodes).")
    parser.add_argument(
        "--episodes",
        default=0,
        help="Number of complete episodes to roll out (overrides --steps).")
    parser.add_argument("--out", default=None, help="Output filename.")
    parser.add_argument(
        "--config",
        default="{}",
        type=json.loads,
        help="Algorithm-specific configuration (e.g. env, hyperparams). "
             "Gets merged with loaded configuration from checkpoint file and "
             "`evaluation_config` settings therein.")
    parser.add_argument(
        "--save-info",
        default=False,
        action="store_true",
        help="Save the info field generated by the step() method, "
             "as well as the action, observations, rewards and done fields.")
    parser.add_argument(
        "--use-shelve",
        default=False,
        action="store_true",
        help="Save rollouts into a python shelf file (will save each episode "
             "as it is generated). An output filename must be set using --out.")
    parser.add_argument(
        "--track-progress",
        default=False,
        action="store_true",
        help="Write progress to a temporary file (updated "
             "after each episode). An output filename must be set using --out; "
             "the progress file will live in the same folder.")

    # save and restore file management
    parser.add_argument(
        "--policy-dir", type=str, help="folder name of the policy.", default="")
    parser.add_argument(
        "--experiment", type=str, help="chosen experiment to reload.", default="")
    parser.add_argument(
        "--ncheckpoint", type=str, help="chosen checkpoint to reload.", default="")

    parser.add_argument(
        "--heuristic-policy", type=bool, help="chosen checkpoint to reload.", default=False)

    parser.add_argument(
        "--static-targets", type=bool, help="chosen checkpoint to reload.", default=False)

    parser.add_argument(
        "--video_dir", type=str, help="chosen folder to save video.", default="")

    parser.add_argument(
        "--horizon", type=int, help="limit of timesteps.", default=40)

    ### Old arguments needs a cleanup

    parser.add_argument("--scenario", type=str, default="simple_spread_assigned",
                        choices=['simple', 'simple_speaker_listener',
                                 'simple_crypto', 'simple_push',
                                 'simple_tag', 'simple_spread', 'simple_adversary', 'simple_spread_assigned',
                                 'matlab_simple_spread_assigned','matlab_simple_spread_assigned_hardcoll', 'matlab_simple_spread_assigned_checkpoints'],
                        help="name of the scenario script")
    parser.add_argument("--max-episode-len", type=int, default=100,
                        help="maximum episode length")
    parser.add_argument("--num-episodes", type=int, default=60000,
                        help="number of episodes")
    parser.add_argument("--num-adversaries", type=int, default=0,
                        help="number of adversaries")
    parser.add_argument("--good-policy", type=str, default="maddpg",
                        help="policy for good agents")
    parser.add_argument("--adv-policy", type=str, default="maddpg",
                        help="policy of adversaries")

    # Core training parameters
    parser.add_argument("--lr", type=float, default=1e-3,
                        help="learning rate for Adam optimizer")
    parser.add_argument("--gamma", type=float, default=0.99,
                        help="discount factor")
    # NOTE: 1 iteration = sample_batch_size * num_workers timesteps * num_envs_per_worker
    parser.add_argument("--sample-batch-size", type=int, default=25,
                        help="number of data points sampled /update /worker")
    parser.add_argument("--train-batch-size", type=int, default=1024,
                        help="number of data points /update")
    parser.add_argument("--n-step", type=int, default=1,
                        help="length of multistep value backup")
    parser.add_argument("--num-units", type=int, default=128,
                        help="number of units in the mlp")
    parser.add_argument("--replay-buffer", type=int, default=1000000,
                        help="size of replay buffer in training")
    parser.add_argument("--seed", type=int, default=100,
                        help="initialization seed for the network weights")

    # Checkpoint
    parser.add_argument("--checkpoint-freq", type=int, default = 10, #75,
                        help="save model once every time this many iterations are completed")
    parser.add_argument("--local-dir", type=str, default="./ray_results",
                        help="path to save checkpoints")
    parser.add_argument("--restore", type=str, default=None,
                        help="directory in which training state and model are loaded")
    parser.add_argument("--in-evaluation", type=bool, default=False, help="trigger evaluation procedure")

    # Parallelism
    #parser.add_argument("--num-workers", type=int, default=0)
    #parser.add_argument("--num-envs-per-worker", type=int, default=1)
    #parser.add_argument("--num-gpus", type=int, default=0)

    parser.add_argument("--num-workers", type=int, default=0)  #0
    parser.add_argument("--num-envs-per-worker", type=int, default=1)  #1
    parser.add_argument("--num-gpus", type=int, default=0)  #0
    #parser.add_argument("--num-cpus-per-worker", type=int, default=1)
    parser.add_argument("--num-gpus-per-worker", type=int, default=0)  #0

    # From the ppo
    parser.add_argument("--stop-iters", type=int, default=100)
    parser.add_argument("--stop-timesteps", type=int, default=160000000)
    # parser.add_argument("--stop-reward", type=float, default=7.99)

    # For rollouts
    parser.add_argument("--stop-iters-rollout", type=int, default=1)
    parser.add_argument("--nagents", type=int, default=1)
    parser.add_argument("--ntargets", type=int, default=1)
    parser.add_argument("--nrobots", type=int, default=1)

    # mode of hand-engineered comm. policy (-1 no hand-engineered)
    parser.add_argument("--mode", type=int, default=-1)
    parser.add_argument("--test", type=int, default=0, choices = [0,1], help="whether we want to test the policy or not")
    parser.add_argument("--test-env", type=int, default=0, choices = [0,1], help="whether we want to act in the test environment or not")
    parser.add_argument("--deterministic", type=int, default=1, choices=[0, 1],
                        help="enable exploration or not during execution")

    return parser