from django.contrib import admin
from .models import TravelRequest, TrTravelRequest, AgentTravelRequest, TravelNotes, Attachment, Others
from contacts.models import Contact, IndividualRole, Zone, Center
from django.core.exceptions import ObjectDoesNotExist
from utils.filters import RelatedDropdownFilter, ChoiceDropdownFilter
from django.shortcuts import render
from django.utils import formats
from datetime import date, timedelta
from ipcaccounts.models import RCOAccountsMaster, VoucherDetails, AccountTypeMaster,EntityMaster,RCOVoucherStatusMaster,ExpensesTypeMaster, VoucherMaster
from django.contrib import messages
import json
from config.models import Configuration, SmartModel
from django.http import HttpResponse, HttpResponseNotFound, Http404,  HttpResponseRedirect
from AasaanUser.models import AasaanUserContact, AasaanUserZone
from django.contrib.auth.models import User
from django.utils import timezone
from .forms import TravelForm, TravelOtherDetailsInlineFormset

class OthersInline(admin.TabularInline):
    formset = TravelOtherDetailsInlineFormset
    model = Others
    extra = 0
    max_num = 10

class AttachmentInline(admin.StackedInline):
    model = Attachment
    extra = 1
    max_num = 5

class TravelNotesInline(admin.StackedInline):
    model = TravelNotes
    extra = 0
    fields = ['note', ]

    def has_change_permission(self, request, obj=None):
        return False

class TravelNotesInline1(admin.StackedInline):

    model = TravelNotes
    extra = 0
    readonly_fields = ['note',]

    fieldsets = [
        ('', {'fields': ('note',), }),
        ('Hidden Fields',
         {'fields': ['created_by',], 'classes': ['hidden']}),
    ]

    def has_add_permission(self, request):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

class BaseTravelAdmin(admin.ModelAdmin):

    date_hierarchy = 'onward_date'
    inlines = [OthersInline, TravelNotesInline1, TravelNotesInline, AttachmentInline]

    def save_related(self, request, form, formsets, change):
        for formset in formsets:
            for fs in formset:
                if isinstance(fs.instance, TravelNotes) and fs.cleaned_data:
                    if fs.instance.pk is None:
                        fs.instance.travel_request = form.instance
                        fs.instance.created_by = request.user.username
                        fs.instance.note = fs.instance.note + ' - created_by : ' + request.user.username + ' created at : ' + timezone.now().strftime(
                            "%b %d %Y %H:%M:%S")
                        fs.instance.save()
        super(BaseTravelAdmin, self).save_related(request, form, formsets, change)

    class Media:
        js = ('/static/aasaan/travels/travels.js',)

class TravelRequestAdmin(BaseTravelAdmin):
    form = TravelForm
    list_display = ('status_flag', 'ticket_number', '__str__', 'source', 'destination', 'onward_date', 'zone', 'status', 'invoice_no', 'created_by')
    list_editable = ('status',)
    list_display_links = ['status_flag', '__str__', 'ticket_number']
    list_filter = ('created',('travel_mode', ChoiceDropdownFilter), ('status', ChoiceDropdownFilter), ('zone', RelatedDropdownFilter), )
    search_fields = ('source', 'destination', 'teacher__first_name', 'teacher__last_name', 'created_by__first_name', 'invoice_no', 'ticket_number')
    fieldsets = (
        ('', {
            'fields': ('ticket_number', 'source', 'destination', 'onward_date', 'travel_mode', 'travel_class', 'zone', 'teacher', 'is_others')

        }),
        ('Booking details', {
            'fields': ('status', 'booked_date', 'invoice_no', 'amount', 'attachments', 'invoice', 'refund_amount'),
            'classes': ('collapse', 'open')
        }),
    )
    filter_horizontal = ('teacher',)

    def create_voucher(self, request, queryset):
        accounts_voucher = RCOAccountsMaster()
        accounts_voucher.account_type = AccountTypeMaster.objects.get(id=3)
        amount = 0
        for tr in list(queryset):
            if tr.status=='VC' or tr.status=='CL' or tr.status=='PD' or tr.status=='IP':
                self.message_user(request, "Voucher already created or processed or Inprogress. Update booking details to create vouchers", level=messages.WARNING)
                return
            accounts_voucher.zone = tr.zone
            amount += tr.amount
            cft = Configuration.objects.get(configuration_key='IPC_ACCOUNTS_TRACKING_CONST')
            data = json.loads(cft.configuration_value)
            prefix = data[accounts_voucher.zone.zone_name]['prefix']
            accounts_voucher.budget_code = 'Teachers budget '+prefix
            accounts_voucher.save()
            bank_voucher = VoucherDetails()
            bank_voucher.voucher_type = 'BV'
            bank_voucher.nature_of_voucher = VoucherMaster.objects.get(id=4)
            bank_voucher.head_of_expenses = ExpensesTypeMaster.objects.get(id=5)
            bank_voucher.expenses_description = 'Booked Teachers Tickets'
            bank_voucher.party_name = 'Kathir Travel Line'
            bank_voucher.amount = amount
            bank_voucher.accounts_master = accounts_voucher
            bank_voucher.save()

            journal_voucher= VoucherDetails()
            journal_voucher.voucher_type = 'JV'
            journal_voucher.nature_of_voucher = VoucherMaster.objects.get(id=3)
            journal_voucher.head_of_expenses = ExpensesTypeMaster.objects.get(id=5)
            journal_voucher.expenses_description = 'Booked tickets for Teachers'
            journal_voucher.party_name = 'Kathir Travel Line'
            journal_voucher.amount = amount
            journal_voucher.accounts_master = accounts_voucher
            journal_voucher.save()

            for tr in list(queryset):
                tr.status = 'VC'
                tr.voucher = accounts_voucher
                tr.save()

        return HttpResponseRedirect('/admin/ipcaccounts/rcoaccountsmaster/'+str(accounts_voucher.id)+'/change/')

    create_voucher.short_description = "Create Vouchers"

    def email_ticket_passanger(self, request, queryset):
        pass

    email_ticket_passanger.short_description = "Email ticket to Teacher"

    def make_email(self, request, queryset):
        pass

    make_email.short_description = "Send Email to Travel Agency"

    def view_passanger_details(self, request, queryset):
        all_tickets = []

        for tr in list(queryset):
            ticket_request = {}
            ticket_request['tracking_no'] = 'TR' + str(tr.id).zfill(6)
            ticket_request['zone'] = tr.zone.zone_name
            ticket_request['source'] = tr.source
            ticket_request['destination'] = tr.destination
            onward_date = formats.date_format(tr.onward_date, "DATE_FORMAT")
            ticket_request['onward_date'] = onward_date
            ticket_request['travel_mode'] = tr.get_travel_mode_display()
            ticket_request['remarks'] = tr.remarks
            traveller_details = list(tr.teacher.all())
            t_list = []
            for index, t in enumerate(traveller_details):
                age = 'Age not known'
                if t.date_of_birth:
                    age = (date.today() - t.date_of_birth) // timedelta(days=365.2425)
                t_dict = {}
                t_dict['SNo'] = str(index + 1)
                t_dict['full_name'] =t.full_name
                t_dict['age'] = t.get_gender_display() + ' - ' + str(age)
                t_dict['primary_mobile'] =t.primary_mobile
                t_list.append(t_dict)
            ticket_request['travellers'] = t_list
            all_tickets.append(ticket_request)

        return render(request,
                      'travels/view_details.html',
                      context={'travel_request':all_tickets})

    view_passanger_details.short_description = "View Passanger Details"

    actions = [make_email, view_passanger_details, create_voucher, email_ticket_passanger]

    def formfield_for_foreignkey(self, db_field, request=None, **kwargs):

        if not request.user.is_superuser and db_field.name == 'zone':
            user_zones = [x.zone.id for x in request.user.aasaanuserzone_set.all()]
            kwargs["queryset"] = Zone.objects.filter(pk__in=user_zones)
        return super(TravelRequestAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

    def formfield_for_manytomany(self, db_field, request=None, **kwargs):
        if db_field.name == 'teacher':

            try:
                teacher_role = IndividualRole.objects.get(role_name='Teacher', role_level='ZO')
                _teacher_list = Contact.objects.filter(individualcontactrolezone__role=teacher_role)
                kwargs["queryset"] = _teacher_list.distinct()
            except ObjectDoesNotExist:
                kwargs["queryset"] = Contact.objects.none()
            return super(TravelRequestAdmin, self).formfield_for_manytomany(db_field, request, **kwargs)

    # filter tickets records based on user permissions
    def get_queryset(self, request):
        qs = super(TravelRequestAdmin, self).get_queryset(request)
        if request.user.is_superuser:
            return qs
        if 'Teacher' in [x.name for x in request.user.groups.all()]:
            return TravelRequest.objects.filter(created_by=request.user)
        else:
            user_zones = [x.zone for x in request.user.aasaanuserzone_set.all()]
            return TravelRequest.objects.filter(zone__in=user_zones)

    def save_model(self, request, obj, form, change):
        if not obj.id:
            obj.created_by = request.user
        obj.save()

    def get_actions(self, request):
        actions = super(TravelRequestAdmin, self).get_actions(request)
        if request.user.is_superuser:
            return actions
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions

class TeachersTravelRequestAdmin(BaseTravelAdmin):
    list_editable = []
    list_filter = []
    list_display = ('status_flag', 'ticket_number', '__str__', 'source', 'destination', 'onward_date', 'status')
    search_fields = []
    list_display_links = ['status_flag', '__str__', 'ticket_number']

    def get_actions(self, request):
        actions = super(TeachersTravelRequestAdmin, self).get_actions(request)
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions

    fieldsets = (
        ('', {
            'fields': ('ticket_number', 'source', 'destination','onward_date', 'travel_mode', 'travel_class', 'zone')
        }),
    )

    def get_queryset(self, request):
        return TravelRequest.objects.filter(created_by=request.user, status__in=['IP', 'BO','BK', 'CL', 'CB'])

    def save_model(self, request, obj, form, change):
        login_user = User.objects.get(username=request.user.username)
        contact = AasaanUserContact.objects.get(user=login_user)
        obj.created_by = request.user
        obj.save()
        obj.teacher.add(contact.contact)

    def get_form(self, request, obj=None, **kwargs):
        login_user = User.objects.get(username=request.user.username)
        try:
            AasaanUserContact.objects.get(user=login_user)
        except AasaanUserContact.DoesNotExist:
            self.message_user(request,
                              "Contact is not mapped to User. Contact aasaan admin",
                              level=messages.WARNING)
        try:
            AasaanUserZone.objects.get(user=login_user)
        except AasaanUserZone.DoesNotExist:
            self.message_user(request,
                              "Zone is not mapped to User. Contact aasaan admin",
                              level=messages.WARNING)

        return super(TeachersTravelRequestAdmin, self).get_form(request, obj=None, **kwargs)

class AgentTravelRequestAdmin(BaseTravelAdmin):
    date_hierarchy = 'created'
    list_display = ('status_flag', 'ticket_number', '__str__', 'source', 'destination', 'onward_date', 'zone', 'status', 'created_by')
    list_editable = ('status',)
    list_display_links = ['status_flag', '__str__', 'ticket_number']
    list_filter = ('onward_date', ('travel_mode', ChoiceDropdownFilter), ('status', ChoiceDropdownFilter),
                   ('zone', RelatedDropdownFilter),)
    search_fields = ('source', 'destination', 'teacher__first_name', 'teacher__last_name', 'created_by__first_name','invoice_no', 'ticket_number')
    fieldsets = (
        ('', {
            'fields': ('ticket_number','source', 'destination', 'onward_date', 'travel_mode', 'travel_class', 'zone', 'remarks')

        }),
        ('Passanger details', {
            'fields': ('teacher', )
        }),
        ('Booking details', {
            'fields': ('status', 'booked_date', 'invoice_no', 'amount', 'attachments', 'invoice', 'refund_amount'),
            'classes': ('collapse', 'open')
        }),
    )
    readonly_fields = ['source', 'destination', 'teacher', 'onward_date', 'travel_mode', 'travel_class', 'zone', 'remarks']

    def get_queryset(self, request):
       return TravelRequest.objects.filter(status__in=['BO', 'BK', 'CL', 'CB'])

    def get_actions(self, request):
        actions = super(AgentTravelRequestAdmin, self).get_actions(request)
        if 'delete_selected' in actions:
            del actions['delete_selected']
        return actions

    def has_delete_permission(self, request, obj=None):
        return False

admin.site.register(TravelRequest, TravelRequestAdmin)
admin.site.register(TrTravelRequest, TeachersTravelRequestAdmin)
admin.site.register(AgentTravelRequest, AgentTravelRequestAdmin)


"""Prediction functions
"""

import io
import logging
from time import sleep
from urllib.parse import urlparse

import boto3 as boto
import httpx
from typing import List

from sync.clients.sync import get_default_client
from sync.models import Platform, PredictionError, Response

logger = logging.getLogger(__name__)

def get_products() -> Response[List[str]]:
    """Get supported platforms

    :return: list of platform names
    :rtype: Response[list[str]]
    """
    response = get_default_client().get_products()
    return Response(**response)

def generate_prediction(
    platform: Platform, cluster_report: dict, eventlog_url: str, preference: str = None
) -> Response[dict]:
    """Create and return prediction

    :param platform: e.g. "aws-emr"
    :type platform: Platform
    :param cluster_report: cluster report
    :type cluster_report: dict
    :param eventlog_url: Apache Spark event log URL
    :type eventlog_url: str
    :param preference: prediction preference, defaults to None
    :type preference: str, optional
    :return: prediction object
    :rtype: Response[dict]
    """
    response = create_prediction(platform, cluster_report, eventlog_url)

    prediction_id = response.result
    if prediction_id:
        return wait_for_prediction(prediction_id, preference)

    return response

def wait_for_prediction(prediction_id: str, preference: str = None) -> Response[dict]:
    """Get a prediction, wait if it's not ready

    :param prediction_id: prediction ID
    :type prediction_id: str
    :param preference: prediction preference, defaults to None
    :type preference: str, optional
    :return: prediction object
    :rtype: Response[dict]
    """
    response = wait_for_final_prediction_status(prediction_id)
    result = response.result
    if result:
        if result == "SUCCESS":
            return get_prediction(prediction_id, preference)

        return Response(error=PredictionError(message="Prediction failed"))

    return response

def get_prediction(prediction_id: str, preference: str = None) -> Response[dict]:
    """Get a prediction, don't wait

    :param prediction_id: prediction ID
    :type prediction_id: str
    :param preference: prediction preference, defaults to None
    :type preference: str, optional
    :return: prediction object
    :rtype: Response[dict]
    """
    response = get_default_client().get_prediction(
        prediction_id, {"preference": preference} if preference else None
    )

    result = response.get("result")
    if result:
        return Response(result=result)

    return Response(**response)

def get_status(prediction_id: str) -> Response[str]:
    """Get prediction status

    :param prediction_id: prediction ID
    :type prediction_id: str
    :return: prediction status, e.g. "SUCCESS"
    :rtype: Response[str]
    """
    response = get_default_client().get_prediction_status(prediction_id)

    result = response.get("result")
    if result:
        return Response(result=result["status"])

    return Response(**response)

def get_predictions(
    product: str = None, project_id: str = None, preference: str = None
) -> Response[List[dict]]:
    """Get predictions

    :param product: platform to filter by, e.g. "aws-emr", defaults to None
    :type product: str, optional
    :param project_id: project to filter by, defaults to None
    :type project_id: str, optional
    :param preference: prediction preference, defaults to None
    :type preference: str, optional
    :return: list of prediction objects
    :rtype: Response[list[dict]]
    """
    params = {}
    if product:
        params["products"] = [product]
    if project_id:
        params["project_id"] = project_id
    if preference:
        params["preference"] = preference

    response = get_default_client().get_predictions(params)

    if response.get("result") is not None:
        return Response(result=response["result"])

    return Response(**response)

def wait_for_final_prediction_status(prediction_id: str) -> Response[str]:
    """Wait for and return terminal prediction status

    :param prediction_id: prediction ID
    :type prediction_id: str
    :return: prediction status, e.g. "SUCCESS"
    :rtype: Response[str]
    """
    response = get_default_client().get_prediction_status(prediction_id)
    while response:
        result = response.get("result")
        if result:
            if result["status"] in ("SUCCESS", "FAILURE"):
                return Response(result=result["status"])
        else:
            return Response(**response)

        logger.info("Waiting for prediction")
        sleep(10)

        response = get_default_client().get_prediction_status(prediction_id)

    return Response(error=PredictionError(message="Failed to get prediction status"))

def create_prediction_with_eventlog_bytes(
    platform: Platform,
    cluster_report: dict,
    eventlog_name: str,
    eventlog_bytes: bytes,
    project_id: str = None,
) -> Response[str]:
    """Creates a prediction giving event log bytes instead of a URL

    :param platform: platform, e.g. "aws-emr"
    :type platform: Platform
    :param cluster_report: cluster report
    :type cluster_report: dict
    :param eventlog_name: name of event log (extension is important)
    :type eventlog_name: str
    :param eventlog_bytes: encoded event log
    :type eventlog_bytes: bytes
    :param project_id: ID of project to which the prediction belongs, defaults to None
    :type project_id: str, optional
    :return: prediction ID
    :rtype: Response[str]
    """
    response = get_default_client().create_prediction(
        {
            "project_id": project_id,
            "product_code": platform,
            "configs": cluster_report,
        }
    )

    if response.get("error"):
        return Response(**response)

    upload_details = response["result"]["upload_details"]
    log_response = httpx.post(
        upload_details["url"],
        data={
            **upload_details["fields"],
            "key": upload_details["fields"]["key"].replace("${filename}", eventlog_name),
        },
        files={"file": io.BytesIO(eventlog_bytes)},
    )
    if not log_response.status_code == httpx.codes.NO_CONTENT:
        return Response(error=PredictionError(message="Failed to upload event log"))

    return Response(result=response["result"]["prediction_id"])

def create_prediction(
    platform: Platform, cluster_report: dict, eventlog_url: str, project_id: str = None
) -> Response[str]:
    """Create prediction

    :param platform: platform, e.g. "aws-emr"
    :type platform: Platform
    :param cluster_report: cluster report
    :type cluster_report: dict
    :param eventlog_url: event log URL
    :type eventlog_url: str
    :param project_id: ID of project to which the prediction belongs, defaults to None
    :type project_id: str, optional
    :return: prediction ID
    :rtype: Response[str]
    """
    scheme = urlparse(eventlog_url).scheme
    if scheme == "s3":
        response = generate_presigned_url(eventlog_url)
        if response.error:
            return response
        eventlog_http_url = response.result
    elif scheme in {"http", "https"}:
        eventlog_http_url = eventlog_url
    else:
        return Response(error=PredictionError(message="Unsupported event log URL scheme"))

    response = get_default_client().create_prediction(
        {
            "project_id": project_id,
            "product_code": platform,
            "eventlog_url": eventlog_http_url,
            "configs": cluster_report,
        }
    )

    if response.get("error"):
        return Response(**response)

    return Response(result=response["result"]["prediction_id"])

def generate_presigned_url(s3_url: str, expires_in_secs: int = 3600) -> Response[str]:
    """Generates presigned HTTP URL for S3 URL

    :param s3_url: URL of object in S3
    :type s3_url: str
    :param expires_in_secs: number of seconds after which presigned URL expires, defaults to 3600
    :type expires_in_secs: int, optional
    :return: presigned URL
    :rtype: Response[str]
    """
    parsed_s3_url = urlparse(s3_url)

    s3 = boto.client("s3")
    return Response(
        result=s3.generate_presigned_url(
            "get_object",
            Params={"Bucket": parsed_s3_url.netloc, "Key": parsed_s3_url.path.lstrip("/")},
            ExpiresIn=expires_in_secs,
        )
    )


from key_value_pair_cache.client import client
import key_value_pair_cache.helper_functions.log_helper as log_helper
import configparser
import os
import sys
from collections import defaultdict
import marshal
import types
import re
from key_value_pair_cache.helper_functions.hash_fnv1a import fnv1a_32
import random
import urllib.request
import threading
import time
from reducer.word_count_reducer import word_count_reducer
from reducer.inverted_index_reducer import inverted_index_reducer
import subprocess

class reducer:
    config = configparser.ConfigParser()
    config.read('config.ini')
    LOG = log_helper._logger("map-reduce reducer")

    def send_reducer_output(self, reducer_output):
        try:
            self.LOG.log(50, 'Sending reducer out put for ' +
                         str(self.reducer_id)+" to key value store")
            msg_key = 'reducer_output' + str(self.reducer_id)
            for list_item in reducer_output:
                msg_val = str(list_item[0]+" "+str(list_item[1])+"\n")
                self.reducer_client.append_key(msg_key, msg_val)
            self.LOG.log(50, 'output for reducer ' +
                         str(self.reducer_id) + 'stored in key value store')
            return True
        except Exception as e:
            self.LOG.log(30, "Exception while running send_reducer_output")
            print(e, sys.exc_info())

    def run_serialized_reducer(self, input_list):
        try:
            self.LOG.log(
                50, 'Running supplied reducer on worker '+str(self.reducer_id))
            # with open(self.reducer_code_serialized, 'rb') as fd:
            #     code_string = fd.read()
            #     code = marshal.loads(code_string)
            #     user_defined_reduce_function = types.FunctionType(
            #         code, globals(), "user_defined_reducer_function"+str(self.reducer_id))
            if self.reducer_code_serialized == 'reducer/word_count_reducer_serialized':
                reducer_output = word_count_reducer(input_list)
            else:
                reducer_output = inverted_index_reducer(input_list)
            # reducer_output = user_defined_reduce_function(input_list)
            self.LOG.log(50, 'Reducer '+str(self.reducer_id) +
                         " has run reducer function")
            return reducer_output
        except Exception as e:
            self.LOG.log(30, "Exception while running run_serialized_reducer")
            print(e, sys.exc_info())

    def get_reducer_data(self):
        try:
            keys = self.reducer_client.get_keys("reducer"+str(self.reducer_id))
            data = ''
            for key in keys:
                data = self.reducer_client.get_key_lines(key)
                self.LOG.log(50, "reducer "+str(self.reducer_id) +
                             " received data "+str(len(data)))
                lines = data.split('\n')[1:]
                input_list = []
                # split from lines to list of tuples
                for line in lines:
                    words = line.split()
                    if len(words) == 2:
                        input_list.append((words[0], words[1]))
                self.LOG.log(50, "reducer "+str(self.reducer_id) +
                             " collected these keys data :"+str(len(input_list)))
                # runs serial reducer script
                reducer_output = self.run_serialized_reducer(input_list)
                self.send_reducer_output(reducer_output)
            if random.randrange(int(self.number_of_reducers)+1) == int(self.number_of_reducers) and self.test_reducer_fail == "True":
                self.LOG.log(30, "Creating an exception in reducer " +
                             str(self.reducer_id)+" for testing")
                raise Exception
            self.finished_checker = False
            self.reducer_client.set_key(
                'reducer_status'+str(self.reducer_id), 'finished')
            return True
        except Exception as e:
            self.LOG.log(30, "Exception while running get_reducer_data")
            print(e, sys.exc_info())
            self.LOG.log(50, "Reducer "+str(self.reducer_id)+" broke down")
            return False

    def get_reducer_id(self):
        # keys = self.reducer_client.get_keys('reducer_status')
        # for key in keys:
        #     val = self.reducer_client.get_key(key)
        #     if val.split(' \r\n')[1] == 'idle':
        #         id = val.split(' \r\n')[0].split()[1][len('reducer_status'):]
        req = urllib.request.Request(
            'http://metadata.google.internal/computeMetadata/v1/instance/hostname', headers={"Metadata-Flavor": "Google"})
        host_name = urllib.request.urlopen(req).read().decode().split('.')
        # host_name = ['reducer0', 'c', 'rishabh-gajra', 'internal']
        id = host_name[0][len('reducer'):]
        self.reducer_client.set_key(
            'reducer_status'+str(id), 'assigned')
        return int(id)

    def send_heartbeat(self):
        heart_beat_client = client(str(self.key_value_ip),
                                   int(self.config['app_config']['KeyValueServerPort']))
        while self.finished_checker:
            heart_beat_client.set_key(
                'reducer_status'+str(self.reducer_id), 'assigned')
            time.sleep(7)
        return True

    def reducer_clean_files(self):
        self.LOG.log(
            50, 'Cleaning previous files for reducer %d' % (self.reducer_id))
        msg_key = 'reducer_output' + str(self.reducer_id)
        self.reducer_client.clean_file_by_line(msg_key)
        return True

    def get_server_ip(self):
        output = subprocess.run(
            "gcloud compute instances describe key-value-server4 --zone=us-central1-a --format='get(networkInterfaces[0].accessConfigs[0].natIP)'", shell=True, check=True, stdout=subprocess.PIPE)
        ip = output.stdout.decode()[:-1]
        return ip

    def __init__(self, reducer_name, reducer_port):
        self.key_value_ip = str(self.get_server_ip())
        self.finished_checker = True
        self.reducer_port = reducer_port
        self.reducer_name = reducer_name
        self.reducer_client = client(str(self.key_value_ip),
                                     int(self.config['app_config']['KeyValueServerPort']))
        self.number_of_mappers = int(self.reducer_client.get_key(
            'NumberOfMappers').split(' ')[3])
        self.number_of_reducers = int(self.reducer_client.get_key(
            'NumberOfReducers').split(' ')[3])
        self.reducer_code_serialized = self.reducer_client.get_key(
            'ReducerCodeSerialized').split(' ')[3][2:]
        self.test_reducer_fail = self.reducer_client.get_key(
            'TestReducerFail').split(' ')[3][2:]
        self.reducer_id = self.get_reducer_id()
        self.LOG.log(50, "Booting up map-reduce reducer with id " +
                     str(self.reducer_id))
        heartbeat_thread = threading.Thread(
            target=self.send_heartbeat, args=(), daemon=True)
        heartbeat_thread.start()
        self.reducer_clean_files()
        self.LOG.log(50, 'reducer '+str(self.reducer_id)+" intialized")
        return


from pytube import YouTube
from pytube import Playlist
#here we give the path where the playlist video will be saved
path="Y:\CoursesUniversity\BBB"
# here we give the path of the playlist
playlist = Playlist('https://www.youtube.com/watch?v=Te67sBlJJMg&list=PLD4dEalVnw4U2YVu2P79dQ_i_eTQ4N-O8')
print('the Urls of  videos in playlist: %s' % len(playlist.video_urls))

for video in playlist.video_urls:
    youtubeObject = YouTube(video)
    youtubeObject=youtubeObject.streams.get_highest_resolution()
    try:
        youtubeObject.download(path)
    except:
        print("Error found")

print("Download is completed successfully")


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from pymongo import mongo_client

#몽고DB
host = "localhost"
port = 27017
client = mongo_client.MongoClient(host, port)
db = client["soobindb"]
col = db["rate"]
col.drop()

path = "C:/PJS/PyAdvanced/day06_crawling/자료실/chromedriver.exe"
driver = webdriver.Chrome(path)

#페이지 크롤링
url = "https://map.kakao.com/"

import time
driver.get(url)

time.sleep(1)
driver.find_element(By.CSS_SELECTOR, "#dimmedLayer").click()

for x in range(0, 10):
    driver.find_element(By.CSS_SELECTOR, "#view\.map > div.MapControlView > div.zoom_control.fold > div > button:nth-child(3)").click()
    time.sleep(0.5)
driver.find_element(By.CSS_SELECTOR, "#search\.keyword\.bounds").click()
time.sleep(1)

el_search = driver.find_element("name", "q")
el_search.send_keys("캠핑장")

el_search.send_keys(Keys.ENTER)
time.sleep(0.1)
driver.find_element(By.CSS_SELECTOR, "#info\.search\.place\.more").click()
time.sleep(0.1)
driver.find_element(By.CSS_SELECTOR, "#info\.search\.page\.no1").click()
time.sleep(0.1)

html = driver.page_source
soup = BeautifulSoup(html, "html.parser")

li = list()
num = 1
id = 1
for z in range(1,35):
    for x in range(1,17):
        name = soup.select_one("#info\.search\.place\.list > li:nth-child("+str(x)+") > div.head_item.clickArea > strong > a.link_name")
        rate = soup.select_one("#info\.search\.place\.list > li:nth-child("+str(x)+") > div.rating.clickArea > span.score > em")
        location = soup.select_one("#info\.search\.place\.list > li:nth-child("+str(x)+") > div.info_item > div.addr > p:nth-child(1)")
        review = soup.select_one("#info\.search\.place\.list > li:nth-child("+str(x)+") > div.rating.clickArea > a > em")
        try:
            #li.append([name["title"], rate.text, location.text])
            dict = {"_id":id, "캠핑장이름":name["title"], "장소":location.text, "평점":float(rate.text), "리뷰수":int(review.text)}
            col.insert_one(dict)
            print("z", z, "x", x, "num", num)
            id += 1
        except TypeError:
            pass
        except ValueError:
            dict = {"_id":id, "캠핑장이름":name["title"], "장소":location.text, "평점":rate.text, "리뷰수":int(review.text)}
            col.insert_one(dict)
            id += 1
    if z == 34:
        if num < 4:
            driver.find_element(By.CSS_SELECTOR, "#info\.search\.page\.no"+str(num+1)+"").click()
            time.sleep(1)
            html = driver.page_source
            soup = BeautifulSoup(html, "html.parser")
        elif num == 4:
            break
    elif z < 34:
        if num < 5:
            driver.find_element(By.CSS_SELECTOR, "#info\.search\.page\.no"+str(num+1)+"").click()
            time.sleep(1)
            html = driver.page_source
            soup = BeautifulSoup(html, "html.parser")
        elif num == 5:
            driver.find_element(By.CSS_SELECTOR, "#info\.search\.page\.next").click()
            time.sleep(1)
            html = driver.page_source
            soup = BeautifulSoup(html, "html.parser")
            num = 0
        num += 1

print(li)


from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    url(r'^$', "BrasserieSite.views.index", name='index'),
    url(r'^NosBiere/', "BrasserieSite.views.NosBiere", name='NosBiere'),
    url(r'^Commande/', "BrasserieSite.views.Commande", name='Commande'),
    url(r'^contact/', "BrasserieSite.views.contact", name='contact'),
    url(r'^accounts/profile/', "BrasserieSite.views.profile", name='profile'),
    url(r'^accounts/logout/$', 'django.contrib.auth.views.logout', {'next_page':'/'}),
    url(r'^BARBANE_DEMENTHE/$', "BrasserieSite.views.BARBANE_DEMENTHE", name='BARBANE_DEMENTHE'),
    url(r'^BARBANE_FLORALE/$', "BrasserieSite.views.BARBANE_FLORALE", name='BARBANE_FLORALE'),
    url(r'^BARBANE_BLONDE/$', "BrasserieSite.views.BARBANE_BLONDE", name='BARBANE_BLONDE'),
    url(r'^BARBANE_NOEL/$', "BrasserieSite.views.BARBANE_NOEL", name='BARBANE_NOEL'),
    url(r'^BARBANE_MONDIALE/$', "BrasserieSite.views.BARBANE_MONDIALE", name='BARBANE_MONDIALE'),
    url(r'^BARBANE_DANS_LES_VIGNES/$', "BrasserieSite.views.BARBANE_DANS_LES_VIGNES", name='BARBANE_DANS_LES_VIGNES'),

    # url(r'^BrasserieSite/', include('BrasserieSite.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^accounts/', include('registration.backends.simple.urls')),
    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),

)


import pandas as pd
from datetime import datetime as dt

curr_search_id = ''

raw_search_weather_dict = {
'search_id': '',
    'city': '',
    'address': '',
    'latitude': 0,
    'longitude': 0,
    'timezone': '',
    'tzoffset': 0,
    'description': '',
    'user_id': 0
}

search_weather_schema = {
    'search_id': str,
    'city': str,
    'address': str,
    'latitude': float,
    'longitude': float,
    'timezone': str,
    'tzoffset': float,
    'description': str,
    'user_id': int
}

def search_weather_decode(key, value, user_id):
    try:

        src_id = str(dt.now().strftime('%Y-%m-%d_%H%M%S'))
        global curr_search_id

        if key == 'address' and len(value) > 0:
            raw_search_weather_dict['search_id'] = src_id + '_' + value.strip().upper().replace(' ', '_')
            raw_search_weather_dict['city'] = value
            curr_search_id = src_id + '_' + value.strip().upper().replace(' ', '_')
            raw_search_weather_dict['user_id'] = user_id

        if key == 'resolvedAddress' and len(value) > 0:
            raw_search_weather_dict['address'] = value

        if key == 'latitude' and len(value) > 0:
            raw_search_weather_dict['latitude'] = value

        if key == 'longitude' and len(value) > 0:
            raw_search_weather_dict['longitude'] = value

        if key == 'timezone' and len(value) > 0:
            raw_search_weather_dict['timezone'] = value

        if key == 'tzoffset' and len(value) > 0 :
            raw_search_weather_dict['tzoffset'] = value

        if key == 'description' and len(value) > 0:
            raw_search_weather_dict['description'] = value

    except Exception as e:
        print('Error in search_weather_decode()', e)

    return raw_search_weather_dict

daily_weather_schema = {
    'day_id': str,
    'search_id': str,
    'day_datetime': 'datetime64',
    'temp_max': float,
    'temp_min': float,
    'temp': float,
    'feels_like': float,
    'dew': float,
    'humidity': float,
    'precip': float,
    'precip_prob': float,
    'precip_cover': float,
    'precip_type': str,
    'snow': float,
    'snowd_depth': float,
    'wind_gust': float,
    'wind_speed': float,
    'wind_dir': float,
    'pressure': float,
    'cloud_cover': float,
    'visibility': float,
    'solar_radiation': float,
    'solar_energy': float,
    'uv_index': float,
    'severe_risk': float,
    'sun_rise': str,
    'sun_set': str,
    'conditions': str,
    'description': str,
}

def daily_weather_decode(day_list):
    try:
        filter_day_list = []
        key_list = list(day_list[0].keys())
        for day in day_list:
            raw_daily_weather_dict = {}
            if 'search_id' in key_list:
                raw_daily_weather_dict['day_id'] = str(day['datetime']) + '_' + day['search_id']
                raw_daily_weather_dict['search_id'] = day['search_id']

            if 'datetime' in key_list:
                raw_daily_weather_dict['day_datetime'] = day['datetime']

            if 'tempmax' in key_list and type(day['tempmax']) != str:
                raw_daily_weather_dict['temp_max'] = day['tempmax']
            else:
                raw_daily_weather_dict['temp_max'] = 0

            if 'tempmin' in key_list and type(day['tempmin']) != str:
                raw_daily_weather_dict['temp_min'] = day['tempmin']
            else:
                raw_daily_weather_dict['temp_min'] = 0

            if 'temp' in key_list and type(day['temp']) != str:
                raw_daily_weather_dict['temp'] = day['temp']
            else:
                raw_daily_weather_dict['temp'] = 0

            if 'feelslike' in key_list and type(day['feelslike']) != str:
                raw_daily_weather_dict['feels_like'] = day['feelslike']
            else:
                raw_daily_weather_dict['feels_like'] = 0

            if 'dew' in key_list and type(day['dew']) != str:
                raw_daily_weather_dict['dew'] = day['dew']
            else:
                raw_daily_weather_dict['dew'] = 0

            if 'humidity' in key_list and type(day['humidity']) != str:
                raw_daily_weather_dict['humidity'] = day['humidity']
            else:
                raw_daily_weather_dict['humidity'] = 0

            if 'precip' in key_list and type(day['precip']) != str:
                raw_daily_weather_dict['precip'] = day['precip']
            else:
                raw_daily_weather_dict['precip'] = 0

            if 'precipprob' in key_list and type(day['precipprob']) != str:
                raw_daily_weather_dict['precip_prob'] = day['precipprob']
            else:
                raw_daily_weather_dict['precip_prob'] = 0

            if 'precipcover' in key_list and type(day['precipcover']) != str:
                raw_daily_weather_dict['precip_cover'] = day['precipcover']
            else:
                raw_daily_weather_dict['precip_cover'] = 0

            if 'preciptype' in key_list :
                raw_daily_weather_dict['precip_type'] = day['preciptype']
            else:
                raw_daily_weather_dict['precip_type'] = '0'

            if 'snow' in key_list and type(day['snow']) != str:
                raw_daily_weather_dict['snow'] = day['snow']
            else:
                raw_daily_weather_dict['snow'] = 0

            if 'snowdepth' in key_list and type(day['snowdepth']) != str:
                raw_daily_weather_dict['snowd_depth'] = day['snowdepth']
            else:
                raw_daily_weather_dict['snowd_depth'] = 0

            if 'windgust' in key_list and type(day['windgust']) != str:
                raw_daily_weather_dict['wind_gust'] = day['windgust']
            else:
                raw_daily_weather_dict['wind_gust'] = 0

            if 'windspeed' in key_list and type(day['windspeed']) != str:
                raw_daily_weather_dict['wind_speed'] = day['windspeed']
            else:
                raw_daily_weather_dict['wind_speed'] = 0

            if 'winddir' in key_list and type(day['winddir']) != str:
                raw_daily_weather_dict['wind_dir'] = day['winddir']
            else:
                raw_daily_weather_dict['wind_dir'] = 0

            if 'pressure' in key_list and type(day['pressure']) != str:
                raw_daily_weather_dict['pressure'] = day['pressure']
            else:
                raw_daily_weather_dict['pressure'] = 0

            if 'cloudcover' in key_list and type(day['cloudcover']) != str:
                raw_daily_weather_dict['cloud_cover'] = day['cloudcover']
            else:
                raw_daily_weather_dict['cloud_cover'] = 0

            if 'visibility' in key_list and type(day['visibility']) != str:
                raw_daily_weather_dict['visibility'] = day['visibility']
            else:
                raw_daily_weather_dict['visibility'] = 0

            if 'solarradiation' in key_list and type(day['solarradiation']) != str:
                raw_daily_weather_dict['solar_radiation'] = day['solarradiation']
            else:
                raw_daily_weather_dict['solar_radiation'] = 0

            if 'solarenergy' in key_list and type(day['solarenergy']) != str:
                raw_daily_weather_dict['solar_energy'] = day['solarenergy']
            else:
                raw_daily_weather_dict['solar_energy'] = 0

            if 'uvindex' in key_list and type(day['uvindex']) != str:
                raw_daily_weather_dict['uv_index'] = day['uvindex']
            else:
                raw_daily_weather_dict['uv_index'] = 0

            if 'severerisk' in key_list and type(day['severerisk']) != str:
                raw_daily_weather_dict['severe_risk'] = day['severerisk']
            else:
                raw_daily_weather_dict['severe_risk'] = 0

            if 'sunrise' in key_list:
                raw_daily_weather_dict['sun_rise'] = day['sunrise']
            else:
                raw_daily_weather_dict['sun_rise'] = ''

            if 'sunset' in key_list:
                raw_daily_weather_dict['sun_set'] = day['sunset']
            else:
                raw_daily_weather_dict['sun_set'] = ''

            if 'conditions' in key_list:
                raw_daily_weather_dict['conditions'] = day['conditions']
            else:
                raw_daily_weather_dict['conditions'] = ''

            if 'description' in key_list:
                raw_daily_weather_dict['description'] = day['description']
            else:
                raw_daily_weather_dict['description'] = ''

            if 'hours' in key_list:
                raw_daily_weather_dict['hours'] = day['hours']

            filter_day_list.append(raw_daily_weather_dict)
        return filter_day_list

    except Exception as e:
        print('Error in daily_weather_decode()', str(e))

hourly_weather_schema = {'hour_id': str,
                         'search_id': str,
                         'day_id': str,
                         'hour_no': str,
                         'hour': str,
                         'temp': float,
                         'feels_like': float,
                         'humidity': float,
                         'dew': float,
                         'precip': float,
                         'precip_prob': float,
                         'snow': float,
                         'snow_depth': float,
                         'precip_type': str,
                         'wind_gust': float,
                         'wind_speed': float,
                         'wind_dir': float,
                         'pressure': float,
                         'visibility': float,
                         'cloud_cover': float,
                         'solar_radiation': float,
                         'solar_energy': float,
                         'uv_index': float,
                         'severe_risk': float,
                         'conditions': str,
                         'icon': str,
                         'stations': str,
                         'source': str}

final_hrs_list = []

def hourly_weather_decode(day):
    try:
        for hour in day['hours']:
            raw_hrs_weather = {}
            if 'datetime' in hour.keys():
                raw_hrs_weather['hour_id'] = hour['hour_id']
                raw_hrs_weather['search_id'] = hour['search_id']
                raw_hrs_weather['day_id'] = hour['day_id']
                raw_hrs_weather['hour_no'] = str(hour['datetime']).split(':')[0]
                raw_hrs_weather['hour'] = hour['datetime']

            if 'temp' in hour.keys():
                raw_hrs_weather['temp'] = hour['temp']

            if 'feelslike' in hour.keys() and type(hour['feelslike']) != str:
                raw_hrs_weather['feels_like'] = hour['feelslike']
            else:
                raw_hrs_weather['feels_like'] = 0

            if 'humidity' in hour.keys() and type(hour['humidity']) != str:
                raw_hrs_weather['humidity'] = hour['humidity']
            else:
                raw_hrs_weather['humidity'] = 0

            if 'dew' in hour.keys() and type(hour['dew']) != str:
                raw_hrs_weather['dew'] = hour['dew']
            else:
                raw_hrs_weather['dew'] = 0

            if 'precip' in hour.keys() and type(hour['precip']) != str:
                raw_hrs_weather['precip'] = hour['precip']
            else:
                raw_hrs_weather['precip'] = 0

            if 'precipprob' in hour.keys() and type(hour['precipprob']) != str:
                raw_hrs_weather['precip_prob'] = hour['precipprob']
            else:
                raw_hrs_weather['precip_prob'] = 0

            if 'snow' in hour.keys() and type(hour['snow']) != str:
                raw_hrs_weather['snow'] = hour['snow']
            else:
                raw_hrs_weather['snow'] = 0

            if 'snowdepth' in hour.keys() and type(hour['snowdepth']) != str:
                raw_hrs_weather['snow_depth'] = hour['snowdepth']
            else:
                raw_hrs_weather['snow_depth'] = 0

            if 'preciptype' in hour.keys() :
                raw_hrs_weather['precip_type'] = hour['preciptype']
            else:
                raw_hrs_weather['precip_type'] = '0'

            if 'windgust' in hour.keys() and type(hour['windgust']) != str:
                raw_hrs_weather['wind_gust'] = hour['windgust']
            else:
                raw_hrs_weather['wind_gust'] = 0

            if 'windspeed' in hour.keys() and type(hour['windspeed']) != str:
                raw_hrs_weather['wind_speed'] = hour['windspeed']
            else:
                raw_hrs_weather['wind_speed'] = 0

            if 'winddir' in hour.keys() and type(hour['winddir']) != str:
                raw_hrs_weather['wind_dir'] = hour['winddir']
            else:
                raw_hrs_weather['wind_dir'] = 0

            if 'pressure' in hour.keys() and type(hour['pressure']) != str:
                raw_hrs_weather['pressure'] = hour['pressure']
            else:
                raw_hrs_weather['pressure'] = 0

            if 'visibility' in hour.keys() and type(hour['visibility']) != str:
                raw_hrs_weather['visibility'] = hour['visibility']
            else:
                raw_hrs_weather['visibility'] = 0

            if 'cloudcover' in hour.keys() and type(hour['cloudcover']) != str:
                raw_hrs_weather['cloud_cover'] = hour['cloudcover']
            else:
                raw_hrs_weather['cloud_cover'] = 0

            if 'solarradiation' in hour.keys() and type(hour['solarradiation']) != str:
                raw_hrs_weather['solar_radiation'] = hour['solarradiation']
            else:
                raw_hrs_weather['solar_radiation'] = 0

            if 'solarenergy' in hour.keys() and type(hour['solarenergy']) != str:
                raw_hrs_weather['solar_energy'] = hour['solarenergy']
            else:
                raw_hrs_weather['solar_energy'] = 0

            if 'uvindex' in hour.keys() and type(hour['uvindex']) != str:
                raw_hrs_weather['uv_index'] = hour['uvindex']
            else:
                raw_hrs_weather['uv_index'] = 0

            if 'severerisk' in hour.keys() and type(hour['severerisk']) != str:
                raw_hrs_weather['severe_risk'] = hour['severerisk']
            else:
                raw_hrs_weather['severe_risk'] = 0

            if 'conditions' in hour.keys():
                raw_hrs_weather['conditions'] = hour['conditions']
            else:
                raw_hrs_weather['conditions'] = ''

            if 'icon' in hour.keys():
                raw_hrs_weather['icon'] = hour['icon']
            else:
                raw_hrs_weather['icon'] = ''

            if 'stations' in hour.keys():
                raw_hrs_weather['stations'] = hour['stations']
            else:
                raw_hrs_weather['stations'] = ''

            if 'source' in hour.keys():
                raw_hrs_weather['source'] = hour['source']
            else:
                raw_hrs_weather['source'] = ''

            final_hrs_list.append(raw_hrs_weather)

        return final_hrs_list
    except Exception as e:
        print('Error in hourly_weather_decode()', str(e))

curr_weather_schema = {'curr_weather_id': str,
                       'search_id': str,
                       'search_hour': str,
                       'temp': float,
                       'feels_like': float,
                       'humidity': float,
                       'dew': float,
                       'precip': float,
                       'precip_prob': float,
                       'snow': float,
                       'snow_depth': float,
                       'precip_type': str,
                       'wind_gust': float,
                       'wind_speed': float,
                       'wind_dir': float,
                       'pressure': float,
                       'visibility': float,
                       'cloud_cover': float,
                       'solar_radiation': float,
                       'solar_energy': float,
                       'uv_index': float,
                       'severe_risk': float,
                       'conditions': str,
                       'icon': str,
                       'stations': str,
                       'source': str,
                       'sun_rise': str,
                       'sun_set': str}

def current_weather_decode(curr_dict):
    try:
        global curr_search_id
        raw_curr_weather = {}
        raw_curr_weather['curr_weather_id'] = curr_dict['datetime'].replace(':', '-') + '_' + curr_search_id
        raw_curr_weather['search_id'] = curr_search_id
        raw_curr_weather['search_hour'] = curr_dict['datetime']

        if 'temp' in curr_dict.keys() and type(curr_dict['temp']) != str:
            raw_curr_weather['temp'] = curr_dict['temp']
        else:
            raw_curr_weather['temp'] = 0

        if 'feelslike' in curr_dict.keys() and type(curr_dict['feelslike']) != str:
            raw_curr_weather['feels_like'] = curr_dict['feelslike']
        else:
            raw_curr_weather['feels_like'] = 0

        if 'humidity' in curr_dict.keys() and type(curr_dict['humidity']) != str:
            raw_curr_weather['humidity'] = curr_dict['humidity']
        else:
            raw_curr_weather['humidity'] = 0

        if 'dew' in curr_dict.keys() and type(curr_dict['dew']) != str:
            raw_curr_weather['dew'] = curr_dict['dew']
        else:
            raw_curr_weather['dew'] = 0

        if 'precip' in curr_dict.keys() and type(curr_dict['precip']) != str:
            raw_curr_weather['precip'] = curr_dict['precip']
        else:
            raw_curr_weather['precip'] = 0

        if 'precipprob' in curr_dict.keys() and type(curr_dict['precipprob']) != str:
            raw_curr_weather['precip_prob'] = curr_dict['precipprob']
        else:
            raw_curr_weather['precip_prob'] = 0

        if 'snow' in curr_dict.keys() and type(curr_dict['snow']) != str:
            raw_curr_weather['snow'] = curr_dict['snow']
        else:
            raw_curr_weather['snow'] = 0

        if 'snowdepth' in curr_dict.keys() and type(curr_dict['snowdepth']) != str:
            raw_curr_weather['snow_depth'] = curr_dict['snowdepth']
        else:
            raw_curr_weather['snow_depth'] = 0

        if 'preciptype' in curr_dict.keys() :
            raw_curr_weather['precip_type'] = curr_dict['preciptype']
        else:
            raw_curr_weather['precip_type'] = '0'

        if 'windgust' in curr_dict.keys() and type(curr_dict['windgust']) != str :
            raw_curr_weather['wind_gust'] = curr_dict['windgust']
        else:
            raw_curr_weather['wind_gust'] = 0

        if 'windspeed' in curr_dict.keys() and type(curr_dict['windspeed']) != str:
            raw_curr_weather['wind_speed'] = curr_dict['windspeed']
        else:
            raw_curr_weather['wind_speed'] = 0

        if 'winddir' in curr_dict.keys() and type(curr_dict['winddir']) != str:
            raw_curr_weather['wind_dir'] = curr_dict['winddir']
        else:
            raw_curr_weather['wind_dir'] = 0

        if 'pressure' in curr_dict.keys() and type(curr_dict['pressure']) != str:
            raw_curr_weather['pressure'] = curr_dict['pressure']
        else:
            raw_curr_weather['pressure'] = 0

        if 'visibility' in curr_dict.keys() and type(curr_dict['visibility']) != str:
            raw_curr_weather['visibility'] = curr_dict['visibility']
        else:
            raw_curr_weather['visibility'] = 0

        if 'cloudcover' in curr_dict.keys() and type(curr_dict['cloudcover']) != str:
            raw_curr_weather['cloud_cover'] = curr_dict['cloudcover']
        else:
            raw_curr_weather['cloud_cover'] = 0

        if 'solarradiation' in curr_dict.keys() and type(curr_dict['solarradiation']) != str:
            raw_curr_weather['solar_radiation'] = curr_dict['solarradiation']
        else:
            raw_curr_weather['solar_radiation'] = 0

        if 'solarenergy' in curr_dict.keys() and type(curr_dict['solarenergy']) != str:
            raw_curr_weather['solar_energy'] = curr_dict['solarenergy']
        else:
            raw_curr_weather['solar_energy'] = 0

        if 'uvindex' in curr_dict.keys() and type(curr_dict['uvindex']) != str:
            raw_curr_weather['uv_index'] = curr_dict['uvindex']
        else:
            raw_curr_weather['uv_index'] = 0

        if 'severerisk' in curr_dict.keys() and type(curr_dict['severerisk']) != str:
            raw_curr_weather['severe_risk'] = curr_dict['severerisk']
        else:
            raw_curr_weather['severe_risk'] = 0

        if 'conditions' in curr_dict.keys():
            raw_curr_weather['conditions'] = curr_dict['conditions']
        else:
            raw_curr_weather['conditions'] = ''

        if 'icon' in curr_dict.keys():
            raw_curr_weather['icon'] = curr_dict['icon']
        else:
            raw_curr_weather['icon'] = ''

        if 'stations' in curr_dict.keys():
            raw_curr_weather['stations'] = curr_dict['stations']
        else:
            raw_curr_weather['stations'] = ''

        if 'source' in curr_dict.keys():
            raw_curr_weather['source'] = curr_dict['source']
        else:
            raw_curr_weather['source'] = ''

        if 'sunrise' in curr_dict.keys():
            raw_curr_weather['sun_rise'] = curr_dict['sunrise']
        else:
            raw_curr_weather['sun_rise'] = ''

        if 'sunset' in curr_dict.keys():
            raw_curr_weather['sun_set'] = curr_dict['sunset']
        else:
            raw_curr_weather['sun_set'] = ''

        return raw_curr_weather
    except Exception as e:
        print('Error in current_weather_decode()', str(e))

def decode_raw_json(raw_data, user_id):
    try:
        filtered_weather_data = {
            'user_id': "",
            'search_id': "",
            'src_city': "",
            'daily_weather': "",
            'hourly_weather': "",
            'current_weather': ""
        }
        src_weather_dict = {}
        # PART : 1 decoding main weather json
        # to create search_id
        for key, value in raw_data.items():
            if key != 'days':
                src_weather_dict = search_weather_decode(key, str(value), user_id)

        # setting search_id
        search_id = src_weather_dict['search_id']

        # converting return dict into dataframe and applying schema for type changed
        search_weather_df = pd.DataFrame([src_weather_dict]).astype(search_weather_schema)

        # PART : 2 Decoding daily weather json
        # decoding daily weather json and adding search_id
        for i in range(len(raw_data['days'])):
            raw_data['days'][i]['search_id'] = search_id

        filter_day_list = daily_weather_decode(raw_data['days'])

        # creating dataframe from raw day weather
        daily_weather_df = pd.DataFrame(filter_day_list, columns=list(daily_weather_schema.keys())).astype(
            daily_weather_schema)
        daily_weather_df.fillna(value=0, inplace=True)

        # PART : 3
        # created hours list to store hourly weather details
        for day in filter_day_list:
            for hour in day['hours']:
                hour['hour_id'] = hour['datetime'] + '_' + day['day_id']
                hour['search_id'] = search_id
                hour['day_id'] = day['day_id']
                hour['hour_no'] = str(hour['datetime'].split(':')[0])

        hours_list = []
        for day in filter_day_list:
            hours_list = hourly_weather_decode(day)

        # converting hours_list into dataframe
        hourly_weather_df = pd.DataFrame(hours_list, columns=list(hourly_weather_schema.keys())).astype(
            hourly_weather_schema)
        hourly_weather_df.fillna(value=0, inplace=True)
        hourly_weather_df.drop_duplicates(inplace=True)

        # setting current weather details
        curr_weather_df = current_weather_decode(raw_data['currentConditions'])
        curr_weather_df = pd.DataFrame([curr_weather_df]).astype(curr_weather_schema)
        curr_weather_df.fillna(value=0, inplace=True)

        filtered_weather_data['user_id'] = user_id
        filtered_weather_data['search_id'] = search_id
        filtered_weather_data['search_city'] = search_weather_df
        filtered_weather_data['daily_weather'] = daily_weather_df
        filtered_weather_data['hourly_weather'] = hourly_weather_df
        filtered_weather_data['current_weather'] = curr_weather_df

        return filtered_weather_data
    except Exception as e:
        print('Error in decode_raw_json()',e)


import numpy as np
import open3d as o3d
import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser("./bin2ply.py")
    parser.add_argument(
        '--root', '-r',
        type=str,
        required=True,
        help='the folder that contains the dataset',
    )
    parser.add_argument(
        '--length', '-l',
        type=int,
        required=True,
        help='the folder of bin file len you wanna convert',
    )

    FLAGS, unparsed = parser.parse_known_args()
    root = FLAGS.root + '/'
    for drive_id in range(FLAGS.length):
        path = root + '%06d.bin' % drive_id
        xyzr = np.fromfile(path, dtype=np.float32).reshape(-1, 4)
        # print(len(xyzr))
        xyz0 = xyzr[:, :3]
        src_keypts = o3d.geometry.PointCloud()
        src_keypts.points = o3d.utility.Vector3dVector(xyz0)
        print(src_keypts)
        o3d.io.write_point_cloud(root + '%06d.ply' % drive_id,src_keypts)


import speech_recognition as sr
from observerx.observerx import ConcreteSubject
import time
import threading

class STTEngine(ConcreteSubject):
    """
    Speech To Text Engine (STT)

    Google API Speech recognition settings
    SpeechRecognition API : https://pypi.org/project/SpeechRecognition/2.1.3
    """

    recognizer = sr.Recognizer()
    microphone = sr.Microphone(sample_rate=32000)
    # energía de audio mínima a considerar para la grabación
    recognizer.energy_threshold = 4000
    # segundos de audio sin hablar antes de que una frase se considere completa
    recognizer.pause_threshold = 0.8
    recognizer.dynamic_energy_threshold = True
    # segundos mínimos de audio hablado antes de que consideremos el audio hablado como una frase; los valores por debajo de esto se ignoran (para filtrar clics y estallidos)
    recognizer.phrase_threshold = 0.8
    # segundos de audio que no habla para mantenerse en ambos lados de la grabación
    recognizer.non_speaking_duration = 0.2
    main_lang = "es-ES"
    keyword_lang = "es"
    keywords = []#[("cristal", 1), ("hey cristal", 1), ("ey cristal", 1), ("christal", 1)]

    def __init__(self):
        super().__init__()

    def run(self, already_activated=False):
        if not isinstance(self.recognizer, sr.Recognizer):
            raise TypeError("`recognizer` must be `Recognizer` instance")
        if not isinstance(self.microphone, sr.Microphone):
            raise TypeError("`microphone` must be `Microphone` instance")
        self.recognize_input(already_activated)

    def stop(self):
        print("Deteniendo...")

    def recognize_input(self, already_activated=False):
        response = STTEngine._recognize_speech_from_mic(already_activated)
        self.stop()
        print(response)
        self.some_business_logic(response)

    @classmethod
    def _recognize_speech_from_mic(cls, already_activated=False):
        print("escuchando")
        response = { "success": True, "error": None, "transcription": None }
        audio = None
        try:
            with cls.microphone as source:
                cls.recognizer.adjust_for_ambient_noise(source, duration=1)
                audio = cls.recognizer.listen(source, timeout=3)
        except sr.WaitTimeoutError:
            pass
        try:
            if audio:
                #with open("microphone-results.wav", "wb") as f:
                #    f.write(audio.get_wav_data())
                #speech_as_text = recognizer.recognize_sphinx(audio, keyword_entries=self.keywords, language=self.keyword_lang)
                response["transcription"] = cls.recognizer.recognize_google(audio, language=cls.main_lang).lower()
                if already_activated == False: #and cls._activation_name_exist(response):
                    response = cls._remove_activation_word(response)
                elif already_activated == True:
                    pass
                else:
                    response["success"] = False
            else:
                response["success"] = False
        except sr.UnknownValueError:
            response["success"] = False
            response["transcription"] = "Función no conocida"
        except sr.RequestError:
            response["success"] = False
            response["transcription"] = "API no disponible en este momento."
        return response

    #@staticmethod
    @classmethod
    def _activation_name_exist(cls, transcript):
        if transcript["transcription"]:
            transcript_words = transcript["transcription"].split()
            names = []
            for name in cls.keywords:
                names.append(name[0])
            return bool(set(transcript_words).intersection(names))
        else:
            return False

    #@staticmethod
    @classmethod
    def _remove_activation_word(cls, transcript):
        for name in cls.keywords:
            count = transcript["transcription"].count(name[0])
            transcript["transcription"] = transcript["transcription"].replace(name[0], "", count)
        transcript["transcription"] = transcript["transcription"].strip()
        return transcript


import torch
import torch.nn as nn
import copy

## (Conditional) Variational Autoencoder class and encoder/decoder class

class VAE(nn.Module):
    def __init__(self, x_dim, y_dim, latent_dim, encoder_layersizes, decoder_layersizes, normparams, conditional=False):
        super().__init__()

        if conditional:
            assert y_dim > 0

        assert type(latent_dim) == int
        assert type(x_dim) == int

        self.latent_dim = latent_dim

        self.encoder = Encoder(x_dim, y_dim, latent_dim, conditional, encoder_layersizes, normparams)
        self.decoder = Decoder(x_dim, y_dim, latent_dim, conditional, decoder_layersizes, normparams)
        self.clamp = [-4, 15]

    def inference(self, z, y=None):
        recon_x = self.decoder(z, y)
        return recon_x

class Encoder(nn.Module):
    def __init__(self, x_dim, y_dim, latent_dim, conditional, encoder_layersizes, normparams):
        """
        Args:
            x_dim (int): e.g., x variable (to be reconstructed) dimension
            y_dim (int): e.g., y variable (if x is conditioned by y) dimension
            latent_dim (int): latent dimension
            conditional (bool): whether to use conditional VAE, if conditional then x is conditioned by y
        """
        super().__init__()

        self.conditional = conditional
        self.normparams = normparams

        layer_sizes = copy.deepcopy(encoder_layersizes)
        layer_sizes.insert(0, x_dim)
        if self.conditional:
            layer_sizes[0] += y_dim

        self.MLP = nn.Sequential()

        for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
            self.MLP.add_module(
                name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
            self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())

        self.liner_means = nn.Linear(layer_sizes[-1], latent_dim)
        self.linear_log_var = nn.Linear(layer_sizes[-1], latent_dim)

class Decoder(nn.Module):
    def __init__(self, x_dim, y_dim, latent_dim, conditional, decoder_layersizes, normparams):
        """
        Args:
            x_dim (int): e.g., x variable (to be reconstructed) dimension
            y_dim (int): e.g., y variable (if x is conditioned by y) dimension
            latent_dim (int): latent dimension
            conditional (bool): whether to use conditional VAE, if conditional then x is conditioned by y
        """
        super().__init__()

        self.conditional = conditional
        self.normparams = normparams

        layer_sizes = copy.deepcopy(decoder_layersizes)
        layer_sizes.insert(0, latent_dim)
        if self.conditional:
            layer_sizes[0] += y_dim

        layer_sizes.append(x_dim)

        self.MLP = nn.Sequential()

        for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
            self.MLP.add_module(
                name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
            if i + 1 < len(layer_sizes[:-1]):
                self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU())
            else:
                self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())

    def forward(self, z, y=None):
        if self.conditional:
            y = idx2onehot(y, 5)
            z = torch.cat((z, y), dim=-1)
        x = self.MLP(z)
        return x

## GAN class definition and corresponding Generator and Discriminator

class GAN(nn.Module):
     def __init__(self, sample_dim, latent_dim, generator_layersizes, discriminator_layersizes):
        super().__init__()

        assert type(sample_dim) == int
        assert type(latent_dim) == int
        self.sample_dim = sample_dim
        self.latent_dim = latent_dim
        self.generator_layersizes = generator_layersizes
        self.discriminator_layersizes = discriminator_layersizes

     def weights_init(self, m):
        classname = m.__class__.__name__
        if classname.find('Conv') != -1:
            nn.init.normal_(m.weight.data, 0.0, 0.02)
        elif classname.find('BatchNorm') != -1:
            nn.init.normal_(m.weight.data, 1.0, 0.02)
            nn.init.constant_(m.bias.data, 0)

     def initialize_generator(self, device):
        generator = Generator(self.latent_dim, self.sample_dim, self.generator_layersizes)
        generator.apply(self.weights_init)
        generator.to(device)
        return generator

     def generate_samples(self, generator, num_samples, latent_dim, device):
        noise = torch.randn(num_samples, latent_dim, device=device)
        generator.eval()
        samples = generator(noise)
        samples = samples.detach().numpy()
        return samples

class Generator(nn.Module):
  """
    DCGan Generator
  """
  def __init__(self, input_dim, out_dim, generator_layersizes):
    super().__init__()

    layer_sizes = copy.deepcopy(generator_layersizes)
    layer_sizes.insert(0, input_dim)
    layer_sizes.append(out_dim)

    self.MLP = nn.Sequential()

    for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
        self.MLP.add_module(
            name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
        if i + 1 < len(layer_sizes[:-1]):
            self.MLP.add_module(name="A{:d}".format(i), module=nn.LeakyReLU())
        else:
            self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())

  def forward(self, x):
    """Forward pass"""
    return self.MLP(x)
    # return self.layers(x)

class Discriminator(nn.Module):
  """
    DCGan Discriminator
  """
  def __init__(self, input_dim, out_dim, discriminator_layersizes):
    super().__init__()

    assert(out_dim==1)
    layer_sizes = copy.deepcopy(discriminator_layersizes)
    layer_sizes.insert(0, input_dim)
    layer_sizes.append(out_dim)

    self.MLP = nn.Sequential()

    for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])):
        self.MLP.add_module(
            name="L{:d}".format(i), module=nn.Linear(in_size, out_size))
        if i + 1 < len(layer_sizes[:-1]):
            self.MLP.add_module(name="A{:d}".format(i), module=nn.LeakyReLU())
            self.MLP.add_module(name="Dropout{:d}".format(i), module=nn.Dropout(0.3))
        else:
            self.MLP.add_module(name="sigmoid", module=nn.Sigmoid())

  def forward(self, x):
    """Forward pass"""
    return self.MLP(x)
    # return self.layers(x)

## Additional functions

def idx2onehot(idx, n):
    assert torch.max(idx).item() < n

    if idx.dim() == 1:
        idx = idx.unsqueeze(1)
    onehot = torch.zeros(idx.size(0), n).to(idx.device)
    onehot.scatter_(1, idx, 1)

    return onehot


import contextlib
import glob
import json
import math
import os
import pickle
import sys
import warnings
from typing import Dict, List

import numpy as np
import optuna
from optuna.trial import FrozenTrial

from simulai.metrics import DeterminationCoeff
from simulai.models import ModelPool
from simulai.regression import EchoStateNetwork
from simulai.utilities import make_temp_directory

from ._cloud_object_storage import CloudObjectStorage

def _compute_damped_residual_block(resblock_damping, esn_input_size, target_ix):
    A = np.zeros((1, esn_input_size), dtype=float)
    A[0, target_ix] = 1 - resblock_damping
    return A

class ObjectiveESNIndependent:
    def __init__(
        self,
        train_data: np.ndarray = None,
        train_data_aux: np.ndarray = None,
        validation_data: np.ndarray = None,
        validation_data_aux: np.ndarray = None,
        target_ix: int = None,
        base_model_path: str = None,
        esn_override_configs: Dict = None,
        reservoir_config_space: Dict = None,
        pool_template: str = None,
    ):
        self.train_data = train_data
        self.train_data_aux = train_data_aux
        self.validation_data = validation_data
        self.validation_data_aux = validation_data_aux
        self.target_ix = target_ix
        self.base_model_path = base_model_path
        self.pool_template = None
        self.set_pool_template(pool_template)

        if esn_override_configs is None:
            esn_override_configs = dict()
        self.esn_override_configs = esn_override_configs

        self.reservoir_config_space = reservoir_config_space
        if self.reservoir_config_space is None:
            self.reservoir_config_space = self._default_reservoir_config_space

        self._pool_ix = ModelPool(
            config={
                "template": pool_template,
                "n_inputs": self._pool_n_inputs,
                "n_auxiliary": self.train_data_aux.shape[1],
                "n_outputs": self.train_data.shape[1],
            },
            model_type="EchoStateNetwork",
        )

        self.instability_penalty = 100

    @staticmethod
    def get_model_ids(base_model_path):
        ids = [
            os.path.basename(os.path.splitext(m)[0])
            for m in glob.glob(os.path.join(base_model_path, "*.pkl"))
        ]
        ids = sorted(ids, key=lambda p: int(p.split("_")[-1]))
        return ids

    def set_pool_template(self, pool_template):
        assert pool_template in [
            "independent_series",
            "no_communication_series",
        ], f"pool_template={self.pool_template} is not recognized"
        self.pool_template = pool_template

    @property
    def _default_reservoir_config_space(self):
        return {
            "radius": [0.1, 1.2],
            "sigma": [0.1, 1.2],
            "exp_beta": [-100, -1],
            "leak_rate": [0, 1],
            "activation": ["tanh"],
            "tau": [0, 1],
            "transformation": ["T1"],
            "resblock_damping": [0.0, 1.0],
            "estimate_bias_transition": [False, True],
            "estimate_linear_transition": [False, True],
            "input_augmented_reservoir": [False, True],
            "model_id": self.get_model_ids(self.base_model_path),
        }

    def optuna_objective(self, trial: optuna.Trial):
        float_vars = [
            "radius",
            "sigma",
            "exp_beta",
            "leak_rate",
            "tau",
            "resblock_damping",
        ]
        float_config = {
            fv: trial.suggest_float(fv, *self.reservoir_config_space[fv])
            for fv in float_vars
            if fv in self.reservoir_config_space
        }

        categorical_vars = [
            "activation",
            "transformation",
            "model_id",
            "estimate_bias_transition",
            "estimate_linear_transition",
            "input_augmented_reservoir",
        ]
        categorical_config = {
            cv: trial.suggest_categorical(cv, self.reservoir_config_space[cv])
            for cv in categorical_vars
            if cv in self.reservoir_config_space
        }

        configs = {**float_config, **categorical_config}
        return self.objective(configs)

    def get_esn_model_instance(
        self,
    ) -> EchoStateNetwork:
        return self._pool_ix.get_model_instance(index=self.target_ix)

    @property
    def _pool_n_inputs(self):
        return self.train_data.shape[1] + self.train_data_aux.shape[1]

    @property
    def _esn_n_inputs(self):
        if self.pool_template == "independent_series":
            return self.train_data.shape[1] + self.train_data_aux.shape[1]
        elif self.pool_template == "no_communication_series":
            return 1 + self.train_data_aux.shape[1]
        else:
            raise RuntimeError(f"pool_template={self.pool_template} is not recognized")

    def check_stability(self):
        esn = self.get_esn_model_instance()
        trainable_variables = esn.trainable_variables

        if self.pool_template == "independent_series":
            self_ix = self.target_ix
        elif self.pool_template == "no_communication_series":
            self_ix = 0
        else:
            raise RuntimeError(f"pool_template={self.pool_template} is not recognized")

        A_ix = 0
        if trainable_variables["A"] is not None:
            A_ix = trainable_variables["A"][0, self_ix]

        aug_ix = 0
        if esn.input_augmented_reservoir:
            aug_ix = trainable_variables["W_out"][0, -esn.number_of_inputs + self_ix]

        return abs(A_ix + aug_ix) < 1

    def _configure_pool(self, configs: Dict, trainable_variables: Dict = None):
        local_config = configs.copy()
        model_id = local_config.pop("model_id")
        if "exp_beta" in local_config:
            local_config["beta"] = 10 ** local_config.pop("exp_beta")
        if "resblock_damping" in local_config:
            local_config["A"] = _compute_damped_residual_block(
                local_config.pop("resblock_damping"), self._esn_n_inputs, self.target_ix
            )
        if trainable_variables is not None:
            local_config.update(trainable_variables)

        esn = EchoStateNetwork.restore(self.base_model_path, model_id)
        merged_configs = {**self.esn_override_configs, **local_config}
        esn.set_parameters(merged_configs)
        esn.set_reference()
        esn.reset()

        self._pool_ix.load_model_instance(esn, index=self.target_ix)

    def _fit(
        self,
    ):
        self._pool_ix.fit(
            input_data=self.train_data[0:-1, :],
            target_data=self.train_data[1:, :],
            auxiliary_data=self.train_data_aux[0:-1, :],
            index=self.target_ix,
        )

    def _extrapolate_target_model(
        self, init_state=None, init_aux=None, next_data=None, next_aux=None, reset=True
    ):
        if reset:
            self._pool_ix.set_reference()
            self._pool_ix.reset()

        extrapolation_data = []
        total_steps = next_aux.shape[0] + 1
        for n in range(total_steps):
            sys.stdout.write(f"\rPredicting step {n+1}/{total_steps}")
            sys.stdout.flush()
            with contextlib.redirect_stdout(None):
                s = self._pool_ix.predict(
                    initial_state=init_state,
                    auxiliary_data=init_aux,
                    index=self.target_ix,
                )

            extrapolation_data.append(s)

            if n < next_aux.shape[0]:
                init_state = next_data[n : n + 1, :].copy()
                init_state[:, self.target_ix : self.target_ix + 1] = s
                init_aux = next_aux[n : n + 1, :]

        sys.stdout.write("\r")
        sys.stdout.flush()

        extrapolation_data = np.vstack(extrapolation_data)

        return extrapolation_data

    def objective(self, configs: Dict):
        self._configure_pool(configs=configs)

        self._fit()

        extrapolation_data = self._extrapolate_target_model(
            init_state=self.train_data[-1:, :],
            init_aux=self.train_data_aux[-1:, :],
            next_data=self.validation_data[:-1, :],
            next_aux=self.validation_data_aux[:-1, :],
            reset=False,
        )

        # This value is intended to be as close as possible to zero
        objective_value = math.sqrt(
            math.pow(
                1
                - DeterminationCoeff()(
                    data=extrapolation_data,
                    reference_data=self.validation_data[
                        :, self.target_ix : self.target_ix + 1
                    ],
                ),
                2,
            )
        )

        is_stable = self.check_stability()
        if not is_stable:
            objective_value += self.instability_penalty

        return objective_value

    def cos_save_callback(
        self,
        study: optuna.Study,
        trial: FrozenTrial,
        cos_config,
        bucket_name,
        dir_path_in_bucket,
    ):
        esn = self.get_esn_model_instance()
        save_trainable_variables(
            cos_config=cos_config,
            bucket_name=bucket_name,
            dir_path_in_bucket=dir_path_in_bucket,
            study_name=study.study_name,
            trial_number=trial.number,
            trainable_variables=esn.trainable_variables,
        )

def _trial_filename(study: str, trial_number: int):
    assert study is not None
    assert trial_number is not None

    return f"study_{study}_trial_{str(trial_number)}.pkl"

def load_trainable_variables(
    cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
):
    success = False
    trainable_variables = {}
    if cos_config is not None:
        cos = CloudObjectStorage.from_config_json(cos_config)
        trial_filename = _trial_filename(study_name, trial_number)
        filename_in_bucket = os.path.join(dir_path_in_bucket, trial_filename)
        success = cos.check_file_exists(bucket_name, filename_in_bucket)
        if success:
            with make_temp_directory() as tmp_dir:
                saved_variables_filename = os.path.join(tmp_dir, trial_filename)
                cos.get_file(bucket_name, filename_in_bucket, saved_variables_filename)
                with open(saved_variables_filename, "rb") as saved_variables_file:
                    trainable_variables = pickle.load(saved_variables_file)

    return success, trainable_variables

def save_trainable_variables(
    cos_config,
    bucket_name,
    dir_path_in_bucket,
    study_name,
    trial_number,
    trainable_variables,
):
    if cos_config is not None:
        cos = CloudObjectStorage.from_config_json(cos_config)
        trial_filename = _trial_filename(study_name, trial_number)
        filename_in_bucket = os.path.join(dir_path_in_bucket, trial_filename)
        with make_temp_directory() as tmp_dir:
            saved_variables_filename = os.path.join(tmp_dir, trial_filename)
            try:
                with open(saved_variables_filename, "wb") as fp:
                    pickle.dump(trainable_variables, fp, protocol=4)
            except Exception as e:
                print(e, e.args)
            cos.put_file(bucket_name, filename_in_bucket, saved_variables_filename)

def optuna_assess_best_solution_ESNIndependent(
    train_data: np.ndarray = None,
    train_data_aux: np.ndarray = None,
    full_data: np.ndarray = None,
    full_data_aux: np.ndarray = None,
    target_ix: int = 0,
    base_model_path: str = None,
    study_name: str = None,
    storage: str = None,
    trial_number: int = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    dir_path_in_bucket: str = None,
    recompute_trainable_variables: bool = False,
    pool_template: str = None,
):
    hyper_search = optuna.load_study(
        study_name=study_name,
        storage=storage,
    )

    if trial_number is None:
        trial = hyper_search.best_trial
        trial_number = trial.number
    else:
        trial = hyper_search._storage.get_trial(
            hyper_search._storage.get_trial_id_from_study_id_trial_number(
                hyper_search._study_id, trial_number
            )
        )

    print(f"Extrapolating trial with number {trial.number}")

    if recompute_trainable_variables:
        print(f"Recomputing trainable variables")
        success_loading = False
        trainable_variables = None
    else:
        success_loading, trainable_variables = load_trainable_variables(
            cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
        )

        print(f"Loading trainable variables success_loading={success_loading}")

    o = ObjectiveESNIndependent(
        train_data=train_data,
        train_data_aux=train_data_aux,
        validation_data=None,
        validation_data_aux=None,
        target_ix=target_ix,
        base_model_path=base_model_path,
        reservoir_config_space=None,
        pool_template=pool_template,
    )

    o._configure_pool(configs=trial.params, trainable_variables=trainable_variables)

    if not success_loading:
        o._fit()
        esn = o.get_esn_model_instance()
        save_trainable_variables(
            cos_config,
            bucket_name,
            dir_path_in_bucket,
            study_name,
            trial_number,
            esn.trainable_variables,
        )

    extrapolation_data = o._extrapolate_target_model(
        init_state=full_data[0:1, :],
        init_aux=full_data_aux[0:1, :],
        next_data=full_data[1:, :],
        next_aux=full_data_aux[1:, :],
        reset=True,
    )

    return extrapolation_data

def optuna_objectiveESNIndependent(
    train_data: np.ndarray = None,
    train_data_aux: np.ndarray = None,
    validation_data: np.ndarray = None,
    validation_data_aux: np.ndarray = None,
    target_ix: int = 0,
    base_model_path: str = None,
    reservoir_config_space: Dict = None,
    n_trials: int = 1,
    study_name: str = None,
    storage: str = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    dir_path_in_bucket: str = None,
    pool_template: str = None,
    do_grid_search: bool = False,
):
    o = ObjectiveESNIndependent(
        train_data=train_data,
        train_data_aux=train_data_aux,
        validation_data=validation_data,
        validation_data_aux=validation_data_aux,
        target_ix=target_ix,
        base_model_path=base_model_path,
        reservoir_config_space=reservoir_config_space,
        pool_template=pool_template,
    )

    sampler = None
    if do_grid_search:
        sampler = optuna.samplers.GridSampler(reservoir_config_space)
    hyper_search = optuna.create_study(
        study_name=study_name,
        sampler=sampler,
        direction="minimize",
        storage=storage,
        load_if_exists=True,
    )

    def saver(s, t):
        o.cos_save_callback(s, t, cos_config, bucket_name, dir_path_in_bucket)

    hyper_search.optimize(o.optuna_objective, n_trials=n_trials, callbacks=[saver])

def optuna_assess_best_joint_solution_ESNIndependent(
    initial_data: np.ndarray = None,
    data_aux: np.ndarray = None,
    base_model_paths: List[str] = None,
    study_names: List[str] = None,
    storage: str = None,
    trial_numbers: List[int] = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    study_paths_in_bucket: List[str] = None,
    compare_data: np.ndarray = None,
    pool_template: str = None,
    hidden_state_startup_burning_steps: int = 0,
):
    _n_inputs = initial_data.shape[1] + data_aux.shape[1]
    if pool_template == "independent_timeseries":
        _esn_n_inputs = _n_inputs
    elif pool_template == "no_communication_series":
        _esn_n_inputs = 1 + data_aux.shape[1]
    else:
        raise RuntimeError(f"Unrecognized pool_template={pool_template}")

    pool = ModelPool(
        config={
            "template": pool_template,
            "n_inputs": _n_inputs,
            "n_auxiliary": data_aux.shape[1],
            "n_outputs": initial_data.shape[1],
        },
        model_type="EchoStateNetwork",
    )

    for ix, study_name in enumerate(study_names):
        hyper_search = optuna.load_study(
            study_name=study_name,
            storage=storage,
        )

        if trial_numbers is not None:
            trial_number = trial_numbers[ix]
        else:
            trial_number = None

        if trial_number is None:
            trial = hyper_search.best_trial
            trial_number = trial.number
        else:
            trial = hyper_search._storage.get_trial(
                hyper_search._storage.get_trial_id_from_study_id_trial_number(
                    hyper_search._study_id, trial_number
                )
            )

        print(
            f"Extrapolating time_series={int(ix)}, trial with number {trial.number} in study={str(study_name)}"
        )

        dir_path_in_bucket = study_paths_in_bucket[ix]
        configs = trial.params
        success_loading, trainable_variables = load_trainable_variables(
            cos_config, bucket_name, dir_path_in_bucket, study_name, trial_number
        )

        if not success_loading:
            raise RuntimeError(
                f"Loading trainable variables success_loading={success_loading}"
            )

        local_config = configs.copy()
        model_id = local_config.pop("model_id")
        if "exp_beta" in local_config:
            local_config["beta"] = 10 ** local_config.pop("exp_beta")
        if "resblock_damping" in local_config:
            local_config["A"] = _compute_damped_residual_block(
                local_config.pop("resblock_damping"), _esn_n_inputs, ix
            )

        local_config.update(trainable_variables)

        esn = EchoStateNetwork.restore(base_model_paths[ix], model_id)
        esn.set_parameters({**local_config})
        esn.set_reference()
        esn.reset()

        pool.load_model_instance(esn, index=ix)

    one_step_ahead_list = []
    for i in range(hidden_state_startup_burning_steps):
        if compare_data is not None and i < compare_data.shape[0]:
            c = compare_data[i : i + 1, :]
        else:
            c = None
        one_step_ahead = pool.predict(
            initial_state=initial_data[i : i + 1, :],
            auxiliary_data=data_aux[i : i + 1, :],
            compare_data=c,
        )
        one_step_ahead_list.append(one_step_ahead)

    if (
        compare_data is not None
        and hidden_state_startup_burning_steps < compare_data.shape[0]
    ):
        c = compare_data[hidden_state_startup_burning_steps:, :]
    else:
        c = None
    extrapolation_data = pool.predict(
        initial_state=initial_data[hidden_state_startup_burning_steps:, :],
        auxiliary_data=data_aux[hidden_state_startup_burning_steps:, :],
        compare_data=c,
    )

    extrapolation_data = np.vstack((*one_step_ahead_list, extrapolation_data))

    return extrapolation_data

def define_reservoir_configs_for_affine_training(
    data_truncation_n: int = None,
    data_aux_truncation_n: int = None,
    cos_config: Dict = None,
    bucket_name: str = None,
    reservoir_config_path: str = None,
    mount_path: str = None,
    pool_template: str = None,
):
    with contextlib.ExitStack() as stack:
        if mount_path is None:
            mount_path = os.getenv("MOUNT_PATH", "")
        if mount_path == "":
            mount_path = stack.enter_context(make_temp_directory())

        base_model_path = os.path.join(mount_path, reservoir_config_path)
        os.makedirs(base_model_path, exist_ok=True)

        if pool_template == "independent_series":
            sub_model_number_of_inputs = (
                data_truncation_n + data_aux_truncation_n
            )  # size of the data
        elif pool_template == "no_communication_series":
            sub_model_number_of_inputs = 1 + data_aux_truncation_n  # size of the data
        else:
            raise RuntimeError(f"Unsupported pool_template={pool_template}")

        rc_config = {
            "reservoir_dim": 0,
            "sparsity_level": 1,
            "number_of_inputs": sub_model_number_of_inputs,
        }
        esn = EchoStateNetwork(**rc_config)
        esn.save(base_model_path, "EchoStateNetwork_0")
        model_filename = os.path.join(base_model_path, "EchoStateNetwork_0" + ".pkl")

        truncation = {
            "data_truncation_n": data_truncation_n,
            "data_aux_truncation_n": data_aux_truncation_n,
            "pool_template": pool_template,
        }
        truncation_filename = os.path.join(base_model_path, "truncation.json")
        with open(truncation_filename, "w") as f_truncation:
            json.dump(truncation, fp=f_truncation, indent=1)

        reservoir_config_space = {
            "estimate_bias_transition": [False, True],
            "estimate_linear_transition": [False, True],
            "model_id": ["EchoStateNetwork_0"],
        }
        reservoir_config_space_filename = os.path.join(
            base_model_path, "reservoir_config_space.json"
        )
        with open(reservoir_config_space_filename, "w") as f_reservoir_config_space:
            json.dump(reservoir_config_space, fp=f_reservoir_config_space, indent=1)

        if cos_config is not None:
            cos = CloudObjectStorage.from_config_json(cos_config)
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/EchoStateNetwork_0.pkl",
                model_filename,
            )
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/truncation.json",
                truncation_filename,
            )
            cos.put_file(
                bucket_name,
                f"{reservoir_config_path}/reservoir_config_space.json",
                reservoir_config_space_filename,
            )